Merge pull request #9533 from Budibase/develop

Develop -> master

.github/ISSUE_TEMPLATE/bug_report.md

@@ -6,6 +6,8 @@ labels: bug
 assignees: ''
 
 ---
+**Checklist**
+- [ ] I have searched budibase discussions and github issues to check if my issue already exists
 
 **Hosting**
 <!-- Delete as appropriate -->

.github/workflows/release-develop.yml

@@ -26,7 +26,7 @@ env:
   FEATURE_PREVIEW_URL: https://budirelease.live
       
 jobs:
-  release:
+  release-images:
     runs-on: ubuntu-latest 
 
     steps:
@@ -50,13 +50,6 @@ jobs:
       - run: yarn build:sdk
       - run: yarn test
 
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-1
-
       - name: Publish budibase packages to NPM
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -152,3 +145,54 @@ jobs:
           webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
           content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
           embed-title: ${{ env.RELEASE_VERSION }}
+
+  release-helm-chart:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        id: helm-install
+
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
+      - name: Build and release helm chart
+        run: |
+          git config user.name "Budibase Helm Bot"
+          git config user.email "<>"
+          git reset --hard
+          git pull
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync
+          echo "Packaging successful"
+          git checkout gh-pages
+          echo "Indexing helm repo"         
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
+          git add -A
+          git commit -m "Helm Release: develop"
+          git push
+
+  trigger-deploy-to-qa-env:
+    needs: [release-helm-chart]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+        with:
+          repository: budibase/budibase-deploys
+          event: deploy-budibase-develop-to-qa
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-selfhost.yml

@@ -67,16 +67,24 @@ jobs:
         uses: azure/setup-helm@v1
         id: helm-install
 
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
       - name: Build and release helm chart
         run: | 
           git config user.name "Budibase Helm Bot"
           git config user.email "<>"
           git reset --hard
           git pull
-          helm package charts/budibase
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
+          echo "Packaging successful"
           git checkout gh-pages
-          mv *.tgz docs
+          echo "Indexing helm repo"
-          helm repo index docs
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
           git add -A
           git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
           git push

.github/workflows/smoke_test.yaml

@@ -18,30 +18,18 @@ jobs:
       - run: yarn
       - run: yarn bootstrap
       - run: yarn build
-      - name: Pull cypress.env.yaml from budibase-infra
+      - name: Pull  from budibase-infra
         run: | 
           curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
           -H 'Accept: application/vnd.github.v3.raw' \
-          -o packages/builder/cypress.env.json \
+          -o 
-          -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json
+          -L 
-          wc -l packages/builder/cypress.env.json
+          wc -l 
-
-      - name: Cypress run
-        id: cypress
-        continue-on-error: true
-        uses: cypress-io/github-action@v2
-        with:
-          record: true
-          install: false
-          tag: nightly
-          command: yarn test:e2e:ci:record
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
       
       - uses: actions/upload-artifact@v3
         with:
           name: Test Reports
-          path: packages/builder/cypress/reports/testReport.html
+          path: 
 
       # TODO: enable once running in QA test env
       # - name: Configure AWS Credentials
@@ -54,11 +42,3 @@ jobs:
       # - name: Upload test results HTML
       #   uses: aws-actions/configure-aws-credentials@v1
       #   run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html
-
-      - name: Cypress Discord Notify
-        run: yarn test:e2e:ci:notify
-        env:
-          CYPRESS_WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }}
-          CYPRESS_OUTCOME: ${{ steps.cypress.outcome }}
-          CYPRESS_DASHBOARD_URL: ${{ steps.cypress.outputs.dashboardUrl }}
-          GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID

.gitignore

@@ -107,3 +107,5 @@ stats.html
 # plugins
 budibase-component
 budibase-datasource
+
+*.iml

.nvmrc

@@ -0,0 +1 @@
+v14.19.3

.python-version

@@ -0,0 +1 @@
+3.11.1

.tool-versions

@@ -0,0 +1,2 @@
+nodejs 14.19.3
+python 3.11.1

.vscode/extensions.json

@@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "esbenp.prettier-vscode",
+        "svelte.svelte-vscode"
+    ]
+}

.vscode/settings.json

@@ -3,9 +3,9 @@
   "editor.codeActionsOnSave": {
     "source.fixAll": true
   },
-    "editor.defaultFormatter": "svelte.svelte-vscode",
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
   "[json]": {
-        "editor.defaultFormatter": "vscode.json-language-features"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[javascript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
@@ -16,4 +16,13 @@
       "<node_internals>/**"
     ]
   },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[dockercompose]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[svelte]": {
+    "editor.defaultFormatter": "svelte.svelte-vscode"
+  }
 }

charts/budibase/Chart.yaml

@@ -11,8 +11,10 @@ sources:
   - https://github.com/Budibase/budibase
   - https://budibase.com
 type: application
-version: 0.2.11
+# populates on packaging
-appVersion: 1.0.214
+version: 0.0.0
+# populates on packaging
+appVersion: 0.0.0
 dependencies:
   - name: couchdb
     version: 3.3.4

charts/budibase/templates/app-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: app-service
@@ -23,6 +20,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.apps.annotations }}
+{{- toYaml .Values.services.apps.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: app-service
@@ -67,6 +67,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -77,13 +79,19 @@ spec:
             secretKeyRef:
               name: {{ template "budibase.fullname" . }}
               key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@@ -131,6 +139,8 @@ spec:
           value: {{ .Values.globals.automationMaxIterations | quote }}
         - name: TENANT_FEATURE_FLAGS
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
         {{ if .Values.globals.bbAdminUserEmail }}
         - name: BB_ADMIN_USER_EMAIL
           value: {{ .Values.globals.bbAdminUserEmail | quote }}

charts/budibase/templates/proxy-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     app.kubernetes.io/name: budibase-proxy
@@ -23,6 +20,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.proxy.annotations }}
+{{- toYaml .Values.services.proxy.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         app.kubernetes.io/name: budibase-proxy

charts/budibase/templates/worker-service-deployment.yaml

@@ -4,9 +4,6 @@ metadata:
   annotations:
     kompose.cmd: kompose convert
     kompose.version: 1.21.0 (992df58d8)
-{{ if .Values.globals.logAnnotations }}
-{{ toYaml .Values.globals.logAnnotations | indent 4 }}
-{{ end }}
   creationTimestamp: null
   labels:
     io.kompose.service: worker-service
@@ -24,6 +21,9 @@ spec:
       annotations:
         kompose.cmd: kompose convert
         kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.worker.annotations }}
+{{- toYaml .Values.services.worker.annotations | indent 8 -}}
+{{ end }}
       creationTimestamp: null
       labels:
         io.kompose.service: worker-service
@@ -68,6 +68,8 @@ spec:
         - name: AWS_REGION
           value: {{ .Values.services.objectStore.region }}
         {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
         - name: MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
@@ -80,11 +82,17 @@ spec:
               key: objectStoreSecret
         - name: MINIO_URL
           value: {{ .Values.services.objectStore.url }}
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
         - name: PLUGIN_BUCKET_NAME
           value: {{ .Values.services.objectStore.pluginBucketName | quote }}
         - name: APPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
           value: {{ .Values.services.objectStore.globalBucketName | quote }}
         - name: BACKUPS_BUCKET_NAME
           value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@@ -138,6 +146,8 @@ spec:
           value: {{ .Values.globals.google.secret | quote }}
         - name: TENANT_FEATURE_FLAGS
           value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
         {{ if .Values.globals.elasticApmEnabled }}
         - name: ELASTIC_APM_ENABLED
           value: {{ .Values.globals.elasticApmEnabled | quote }}

charts/budibase/values.yaml

@@ -22,12 +22,6 @@ serviceAccount:
 
 podAnnotations: {}
 
-# logAnnotations:
-#   co.elastic.logs/multiline.type: pattern
-#   co.elastic.logs/multiline.pattern: '^[[:space:]]'
-#   co.elastic.logs/multiline.negate: false
-#   co.elastic.logs/multiline.match: after
-
 podSecurityContext:
   {}
   # fsGroup: 2000
@@ -82,7 +76,7 @@ affinity: {}
 globals:
   appVersion: "latest"
   budibaseEnv: PRODUCTION
-  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS"
+  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
   enableAnalytics: "1"
   sentryDSN: ""
   posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
@@ -130,6 +124,10 @@ services:
       minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
       couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
     resources: {}
+#    annotations:
+#      co.elastic.logs/module: nginx
+#      co.elastic.logs/fileset.stdout: access
+#      co.elastic.logs/fileset.stderr: error
 
   apps:
     port: 4002
@@ -137,11 +135,20 @@ services:
     logLevel: info
     resources: {}
 #    nodeDebug: "" # set the value of NODE_DEBUG
-
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
   worker:
     port: 4003
     replicaCount: 1
     resources: {}
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
 
   couchdb:
     enabled: true
@@ -172,6 +179,7 @@ services:
     resources: {}
 
   objectStore:
+    # Set to false if using another object store such as S3
     minio: true
     browser: true
     port: 9000
@@ -187,6 +195,13 @@ services:
     ##   set, choosing the default provisioner.
     storageClass: ""
     resources: {}
+    cloudfront:
+      # Set the url of a distribution to enable cloudfront
+      cdn: ""
+      # ID of public key stored in cloudfront
+      publicKeyId: ""
+      # Base64 encoded private key for the above public key
+      privateKey64: ""
 
 # Override values in couchDB subchart
 couchdb:

docs/CONTRIBUTING.md

@@ -9,7 +9,6 @@ From opening a bug report to creating a pull request: every contribution is appr
 - [Glossary of Terms](#glossary-of-terms)
 - [Contributing to Budibase](#contributing-to-budibase)
 
-
 ## Not Sure Where to Start?
 
 Budibase is a low-code web application builder that creates svelte-based web applications.
@@ -77,24 +76,51 @@ Component libraries are collections of components as well as the definition of t
 
 ## Contributing to Budibase
 
-* Please maintain the existing code style. 
+- Please maintain the existing code style.
 
-* Please try to keep your commits small and focused.
+- Please try to keep your commits small and focused.
 
-* Please write tests.
+- Please write tests.
 
-* If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
+- If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
 
-* Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.
+- Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.
 
 ### Getting Started For Contributors
+
 #### 1. Prerequisites
 
-NodeJS Version `14.x.x`
+- NodeJS version `14.x.x`
+- Python version `3.x`
 
-*yarn -* `npm install -g yarn`
+### Using asdf (recommended)
 
-*jest* - `npm install -g jest`
+Asdf is a package manager that allows managing multiple dependencies.
+
+You can install them following any of the steps described below:
+
+- Install using script (only for mac users):
+
+`./scripts/install-contributor-dependencies.sh`
+
+- Or, manually:
+
+  - Installation steps: https://asdf-vm.com/guide/getting-started.html
+  - asdf plugin add nodejs
+  - asdf plugin add python
+  - npm install -g yarn
+
+### Using NVM and pyenv
+
+- NVM:
+  - Install: https://github.com/nvm-sh/nvm#installing-and-updating
+  - Setup: `nvm use`
+- Pyenv:
+
+  - Install: https://github.com/pyenv/pyenv#installation
+  - Setup: `pyenv install -v 3.7.2`
+
+- _yarn -_ `npm install -g yarn`
 
 #### 2. Clone this repository
 
@@ -172,30 +198,38 @@ A combination of environment variables controls the mode budibase runs in.
 Yarn commands can be used to mimic the different modes as described in the sections below:
 
 #### Self Hosted
+
 The default mode. A single tenant installation with no usage restrictions.
 
 To enable this mode, use:
+
 ```
 yarn mode:self
 ```
 
 #### Cloud
+
 The cloud mode, with account portal turned off.
 
 To enable this mode, use:
+
 ```
 yarn mode:cloud
 ```
+
 #### Cloud & Account
+
 The cloud mode, with account portal turned on. This is a replica of the mode that runs at https://budibase.app
 
-
 To enable this mode, use:
+
 ```
 yarn mode:account
 ```
+
 ### CI
- An overview of the CI pipelines can be found [here](../.github/workflows/README.md)
+
+An overview of the CI pipelines can be found [here](../.github/workflows/README.md)
 
 ### Pro
 
@@ -214,6 +248,7 @@ The `yarn bootstrap` command can be used to replace the NPM supplied dependency
 ### Troubleshooting
 
 Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation.
+
 ### Running tests
 
 #### End-to-end Tests
@@ -226,12 +261,11 @@ yarn test:e2e
 
 Or if you are in the builder you can run `yarn cy:test`.
 
-
 ### Other Useful Information
 
-* The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).
+- The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).
 
-* This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).
+- This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).
 
-* We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
+- We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
   Please read this if you are unfamiliar with it.

hosting/docker-compose.dev.yaml

@@ -6,7 +6,8 @@ services:
   minio-service:
     container_name: budi-minio-dev
     restart: on-failure
-    image: minio/minio
+    # Last version that supports the "fs" backend
+    image: minio/minio:RELEASE.2022-10-24T18-35-07Z
     volumes:
       - minio_data:/data
     ports:

hosting/nginx.dev.conf

@@ -186,6 +186,26 @@ http {
       proxy_pass      http://minio-service:9000;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://minio-service:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/proxy/nginx.prod.conf

@@ -202,6 +202,26 @@ http {
       proxy_pass      $minio;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      $minio;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/single/nginx/nginx-default-site.conf

@@ -98,14 +98,36 @@ server {
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
 
       proxy_connect_timeout 300;
       proxy_http_version 1.1;
       proxy_set_header Connection "";
       chunked_transfer_encoding off;
+
       proxy_pass      http://127.0.0.1:9000;
     }
 
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://127.0.0.1:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
     client_header_timeout 60;
     client_body_timeout   60;
     keepalive_timeout     60;

hosting/single/runner.sh

@@ -10,7 +10,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS"
+[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
 [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
 [[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.26",
+  "version": "2.2.27-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -25,6 +25,7 @@
     "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
     "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
     "build:sdk": "lerna run build:sdk",
     "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
     "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
@@ -44,16 +45,12 @@
     "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test && yarn test:pro",
     "test:pro": "bash scripts/pro/test.sh",
-    "lint:eslint": "eslint packages",
+    "lint:eslint": "eslint packages && eslint qa-core",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
     "lint": "yarn run lint:eslint && yarn run lint:prettier",
     "lint:fix:eslint": "eslint --fix packages qa-core",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
-    "test:e2e": "lerna run cy:test --stream",
-    "test:e2e:ci": "lerna run cy:ci --stream",
-    "test:e2e:ci:record": "lerna run cy:ci:record --stream",
-    "test:e2e:ci:notify": "lerna run cy:ci:notify",
     "build:specs": "lerna run specs",
     "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
     "build:docker:pre": "lerna run build && lerna run predocker",

packages/backend-core/__mocks__/aws-sdk.ts

@@ -3,7 +3,10 @@ const mockS3 = {
   deleteObject: jest.fn().mockReturnThis(),
   deleteObjects: jest.fn().mockReturnThis(),
   createBucket: jest.fn().mockReturnThis(),
-  listObjects: jest.fn().mockReturnThis(),
+  listObject: jest.fn().mockReturnThis(),
+  getSignedUrl: jest.fn((operation: string, params: any) => {
+    return `http://s3.example.com/${params.Bucket}/${params.Key}`
+  }),
   promise: jest.fn().mockReturnThis(),
   catch: jest.fn(),
 }

packages/backend-core/jest.config.ts

@@ -6,6 +6,9 @@ const config: Config.InitialOptions = {
   setupFiles: ["./tests/jestSetup.ts"],
   collectCoverageFrom: ["src/**/*.{js,ts}"],
   coverageReporters: ["lcov", "json", "clover"],
+  transform: {
+    "^.+\\.ts?$": "@swc/jest",
+  },
 }
 
 if (!process.env.CI) {

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.26",
+  "version": "2.2.27-alpha.0",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -15,24 +15,28 @@
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
     "build": "tsc -p tsconfig.build.json",
+    "build:pro": "../../scripts/pro/build.sh",
+    "postbuild": "yarn run build:pro",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "^2.2.26",
+    "@budibase/types": "2.2.27-alpha.0",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
+    "aws-cloudfront-sign": "2.2.0",
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.0.1",
     "bcryptjs": "2.4.3",
     "bull": "4.10.1",
+    "correlation-id": "4.0.0",
     "dotenv": "16.0.1",
     "emitter-listener": "1.1.2",
     "ioredis": "4.28.0",
     "joi": "17.6.0",
-    "jsonwebtoken": "8.5.1",
+    "jsonwebtoken": "9.0.0",
     "koa-passport": "4.1.4",
     "lodash": "4.17.21",
     "lodash.isarguments": "3.1.0",
@@ -53,19 +57,23 @@
     "zlib": "1.0.5"
   },
   "devDependencies": {
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
     "@types/chance": "1.1.3",
     "@types/ioredis": "4.28.0",
     "@types/jest": "27.5.1",
     "@types/koa": "2.13.4",
+    "@types/koa-pino-logger": "3.0.0",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
+    "@types/pino-http": "5.8.1",
     "@types/pouchdb": "6.4.0",
     "@types/redlock": "4.0.3",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
-    "chance": "1.1.3",
+    "chance": "1.1.8",
     "ioredis-mock": "5.8.0",
     "jest": "28.1.1",
     "koa": "2.13.4",

packages/backend-core/src/cache/base/index.ts

@@ -7,7 +7,7 @@ function generateTenantKey(key: string) {
   return `${key}:${tenantId}`
 }
 
-export = class BaseCache {
+export default class BaseCache {
   client: Client | undefined
 
   constructor(client: Client | undefined = undefined) {

packages/backend-core/src/cache/generic.ts

@@ -1,6 +1,6 @@
 const BaseCache = require("./base")
 
-const GENERIC = new BaseCache()
+const GENERIC = new BaseCache.default()
 
 export enum CacheKey {
   CHECKLIST = "checklist",

packages/backend-core/src/cloud/api.ts

@@ -1,6 +1,7 @@
 import fetch from "node-fetch"
+import * as logging from "../logging"
 
-export = class API {
+export default class API {
   host: string
 
   constructor(host: string) {
@@ -22,6 +23,9 @@ export = class API {
 
     let json = options.headers["Content-Type"] === "application/json"
 
+    // add x-budibase-correlation-id header
+    logging.correlation.setHeader(options.headers)
+
     const requestOptions = {
       method: method,
       body: json ? JSON.stringify(options.body) : options.body,

packages/backend-core/src/constants/db.ts

@@ -77,6 +77,7 @@ export const StaticDatabases = {
       apiKeys: "apikeys",
       usageQuota: "usage_quota",
       licenseInfo: "license_info",
+      environmentVariables: "environmentvariables",
     },
   },
   // contains information about tenancy and so on

packages/backend-core/src/constants/misc.ts

@@ -22,6 +22,7 @@ export enum Header {
   TENANT_ID = "x-budibase-tenant-id",
   TOKEN = "x-budibase-token",
   CSRF_TOKEN = "x-csrf-token",
+  CORRELATION_ID = "x-budibase-correlation-id",
 }
 
 export enum GlobalRole {

packages/backend-core/src/context/Context.ts

@@ -1,17 +1,14 @@
 import { AsyncLocalStorage } from "async_hooks"
+import { ContextMap } from "./mainContext"
 
 export default class Context {
-  static storage = new AsyncLocalStorage<Record<string, any>>()
+  static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: Record<string, any>, func: any) {
+  static run(context: ContextMap, func: any) {
     return Context.storage.run(context, () => func())
   }
 
-  static get(): Record<string, any> {
+  static get(): ContextMap {
-    return Context.storage.getStore() as Record<string, any>
+    return Context.storage.getStore() as ContextMap
-  }
-
-  static set(context: Record<string, any>) {
-    Context.storage.enterWith(context)
   }
 }

packages/backend-core/src/context/mainContext.ts

@@ -2,7 +2,7 @@
 // store an app ID to pretend there is a context
 import env from "../environment"
 import Context from "./Context"
-import { getDevelopmentAppID, getProdAppID } from "../db/conversions"
+import * as conversions from "../db/conversions"
 import { getDB } from "../db/db"
 import {
   DocumentType,
@@ -16,6 +16,7 @@ export type ContextMap = {
   tenantId?: string
   appId?: string
   identity?: IdentityContext
+  environmentVariables?: Record<string, string>
 }
 
 let TEST_APP_ID: string | null = null
@@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) {
   }
 }
 
-function updateContext(updates: ContextMap) {
+function updateContext(updates: ContextMap): ContextMap {
   let context: ContextMap
   try {
     context = Context.get()
@@ -120,16 +121,24 @@ export async function doInTenant(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(appId: string, task: any): Promise<any> {
+export async function doInAppContext(
-  if (!appId) {
+  appId: string | null,
+  task: any
+): Promise<any> {
+  if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
 
+  let updates: ContextMap
+  if (!appId) {
+    updates = { appId: "" }
+  } else {
     const tenantId = getTenantIDFromAppID(appId)
-  const updates: ContextMap = { appId }
+    updates = { appId }
     if (tenantId) {
       updates.tenantId = tenantId
     }
+  }
   return newContext(updates, task)
 }
 
@@ -181,25 +190,33 @@ export function getAppId(): string | undefined {
   }
 }
 
-export function updateTenantId(tenantId?: string) {
+export const getProdAppId = () => {
-  let context: ContextMap = updateContext({
+  const appId = getAppId()
-    tenantId,
+  if (!appId) {
-  })
+    throw new Error("Could not get appId")
-  Context.set(context)
+  }
+  return conversions.getProdAppID(appId)
 }
 
-export function updateAppId(appId: string) {
+export function doInEnvironmentContext(
-  let context: ContextMap = updateContext({
+  values: Record<string, string>,
-    appId,
+  task: any
-  })
+) {
-  try {
+  if (!values) {
-    Context.set(context)
+    throw new Error("Must supply environment variables.")
-  } catch (err) {
-    if (env.isTest()) {
-      TEST_APP_ID = appId
-    } else {
-      throw err
   }
+  const updates = {
+    environmentVariables: values,
+  }
+  return newContext(updates, task)
+}
+
+export function getEnvironmentVariables() {
+  const context = Context.get()
+  if (!context.environmentVariables) {
+    return null
+  } else {
+    return context.environmentVariables
   }
 }
 
@@ -229,7 +246,7 @@ export function getProdAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve prod DB - no app ID.")
   }
-  return getDB(getProdAppID(appId), opts)
+  return getDB(conversions.getProdAppID(appId), opts)
 }
 
 /**
@@ -241,5 +258,5 @@ export function getDevAppDB(opts?: any): Database {
   if (!appId) {
     throw new Error("Unable to retrieve dev DB - no app ID.")
   }
-  return getDB(getDevelopmentAppID(appId), opts)
+  return getDB(conversions.getDevelopmentAppID(appId), opts)
 }

packages/backend-core/src/context/tests/index.spec.ts

@@ -1,7 +1,7 @@
 require("../../../tests")
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-const env = require("../../environment")
+import env from "../../environment"
 
 describe("context", () => {
   describe("doInTenant", () => {
@@ -26,7 +26,7 @@ describe("context", () => {
 
       it("fails when no tenant id is set", () => {
         const test = () => {
-          let error
+          let error: any
           try {
             context.getTenantId()
           } catch (e) {
@@ -45,7 +45,7 @@ describe("context", () => {
 
       it("fails when no tenant db is set", () => {
         const test = () => {
-          let error
+          let error: any
           try {
             context.getGlobalDB()
           } catch (e) {

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -15,26 +15,10 @@ import { getCouchInfo } from "./connections"
 import { directCouchCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
+import { newid } from "../../newid"
 
-export class DatabaseImpl implements Database {
+function buildNano(couchInfo: { url: string; cookie: string }) {
-  public readonly name: string
+  return Nano({
-  private static nano: Nano.ServerScope
-  private readonly pouchOpts: DatabaseOpts
-
-  constructor(dbName?: string, opts?: DatabaseOpts) {
-    if (dbName == null) {
-      throw new Error("Database name cannot be undefined.")
-    }
-    this.name = dbName
-    this.pouchOpts = opts || {}
-    if (!DatabaseImpl.nano) {
-      DatabaseImpl.init()
-    }
-  }
-
-  static init() {
-    const couchInfo = getCouchInfo()
-    DatabaseImpl.nano = Nano({
     url: couchInfo.url,
     requestDefaults: {
       headers: {
@@ -43,6 +27,43 @@ export class DatabaseImpl implements Database {
     },
     parseUrl: false,
   })
+}
+
+export function DatabaseWithConnection(
+  dbName: string,
+  connection: string,
+  opts?: DatabaseOpts
+) {
+  if (!connection) {
+    throw new Error("Must provide connection details")
+  }
+  return new DatabaseImpl(dbName, opts, connection)
+}
+
+export class DatabaseImpl implements Database {
+  public readonly name: string
+  private static nano: Nano.ServerScope
+  private readonly instanceNano?: Nano.ServerScope
+  private readonly pouchOpts: DatabaseOpts
+
+  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
+    if (dbName == null) {
+      throw new Error("Database name cannot be undefined.")
+    }
+    this.name = dbName
+    this.pouchOpts = opts || {}
+    if (connection) {
+      const couchInfo = getCouchInfo(connection)
+      this.instanceNano = buildNano(couchInfo)
+    }
+    if (!DatabaseImpl.nano) {
+      DatabaseImpl.init()
+    }
+  }
+
+  static init() {
+    const couchInfo = getCouchInfo()
+    DatabaseImpl.nano = buildNano(couchInfo)
   }
 
   async exists() {
@@ -50,6 +71,10 @@ export class DatabaseImpl implements Database {
     return response.status === 200
   }
 
+  private nano() {
+    return this.instanceNano || DatabaseImpl.nano
+  }
+
   async checkSetup() {
     let shouldCreate = !this.pouchOpts?.skip_setup
     // check exists in a lightweight fashion
@@ -58,9 +83,9 @@ export class DatabaseImpl implements Database {
       throw new Error("DB does not exist")
     }
     if (!exists) {
-      await DatabaseImpl.nano.db.create(this.name)
+      await this.nano().db.create(this.name)
     }
-    return DatabaseImpl.nano.db.use(this.name)
+    return this.nano().db.use(this.name)
   }
 
   private async updateOutput(fnc: any) {
@@ -101,6 +126,13 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.destroy(_id, _rev))
   }
 
+  async post(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      document._id = newid()
+    }
+    return this.put(document, opts)
+  }
+
   async put(document: AnyDocument, opts?: DatabasePutOpts) {
     if (!document._id) {
       throw new Error("Cannot store document without _id field.")
@@ -146,7 +178,7 @@ export class DatabaseImpl implements Database {
 
   async destroy() {
     try {
-      await DatabaseImpl.nano.db.destroy(this.name)
+      await this.nano().db.destroy(this.name)
     } catch (err: any) {
       // didn't exist, don't worry
       if (err.statusCode === 404) {

packages/backend-core/src/db/couch/connections.ts

@@ -1,7 +1,7 @@
 import env from "../../environment"
 
-export const getCouchInfo = () => {
+export const getCouchInfo = (connection?: string) => {
-  const urlInfo = getUrlInfo()
+  const urlInfo = getUrlInfo(connection)
   let username
   let password
   if (env.COUCH_DB_USERNAME) {

packages/backend-core/src/db/tests/utils.spec.ts

@@ -5,18 +5,13 @@ const {
   isDevAppID,
   isProdAppID,
 } = require("../conversions")
-const {
+const { generateAppID, getPlatformUrl, getScopedConfig } = require("../utils")
-  generateAppID,
-  getPlatformUrl,
-  getScopedConfig
-} = require("../utils")
 const tenancy = require("../../tenancy")
 const { Config, DEFAULT_TENANT_ID } = require("../../constants")
-const env = require("../../environment")
+import env from "../../environment"
 
 describe("utils", () => {
   describe("app ID manipulation", () => {
-
     function getID() {
       const appId = generateAppID()
       const split = appId.split("_")
@@ -81,8 +76,8 @@ const setDbPlatformUrl = async () => {
     _id: "config_settings",
     type: Config.SETTINGS,
     config: {
-      platformUrl: DB_URL
+      platformUrl: DB_URL,
-    }
+    },
   })
 }
 
@@ -92,7 +87,7 @@ const clearSettingsConfig = async () => {
     try {
       const config = await db.get("config_settings")
       await db.remove("config_settings", config._rev)
-    } catch (e) {
+    } catch (e: any) {
       if (e.status !== 404) {
         throw e
       }
@@ -102,7 +97,6 @@ const clearSettingsConfig = async () => {
 
 describe("getPlatformUrl", () => {
   describe("self host", () => {
-
     beforeEach(async () => {
       env._set("SELF_HOST", 1)
       await clearSettingsConfig()
@@ -132,7 +126,6 @@ describe("getPlatformUrl", () => {
     })
   })
 
-
   describe("cloud", () => {
     const TENANT_AWARE_URL = "http://default.env.com"
 
@@ -169,7 +162,6 @@ describe("getPlatformUrl", () => {
 
 describe("getScopedConfig", () => {
   describe("settings config", () => {
-
     beforeEach(async () => {
       env._set("SELF_HOSTED", 1)
       env._set("PLATFORM_URL", "")

packages/backend-core/src/db/utils.ts

@@ -14,7 +14,7 @@ import { doWithDB, allDbs, directCouchAllDbs } from "./db"
 import { getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "./conversions"
 import * as events from "../events"
-import { App, Database, ConfigType } from "@budibase/types"
+import { App, Database, ConfigType, isSettingsConfig } from "@budibase/types"
 
 /**
  * Generates a new app ID.
@@ -489,18 +489,12 @@ export const getScopedFullConfig = async function (
 
   // custom logic for settings doc
   if (type === ConfigType.SETTINGS) {
-    if (scopedConfig && scopedConfig.doc) {
+    if (!scopedConfig || !scopedConfig.doc) {
-      // overrides affected by environment variables
-      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
-        tenantAware: true,
-      })
-      scopedConfig.doc.config.analyticsEnabled =
-        await events.analytics.enabled()
-    } else {
       // defaults
       scopedConfig = {
         doc: {
           _id: generateConfigID({ type, user, workspace }),
+          type: ConfigType.SETTINGS,
           config: {
             platformUrl: await getPlatformUrl({ tenantAware: true }),
             analyticsEnabled: await events.analytics.enabled(),
@@ -508,6 +502,16 @@ export const getScopedFullConfig = async function (
         },
       }
     }
+
+    // will always be true - use assertion function to get type access
+    if (isSettingsConfig(scopedConfig.doc)) {
+      // overrides affected by environment
+      scopedConfig.doc.config.platformUrl = await getPlatformUrl({
+        tenantAware: true,
+      })
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
+    }
   }
 
   return scopedConfig && scopedConfig.doc

packages/backend-core/src/environment.ts

@@ -1,9 +1,13 @@
 function isTest() {
-  return (
+  return isCypress() || isJest()
-    process.env.NODE_ENV === "jest" ||
+}
-    process.env.NODE_ENV === "cypress" ||
+
-    process.env.JEST_WORKER_ID != null
+function isJest() {
-  )
+  return !!(process.env.NODE_ENV === "jest" || process.env.JEST_WORKER_ID)
+}
+
+function isCypress() {
+  return process.env.NODE_ENV === "cypress"
 }
 
 function isDev() {
@@ -21,15 +25,19 @@ const DefaultBucketName = {
   APPS: "prod-budi-app-assets",
   TEMPLATES: "templates",
   GLOBAL: "global",
-  CLOUD: "prod-budi-tenant-uploads",
   PLUGINS: "plugins",
 }
 
 const environment = {
   isTest,
+  isJest,
   isDev,
+  isProd: () => {
+    return !isDev()
+  },
   JS_BCRYPT: process.env.JS_BCRYPT,
   JWT_SECRET: process.env.JWT_SECRET,
+  ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
@@ -42,6 +50,7 @@ const environment = {
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_REGION: process.env.AWS_REGION,
   MINIO_URL: process.env.MINIO_URL,
+  MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL:
@@ -54,6 +63,9 @@ const environment = {
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
+  CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,
+  CLOUDFRONT_PRIVATE_KEY_64: process.env.CLOUDFRONT_PRIVATE_KEY_64,
+  CLOUDFRONT_PUBLIC_KEY_ID: process.env.CLOUDFRONT_PUBLIC_KEY_ID,
   BACKUPS_BUCKET_NAME:
     process.env.BACKUPS_BUCKET_NAME || DefaultBucketName.BACKUPS,
   APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || DefaultBucketName.APPS,
@@ -61,12 +73,9 @@ const environment = {
     process.env.TEMPLATES_BUCKET_NAME || DefaultBucketName.TEMPLATES,
   GLOBAL_BUCKET_NAME:
     process.env.GLOBAL_BUCKET_NAME || DefaultBucketName.GLOBAL,
-  GLOBAL_CLOUD_BUCKET_NAME:
-    process.env.GLOBAL_CLOUD_BUCKET_NAME || DefaultBucketName.CLOUD,
   PLUGIN_BUCKET_NAME:
     process.env.PLUGIN_BUCKET_NAME || DefaultBucketName.PLUGINS,
   USE_COUCH: process.env.USE_COUCH || true,
-  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
   SERVICE: process.env.SERVICE || "budibase",
   LOG_LEVEL: process.env.LOG_LEVEL,
@@ -87,6 +96,11 @@ for (let [key, value] of Object.entries(environment)) {
     // @ts-ignore
     environment[key] = 0
   }
+  // handle the edge case of "false" to disable an environment variable
+  if (value === "false") {
+    // @ts-ignore
+    environment[key] = 0
+  }
 }
 
-export = environment
+export default environment

packages/backend-core/src/errors/errors.ts

@@ -0,0 +1,37 @@
+import * as licensing from "./licensing"
+
+// combine all error codes into single object
+
+export const codes = {
+  ...licensing.codes,
+}
+
+// combine all error types
+export const types = [licensing.type]
+
+// combine all error contexts
+const context = {
+  ...licensing.context,
+}
+
+// derive a public error message using codes, types and any custom contexts
+export const getPublicError = (err: any) => {
+  let error
+  if (err.code || err.type) {
+    // add generic error information
+    error = {
+      code: err.code,
+      type: err.type,
+    }
+
+    if (err.code && context[err.code]) {
+      error = {
+        ...error,
+        // get any additional context from this error
+        ...context[err.code](err),
+      }
+    }
+  }
+
+  return error
+}

packages/backend-core/src/errors/index.ts

@@ -1,47 +1,3 @@
-import { HTTPError } from "./http"
+export * from "./errors"
-import { UsageLimitError, FeatureDisabledError } from "./licensing"
+export { UsageLimitError, FeatureDisabledError } from "./licensing"
-import * as licensing from "./licensing"
+export { HTTPError } from "./http"
-
-const codes = {
-  ...licensing.codes,
-}
-
-const types = [licensing.type]
-
-const context = {
-  ...licensing.context,
-}
-
-const getPublicError = (err: any) => {
-  let error
-  if (err.code || err.type) {
-    // add generic error information
-    error = {
-      code: err.code,
-      type: err.type,
-    }
-
-    if (err.code && context[err.code]) {
-      error = {
-        ...error,
-        // get any additional context from this error
-        ...context[err.code](err),
-      }
-    }
-  }
-
-  return error
-}
-
-const pkg = {
-  codes,
-  types,
-  errors: {
-    UsageLimitError,
-    FeatureDisabledError,
-    HTTPError,
-  },
-  getPublicError,
-}
-
-export = pkg

packages/backend-core/src/events/events.ts

@@ -1,6 +1,6 @@
 import { Event } from "@budibase/types"
 import { processors } from "./processors"
-import * as identification from "./identification"
+import identification from "./identification"
 import * as backfill from "./backfill"
 
 export const publishEvent = async (

packages/backend-core/src/events/identification.ts

@@ -33,7 +33,7 @@ const pkg = require("../../package.json")
  * - tenant
  * - installation
  */
-export const getCurrentIdentity = async (): Promise<Identity> => {
+const getCurrentIdentity = async (): Promise<Identity> => {
   let identityContext = identityCtx.getIdentity()
   const environment = getDeploymentEnvironment()
 
@@ -94,7 +94,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
   }
 }
 
-export const identifyInstallationGroup = async (
+const identifyInstallationGroup = async (
   installId: string,
   timestamp?: string | number
 ): Promise<void> => {
@@ -118,7 +118,7 @@ export const identifyInstallationGroup = async (
   await identify({ ...group, id: `$${type}_${id}` }, timestamp)
 }
 
-export const identifyTenantGroup = async (
+const identifyTenantGroup = async (
   tenantId: string,
   account: Account | undefined,
   timestamp?: string | number
@@ -156,7 +156,7 @@ export const identifyTenantGroup = async (
   await identify({ ...group, id: `$${type}_${id}` }, timestamp)
 }
 
-export const identifyUser = async (
+const identifyUser = async (
   user: User,
   account: CloudAccount | undefined,
   timestamp?: string | number
@@ -191,7 +191,7 @@ export const identifyUser = async (
   await identify(identity, timestamp)
 }
 
-export const identifyAccount = async (account: Account) => {
+const identifyAccount = async (account: Account) => {
   let id = account.accountId
   const tenantId = account.tenantId
   let type = IdentityType.USER
@@ -224,17 +224,11 @@ export const identifyAccount = async (account: Account) => {
   await identify(identity)
 }
 
-export const identify = async (
+const identify = async (identity: Identity, timestamp?: string | number) => {
-  identity: Identity,
-  timestamp?: string | number
-) => {
   await processors.identify(identity, timestamp)
 }
 
-export const identifyGroup = async (
+const identifyGroup = async (group: Group, timestamp?: string | number) => {
-  group: Group,
-  timestamp?: string | number
-) => {
   await processors.identifyGroup(group, timestamp)
 }
 
@@ -250,7 +244,7 @@ const getHostingFromEnv = () => {
   return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
 }
 
-export const getInstallationId = async () => {
+const getInstallationId = async () => {
   if (isAccountPortal()) {
     return "account-portal"
   }
@@ -300,3 +294,14 @@ const formatDistinctId = (id: string, type: IdentityType) => {
     return id
   }
 }
+
+export default {
+  getCurrentIdentity,
+  identifyInstallationGroup,
+  identifyTenantGroup,
+  identifyUser,
+  identifyAccount,
+  identify,
+  identifyGroup,
+  getInstallationId,
+}

packages/backend-core/src/events/index.ts

@@ -1,7 +1,7 @@
 export * from "./publishers"
 export * as processors from "./processors"
 export * as analytics from "./analytics"
-export * as identification from "./identification"
+export { default as identification } from "./identification"
 export * as backfillCache from "./backfill"
 
 import { processors } from "./processors"

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -23,7 +23,7 @@ export default class LoggingProcessor implements EventProcessor {
       return
     }
     let timestampString = getTimestampString(timestamp)
-    let message = `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+    let message = `[audit] [identityType=${identity.type}] ${timestampString} ${event} `
     if (env.isDev()) {
       message = message + `[debug: [properties=${JSON.stringify(properties)}] ]`
     }

packages/backend-core/src/events/publishers/account.ts

@@ -7,23 +7,29 @@ import {
   AccountVerifiedEvent,
 } from "@budibase/types"
 
-export async function created(account: Account) {
+async function created(account: Account) {
   const properties: AccountCreatedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_CREATED, properties)
 }
 
-export async function deleted(account: Account) {
+async function deleted(account: Account) {
   const properties: AccountDeletedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_DELETED, properties)
 }
 
-export async function verified(account: Account) {
+async function verified(account: Account) {
   const properties: AccountVerifiedEvent = {
     tenantId: account.tenantId,
   }
   await publishEvent(Event.ACCOUNT_VERIFIED, properties)
 }
+
+export default {
+  created,
+  deleted,
+  verified,
+}

packages/backend-core/src/events/publishers/app.ts

@@ -15,7 +15,7 @@ import {
   AppExportedEvent,
 } from "@budibase/types"
 
-export const created = async (app: App, timestamp?: string | number) => {
+const created = async (app: App, timestamp?: string | number) => {
   const properties: AppCreatedEvent = {
     appId: app.appId,
     version: app.version,
@@ -23,7 +23,7 @@ export const created = async (app: App, timestamp?: string | number) => {
   await publishEvent(Event.APP_CREATED, properties, timestamp)
 }
 
-export async function updated(app: App) {
+async function updated(app: App) {
   const properties: AppUpdatedEvent = {
     appId: app.appId,
     version: app.version,
@@ -31,35 +31,35 @@ export async function updated(app: App) {
   await publishEvent(Event.APP_UPDATED, properties)
 }
 
-export async function deleted(app: App) {
+async function deleted(app: App) {
   const properties: AppDeletedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_DELETED, properties)
 }
 
-export async function published(app: App, timestamp?: string | number) {
+async function published(app: App, timestamp?: string | number) {
   const properties: AppPublishedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
 }
 
-export async function unpublished(app: App) {
+async function unpublished(app: App) {
   const properties: AppUnpublishedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_UNPUBLISHED, properties)
 }
 
-export async function fileImported(app: App) {
+async function fileImported(app: App) {
   const properties: AppFileImportedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_FILE_IMPORTED, properties)
 }
 
-export async function templateImported(app: App, templateKey: string) {
+async function templateImported(app: App, templateKey: string) {
   const properties: AppTemplateImportedEvent = {
     appId: app.appId,
     templateKey,
@@ -67,7 +67,7 @@ export async function templateImported(app: App, templateKey: string) {
   await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
 }
 
-export async function versionUpdated(
+async function versionUpdated(
   app: App,
   currentVersion: string,
   updatedToVersion: string
@@ -80,7 +80,7 @@ export async function versionUpdated(
   await publishEvent(Event.APP_VERSION_UPDATED, properties)
 }
 
-export async function versionReverted(
+async function versionReverted(
   app: App,
   currentVersion: string,
   revertedToVersion: string
@@ -93,16 +93,30 @@ export async function versionReverted(
   await publishEvent(Event.APP_VERSION_REVERTED, properties)
 }
 
-export async function reverted(app: App) {
+async function reverted(app: App) {
   const properties: AppRevertedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_REVERTED, properties)
 }
 
-export async function exported(app: App) {
+async function exported(app: App) {
   const properties: AppExportedEvent = {
     appId: app.appId,
   }
   await publishEvent(Event.APP_EXPORTED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  published,
+  unpublished,
+  fileImported,
+  templateImported,
+  versionUpdated,
+  versionReverted,
+  reverted,
+  exported,
+}

packages/backend-core/src/events/publishers/auth.ts

@@ -12,7 +12,7 @@ import {
 } from "@budibase/types"
 import { identification } from ".."
 
-export async function login(source: LoginSource) {
+async function login(source: LoginSource) {
   const identity = await identification.getCurrentIdentity()
   const properties: LoginEvent = {
     userId: identity.id,
@@ -21,7 +21,7 @@ export async function login(source: LoginSource) {
   await publishEvent(Event.AUTH_LOGIN, properties)
 }
 
-export async function logout() {
+async function logout() {
   const identity = await identification.getCurrentIdentity()
   const properties: LogoutEvent = {
     userId: identity.id,
@@ -29,30 +29,39 @@ export async function logout() {
   await publishEvent(Event.AUTH_LOGOUT, properties)
 }
 
-export async function SSOCreated(type: SSOType, timestamp?: string | number) {
+async function SSOCreated(type: SSOType, timestamp?: string | number) {
   const properties: SSOCreatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp)
 }
 
-export async function SSOUpdated(type: SSOType) {
+async function SSOUpdated(type: SSOType) {
   const properties: SSOUpdatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_UPDATED, properties)
 }
 
-export async function SSOActivated(type: SSOType, timestamp?: string | number) {
+async function SSOActivated(type: SSOType, timestamp?: string | number) {
   const properties: SSOActivatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp)
 }
 
-export async function SSODeactivated(type: SSOType) {
+async function SSODeactivated(type: SSOType) {
   const properties: SSODeactivatedEvent = {
     type,
   }
   await publishEvent(Event.AUTH_SSO_DEACTIVATED, properties)
 }
+
+export default {
+  login,
+  logout,
+  SSOCreated,
+  SSOUpdated,
+  SSOActivated,
+  SSODeactivated,
+}

packages/backend-core/src/events/publishers/automation.ts

@@ -12,10 +12,7 @@ import {
   AutomationsRunEvent,
 } from "@budibase/types"
 
-export async function created(
+async function created(automation: Automation, timestamp?: string | number) {
-  automation: Automation,
-  timestamp?: string | number
-) {
   const properties: AutomationCreatedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -25,7 +22,7 @@ export async function created(
   await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
 }
 
-export async function triggerUpdated(automation: Automation) {
+async function triggerUpdated(automation: Automation) {
   const properties: AutomationTriggerUpdatedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -35,7 +32,7 @@ export async function triggerUpdated(automation: Automation) {
   await publishEvent(Event.AUTOMATION_TRIGGER_UPDATED, properties)
 }
 
-export async function deleted(automation: Automation) {
+async function deleted(automation: Automation) {
   const properties: AutomationDeletedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -45,7 +42,7 @@ export async function deleted(automation: Automation) {
   await publishEvent(Event.AUTOMATION_DELETED, properties)
 }
 
-export async function tested(automation: Automation) {
+async function tested(automation: Automation) {
   const properties: AutomationTestedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -55,14 +52,14 @@ export async function tested(automation: Automation) {
   await publishEvent(Event.AUTOMATION_TESTED, properties)
 }
 
-export const run = async (count: number, timestamp?: string | number) => {
+const run = async (count: number, timestamp?: string | number) => {
   const properties: AutomationsRunEvent = {
     count,
   }
   await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
 }
 
-export async function stepCreated(
+async function stepCreated(
   automation: Automation,
   step: AutomationStep,
   timestamp?: string | number
@@ -78,10 +75,7 @@ export async function stepCreated(
   await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
 }
 
-export async function stepDeleted(
+async function stepDeleted(automation: Automation, step: AutomationStep) {
-  automation: Automation,
-  step: AutomationStep
-) {
   const properties: AutomationStepDeletedEvent = {
     appId: automation.appId,
     automationId: automation._id as string,
@@ -92,3 +86,13 @@ export async function stepDeleted(
   }
   await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
 }
+
+export default {
+  created,
+  triggerUpdated,
+  deleted,
+  tested,
+  run,
+  stepCreated,
+  stepDeleted,
+}

packages/backend-core/src/events/publishers/backfill.ts

@@ -8,18 +8,18 @@ import {
   InstallationBackfillSucceededEvent,
   InstallationBackfillFailedEvent,
 } from "@budibase/types"
-const env = require("../../environment")
+import env from "../../environment"
 
 const shouldSkip = !env.SELF_HOSTED && !env.isDev()
 
-export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+async function appSucceeded(properties: AppBackfillSucceededEvent) {
   if (shouldSkip) {
     return
   }
   await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function appFailed(error: any) {
+async function appFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -29,16 +29,14 @@ export async function appFailed(error: any) {
   await publishEvent(Event.APP_BACKFILL_FAILED, properties)
 }
 
-export async function tenantSucceeded(
+async function tenantSucceeded(properties: TenantBackfillSucceededEvent) {
-  properties: TenantBackfillSucceededEvent
-) {
   if (shouldSkip) {
     return
   }
   await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function tenantFailed(error: any) {
+async function tenantFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -48,7 +46,7 @@ export async function tenantFailed(error: any) {
   await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
 }
 
-export async function installationSucceeded() {
+async function installationSucceeded() {
   if (shouldSkip) {
     return
   }
@@ -56,7 +54,7 @@ export async function installationSucceeded() {
   await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
 }
 
-export async function installationFailed(error: any) {
+async function installationFailed(error: any) {
   if (shouldSkip) {
     return
   }
@@ -65,3 +63,12 @@ export async function installationFailed(error: any) {
   }
   await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
 }
+
+export default {
+  appSucceeded,
+  appFailed,
+  tenantSucceeded,
+  tenantFailed,
+  installationSucceeded,
+  installationFailed,
+}

packages/backend-core/src/events/publishers/backup.ts

@@ -8,7 +8,7 @@ import {
 } from "@budibase/types"
 import { publishEvent } from "../events"
 
-export async function appBackupRestored(backup: AppBackup) {
+async function appBackupRestored(backup: AppBackup) {
   const properties: AppBackupRestoreEvent = {
     appId: backup.appId,
     restoreId: backup._id!,
@@ -18,7 +18,7 @@ export async function appBackupRestored(backup: AppBackup) {
   await publishEvent(Event.APP_BACKUP_RESTORED, properties)
 }
 
-export async function appBackupTriggered(
+async function appBackupTriggered(
   appId: string,
   backupId: string,
   type: AppBackupType,
@@ -32,3 +32,8 @@ export async function appBackupTriggered(
   }
   await publishEvent(Event.APP_BACKUP_TRIGGERED, properties)
 }
+
+export default {
+  appBackupRestored,
+  appBackupTriggered,
+}

packages/backend-core/src/events/publishers/datasource.ts

@@ -14,10 +14,7 @@ function isCustom(datasource: Datasource) {
   return !sources.includes(datasource.source)
 }
 
-export async function created(
+async function created(datasource: Datasource, timestamp?: string | number) {
-  datasource: Datasource,
-  timestamp?: string | number
-) {
   const properties: DatasourceCreatedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -26,7 +23,7 @@ export async function created(
   await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
 }
 
-export async function updated(datasource: Datasource) {
+async function updated(datasource: Datasource) {
   const properties: DatasourceUpdatedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -35,7 +32,7 @@ export async function updated(datasource: Datasource) {
   await publishEvent(Event.DATASOURCE_UPDATED, properties)
 }
 
-export async function deleted(datasource: Datasource) {
+async function deleted(datasource: Datasource) {
   const properties: DatasourceDeletedEvent = {
     datasourceId: datasource._id as string,
     source: datasource.source,
@@ -43,3 +40,9 @@ export async function deleted(datasource: Datasource) {
   }
   await publishEvent(Event.DATASOURCE_DELETED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+}

packages/backend-core/src/events/publishers/email.ts

@@ -1,12 +1,17 @@
 import { publishEvent } from "../events"
 import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
 
-export async function SMTPCreated(timestamp?: string | number) {
+async function SMTPCreated(timestamp?: string | number) {
   const properties: SMTPCreatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
 }
 
-export async function SMTPUpdated() {
+async function SMTPUpdated() {
   const properties: SMTPUpdatedEvent = {}
   await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
 }
+
+export default {
+  SMTPCreated,
+  SMTPUpdated,
+}

packages/backend-core/src/events/publishers/environmentVariable.ts

@@ -0,0 +1,38 @@
+import {
+  Event,
+  EnvironmentVariableCreatedEvent,
+  EnvironmentVariableDeletedEvent,
+  EnvironmentVariableUpgradePanelOpenedEvent,
+} from "@budibase/types"
+import { publishEvent } from "../events"
+
+async function created(name: string, environments: string[]) {
+  const properties: EnvironmentVariableCreatedEvent = {
+    name,
+    environments,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_CREATED, properties)
+}
+
+async function deleted(name: string) {
+  const properties: EnvironmentVariableDeletedEvent = {
+    name,
+  }
+  await publishEvent(Event.ENVIRONMENT_VARIABLE_DELETED, properties)
+}
+
+async function upgradePanelOpened(userId: string) {
+  const properties: EnvironmentVariableUpgradePanelOpenedEvent = {
+    userId,
+  }
+  await publishEvent(
+    Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED,
+    properties
+  )
+}
+
+export default {
+  created,
+  deleted,
+  upgradePanelOpened,
+}

packages/backend-core/src/events/publishers/group.ts

@@ -11,28 +11,28 @@ import {
   UserGroupRoles,
 } from "@budibase/types"
 
-export async function created(group: UserGroup, timestamp?: number) {
+async function created(group: UserGroup, timestamp?: number) {
   const properties: GroupCreatedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_CREATED, properties, timestamp)
 }
 
-export async function updated(group: UserGroup) {
+async function updated(group: UserGroup) {
   const properties: GroupUpdatedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_UPDATED, properties)
 }
 
-export async function deleted(group: UserGroup) {
+async function deleted(group: UserGroup) {
   const properties: GroupDeletedEvent = {
     groupId: group._id as string,
   }
   await publishEvent(Event.USER_GROUP_DELETED, properties)
 }
 
-export async function usersAdded(count: number, group: UserGroup) {
+async function usersAdded(count: number, group: UserGroup) {
   const properties: GroupUsersAddedEvent = {
     count,
     groupId: group._id as string,
@@ -40,7 +40,7 @@ export async function usersAdded(count: number, group: UserGroup) {
   await publishEvent(Event.USER_GROUP_USERS_ADDED, properties)
 }
 
-export async function usersDeleted(count: number, group: UserGroup) {
+async function usersDeleted(count: number, group: UserGroup) {
   const properties: GroupUsersDeletedEvent = {
     count,
     groupId: group._id as string,
@@ -48,7 +48,7 @@ export async function usersDeleted(count: number, group: UserGroup) {
   await publishEvent(Event.USER_GROUP_USERS_REMOVED, properties)
 }
 
-export async function createdOnboarding(groupId: string) {
+async function createdOnboarding(groupId: string) {
   const properties: GroupAddedOnboardingEvent = {
     groupId: groupId,
     onboarding: true,
@@ -56,9 +56,19 @@ export async function createdOnboarding(groupId: string) {
   await publishEvent(Event.USER_GROUP_ONBOARDING, properties)
 }
 
-export async function permissionsEdited(roles: UserGroupRoles) {
+async function permissionsEdited(roles: UserGroupRoles) {
   const properties: UserGroupRoles = {
     ...roles,
   }
   await publishEvent(Event.USER_GROUP_PERMISSIONS_EDITED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  usersAdded,
+  usersDeleted,
+  createdOnboarding,
+  permissionsEdited,
+}

packages/backend-core/src/events/publishers/index.ts

@@ -1,22 +1,23 @@
-export * as account from "./account"
+export { default as account } from "./account"
-export * as app from "./app"
+export { default as app } from "./app"
-export * as auth from "./auth"
+export { default as auth } from "./auth"
-export * as automation from "./automation"
+export { default as automation } from "./automation"
-export * as datasource from "./datasource"
+export { default as datasource } from "./datasource"
-export * as email from "./email"
+export { default as email } from "./email"
-export * as license from "./license"
+export { default as license } from "./license"
-export * as layout from "./layout"
+export { default as layout } from "./layout"
-export * as org from "./org"
+export { default as org } from "./org"
-export * as query from "./query"
+export { default as query } from "./query"
-export * as role from "./role"
+export { default as role } from "./role"
-export * as screen from "./screen"
+export { default as screen } from "./screen"
-export * as rows from "./rows"
+export { default as rows } from "./rows"
-export * as table from "./table"
+export { default as table } from "./table"
-export * as serve from "./serve"
+export { default as serve } from "./serve"
-export * as user from "./user"
+export { default as user } from "./user"
-export * as view from "./view"
+export { default as view } from "./view"
-export * as installation from "./installation"
+export { default as installation } from "./installation"
-export * as backfill from "./backfill"
+export { default as backfill } from "./backfill"
-export * as group from "./group"
+export { default as group } from "./group"
-export * as plugin from "./plugin"
+export { default as plugin } from "./plugin"
-export * as backup from "./backup"
+export { default as backup } from "./backup"
+export { default as environmentVariable } from "./environmentVariable"

packages/backend-core/src/events/publishers/installation.ts

@@ -1,14 +1,14 @@
 import { publishEvent } from "../events"
 import { Event, VersionCheckedEvent, VersionChangeEvent } from "@budibase/types"
 
-export async function versionChecked(version: string) {
+async function versionChecked(version: string) {
   const properties: VersionCheckedEvent = {
     currentVersion: version,
   }
   await publishEvent(Event.INSTALLATION_VERSION_CHECKED, properties)
 }
 
-export async function upgraded(from: string, to: string) {
+async function upgraded(from: string, to: string) {
   const properties: VersionChangeEvent = {
     from,
     to,
@@ -17,7 +17,7 @@ export async function upgraded(from: string, to: string) {
   await publishEvent(Event.INSTALLATION_VERSION_UPGRADED, properties)
 }
 
-export async function downgraded(from: string, to: string) {
+async function downgraded(from: string, to: string) {
   const properties: VersionChangeEvent = {
     from,
     to,
@@ -25,7 +25,14 @@ export async function downgraded(from: string, to: string) {
   await publishEvent(Event.INSTALLATION_VERSION_DOWNGRADED, properties)
 }
 
-export async function firstStartup() {
+async function firstStartup() {
   const properties = {}
   await publishEvent(Event.INSTALLATION_FIRST_STARTUP, properties)
 }
+
+export default {
+  versionChecked,
+  upgraded,
+  downgraded,
+  firstStartup,
+}

packages/backend-core/src/events/publishers/layout.ts

@@ -6,16 +6,21 @@ import {
   LayoutDeletedEvent,
 } from "@budibase/types"
 
-export async function created(layout: Layout, timestamp?: string | number) {
+async function created(layout: Layout, timestamp?: string | number) {
   const properties: LayoutCreatedEvent = {
     layoutId: layout._id as string,
   }
   await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
 }
 
-export async function deleted(layoutId: string) {
+async function deleted(layoutId: string) {
   const properties: LayoutDeletedEvent = {
     layoutId,
   }
   await publishEvent(Event.LAYOUT_DELETED, properties)
 }
+
+export default {
+  created,
+  deleted,
+}

packages/backend-core/src/events/publishers/license.ts

@@ -13,7 +13,7 @@ import {
   LicensePaymentRecoveredEvent,
 } from "@budibase/types"
 
-export async function tierChanged(account: Account, from: number, to: number) {
+async function tierChanged(account: Account, from: number, to: number) {
   const properties: LicenseTierChangedEvent = {
     accountId: account.accountId,
     to,
@@ -22,11 +22,7 @@ export async function tierChanged(account: Account, from: number, to: number) {
   await publishEvent(Event.LICENSE_TIER_CHANGED, properties)
 }
 
-export async function planChanged(
+async function planChanged(account: Account, from: PlanType, to: PlanType) {
-  account: Account,
-  from: PlanType,
-  to: PlanType
-) {
   const properties: LicensePlanChangedEvent = {
     accountId: account.accountId,
     to,
@@ -35,44 +31,55 @@ export async function planChanged(
   await publishEvent(Event.LICENSE_PLAN_CHANGED, properties)
 }
 
-export async function activated(account: Account) {
+async function activated(account: Account) {
   const properties: LicenseActivatedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_ACTIVATED, properties)
 }
 
-export async function checkoutOpened(account: Account) {
+async function checkoutOpened(account: Account) {
   const properties: LicenseCheckoutOpenedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_CHECKOUT_OPENED, properties)
 }
 
-export async function checkoutSuccess(account: Account) {
+async function checkoutSuccess(account: Account) {
   const properties: LicenseCheckoutSuccessEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_CHECKOUT_SUCCESS, properties)
 }
 
-export async function portalOpened(account: Account) {
+async function portalOpened(account: Account) {
   const properties: LicensePortalOpenedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PORTAL_OPENED, properties)
 }
 
-export async function paymentFailed(account: Account) {
+async function paymentFailed(account: Account) {
   const properties: LicensePaymentFailedEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PAYMENT_FAILED, properties)
 }
 
-export async function paymentRecovered(account: Account) {
+async function paymentRecovered(account: Account) {
   const properties: LicensePaymentRecoveredEvent = {
     accountId: account.accountId,
   }
   await publishEvent(Event.LICENSE_PAYMENT_RECOVERED, properties)
 }
+
+export default {
+  tierChanged,
+  planChanged,
+  activated,
+  checkoutOpened,
+  checkoutSuccess,
+  portalOpened,
+  paymentFailed,
+  paymentRecovered,
+}

packages/backend-core/src/events/publishers/org.ts

@@ -1,29 +1,37 @@
 import { publishEvent } from "../events"
 import { Event } from "@budibase/types"
 
-export async function nameUpdated(timestamp?: string | number) {
+async function nameUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_NAME_UPDATED, properties, timestamp)
 }
 
-export async function logoUpdated(timestamp?: string | number) {
+async function logoUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_LOGO_UPDATED, properties, timestamp)
 }
 
-export async function platformURLUpdated(timestamp?: string | number) {
+async function platformURLUpdated(timestamp?: string | number) {
   const properties = {}
   await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties, timestamp)
 }
 
 // TODO
 
-export async function analyticsOptOut() {
+async function analyticsOptOut() {
   const properties = {}
   await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
 }
 
-export async function analyticsOptIn() {
+async function analyticsOptIn() {
   const properties = {}
   await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
 }
+
+export default {
+  nameUpdated,
+  logoUpdated,
+  platformURLUpdated,
+  analyticsOptOut,
+  analyticsOptIn,
+}

packages/backend-core/src/events/publishers/plugin.ts

@@ -7,7 +7,7 @@ import {
   PluginInitEvent,
 } from "@budibase/types"
 
-export async function init(plugin: Plugin) {
+async function init(plugin: Plugin) {
   const properties: PluginInitEvent = {
     type: plugin.schema.type,
     name: plugin.name,
@@ -17,7 +17,7 @@ export async function init(plugin: Plugin) {
   await publishEvent(Event.PLUGIN_INIT, properties)
 }
 
-export async function imported(plugin: Plugin) {
+async function imported(plugin: Plugin) {
   const properties: PluginImportedEvent = {
     pluginId: plugin._id as string,
     type: plugin.schema.type,
@@ -29,7 +29,7 @@ export async function imported(plugin: Plugin) {
   await publishEvent(Event.PLUGIN_IMPORTED, properties)
 }
 
-export async function deleted(plugin: Plugin) {
+async function deleted(plugin: Plugin) {
   const properties: PluginDeletedEvent = {
     pluginId: plugin._id as string,
     type: plugin.schema.type,
@@ -39,3 +39,9 @@ export async function deleted(plugin: Plugin) {
   }
   await publishEvent(Event.PLUGIN_DELETED, properties)
 }
+
+export default {
+  init,
+  imported,
+  deleted,
+}

packages/backend-core/src/events/publishers/query.ts

@@ -13,7 +13,7 @@ import {
 
 /* eslint-disable */
 
-export const created = async (
+const created = async (
   datasource: Datasource,
   query: Query,
   timestamp?: string | number
@@ -27,7 +27,7 @@ export const created = async (
   await publishEvent(Event.QUERY_CREATED, properties, timestamp)
 }
 
-export const updated = async (datasource: Datasource, query: Query) => {
+const updated = async (datasource: Datasource, query: Query) => {
   const properties: QueryUpdatedEvent = {
     queryId: query._id as string,
     datasourceId: datasource._id as string,
@@ -37,7 +37,7 @@ export const updated = async (datasource: Datasource, query: Query) => {
   await publishEvent(Event.QUERY_UPDATED, properties)
 }
 
-export const deleted = async (datasource: Datasource, query: Query) => {
+const deleted = async (datasource: Datasource, query: Query) => {
   const properties: QueryDeletedEvent = {
     queryId: query._id as string,
     datasourceId: datasource._id as string,
@@ -47,7 +47,7 @@ export const deleted = async (datasource: Datasource, query: Query) => {
   await publishEvent(Event.QUERY_DELETED, properties)
 }
 
-export const imported = async (
+const imported = async (
   datasource: Datasource,
   importSource: any,
   count: any
@@ -61,14 +61,14 @@ export const imported = async (
   await publishEvent(Event.QUERY_IMPORT, properties)
 }
 
-export const run = async (count: number, timestamp?: string | number) => {
+const run = async (count: number, timestamp?: string | number) => {
   const properties: QueriesRunEvent = {
     count,
   }
   await publishEvent(Event.QUERIES_RUN, properties, timestamp)
 }
 
-export const previewed = async (datasource: Datasource, query: Query) => {
+const previewed = async (datasource: Datasource, query: Query) => {
   const properties: QueryPreviewedEvent = {
     queryId: query._id,
     datasourceId: datasource._id as string,
@@ -77,3 +77,12 @@ export const previewed = async (datasource: Datasource, query: Query) => {
   }
   await publishEvent(Event.QUERY_PREVIEWED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  imported,
+  run,
+  previewed,
+}

packages/backend-core/src/events/publishers/role.ts

@@ -10,7 +10,7 @@ import {
   User,
 } from "@budibase/types"
 
-export async function created(role: Role, timestamp?: string | number) {
+async function created(role: Role, timestamp?: string | number) {
   const properties: RoleCreatedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -19,7 +19,7 @@ export async function created(role: Role, timestamp?: string | number) {
   await publishEvent(Event.ROLE_CREATED, properties, timestamp)
 }
 
-export async function updated(role: Role) {
+async function updated(role: Role) {
   const properties: RoleUpdatedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -28,7 +28,7 @@ export async function updated(role: Role) {
   await publishEvent(Event.ROLE_UPDATED, properties)
 }
 
-export async function deleted(role: Role) {
+async function deleted(role: Role) {
   const properties: RoleDeletedEvent = {
     roleId: role._id as string,
     permissionId: role.permissionId,
@@ -37,7 +37,7 @@ export async function deleted(role: Role) {
   await publishEvent(Event.ROLE_DELETED, properties)
 }
 
-export async function assigned(user: User, roleId: string, timestamp?: number) {
+async function assigned(user: User, roleId: string, timestamp?: number) {
   const properties: RoleAssignedEvent = {
     userId: user._id as string,
     roleId,
@@ -45,10 +45,18 @@ export async function assigned(user: User, roleId: string, timestamp?: number) {
   await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp)
 }
 
-export async function unassigned(user: User, roleId: string) {
+async function unassigned(user: User, roleId: string) {
   const properties: RoleUnassignedEvent = {
     userId: user._id as string,
     roleId,
   }
   await publishEvent(Event.ROLE_UNASSIGNED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  assigned,
+  unassigned,
+}

packages/backend-core/src/events/publishers/rows.ts

@@ -3,28 +3,27 @@ import {
   Event,
   RowsImportedEvent,
   RowsCreatedEvent,
-  RowImportFormat,
   Table,
 } from "@budibase/types"
 
 /* eslint-disable */
 
-export const created = async (count: number, timestamp?: string | number) => {
+const created = async (count: number, timestamp?: string | number) => {
   const properties: RowsCreatedEvent = {
     count,
   }
   await publishEvent(Event.ROWS_CREATED, properties, timestamp)
 }
 
-export const imported = async (
+const imported = async (table: Table, count: number) => {
-  table: Table,
-  format: RowImportFormat,
-  count: number
-) => {
   const properties: RowsImportedEvent = {
     tableId: table._id as string,
-    format,
     count,
   }
   await publishEvent(Event.ROWS_IMPORTED, properties)
 }
+
+export default {
+  created,
+  imported,
+}

packages/backend-core/src/events/publishers/screen.ts

@@ -6,7 +6,7 @@ import {
   ScreenDeletedEvent,
 } from "@budibase/types"
 
-export async function created(screen: Screen, timestamp?: string | number) {
+async function created(screen: Screen, timestamp?: string | number) {
   const properties: ScreenCreatedEvent = {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
@@ -15,7 +15,7 @@ export async function created(screen: Screen, timestamp?: string | number) {
   await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
 }
 
-export async function deleted(screen: Screen) {
+async function deleted(screen: Screen) {
   const properties: ScreenDeletedEvent = {
     layoutId: screen.layoutId,
     screenId: screen._id as string,
@@ -23,3 +23,8 @@ export async function deleted(screen: Screen) {
   }
   await publishEvent(Event.SCREEN_DELETED, properties)
 }
+
+export default {
+  created,
+  deleted,
+}

packages/backend-core/src/events/publishers/serve.ts

@@ -7,14 +7,14 @@ import {
   AppServedEvent,
 } from "@budibase/types"
 
-export async function servedBuilder(timezone: string) {
+async function servedBuilder(timezone: string) {
   const properties: BuilderServedEvent = {
     timezone,
   }
   await publishEvent(Event.SERVED_BUILDER, properties)
 }
 
-export async function servedApp(app: App, timezone: string) {
+async function servedApp(app: App, timezone: string) {
   const properties: AppServedEvent = {
     appVersion: app.version,
     timezone,
@@ -22,7 +22,7 @@ export async function servedApp(app: App, timezone: string) {
   await publishEvent(Event.SERVED_APP, properties)
 }
 
-export async function servedAppPreview(app: App, timezone: string) {
+async function servedAppPreview(app: App, timezone: string) {
   const properties: AppPreviewServedEvent = {
     appId: app.appId,
     appVersion: app.version,
@@ -30,3 +30,9 @@ export async function servedAppPreview(app: App, timezone: string) {
   }
   await publishEvent(Event.SERVED_APP_PREVIEW, properties)
 }
+
+export default {
+  servedBuilder,
+  servedApp,
+  servedAppPreview,
+}

packages/backend-core/src/events/publishers/table.ts

@@ -2,7 +2,6 @@ import { publishEvent } from "../events"
 import {
   Event,
   TableExportFormat,
-  TableImportFormat,
   Table,
   TableCreatedEvent,
   TableUpdatedEvent,
@@ -11,28 +10,28 @@ import {
   TableImportedEvent,
 } from "@budibase/types"
 
-export async function created(table: Table, timestamp?: string | number) {
+async function created(table: Table, timestamp?: string | number) {
   const properties: TableCreatedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_CREATED, properties, timestamp)
 }
 
-export async function updated(table: Table) {
+async function updated(table: Table) {
   const properties: TableUpdatedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_UPDATED, properties)
 }
 
-export async function deleted(table: Table) {
+async function deleted(table: Table) {
   const properties: TableDeletedEvent = {
     tableId: table._id as string,
   }
   await publishEvent(Event.TABLE_DELETED, properties)
 }
 
-export async function exported(table: Table, format: TableExportFormat) {
+async function exported(table: Table, format: TableExportFormat) {
   const properties: TableExportedEvent = {
     tableId: table._id as string,
     format,
@@ -40,10 +39,17 @@ export async function exported(table: Table, format: TableExportFormat) {
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
 
-export async function imported(table: Table, format: TableImportFormat) {
+async function imported(table: Table) {
   const properties: TableImportedEvent = {
     tableId: table._id as string,
-    format,
   }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  exported,
+  imported,
+}

packages/backend-core/src/events/publishers/user.ts

@@ -13,32 +13,40 @@ import {
   UserPermissionAssignedEvent,
   UserPermissionRemovedEvent,
   UserUpdatedEvent,
+  UserOnboardingEvent,
 } from "@budibase/types"
 
-export async function created(user: User, timestamp?: number) {
+async function created(user: User, timestamp?: number) {
   const properties: UserCreatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_CREATED, properties, timestamp)
 }
 
-export async function updated(user: User) {
+async function updated(user: User) {
   const properties: UserUpdatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_UPDATED, properties)
 }
 
-export async function deleted(user: User) {
+async function deleted(user: User) {
   const properties: UserDeletedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_DELETED, properties)
 }
 
+export async function onboardingComplete(user: User) {
+  const properties: UserOnboardingEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_ONBOARDING_COMPLETE, properties)
+}
+
 // PERMISSIONS
 
-export async function permissionAdminAssigned(user: User, timestamp?: number) {
+async function permissionAdminAssigned(user: User, timestamp?: number) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
   }
@@ -49,17 +57,14 @@ export async function permissionAdminAssigned(user: User, timestamp?: number) {
   )
 }
 
-export async function permissionAdminRemoved(user: User) {
+async function permissionAdminRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
 }
 
-export async function permissionBuilderAssigned(
+async function permissionBuilderAssigned(user: User, timestamp?: number) {
-  user: User,
-  timestamp?: number
-) {
   const properties: UserPermissionAssignedEvent = {
     userId: user._id as string,
   }
@@ -70,7 +75,7 @@ export async function permissionBuilderAssigned(
   )
 }
 
-export async function permissionBuilderRemoved(user: User) {
+async function permissionBuilderRemoved(user: User) {
   const properties: UserPermissionRemovedEvent = {
     userId: user._id as string,
   }
@@ -79,12 +84,12 @@ export async function permissionBuilderRemoved(user: User) {
 
 // INVITE
 
-export async function invited() {
+async function invited() {
   const properties: UserInvitedEvent = {}
   await publishEvent(Event.USER_INVITED, properties)
 }
 
-export async function inviteAccepted(user: User) {
+async function inviteAccepted(user: User) {
   const properties: UserInviteAcceptedEvent = {
     userId: user._id as string,
   }
@@ -93,30 +98,47 @@ export async function inviteAccepted(user: User) {
 
 // PASSWORD
 
-export async function passwordForceReset(user: User) {
+async function passwordForceReset(user: User) {
   const properties: UserPasswordForceResetEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
 }
 
-export async function passwordUpdated(user: User) {
+async function passwordUpdated(user: User) {
   const properties: UserPasswordUpdatedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
 }
 
-export async function passwordResetRequested(user: User) {
+async function passwordResetRequested(user: User) {
   const properties: UserPasswordResetRequestedEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
 }
 
-export async function passwordReset(user: User) {
+async function passwordReset(user: User) {
   const properties: UserPasswordResetEvent = {
     userId: user._id as string,
   }
   await publishEvent(Event.USER_PASSWORD_RESET, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  permissionAdminAssigned,
+  permissionAdminRemoved,
+  permissionBuilderAssigned,
+  permissionBuilderRemoved,
+  onboardingComplete,
+  invited,
+  inviteAccepted,
+  passwordForceReset,
+  passwordUpdated,
+  passwordResetRequested,
+  passwordReset,
+}

packages/backend-core/src/events/publishers/view.ts

@@ -19,28 +19,28 @@ import {
 
 /* eslint-disable */
 
-export async function created(view: View, timestamp?: string | number) {
+async function created(view: View, timestamp?: string | number) {
   const properties: ViewCreatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_CREATED, properties, timestamp)
 }
 
-export async function updated(view: View) {
+async function updated(view: View) {
   const properties: ViewUpdatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_UPDATED, properties)
 }
 
-export async function deleted(view: View) {
+async function deleted(view: View) {
   const properties: ViewDeletedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_DELETED, properties)
 }
 
-export async function exported(table: Table, format: TableExportFormat) {
+async function exported(table: Table, format: TableExportFormat) {
   const properties: ViewExportedEvent = {
     tableId: table._id as string,
     format,
@@ -48,31 +48,28 @@ export async function exported(table: Table, format: TableExportFormat) {
   await publishEvent(Event.VIEW_EXPORTED, properties)
 }
 
-export async function filterCreated(view: View, timestamp?: string | number) {
+async function filterCreated(view: View, timestamp?: string | number) {
   const properties: ViewFilterCreatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp)
 }
 
-export async function filterUpdated(view: View) {
+async function filterUpdated(view: View) {
   const properties: ViewFilterUpdatedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_UPDATED, properties)
 }
 
-export async function filterDeleted(view: View) {
+async function filterDeleted(view: View) {
   const properties: ViewFilterDeletedEvent = {
     tableId: view.tableId,
   }
   await publishEvent(Event.VIEW_FILTER_DELETED, properties)
 }
 
-export async function calculationCreated(
+async function calculationCreated(view: View, timestamp?: string | number) {
-  view: View,
-  timestamp?: string | number
-) {
   const properties: ViewCalculationCreatedEvent = {
     tableId: view.tableId,
     calculation: view.calculation as ViewCalculation,
@@ -80,7 +77,7 @@ export async function calculationCreated(
   await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp)
 }
 
-export async function calculationUpdated(view: View) {
+async function calculationUpdated(view: View) {
   const properties: ViewCalculationUpdatedEvent = {
     tableId: view.tableId,
     calculation: view.calculation as ViewCalculation,
@@ -88,10 +85,23 @@ export async function calculationUpdated(view: View) {
   await publishEvent(Event.VIEW_CALCULATION_UPDATED, properties)
 }
 
-export async function calculationDeleted(existingView: View) {
+async function calculationDeleted(existingView: View) {
   const properties: ViewCalculationDeletedEvent = {
     tableId: existingView.tableId,
     calculation: existingView.calculation as ViewCalculation,
   }
   await publishEvent(Event.VIEW_CALCULATION_DELETED, properties)
 }
+
+export default {
+  created,
+  updated,
+  deleted,
+  exported,
+  filterCreated,
+  filterUpdated,
+  filterDeleted,
+  calculationCreated,
+  calculationUpdated,
+  calculationDeleted,
+}

packages/backend-core/src/featureFlags/index.ts

@@ -6,7 +6,7 @@ import * as tenancy from "../tenancy"
  * The env var is formatted as:
  *  tenant1:feature1:feature2,tenant2:feature1
  */
-function getFeatureFlags() {
+export function buildFeatureFlags() {
   if (!env.TENANT_FEATURE_FLAGS) {
     return
   }
@@ -27,8 +27,6 @@ function getFeatureFlags() {
   return tenantFeatureFlags
 }
 
-const TENANT_FEATURE_FLAGS = getFeatureFlags()
-
 export function isEnabled(featureFlag: string) {
   const tenantId = tenancy.getTenantId()
   const flags = getTenantFeatureFlags(tenantId)
@@ -36,18 +34,36 @@ export function isEnabled(featureFlag: string) {
 }
 
 export function getTenantFeatureFlags(tenantId: string) {
-  const flags = []
+  let flags: string[] = []
+  const envFlags = buildFeatureFlags()
+  if (envFlags) {
+    const globalFlags = envFlags["*"]
+    const tenantFlags = envFlags[tenantId] || []
 
-  if (TENANT_FEATURE_FLAGS) {
+    // Explicitly exclude tenants from global features if required.
-    const globalFlags = TENANT_FEATURE_FLAGS["*"]
+    // Prefix the tenant flag with '!'
-    const tenantFlags = TENANT_FEATURE_FLAGS[tenantId]
+    const tenantOverrides = tenantFlags.reduce(
+      (acc: string[], flag: string) => {
+        if (flag.startsWith("!")) {
+          let stripped = flag.substring(1)
+          acc.push(stripped)
+        }
+        return acc
+      },
+      []
+    )
 
     if (globalFlags) {
       flags.push(...globalFlags)
     }
-    if (tenantFlags) {
+    if (tenantFlags.length) {
       flags.push(...tenantFlags)
     }
+
+    // Purge any tenant specific overrides
+    flags = flags.filter(flag => {
+      return tenantOverrides.indexOf(flag) == -1 && !flag.startsWith("!")
+    })
   }
 
   return flags
@@ -57,4 +73,5 @@ export enum TenantFeatureFlag {
   LICENSING = "LICENSING",
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   USER_GROUPS = "USER_GROUPS",
+  ONBOARDING_TOUR = "ONBOARDING_TOUR",
 }

packages/backend-core/src/featureFlags/tests/featureFlags.spec.ts

@@ -0,0 +1,85 @@
+import {
+  TenantFeatureFlag,
+  buildFeatureFlags,
+  getTenantFeatureFlags,
+} from "../"
+import env from "../../environment"
+
+const { ONBOARDING_TOUR, LICENSING, USER_GROUPS } = TenantFeatureFlag
+
+describe("featureFlags", () => {
+  beforeEach(() => {
+    env._set("TENANT_FEATURE_FLAGS", "")
+  })
+
+  it("Should return no flags when the TENANT_FEATURE_FLAG is empty", async () => {
+    let features = buildFeatureFlags()
+    expect(features).toBeUndefined()
+  })
+
+  it("Should generate a map of global and named tenant feature flags from the env value", async () => {
+    env._set(
+      "TENANT_FEATURE_FLAGS",
+      `*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR},tenant2:${USER_GROUPS},tenant1:${LICENSING}`
+    )
+
+    const parsedFlags: Record<string, string[]> = {
+      "*": [ONBOARDING_TOUR],
+      tenant1: [`!${ONBOARDING_TOUR}`, LICENSING],
+      tenant2: [USER_GROUPS],
+    }
+
+    let features = buildFeatureFlags()
+
+    expect(features).toBeDefined()
+    expect(features).toEqual(parsedFlags)
+  })
+
+  it("Should add feature flag flag only to explicitly configured tenant", async () => {
+    env._set(
+      "TENANT_FEATURE_FLAGS",
+      `*:${LICENSING},*:${USER_GROUPS},tenant1:${ONBOARDING_TOUR}`
+    )
+
+    let tenant1Flags = getTenantFeatureFlags("tenant1")
+    let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+    expect(tenant1Flags).toBeDefined()
+    expect(tenant1Flags).toEqual([LICENSING, USER_GROUPS, ONBOARDING_TOUR])
+
+    expect(tenant2Flags).toBeDefined()
+    expect(tenant2Flags).toEqual([LICENSING, USER_GROUPS])
+  })
+})
+
+it("Should exclude tenant1 from global feature flag", async () => {
+  env._set(
+    "TENANT_FEATURE_FLAGS",
+    `*:${LICENSING},*:${ONBOARDING_TOUR},tenant1:!${ONBOARDING_TOUR}`
+  )
+
+  let tenant1Flags = getTenantFeatureFlags("tenant1")
+  let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+  expect(tenant1Flags).toBeDefined()
+  expect(tenant1Flags).toEqual([LICENSING])
+
+  expect(tenant2Flags).toBeDefined()
+  expect(tenant2Flags).toEqual([LICENSING, ONBOARDING_TOUR])
+})
+
+it("Should explicitly add flags to configured tenants only", async () => {
+  env._set(
+    "TENANT_FEATURE_FLAGS",
+    `tenant1:${ONBOARDING_TOUR},tenant1:${LICENSING},tenant2:${LICENSING}`
+  )
+
+  let tenant1Flags = getTenantFeatureFlags("tenant1")
+  let tenant2Flags = getTenantFeatureFlags("tenant2")
+
+  expect(tenant1Flags).toBeDefined()
+  expect(tenant1Flags).toEqual([ONBOARDING_TOUR, LICENSING])
+
+  expect(tenant2Flags).toBeDefined()
+  expect(tenant2Flags).toEqual([LICENSING])
+})

packages/backend-core/src/index.ts

@@ -1,68 +1,42 @@
-import errors from "./errors"
+export * as events from "./events"
-const errorClasses = errors.errors
+export * as migrations from "./migrations"
-import * as events from "./events"
+export * as users from "./users"
-import * as migrations from "./migrations"
+export * as roles from "./security/roles"
-import * as users from "./users"
+export * as permissions from "./security/permissions"
-import * as roles from "./security/roles"
+export * as accounts from "./cloud/accounts"
-import * as permissions from "./security/permissions"
+export * as installation from "./installation"
-import * as accounts from "./cloud/accounts"
+export * as tenancy from "./tenancy"
-import * as installation from "./installation"
+export * as featureFlags from "./featureFlags"
-import env from "./environment"
+export * as sessions from "./security/sessions"
-import * as tenancy from "./tenancy"
+export * as deprovisioning from "./context/deprovision"
-import * as featureFlags from "./featureFlags"
+export * as auth from "./auth"
-import * as sessions from "./security/sessions"
+export * as constants from "./constants"
-import * as deprovisioning from "./context/deprovision"
+export * as logging from "./logging"
-import * as auth from "./auth"
+export * as middleware from "./middleware"
-import * as constants from "./constants"
+export * as plugins from "./plugin"
-import * as logging from "./logging"
+export * as encryption from "./security/encryption"
-import * as pino from "./pino"
+export * as queue from "./queue"
-import * as middleware from "./middleware"
+export * as db from "./db"
-import * as plugins from "./plugin"
+export * as context from "./context"
-import * as encryption from "./security/encryption"
+export * as cache from "./cache"
-import * as queue from "./queue"
+export * as objectStore from "./objectStore"
-import * as db from "./db"
+export * as redis from "./redis"
-import * as context from "./context"
+export * as utils from "./utils"
-import * as cache from "./cache"
+export * as errors from "./errors"
-import * as objectStore from "./objectStore"
+export { default as env } from "./environment"
-import * as redis from "./redis"
-import * as utils from "./utils"
 
-const init = (opts: any = {}) => {
+// expose error classes directly
+export * from "./errors"
+
+// expose constants directly
+export * from "./constants"
+
+// expose inner locks from redis directly
+import * as redis from "./redis"
+export const locks = redis.redlock
+
+// expose package init function
+import * as db from "./db"
+export const init = (opts: any = {}) => {
   db.init(opts.db)
 }
-
-const core = {
-  init,
-  db,
-  ...constants,
-  redis,
-  locks: redis.redlock,
-  objectStore,
-  utils,
-  users,
-  cache,
-  auth,
-  constants,
-  migrations,
-  env,
-  accounts,
-  tenancy,
-  context,
-  featureFlags,
-  events,
-  sessions,
-  deprovisioning,
-  installation,
-  errors,
-  logging,
-  roles,
-  plugins,
-  ...pino,
-  ...errorClasses,
-  middleware,
-  encryption,
-  queue,
-  permissions,
-}
-
-export = core

packages/backend-core/src/logging.ts

@@ -1,3 +1,9 @@
+import { Header } from "./constants"
+import env from "./environment"
+const correlator = require("correlation-id")
+import { Options } from "pino-http"
+import { IncomingMessage } from "http"
+
 const NonErrors = ["AccountError"]
 
 function isSuppressed(e?: any) {
@@ -29,8 +35,26 @@ export function logWarn(message: string) {
   console.warn(`bb-warn: ${message}`)
 }
 
-export default {
+export function pinoSettings(): Options {
-  logAlert,
+  return {
-  logAlertWithInfo,
+    prettyPrint: {
-  logWarn,
+      levelFirst: true,
+    },
+    genReqId: correlator.getId,
+    level: env.LOG_LEVEL || "error",
+    autoLogging: {
+      ignore: (req: IncomingMessage) => !!req.url?.includes("/health"),
+    },
+  }
+}
+
+const setCorrelationHeader = (headers: any) => {
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    headers[Header.CORRELATION_ID] = correlationId
+  }
+}
+
+export const correlation = {
+  setHeader: setCorrelationHeader,
 }

packages/backend-core/src/middleware/adminOnly.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.admin || !ctx.user.admin.global)

packages/backend-core/src/middleware/auditLog.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext | any, next: any) => {
+export default async (ctx: BBContext | any, next: any) => {
   // Placeholder for audit log middleware
   return next()
 }

packages/backend-core/src/middleware/authenticated.ts

@@ -66,7 +66,7 @@ async function checkApiKey(apiKey: string, populateUser?: Function) {
  * The tenancy modules should not be used here and it should be assumed that the tenancy context
  * has not yet been populated.
  */
-export = function (
+export default function (
   noAuthPatterns: EndpointMatcher[] = [],
   opts: { publicAllowed?: boolean; populateUser?: Function } = {
     publicAllowed: false,

packages/backend-core/src/middleware/builderOnly.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)

packages/backend-core/src/middleware/builderOrAdmin.ts

@@ -1,6 +1,6 @@
 import { BBContext } from "@budibase/types"
 
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   if (
     !ctx.internal &&
     (!ctx.user || !ctx.user.builder || !ctx.user.builder.global) &&

packages/backend-core/src/middleware/csrf.ts

@@ -32,7 +32,7 @@ const INCLUDED_CONTENT_TYPES = [
  * https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#synchronizer-token-pattern
  *
  */
-export = function (
+export default function (
   opts: { noCsrfPatterns: EndpointMatcher[] } = { noCsrfPatterns: [] }
 ) {
   const noCsrfOptions = buildMatcherRegex(opts.noCsrfPatterns)

packages/backend-core/src/middleware/index.ts

@@ -1,38 +1,19 @@
-import * as jwt from "./passport/jwt"
+export * as jwt from "./passport/jwt"
-import * as local from "./passport/local"
+export * as local from "./passport/local"
-import * as google from "./passport/google"
+export * as google from "./passport/google"
-import * as oidc from "./passport/oidc"
+export * as oidc from "./passport/oidc"
-import { authError, ssoCallbackUrl } from "./passport/utils"
-import authenticated from "./authenticated"
-import auditLog from "./auditLog"
-import tenancy from "./tenancy"
-import internalApi from "./internalApi"
 import * as datasourceGoogle from "./passport/datasource/google"
-import csrf from "./csrf"
+export const datasource = {
-import adminOnly from "./adminOnly"
-import builderOrAdmin from "./builderOrAdmin"
-import builderOnly from "./builderOnly"
-import * as joiValidator from "./joi-validator"
-
-const pkg = {
-  google,
-  oidc,
-  jwt,
-  local,
-  authenticated,
-  auditLog,
-  tenancy,
-  authError,
-  internalApi,
-  ssoCallbackUrl,
-  datasource: {
   google: datasourceGoogle,
-  },
-  csrf,
-  adminOnly,
-  builderOnly,
-  builderOrAdmin,
-  joiValidator,
 }
-
+export { authError, ssoCallbackUrl } from "./passport/utils"
-export = pkg
+export { default as authenticated } from "./authenticated"
+export { default as auditLog } from "./auditLog"
+export { default as tenancy } from "./tenancy"
+export { default as internalApi } from "./internalApi"
+export { default as csrf } from "./csrf"
+export { default as adminOnly } from "./adminOnly"
+export { default as builderOrAdmin } from "./builderOrAdmin"
+export { default as builderOnly } from "./builderOnly"
+export { default as logging } from "./logging"
+export * as joiValidator from "./joi-validator"

packages/backend-core/src/middleware/internalApi.ts

@@ -5,7 +5,7 @@ import { BBContext } from "@budibase/types"
 /**
  * API Key only endpoint.
  */
-export = async (ctx: BBContext, next: any) => {
+export default async (ctx: BBContext, next: any) => {
   const apiKey = ctx.request.headers[Header.API_KEY]
   if (apiKey !== env.INTERNAL_API_KEY) {
     ctx.throw(403, "Unauthorized")

packages/backend-core/src/middleware/logging.ts

@@ -0,0 +1,88 @@
+const correlator = require("correlation-id")
+import { Header } from "../constants"
+import { v4 as uuid } from "uuid"
+import * as context from "../context"
+
+const debug = console.warn
+const trace = console.trace
+const log = console.log
+const info = console.info
+const warn = console.warn
+const error = console.error
+
+const getTenantId = () => {
+  let tenantId
+  try {
+    tenantId = context.getTenantId()
+  } catch (e: any) {
+    // do nothing
+  }
+  return tenantId
+}
+
+const getAppId = () => {
+  let appId
+  try {
+    appId = context.getAppId()
+  } catch (e) {
+    // do nothing
+  }
+  return appId
+}
+
+const getIdentityId = () => {
+  let identityId
+  try {
+    const identity = context.getIdentity()
+    identityId = identity?._id
+  } catch (e) {
+    // do nothing
+  }
+  return identityId
+}
+
+const print = (fn: any, data: any[]) => {
+  let message = ""
+
+  const correlationId = correlator.getId()
+  if (correlationId) {
+    message = message + `[correlationId=${correlator.getId()}]`
+  }
+
+  const tenantId = getTenantId()
+  if (tenantId) {
+    message = message + ` [tenantId=${tenantId}]`
+  }
+
+  const appId = getAppId()
+  if (appId) {
+    message = message + ` [appId=${appId}]`
+  }
+
+  const identityId = getIdentityId()
+  if (identityId) {
+    message = message + ` [identityId=${identityId}]`
+  }
+
+  fn(message, data)
+}
+
+const logging = (ctx: any, next: any) => {
+  // use the provided correlation id header if present
+  let correlationId = ctx.headers[Header.CORRELATION_ID]
+  if (!correlationId) {
+    correlationId = uuid()
+  }
+
+  return correlator.withId(correlationId, () => {
+    console.debug = data => print(debug, data)
+    console.trace = data => print(trace, data)
+    console.log = data => print(log, data)
+    console.info = data => print(info, data)
+    console.warn = data => print(warn, data)
+    console.error = data => print(error, data)
+    return next()
+  })
+}
+
+export default logging

packages/backend-core/src/middleware/passport/oidc.ts

@@ -2,7 +2,6 @@ import fetch from "node-fetch"
 import { authenticateThirdParty, SaveUserFunction } from "./third-party-common"
 import { ssoCallbackUrl } from "./utils"
 import {
-  Config,
   ConfigType,
   OIDCInnerCfg,
   Database,

packages/backend-core/src/middleware/tenancy.ts

@@ -8,7 +8,7 @@ import {
   TenantResolutionStrategy,
 } from "@budibase/types"
 
-export = function (
+export default function (
   allowQueryStringPatterns: EndpointMatcher[],
   noTenancyPatterns: EndpointMatcher[],
   opts: { noTenancyRequired?: boolean } = { noTenancyRequired: false }

packages/backend-core/src/migrations/migrations.ts

@@ -88,7 +88,7 @@ export const runMigration = async (
 
     await doWithDB(dbName, async (db: any) => {
       try {
-        const doc = await exports.getMigrationsDoc(db)
+        const doc = await getMigrationsDoc(db)
 
         // the migration has already been run
         if (doc[migrationName]) {

packages/backend-core/src/objectStore/buckets/app.ts

@@ -0,0 +1,40 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+/**
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
+ * @return {string} The URL to be inserted into appPackage response or server rendered
+ * app index file.
+ */
+export const clientLibraryUrl = (appId: string, version: string) => {
+  if (env.isProd()) {
+    let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js`
+    if (env.CLOUDFRONT_CDN) {
+      // append app version to bust the cache
+      if (version) {
+        file += `?v=${version}`
+      }
+      // don't need to use presigned for client with cloudfront
+      // file is public
+      return cloudfront.getUrl(file)
+    } else {
+      return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    }
+  } else {
+    return `/api/assets/client`
+  }
+}
+
+export const getAppFileUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
+  }
+}

packages/backend-core/src/objectStore/buckets/global.ts

@@ -0,0 +1,29 @@
+import env from "../../environment"
+import * as tenancy from "../../tenancy"
+import * as objectStore from "../objectStore"
+import * as cloudfront from "../cloudfront"
+
+// URLs
+
+export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
+  let file = getGlobalFileS3Key(type, name)
+  if (env.CLOUDFRONT_CDN) {
+    if (etag) {
+      file = `${file}?etag=${etag}`
+    }
+    return cloudfront.getPresignedUrl(file)
+  } else {
+    return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
+  }
+}
+
+// KEYS
+
+export const getGlobalFileS3Key = (type: string, name: string) => {
+  let file = `${type}/${name}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    file = `${tenantId}/${file}`
+  }
+  return file
+}

packages/backend-core/src/objectStore/buckets/index.ts

@@ -0,0 +1,3 @@
+export * from "./app"
+export * from "./global"
+export * from "./plugins"

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -0,0 +1,71 @@
+import env from "../../environment"
+import * as objectStore from "../objectStore"
+import * as tenancy from "../../tenancy"
+import * as cloudfront from "../cloudfront"
+import { Plugin } from "@budibase/types"
+
+// URLS
+
+export const enrichPluginURLs = (plugins: Plugin[]) => {
+  if (!plugins || !plugins.length) {
+    return []
+  }
+  return plugins.map(plugin => {
+    const jsUrl = getPluginJSUrl(plugin)
+    const iconUrl = getPluginIconUrl(plugin)
+    return { ...plugin, jsUrl, iconUrl }
+  })
+}
+
+const getPluginJSUrl = (plugin: Plugin) => {
+  const s3Key = getPluginJSKey(plugin)
+  return getPluginUrl(s3Key)
+}
+
+const getPluginIconUrl = (plugin: Plugin): string | undefined => {
+  const s3Key = getPluginIconKey(plugin)
+  if (!s3Key) {
+    return
+  }
+  return getPluginUrl(s3Key)
+}
+
+const getPluginUrl = (s3Key: string) => {
+  if (env.CLOUDFRONT_CDN) {
+    return cloudfront.getPresignedUrl(s3Key)
+  } else {
+    return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
+  }
+}
+
+// S3 KEYS
+
+export const getPluginJSKey = (plugin: Plugin) => {
+  return getPluginS3Key(plugin, "plugin.min.js")
+}
+
+export const getPluginIconKey = (plugin: Plugin) => {
+  // stored iconUrl is deprecated - hardcode to icon.svg in this case
+  const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName
+  if (!iconFileName) {
+    return
+  }
+  return getPluginS3Key(plugin, iconFileName)
+}
+
+const getPluginS3Key = (plugin: Plugin, fileName: string) => {
+  const s3Key = getPluginS3Dir(plugin.name)
+  return `${s3Key}/${fileName}`
+}
+
+export const getPluginS3Dir = (pluginName: string) => {
+  let s3Key = `${pluginName}`
+  if (env.MULTI_TENANCY) {
+    const tenantId = tenancy.getTenantId()
+    s3Key = `${tenantId}/${s3Key}`
+  }
+  if (env.CLOUDFRONT_CDN) {
+    s3Key = `plugins/${s3Key}`
+  }
+  return s3Key
+}

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -0,0 +1,171 @@
+import * as app from "../app"
+import { getAppFileUrl } from "../app"
+import { testEnv } from "../../../../tests"
+
+describe("app", () => {
+  beforeEach(() => {
+    testEnv.nodeJest()
+  })
+
+  describe("clientLibraryUrl", () => {
+    function getClientUrl() {
+      return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url in dev", () => {
+        testEnv.nodeDev()
+        const url = getClientUrl()
+        expect(url).toBe("/api/assets/client")
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getClientUrl()
+        expect(url).toBe(
+          "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+        )
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url in dev", async () => {
+        testEnv.nodeDev()
+        await testEnv.withTenant(tenantId => {
+          const url = getClientUrl()
+          expect(url).toBe("/api/assets/client")
+        })
+      })
+
+      it("gets url with embedded minio", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withMinio()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withS3()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        await testEnv.withTenant(tenantId => {
+          testEnv.withCloudfront()
+          const url = getClientUrl()
+          expect(url).toBe(
+            "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0"
+          )
+        })
+      })
+    })
+  })
+
+  describe("getAppFileUrl", () => {
+    function getAppFileUrl() {
+      return app.getAppFileUrl("app_123/attachments/image.jpeg")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getAppFileUrl()
+        expect(url).toBe(
+          "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getAppFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/app_123/attachments/image.jpeg?")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          expect(url).toBe(
+            "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getAppFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              "http://cf.example.com/app_123/attachments/image.jpeg?"
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/global.spec.ts

@@ -0,0 +1,74 @@
+import * as global from "../global"
+import { testEnv } from "../../../../tests"
+
+describe("global", () => {
+  describe("getGlobalFileUrl", () => {
+    function getGlobalFileUrl() {
+      return global.getGlobalFileUrl("settings", "logoUrl", "etag")
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("/files/signed/global/settings/logoUrl")
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const url = getGlobalFileUrl()
+        expect(url).toBe("http://s3.example.com/global/settings/logoUrl")
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const url = getGlobalFileUrl()
+        // omit rest of signed params
+        expect(
+          url.includes("http://cf.example.com/settings/logoUrl?etag=etag&")
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(`/files/signed/global/${tenantId}/settings/logoUrl`)
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          expect(url).toBe(
+            `http://s3.example.com/global/${tenantId}/settings/logoUrl`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const url = getGlobalFileUrl()
+          // omit rest of signed params
+          expect(
+            url.includes(
+              `http://cf.example.com/${tenantId}/settings/logoUrl?etag=etag&`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/buckets/tests/plugins.spec.ts

@@ -0,0 +1,110 @@
+import * as plugins from "../plugins"
+import { structures, testEnv } from "../../../../tests"
+
+describe("plugins", () => {
+  describe("enrichPluginURLs", () => {
+    const plugin = structures.plugins.plugin()
+
+    function getEnrichedPluginUrls() {
+      const enriched = plugins.enrichPluginURLs([plugin])[0]
+      return {
+        jsUrl: enriched.jsUrl!,
+        iconUrl: enriched.iconUrl!,
+      }
+    }
+
+    describe("single tenant", () => {
+      beforeAll(() => {
+        testEnv.singleTenant()
+      })
+
+      it("gets url with embedded minio", () => {
+        testEnv.withMinio()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `/files/signed/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with custom S3", () => {
+        testEnv.withS3()
+        const urls = getEnrichedPluginUrls()
+        expect(urls.jsUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/plugin.min.js`
+        )
+        expect(urls.iconUrl).toBe(
+          `http://s3.example.com/plugins/${plugin.name}/icon.svg`
+        )
+      })
+
+      it("gets url with cloudfront + s3", () => {
+        testEnv.withCloudfront()
+        const urls = getEnrichedPluginUrls()
+        // omit rest of signed params
+        expect(
+          urls.jsUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/plugin.min.js?`
+          )
+        ).toBe(true)
+        expect(
+          urls.iconUrl.includes(
+            `http://cf.example.com/plugins/${plugin.name}/icon.svg?`
+          )
+        ).toBe(true)
+      })
+    })
+
+    describe("multi tenant", () => {
+      beforeAll(() => {
+        testEnv.multiTenant()
+      })
+
+      it("gets url with embedded minio", async () => {
+        testEnv.withMinio()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `/files/signed/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with custom S3", async () => {
+        testEnv.withS3()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          expect(urls.jsUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js`
+          )
+          expect(urls.iconUrl).toBe(
+            `http://s3.example.com/plugins/${tenantId}/${plugin.name}/icon.svg`
+          )
+        })
+      })
+
+      it("gets url with cloudfront + s3", async () => {
+        testEnv.withCloudfront()
+        await testEnv.withTenant(tenantId => {
+          const urls = getEnrichedPluginUrls()
+          // omit rest of signed params
+          expect(
+            urls.jsUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/plugin.min.js?`
+            )
+          ).toBe(true)
+          expect(
+            urls.iconUrl.includes(
+              `http://cf.example.com/plugins/${tenantId}/${plugin.name}/icon.svg?`
+            )
+          ).toBe(true)
+        })
+      })
+    })
+  })
+})

packages/backend-core/src/objectStore/cloudfront.ts

@@ -0,0 +1,41 @@
+import env from "../environment"
+const cfsign = require("aws-cloudfront-sign")
+
+let PRIVATE_KEY: string | undefined
+
+function getPrivateKey() {
+  if (!env.CLOUDFRONT_PRIVATE_KEY_64) {
+    throw new Error("CLOUDFRONT_PRIVATE_KEY_64 is not set")
+  }
+
+  if (PRIVATE_KEY) {
+    return PRIVATE_KEY
+  }
+
+  PRIVATE_KEY = Buffer.from(env.CLOUDFRONT_PRIVATE_KEY_64, "base64").toString(
+    "utf-8"
+  )
+
+  return PRIVATE_KEY
+}
+
+const getCloudfrontSignParams = () => {
+  return {
+    keypairId: env.CLOUDFRONT_PUBLIC_KEY_ID,
+    privateKeyString: getPrivateKey(),
+    expireTime: new Date().getTime() + 1000 * 60 * 60, // 1 hour
+  }
+}
+
+export const getPresignedUrl = (s3Key: string) => {
+  const url = getUrl(s3Key)
+  return cfsign.getSignedUrl(url, getCloudfrontSignParams())
+}
+
+export const getUrl = (s3Key: string) => {
+  let prefix = "/"
+  if (s3Key.startsWith("/")) {
+    prefix = ""
+  }
+  return `${env.CLOUDFRONT_CDN}${prefix}${s3Key}`
+}

packages/backend-core/src/objectStore/index.ts

@@ -1,2 +1,3 @@
 export * from "./objectStore"
 export * from "./utils"
+export * from "./buckets"

packages/backend-core/src/objectStore/objectStore.ts

@@ -8,7 +8,7 @@ import { promisify } from "util"
 import { join } from "path"
 import fs from "fs"
 import env from "../environment"
-import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
+import { budibaseTempDir } from "./utils"
 import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 
@@ -26,7 +26,7 @@ type UploadParams = {
   bucket: string
   filename: string
   path: string
-  type?: string
+  type?: string | null
   // can be undefined, we will remove it
   metadata?: {
     [key: string]: string | undefined
@@ -41,6 +41,7 @@ const CONTENT_TYPE_MAP: any = {
   json: "application/json",
   gz: "application/gzip",
 }
+
 const STRING_CONTENT_TYPES = [
   CONTENT_TYPE_MAP.html,
   CONTENT_TYPE_MAP.css,
@@ -58,35 +59,17 @@ export function sanitizeBucket(input: string) {
   return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }
 
-function publicPolicy(bucketName: string) {
-  return {
-    Version: "2012-10-17",
-    Statement: [
-      {
-        Effect: "Allow",
-        Principal: {
-          AWS: ["*"],
-        },
-        Action: "s3:GetObject",
-        Resource: [`arn:aws:s3:::${bucketName}/*`],
-      },
-    ],
-  }
-}
-
-const PUBLIC_BUCKETS = [
-  ObjectStoreBuckets.APPS,
-  ObjectStoreBuckets.GLOBAL,
-  ObjectStoreBuckets.PLUGINS,
-]
-
 /**
  * Gets a connection to the object store using the S3 SDK.
  * @param {string} bucket the name of the bucket which blobs will be uploaded/retrieved from.
+ * @param {object} opts configuration for the object store.
  * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-export const ObjectStore = (bucket: string) => {
+export const ObjectStore = (
+  bucket: string,
+  opts: { presigning: boolean } = { presigning: false }
+) => {
   const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
@@ -100,9 +83,20 @@ export const ObjectStore = (bucket: string) => {
       Bucket: sanitizeBucket(bucket),
     }
   }
+
+  // custom S3 is in use i.e. minio
   if (env.MINIO_URL) {
+    if (opts.presigning && env.MINIO_ENABLED) {
+      // IMPORTANT: Signed urls will inspect the host header of the request.
+      // Normally a signed url will need to be generated with a specified host in mind.
+      // To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      // use a predefined host. The host 'minio-service' is also forwarded to minio requests via nginx
+      config.endpoint = "minio-service"
+    } else {
       config.endpoint = env.MINIO_URL
     }
+  }
+
   return new AWS.S3(config)
 }
 
@@ -135,16 +129,6 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => {
         await promises[bucketName]
         delete promises[bucketName]
       }
-      // public buckets are quite hidden in the system, make sure
-      // no bucket is set accidentally
-      if (PUBLIC_BUCKETS.includes(bucketName)) {
-        await client
-          .putBucketPolicy({
-            Bucket: bucketName,
-            Policy: JSON.stringify(publicPolicy(bucketName)),
-          })
-          .promise()
-      }
     } else {
       throw new Error("Unable to write to object store bucket.")
     }
@@ -274,6 +258,36 @@ export const listAllObjects = async (bucketName: string, path: string) => {
   return objects
 }
 
+/**
+ * Generate a presigned url with a default TTL of 1 hour
+ */
+export const getPresignedUrl = (
+  bucketName: string,
+  key: string,
+  durationSeconds: number = 3600
+) => {
+  const objectStore = ObjectStore(bucketName, { presigning: true })
+  const params = {
+    Bucket: sanitizeBucket(bucketName),
+    Key: sanitizeKey(key),
+    Expires: durationSeconds,
+  }
+  const url = objectStore.getSignedUrl("getObject", params)
+
+  if (!env.MINIO_ENABLED) {
+    // return the full URL to the client
+    return url
+  } else {
+    // return the path only to the client
+    // use the presigned url route to ensure the static
+    // hostname will be used in the request
+    const signedUrl = new URL(url)
+    const path = signedUrl.pathname
+    const query = signedUrl.search
+    return `/files/signed${path}${query}`
+  }
+}
+
 /**
  * Same as retrieval function but puts to a temporary file.
  */
@@ -315,9 +329,9 @@ export const deleteFile = async (bucketName: string, filepath: string) => {
   await makeSureBucketExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
-    Key: filepath,
+    Key: sanitizeKey(filepath),
   }
-  return objectStore.deleteObject(params)
+  return objectStore.deleteObject(params).promise()
 }
 
 export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
@@ -326,7 +340,7 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => {
   const params = {
     Bucket: bucketName,
     Delete: {
-      Objects: filepaths.map((path: any) => ({ Key: path })),
+      Objects: filepaths.map((path: any) => ({ Key: sanitizeKey(path) })),
     },
   }
   return objectStore.deleteObjects(params).promise()

packages/backend-core/src/objectStore/utils.ts

@@ -14,7 +14,6 @@ export const ObjectStoreBuckets = {
   APPS: env.APPS_BUCKET_NAME,
   TEMPLATES: env.TEMPLATES_BUCKET_NAME,
   GLOBAL: env.GLOBAL_BUCKET_NAME,
-  GLOBAL_CLOUD: env.GLOBAL_CLOUD_BUCKET_NAME,
   PLUGINS: env.PLUGIN_BUCKET_NAME,
 }
 

packages/backend-core/src/pino.ts

@@ -1,13 +0,0 @@
-import env from "./environment"
-
-export function pinoSettings() {
-  return {
-    prettyPrint: {
-      levelFirst: true,
-    },
-    level: env.LOG_LEVEL || "error",
-    autoLogging: {
-      ignore: (req: { url: string }) => req.url.includes("/health"),
-    },
-  }
-}

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -137,4 +137,4 @@ class InMemoryQueue {
   }
 }
 
-export = InMemoryQueue
+export default InMemoryQueue

packages/backend-core/src/redis/redis.ts

@@ -276,4 +276,4 @@ class RedisWrapper {
   }
 }
 
-export = RedisWrapper
+export default RedisWrapper