Merge branch 'develop' into merge_create_user_modals

# Conflicts: # packages/builder/src/pages/builder/portal/manage/users/_components/BasicOnboardingModal.svelte
2022-06-17 14:23:36 +02:00 · 2022-06-17 14:23:36 +02:00 · 4799af1df6
parent 1212deb043 7050ffc2cb
commit 4799af1df6
764 changed files with 34803 additions and 13428 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,9 @@
+packages/server/node_modules
+packages/builder
+packages/frontend-core
+packages/backend-core
+packages/worker/node_modules
+packages/cli
+packages/client
+packages/bbui
+packages/string-templates
--- a/.eslintignore
+++ b/.eslintignore
@ -6,4 +6,5 @@ packages/server/coverage
 packages/server/client
 packages/builder/.routify
 packages/builder/cypress/support/queryLevelTransformerFunction.js
-packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
+packages/builder/cypress/support/queryLevelTransformerFunctionWithData.js
+packages/builder/cypress/reports
--- a/.github/CONTRIBUTING.md
+++ b/.github/CONTRIBUTING.md
@ -93,6 +93,8 @@ then `cd ` into your local copy.

 #### 3.  Install and Build

+| **NOTE**: On Windows, all yarn commands must be executed on a bash shell (e.g. git bash)
+
 To develop the Budibase platform you'll need [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) installed.

 ##### Quick method
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@ -7,6 +7,15 @@ assignees: ''

 ---

+**Hosting**
+<!-- Delete as appropriate -->
+- Self
+  - Method: <method> <!-- One of: k8s, docker single image, docker compose, digital ocean: -->
+  - Budibase Version: <version> <!-- e.g. 1.0.105 -->
+  - App Version: <version> <!-- Indicate app version if bug is related to an application -->
+- Cloud
+  - Tenant ID: <tenantId> <!-- shown in URL as <tenantID>.budibase.app -->
+
 **Describe the bug**
 A clear and concise description of what the bug is.

--- a/.github/stale.yml
+++ b/.github/stale.yml
@ -14,7 +14,6 @@ staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable
 markComment: >
  This issue has been automatically marked as stale because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
+  recent activity.
 # Comment to post when closing a stale issue. Set to `false` to disable
 closeComment: false
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -11,6 +11,13 @@ on:
    branches:
      - master
      - develop 
+      - release 
+ workflow_dispatch:
+
+env:
+  BRANCH: ${{ github.event.pull_request.head.ref }}
+  BASE_BRANCH: ${{ github.event.pull_request.base.ref}}
+  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

 jobs:
  build:
@ -27,6 +34,10 @@ jobs:
      uses: actions/setup-node@v1
      with:
        node-version: ${{ matrix.node-version }}
+
+    - name: Install Pro
+      run: yarn install:pro $BRANCH $BASE_BRANCH
+
    - run: yarn
    - run: yarn bootstrap
    - run: yarn lint
--- a/.github/workflows/deploy-cloud.yaml
+++ b/.github/workflows/deploy-cloud.yaml
@ -66,7 +66,7 @@ jobs:
          config-files: values.production.yaml
          chart-path: charts/budibase
          namespace: budibase
-          values: globals.appVersion=v${{ env.RELEASE_VERSION }}
+          values: globals.appVersion=v${{ env.RELEASE_VERSION }},services.couchdb.url=${{ secrets.PRODUCTION_COUCHDB_URL }},services.couchdb.password=${{ secrets.PRODUCTION_COUCHDB_PASSWORD }}
          name: budibase-prod

      - name: Discord Webhook Action
--- a/.github/workflows/deploy-preprod.yml
+++ b/.github/workflows/deploy-preprod.yml
@ -4,9 +4,7 @@ on:
 workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}

 jobs:
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -1,9 +1,10 @@
 name: Budibase Release Staging
+concurrency: release-develop

 on: 
 push:
    branches:
-      - develop
+      - release
    paths:
      - '.aws/**'
      - '.github/**'
@ -14,21 +15,32 @@ on:
      - 'yarn.lock'
      - 'package.json'
      - 'yarn.lock'
+ workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      
 jobs:
  release:
    runs-on: ubuntu-latest 

    steps:
+      # - name: Fail if branch is not develop
+      #   if: github.ref != 'refs/heads/develop' 
+      #   run: | 
+      #     echo "Ref is not develop, you must run this job from develop."
+      #     exit 1
      - uses: actions/checkout@v2
      - uses: actions/setup-node@v1
        with:
          node-version: 14.x
+
+      - name: Install Pro
+        run: yarn install:pro develop
+
      - run: yarn 
      - run: yarn bootstrap 
      - run: yarn lint 
@ -46,9 +58,9 @@ jobs:
        env:
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
        run: | 
-          # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
-          git config user.name "Budibase Staging Release Bot"
-          git config user.email "<>"
+          # setup the username and email.
+          git config --global user.name "Budibase Staging Release Bot"
+          git config --global user.email "<>"
          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc 
          yarn release:develop

@ -60,3 +72,56 @@ jobs:
        env:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+
+      - name: Get the latest budibase release version
+        id: version
+        run: | 
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - name: Tag and release Proxy service docker image
+        run: | 
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:proxy:release
+          docker tag proxy-service budibase/proxy:$RELEASE_TAG
+          docker push budibase/proxy:$RELEASE_TAG
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          RELEASE_TAG: k8s-release
+
+      - name: Pull values.yaml from budibase-infra
+        run: | 
+          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
+          -H 'Accept: application/vnd.github.v3.raw' \
+          -o values.release.yaml \
+          -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-release/values.yaml
+          wc -l values.release.yaml
+
+      - name: Deploy to Release Environment
+        uses: glopezep/helm@v1.7.1
+        with:
+          release: budibase-release
+          namespace: budibase
+          chart: charts/budibase
+          token: ${{ github.token }}
+          helm: helm3
+          values: |
+            globals: 
+              appVersion: develop
+            ingress:
+              enabled: true
+              nginx: true
+          value-files: >-
+            [
+              "values.release.yaml"
+            ]
+        env:
+          KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
+
+      - name: Discord Webhook Action
+        uses: tsickert/discord-webhook@v4.0.0
+        with:
+          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
+          embed-title: ${{ env.RELEASE_VERSION }}
--- a/.github/workflows/release-selfhost.yml
+++ b/.github/workflows/release-selfhost.yml
@ -87,3 +87,10 @@ jobs:
            packages/cli/build/cli-macos 
            packages/server/specs/openapi.yaml
            packages/server/specs/openapi.json
+
+      - name: Discord Webhook Action
+        uses: tsickert/discord-webhook@v4.0.0
+        with:
+          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host."
+          embed-title: ${{ env.RELEASE_VERSION }}
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -1,4 +1,5 @@
 name: Budibase Release
+concurrency: release

 on:
 push:
@ -14,22 +15,33 @@ on:
      - 'yarn.lock'
      - 'package.json'
      - 'yarn.lock'
+ workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

 jobs:
  release:
    runs-on: ubuntu-latest

    steps:
+      - name: Fail if branch is not master
+        if: github.ref != 'refs/heads/master' 
+        run: | 
+          echo "Ref is not master, you must run this job from master."
+          exit 1
      - uses: actions/checkout@v2
      - uses: actions/setup-node@v1
        with:
          node-version: 14.x
+
+      - name: Install Pro
+        run: yarn install:pro master
+
      - run: yarn
      - run: yarn bootstrap
      - run: yarn lint
@ -48,8 +60,8 @@ jobs:
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
        run: |
          # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
-          git config user.name "Budibase Release Bot"
-          git config user.email "<>"
+          git config --global user.name "Budibase Release Bot"
+          git config --global user.email "<>"
          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
          yarn release

@ -66,3 +78,57 @@ jobs:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
          BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
+      
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+
+      - name: Tag and release Proxy service docker image
+        run: | 
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:proxy:preprod
+          docker tag proxy-service budibase/proxy:$PREPROD_TAG
+          docker push budibase/proxy:$PREPROD_TAG
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          PREPROD_TAG: k8s-preprod
+
+      - name: Pull values.yaml from budibase-infra
+        run: | 
+          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
+          -H 'Accept: application/vnd.github.v3.raw' \
+          -o values.preprod.yaml \
+          -L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml
+          wc -l values.preprod.yaml
+
+      - name: Deploy to Preprod Environment
+        uses: glopezep/helm@v1.7.1
+        with:
+          release: budibase-preprod
+          namespace: budibase
+          chart: charts/budibase
+          token: ${{ github.token }}
+          helm: helm3
+          values: |
+            globals: 
+              appVersion: ${{ steps.previoustag.outputs.tag }}
+            ingress:
+              enabled: true
+              nginx: true
+          value-files: >-
+            [
+              "values.preprod.yaml"
+            ]
+        env:
+          KUBECONFIG_FILE: '${{ secrets.PREPROD_KUBECONFIG }}'
+
+      - name: Discord Webhook Action
+        uses: tsickert/discord-webhook@v4.0.0
+        with:
+          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
+          content: "Preprod Deployment Complete: ${{ steps.previoustag.outputs.tag }} deployed to Budibase Pre-prod."
+          embed-title: ${{ steps.previoustag.outputs.tag }}
--- a/.github/workflows/smoke_test.yaml
+++ b/.github/workflows/smoke_test.yaml
@ -2,6 +2,8 @@ name: Budibase Smoke Test

 on:
 workflow_dispatch:
+ schedule:
+  - cron: "0 5 * * *" # every day at 5AM

 jobs:
  release:
@ -23,24 +25,28 @@ jobs:
          -o packages/builder/cypress.env.json \
          -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json
          wc -l packages/builder/cypress.env.json
-      - run: yarn test:e2e:ci
-        env:
-          CI: true
-          name: Budibase CI
-      
-      # TODO: upload recordings to s3
-      # - name: Configure AWS Credentials
-      #   uses: aws-actions/configure-aws-credentials@v1
-      #   with:
-      #     aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-      #     aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      #     aws-region: eu-west-1
-        
-      # TODO look at cypress reporters
-      # - name: Discord Webhook Action
-      #   uses: tsickert/discord-webhook@v4.0.0
-      #   with:
-      #     webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
-      #     content: "Production Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Cloud."
-      #     embed-title: ${{ env.RELEASE_VERSION }}

+      - name: Cypress run
+        id: cypress
+        continue-on-error: true
+        uses: cypress-io/github-action@v2
+        with:
+          record: true
+          install: false
+          tag: nightly
+          command: yarn test:e2e:ci:record
+        env:
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+      
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Test Reports
+          path: packages/builder/cypress/reports/testReport.html
+
+      - name: Cypress Discord Notify
+        run: yarn test:e2e:ci:notify
+        env:
+          CYPRESS_WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }}
+          CYPRESS_OUTCOME: ${{ steps.cypress.outcome }}
+          CYPRESS_DASHBOARD_URL: ${{ steps.cypress.outputs.dashboardUrl }}
+          GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
--- a/.gitignore
+++ b/.gitignore
@ -97,5 +97,9 @@ hosting/proxy/.generated-nginx.prod.conf

 bin/
 hosting/.generated*
-packages/builder/cypress.env.json
-stats.html
+packages/builder/cypress.env.json
+packages/builder/cypress/reports
+stats.html
+
+# TypeScript cache
+*.tsbuildinfo
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -22,9 +22,16 @@
            "name": "Budibase Worker",
            "type": "node",
            "request": "launch",
-            "program": "${workspaceFolder}/packages/worker/src/index.js",
+            "runtimeArgs": [
+                "--nolazy",
+                "-r",
+                "ts-node/register/transpile-only"
+            ],
+            "args": [
+                "${workspaceFolder}/packages/worker/src/index.ts"
+            ],
            "cwd": "${workspaceFolder}/packages/worker"
-        }
+        },
    ],
    "compounds": [
        {
--- a/charts/budibase/Chart.yaml
+++ b/charts/budibase/Chart.yaml
@ -11,11 +11,11 @@ sources:
  - https://github.com/Budibase/budibase
  - https://budibase.com
 type: application
-version: 0.2.8
+version: 0.2.10
 appVersion: 1.0.48
 dependencies:
  - name: couchdb
-    version: 3.3.4
+    version: 3.6.1
    repository: https://apache.github.io/couchdb-helm
    condition: services.couchdb.enabled
  - name: ingress-nginx
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@ -28,12 +28,15 @@ spec:
      - env:
        - name: BUDIBASE_ENVIRONMENT
          value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
        - name: COUCH_DB_URL
          {{ if .Values.services.couchdb.url }}
          value: {{ .Values.services.couchdb.url }}
          {{ else }}
          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
          {{ end }}
+        {{ if .Values.services.couchdb.enabled }}
        - name: COUCH_DB_USER
          valueFrom:
            secretKeyRef:
@ -44,6 +47,7 @@ spec:
            secretKeyRef:
              name: {{ template "couchdb.fullname" . }}
              key: adminPassword
+        {{ end }}
        - name: ENABLE_ANALYTICS
          value: {{ .Values.globals.enableAnalytics | quote }}
        - name: INTERNAL_API_KEY
@ -76,6 +80,10 @@ spec:
          value: {{ .Values.services.objectStore.url }}
        - name: PORT
          value: {{ .Values.services.apps.port | quote }}
+        {{ if .Values.services.worker.publicApiRateLimitPerSecond }}
+        - name: API_REQ_LIMIT_PER_SEC
+          value: {{ .Values.globals.apps.publicApiRateLimitPerSecond | quote }}
+        {{ end }}
        - name: MULTI_TENANCY
          value: {{ .Values.globals.multiTenancy | quote }}
        - name: LOG_LEVEL
@ -98,10 +106,6 @@ spec:
          value: http://worker-service:{{ .Values.services.worker.port }}
        - name: PLATFORM_URL
          value: {{ .Values.globals.platformUrl | quote }}
-        - name: USE_QUOTAS
-          value: {{ .Values.globals.useQuotas | quote }}
-        - name: EXCLUDE_QUOTAS_TENANTS
-          value: {{ .Values.globals.excludeQuotasTenants | quote }}
        - name: ACCOUNT_PORTAL_URL
          value: {{ .Values.globals.accountPortalUrl | quote }}
        - name: ACCOUNT_PORTAL_API_KEY
@ -110,12 +114,35 @@ spec:
          value: {{ .Values.globals.cookieDomain | quote }}
        - name: HTTP_MIGRATIONS
          value: {{ .Values.globals.httpMigrations | quote }}
+        - name: GOOGLE_CLIENT_ID
+          value: {{ .Values.globals.google.clientId | quote }}
+        - name: GOOGLE_CLIENT_SECRET
+          value: {{ .Values.globals.google.secret | quote }}
+        - name: AUTOMATION_MAX_ITERATIONS
+          value: {{ .Values.globals.automationMaxIterations | quote }}
+        - name: TENANT_FEATURE_FLAGS
+          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+
        image: budibase/apps:{{ .Values.globals.appVersion }}
        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.apps.port }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
        name: bbapps
        ports:
        - containerPort: {{ .Values.services.apps.port }}
        resources: {}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
      restartPolicy: Always
      serviceAccountName: ""
 status: {}
--- a/charts/budibase/templates/couchdb-backup.yaml
+++ b/charts/budibase/templates/couchdb-backup.yaml
@ -39,5 +39,13 @@ spec:
        imagePullPolicy: Always
        name: couchdb-backup
        resources: {}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
 status: {}
 {{- end }}
--- a/charts/budibase/templates/minio-data-persistentvolumeclaim.yaml
+++ b/charts/budibase/templates/minio-data-persistentvolumeclaim.yaml
@ -12,5 +12,8 @@ spec:
  resources:
    requests:
      storage: {{ .Values.services.objectStore.storage }}
+  {{ if .Values.services.objectStore.storageClass }}
+  storageClassName: {{ .Values.services.objectStore.storageClass }}
+  {{- end }}
 status: {}
-{{- end }}
+{{- end }}
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@ -60,6 +60,14 @@ spec:
        volumeMounts:
        - mountPath: /data
          name: minio-data
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
      restartPolicy: Always
      serviceAccountName: ""
      volumes:
--- a/charts/budibase/templates/proxy-service-deployment.yaml
+++ b/charts/budibase/templates/proxy-service-deployment.yaml
@ -32,6 +32,14 @@ spec:
        - containerPort: {{ .Values.services.proxy.port }}
        resources: {}
        volumeMounts:
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
      restartPolicy: Always
      serviceAccountName: ""
      volumes:
--- a/charts/budibase/templates/redis-data-persistentvolumeclaim.yaml
+++ b/charts/budibase/templates/redis-data-persistentvolumeclaim.yaml
@ -12,5 +12,8 @@ spec:
  resources:
    requests:
      storage: {{ .Values.services.redis.storage }}
+  {{ if .Values.services.redis.storageClass }}
+  storageClassName: {{ .Values.services.redis.storageClass }}
+  {{ end }}
 status: {}
-{{- end }}
+{{- end }}
--- a/charts/budibase/templates/redis-service-deployment.yaml
+++ b/charts/budibase/templates/redis-service-deployment.yaml
@ -39,6 +39,14 @@ spec:
        volumeMounts:
        - mountPath: /data
          name: redis-data
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
      restartPolicy: Always
      serviceAccountName: ""
      volumes:
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@ -27,8 +27,11 @@ spec:
    spec:
      containers:
      - env:
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
        - name: CLUSTER_PORT
          value: {{ .Values.services.worker.port | quote }}
+        {{ if .Values.services.couchdb.enabled }}
        - name: COUCH_DB_USER
          valueFrom:
            secretKeyRef:
@ -39,6 +42,7 @@ spec:
            secretKeyRef:
              name: {{ template "couchdb.fullname" . }}
              key: adminPassword
+        {{ end }}
        - name: COUCH_DB_URL
          {{ if .Values.services.couchdb.url }}
          value: {{ .Values.services.couchdb.url }}
@ -89,6 +93,10 @@ spec:
          value: {{ .Values.globals.selfHosted | quote }}
        - name: SENTRY_DSN
          value: {{ .Values.globals.sentryDSN }}
+        - name: ENABLE_ANALYTICS
+          value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: POSTHOG_TOKEN
+          value: {{ .Values.globals.posthogToken }}
        - name: ACCOUNT_PORTAL_URL
          value: {{ .Values.globals.accountPortalUrl | quote }}
        - name: ACCOUNT_PORTAL_API_KEY
@ -117,10 +125,24 @@ spec:
          value: {{ .Values.globals.google.secret | quote }}
        image: budibase/worker:{{ .Values.globals.appVersion }}
        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.worker.port }}
+          initialDelaySeconds: 5
+          periodSeconds: 5
        name: bbworker
        ports:
        - containerPort: {{ .Values.services.worker.port }}
        resources: {}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
      restartPolicy: Always
      serviceAccountName: ""
 status: {}
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@ -47,6 +47,8 @@ ingress:
  className: ""
  annotations:
    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/client-max-body-size: 150M
+    nginx.ingress.kubernetes.io/proxy-body-size: 50m
  hosts:
    - host: # change if using custom domain
      paths:
@ -87,22 +89,21 @@ affinity: {}
 globals:
  appVersion: "latest"
  budibaseEnv: PRODUCTION
-  enableAnalytics: true
+  enableAnalytics: "1"
  sentryDSN: ""
-  posthogToken: ""
+  posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS"
  logLevel: info
  selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
  multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
-  useQuotas: "0"
-  excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
  accountPortalUrl: ""
  accountPortalApiKey: ""
  cookieDomain: ""
  platformUrl: ""
  httpMigrations: "0"
  google:
-    clientId: "" 
+    clientId: ""
    secret: ""
+  automationMaxIterations: "500"

  createSecrets: true # creates an internal API key, JWT secrets and redis password for you

@ -150,6 +151,11 @@ services:
    url: "" # only change if pointing to existing redis cluster and enabled: false
    password: "budibase" # recommended to override if using built-in redis
    storage: 100Mi
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.
+    storageClass: ""

  objectStore:
    minio: true
@ -161,6 +167,11 @@ services:
    region: "" # AWS_REGION if using S3 or existing minio secret
    url: "http://minio-service:9000" # only change if pointing to existing minio cluster or S3 and minio: false
    storage: 100Mi
+    ## If defined, storageClassName: <storageClass>
+    ## If set to "-", storageClassName: "", which disables dynamic provisioning
+    ## If undefined (the default) or set to null, no storageClassName spec is
+    ##   set, choosing the default provisioner.
+    storageClass: ""

 # Override values in couchDB subchart
 couchdb:
@ -204,7 +215,7 @@ couchdb:
  ## The CouchDB image
  image:
    repository: couchdb
-    tag: 3.1.0
+    tag: 3.2.1
    pullPolicy: IfNotPresent

  ## Experimental integration with Lucene-powered fulltext search
@ -230,6 +241,8 @@ couchdb:
  ## Optional tolerations
  tolerations: []

+  affinity: {}
+
  service:
    # annotations:
    enabled: true
--- a/examples/nextjs-api-sales/yarn.lock
+++ b/examples/nextjs-api-sales/yarn.lock
@ -1894,9 +1894,9 @@ minimist-options@4.1.0:
    kind-of "^6.0.3"

 minimist@^1.2.0:
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==

 minipass-collect@^1.0.2:
  version "1.0.2"
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@ -27,6 +27,7 @@ services:
    image: nginx:latest
    volumes:
      - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf
+      - ./proxy/error.html:/usr/share/nginx/html/error.html
    ports:
      - "${MAIN_PORT}:10000"
    depends_on:
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@ -117,7 +117,6 @@ services:
    labels:
      - "com.centurylinklabs.watchtower.enable=false"

-
 volumes:
  couchdb3_data:
    driver: local
--- a/hosting/nginx.dev.conf.hbs
+++ b/hosting/nginx.dev.conf.hbs
@ -28,6 +28,12 @@ http {
    ignore_invalid_headers off;
    proxy_buffering off;

+    error_page 502 503 504 /error.html;
+    location = /error.html {
+      root /usr/share/nginx/html; 
+      internal;
+    }
+
    location /db/ {
      proxy_pass      http://couchdb-service:5984;
      rewrite ^/db/(.*)$ /$1 break;
--- a/hosting/nginx.prod.conf.hbs
+++ b/hosting/nginx.prod.conf.hbs
@ -30,7 +30,7 @@ http {
  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent" "$http_x_forwarded_for"';
-  
+
  map $http_upgrade $connection_upgrade {
    default     "upgrade";
  }
@ -42,13 +42,31 @@ http {
    client_max_body_size 1000m;
    ignore_invalid_headers off;
    proxy_buffering off;
-    # port_in_redirect off;
+
+    set $csp_default "default-src 'self'";
+    set $csp_script "script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io";
+    set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
+    set $csp_object "object-src 'none'";
+    set $csp_base_uri "base-uri 'self'";
+    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
+    set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
+    set $csp_frame "frame-src 'self' https:";
+    set $csp_img "img-src http: https: data: blob:";
+    set $csp_manifest "manifest-src 'self'";
+    set $csp_media "media-src 'self' https://js.intercomcdn.com";
+    set $csp_worker "worker-src 'none'";
+
+    error_page 502 503 504 /error.html;
+    location = /error.html {
+      root /usr/share/nginx/html;
+      internal;
+    }

    # Security Headers
    add_header X-Frame-Options SAMEORIGIN always;
    add_header X-Content-Type-Options nosniff always;
    add_header X-XSS-Protection "1; mode=block" always;
-    add_header Content-Security-Policy  "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.budi.live https://js.intercomcdn.com https://widget.intercom.io; style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com; object-src 'none'; base-uri 'self'; connect-src 'self' https://api-iam.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io ; font-src 'self' data https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com; frame-src 'self' https:; img-src http: https: data; manifest-src 'self'; media-src 'self'; worker-src 'none';" always;
+    add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;

    # upstreams
    set $apps {{ apps }};
@ -136,4 +154,4 @@ http {
    gzip_comp_level 6;
    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
  }
-}
+}
--- a/hosting/portainer/template.json
+++ b/hosting/portainer/template.json
@ -0,0 +1,94 @@
+{
+  "version": "2",
+  "templates": [
+    {
+      "type": 3,
+      "title": "Budibase",
+      "categories": ["Tools"],
+      "description": "Build modern business apps in minutes",
+      "logo": "https://budibase.com/favicon.ico",
+      "platform": "linux",
+      "repository": {
+        "url": "https://github.com/Budibase/budibase",
+        "stackfile": "hosting/docker-compose.yaml"
+      },
+      "env": [
+        {
+          "name": "MAIN_PORT",
+          "label": "Main port",
+          "default": "10000"
+        },
+        {
+          "name": "JWT_SECRET",
+          "label": "JWT secret",
+          "default": "change-me"
+        },
+        {
+          "name": "MINIO_ACCESS_KEY",
+          "label": "MinIO access key",
+          "default": "change-me"
+        },
+        {
+          "name": "MINIO_SECRET_KEY",
+          "label": "MinIO secret key",
+          "default": "change-me"
+        },
+        {
+          "name": "COUCH_DB_USER",
+          "default": "budibase",
+          "preset": true
+        },
+        {
+          "name": "COUCH_DB_PASSWORD",
+          "label": "Couch DB password",
+          "default": "change-me"
+        },
+        {
+          "name": "REDIS_PASSWORD",
+          "label": "Redis password",
+          "default": "change-me"
+        },
+        {
+          "name": "INTERNAL_API_KEY",
+          "label": "Internal API key",
+          "default": "change-me"
+        },
+        {
+          "name": "APP_PORT",
+          "default": "4002",
+          "preset": true
+        },
+        {
+          "name": "WORKER_PORT",
+          "default": "4003",
+          "preset": true
+        },
+        {
+          "name": "MINIO_PORT",
+          "default": "4004",
+          "preset": true
+        },
+        {
+          "name": "COUCH_DB_PORT",
+          "default": "4005",
+          "preset": true
+        },
+        {
+          "name": "REDIS_PORT",
+          "default": "6379",
+          "preset": true
+        },
+        {
+          "name": "WATCHTOWER_PORT",
+          "default": "6161",
+          "preset": true
+        },
+        {
+          "name": "BUDIBASE_ENVIRONMENT",
+          "default": "PRODUCTION",
+          "preset": true
+        }
+      ]
+    }
+  ]
+}
--- a/hosting/proxy/Dockerfile
+++ b/hosting/proxy/Dockerfile
@ -1,2 +1,3 @@
 FROM nginx:latest
-COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
+COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
+COPY error.html /usr/share/nginx/html/error.html
--- a/hosting/proxy/error.html
+++ b/hosting/proxy/error.html
@ -0,0 +1,175 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <title>Budibase</title>
+    <meta name="viewport" content="width=device-width,initial-scale=1">  
+  </head>
+
+  <script>
+    function checkStatusButton() {
+      if (window.location.href.includes("budibase.app")) {
+        var button = document.getElementById("statusButton")
+        button.removeAttribute("hidden")
+      }
+    }
+
+    function goToStatus() {
+      window.location.href = "https://status.budibase.com";
+    }
+    function goHome() {
+      window.location.href = window.location.origin;
+    }
+    function getStatus() {
+      var http = new XMLHttpRequest()
+      var url = window.location.href
+      http.open('GET', url, true)
+      http.send()
+      http.onreadystatechange = (e) => {
+        var status = http.status
+        document.getElementById("status").innerHTML = status
+
+        var message
+        if (status === 502) {
+          message = "Bad gateway. Please try again later."
+        } else if (status === 503) {
+          message = "Service Unavailable. Please try again later."
+        } else if (status === 504) {
+          message = "Gateway timeout. Please try again later."
+        } else {
+          message = "Please try again later."
+        }
+
+        document.getElementById("message").innerHTML = message
+      }
+    }
+
+    window.onload = function() {
+      checkStatusButton()
+      getStatus()
+    };
+   
+  </script>
+
+  <style>
+
+    :root {
+      --spectrum-global-color-gray-600: rgb(144,144,144);
+      --spectrum-global-color-gray-900: rgb(255,255,255);
+      --spectrum-global-color-gray-800: rgb(227,227,227);
+      --spectrum-global-color-static-blue-600: rgb(20,115,230);
+      --spectrum-global-color-static-blue-hover: rgb(	18, 103, 207);
+    }
+
+    html, body {
+      background-color: #1a1a1a;
+      padding: 0;
+      margin: 0;
+      overflow: hidden;
+      color: #e7e7e7;
+      font-family: 'Roboto', sans-serif;
+    } 
+    button {
+      color: #e7e7e7;
+      font-family: 'Roboto', sans-serif;
+      border: none;
+      font-size: 15px;
+      border-radius: 15px;
+      padding: 8px 22px;
+    }
+    button:hover {
+      cursor: pointer;
+    }
+    .main {
+      height: 100vh;
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      justify-content: center;
+    }
+    .info {
+      display: flex;
+      flex-direction: column;
+      align-items: left;
+    }
+
+    @media only screen and (max-width: 600px) {
+      .info {
+        align-items: center;
+      }
+    }
+
+    .status {
+      color: var(--spectrum-global-color-gray-600)
+    }
+    .title {
+      font-weight: 400;
+      color: var(--spectrum-global-color-gray-900)
+    }
+    .message {
+      font-weight: 200;
+      color: var(--spectrum-global-color-gray-800)
+    }
+    .buttons {
+      display: flex;
+      flex-direction: row;
+      margin-top: 15px;
+    }
+    .homeButton {
+      background-color: var(--spectrum-global-color-static-blue-600);
+    }
+    .homeButton:hover {
+      background-color: var(--spectrum-global-color-static-blue-hover);
+    }
+    .statusButton {
+      background-color: transparent;
+      margin-left: 20px;
+      border: none;
+    }
+    .hero {
+      height: 160px;
+      width: 160px;
+      margin-right: 80px;
+    }
+    .content {
+      display: flex;
+      flex-direction: row;
+      align-items: flex-end;
+      justify-content: center;
+    }
+
+    @media only screen and (max-width: 600px) {
+      .content {
+        flex-direction: column;
+      }
+    }
+  </style>
+
+  <script src="">
+  </script>
+
+  <body>
+    <div class="main">
+      <div class="content">
+        <div class="hero">
+          <img src="https://raw.githubusercontent.com/Budibase/budibase/master/packages/builder/assets/bb-space-man.svg" alt="Budibase Logo">
+        </div>
+        <div class="info">
+          <div>
+            <h4 id="status" class="status"></h4>
+            <h1 class="title">
+              Houston we have a problem!
+            </h1>
+            <h3 id="message" class="message">
+            </h3>
+          </div>
+          <div class="buttons">
+            <button class="homeButton" onclick=goHome()>Return home</button> 
+            <button id="statusButton" class="statusButton" hidden="true" onclick=goToStatus()>Check out status</button> 
+          </div>
+        </div>
+      </div>
+    </div>
+  </body>
+
+</html>
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@ -0,0 +1,99 @@
+FROM couchdb
+
+ENV DEPLOYMENT_ENVIRONMENT=docker
+ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
+ENV COUCHDB_PASSWORD=budibase
+ENV COUCHDB_USER=budibase
+ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984
+ENV BUDIBASE_ENVIRONMENT=PRODUCTION
+ENV MINIO_URL=http://localhost:9000
+ENV REDIS_URL=localhost:6379
+ENV WORKER_URL=http://localhost:4002
+ENV INTERNAL_API_KEY=budibase
+ENV JWT_SECRET=testsecret
+ENV MINIO_ACCESS_KEY=budibase
+ENV MINIO_SECRET_KEY=budibase
+ENV SELF_HOSTED=1
+ENV CLUSTER_PORT=10000
+ENV REDIS_PASSWORD=budibase
+ENV ARCHITECTURE=amd
+ENV APP_PORT=4001
+ENV WORKER_PORT=4002
+
+RUN apt-get update
+RUN apt-get install software-properties-common wget nginx -y
+RUN apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main'
+RUN apt-get update
+
+# setup nginx
+ADD hosting/single/nginx.conf /etc/nginx
+RUN mkdir /etc/nginx/logs
+RUN useradd www
+RUN touch /etc/nginx/logs/error.log
+RUN touch /etc/nginx/logs/nginx.pid
+
+# install java
+RUN apt-get install openjdk-8-jdk -y
+
+# setup nodejs
+WORKDIR /nodejs
+RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh
+RUN bash /tmp/nodesource_setup.sh
+RUN apt-get install nodejs
+RUN npm install --global yarn
+RUN npm install --global pm2
+
+# setup redis
+RUN apt install redis-server -y
+
+# setup server
+WORKDIR /app
+ADD packages/server .
+RUN ls -al
+RUN yarn
+RUN yarn build
+# Install client for oracle datasource
+RUN apt-get install unzip libaio1
+RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh
+
+# setup worker
+WORKDIR /worker
+ADD packages/worker .
+RUN yarn
+RUN yarn build
+
+# setup clouseau
+WORKDIR /
+RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip
+RUN unzip clouseau-2.21.0-dist.zip
+RUN mv clouseau-2.21.0 /opt/clouseau
+RUN rm clouseau-2.21.0-dist.zip
+
+WORKDIR /opt/clouseau
+RUN mkdir ./bin
+ADD hosting/single/clouseau ./bin/
+ADD hosting/single/log4j.properties .
+ADD hosting/single/clouseau.ini .
+RUN chmod +x ./bin/clouseau
+
+# setup CouchDB
+WORKDIR /opt/couchdb
+ADD hosting/single/vm.args ./etc/
+
+# setup minio
+WORKDIR /minio
+RUN wget https://dl.min.io/server/minio/release/linux-${ARCHITECTURE}64/minio
+RUN chmod +x minio
+
+# setup runner file
+WORKDIR /
+ADD hosting/single/runner.sh .
+RUN chmod +x ./runner.sh
+
+EXPOSE 10000
+VOLUME /opt/couchdb/data
+VOLUME /minio
+
+# must set this just before running
+ENV NODE_ENV=production
+CMD ["./runner.sh"]
--- a/hosting/single/clouseau
+++ b/hosting/single/clouseau
@ -0,0 +1,12 @@
+#!/bin/sh
+/usr/bin/java -server \
+     -Xmx2G \
+     -Dsun.net.inetaddr.ttl=30 \
+     -Dsun.net.inetaddr.negative.ttl=30 \
+     -Dlog4j.configuration=file:/opt/clouseau/log4j.properties \
+     -XX:OnOutOfMemoryError="kill -9 %p" \
+     -XX:+UseConcMarkSweepGC \
+     -XX:+CMSParallelRemarkEnabled \
+     -classpath '/opt/clouseau/*' \
+     com.cloudant.clouseau.Main \
+     /opt/clouseau/clouseau.ini
--- a/hosting/single/clouseau.ini
+++ b/hosting/single/clouseau.ini
@ -0,0 +1,13 @@
+[clouseau]
+
+; the name of the Erlang node created by the service, leave this unchanged
+name=clouseau@127.0.0.1
+
+; set this to the same distributed Erlang cookie used by the CouchDB nodes
+cookie=monster
+
+; the path where you would like to store the search index files
+dir=/opt/couchdb/data/search
+
+; the number of search indexes that can be open simultaneously
+max_indexes_open=500
--- a/hosting/single/log4j.properties
+++ b/hosting/single/log4j.properties
@ -0,0 +1,4 @@
+log4j.rootLogger=debug, CONSOLE
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} %c [%p] %m%n
--- a/hosting/single/nginx.conf
+++ b/hosting/single/nginx.conf
@ -0,0 +1,116 @@
+user                 www www;
+error_log            /etc/nginx/logs/error.log;
+pid                  /etc/nginx/logs/nginx.pid;
+worker_processes     auto;
+worker_rlimit_nofile 8192;
+
+events {
+  worker_connections  1024;
+}
+
+http {
+  limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
+  proxy_set_header Host $host;
+  charset utf-8;
+  sendfile on;
+  tcp_nopush on;
+  tcp_nodelay on;
+  server_tokens off;
+  types_hash_max_size 2048;
+
+  # buffering
+  client_header_buffer_size 1k;
+  client_max_body_size 20M;
+  ignore_invalid_headers off;
+  proxy_buffering off;
+
+  log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+  map $http_upgrade $connection_upgrade {
+    default     "upgrade";
+  }
+
+  server {
+    listen       10000 default_server;
+    listen  [::]:10000 default_server;
+    server_name  _;
+    client_max_body_size 1000m;
+    ignore_invalid_headers off;
+    proxy_buffering off;
+    # port_in_redirect off;
+
+    location /app {
+      proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location = / {
+      proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location ~ ^/(builder|app_) {
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+      proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location ~ ^/api/(system|admin|global)/ {
+      proxy_pass      http://127.0.0.1:4002;
+    }
+
+    location /worker/ {
+      proxy_pass      http://127.0.0.1:4002;
+      rewrite ^/worker/(.*)$ /$1 break;
+    }
+
+    location /api/ {
+      # calls to the API are rate limited with bursting
+      limit_req zone=ratelimit burst=20 nodelay;
+
+      # 120s timeout on API requests
+      proxy_read_timeout 120s;
+      proxy_connect_timeout 120s;
+      proxy_send_timeout 120s;
+
+      proxy_http_version  1.1;
+      proxy_set_header    Connection          $connection_upgrade;
+      proxy_set_header    Upgrade             $http_upgrade;
+      proxy_set_header    X-Real-IP           $remote_addr;
+      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+
+      proxy_pass      http://127.0.0.1:4001;
+    }
+
+    location /db/ {
+      proxy_pass      http://127.0.0.1:5984;
+      rewrite ^/db/(.*)$ /$1 break;
+    }
+
+    location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+      proxy_pass      http://127.0.0.1:9000;
+    }
+
+    client_header_timeout 60;
+    client_body_timeout   60;
+    keepalive_timeout     60;
+
+    # gzip
+    gzip on;
+    gzip_vary on;
+    gzip_proxied any;
+    gzip_comp_level 6;
+    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
+  }
+}
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@ -0,0 +1,16 @@
+redis-server --requirepass $REDIS_PASSWORD &
+/opt/clouseau/bin/clouseau &
+/minio/minio server /minio &
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/etc/init.d/nginx restart
+pushd app
+pm2 start --name app "yarn run:docker"
+popd
+pushd worker
+pm2 start --name worker "yarn run:docker"
+popd
+sleep 10
+URL=http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984
+curl -X PUT ${URL}/_users
+curl -X PUT ${URL}/_replicator
+sleep infinity
--- a/hosting/single/vm.args
+++ b/hosting/single/vm.args
@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+# erlang cookie for clouseau security
+-name couchdb@127.0.0.1
+-setcookie monster
+
+# Ensure that the Erlang VM listens on a known port
+-kernel inet_dist_listen_min 9100
+-kernel inet_dist_listen_max 9100
+
+# Tell kernel and SASL not to log anything
+-kernel error_logger silent
+-sasl sasl_error_logger false
+
+# Use kernel poll functionality if supported by emulator
+K true
+
+# Start a pool of asynchronous IO threads
+A 16
+
+# Comment this line out to enable the interactive Erlang shell on startup
+Bd -noinput
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "1.0.98-alpha.1",
+  "version": "1.0.206",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/package.json
+++ b/package.json
@ -3,6 +3,8 @@
  "private": true,
  "devDependencies": {
    "@rollup/plugin-json": "^4.0.2",
+    "@types/mongodb": "3.6.3",
+    "@typescript-eslint/parser": "4.28.0",
    "babel-eslint": "^10.0.3",
    "eslint": "^7.28.0",
    "eslint-plugin-cypress": "^2.11.3",
@ -16,29 +18,28 @@
    "rimraf": "^3.0.2",
    "rollup-plugin-replace": "^2.2.0",
    "svelte": "^3.38.2",
-    "@typescript-eslint/parser": "4.28.0",
    "typescript": "4.5.5"
  },
  "scripts": {
    "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "lerna link && lerna bootstrap",
+    "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
    "build": "lerna run build",
-    "publishdev": "lerna run publishdev",
-    "publishnpm": "yarn build && lerna publish --force-publish",
-    "release": "lerna publish patch --yes --force-publish",
-    "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop",
+    "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+    "release": "lerna publish patch --yes --force-publish && yarn release:pro",
+    "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop",
+    "release:pro": "bash scripts/pro/release.sh",
+    "release:pro:develop": "bash scripts/pro/release.sh develop",
    "restore": "yarn run clean && yarn run bootstrap && yarn run build",
    "nuke": "yarn run nuke:packages && yarn run nuke:docker",
    "nuke:packages": "yarn run restore",
    "nuke:docker": "lerna run --parallel dev:stack:nuke",
    "clean": "lerna clean",
-    "kill-port": "kill-port 4001",
    "kill-builder": "kill-port 3000",
    "kill-server": "kill-port 4001 4002",
    "kill-all": "yarn run kill-builder && yarn run kill-server",
    "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
+    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
    "test": "lerna run test",
    "lint:eslint": "eslint packages",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
@ -48,16 +49,21 @@
    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
    "test:e2e": "lerna run cy:test --stream",
    "test:e2e:ci": "lerna run cy:ci --stream",
+    "test:e2e:ci:record": "lerna run cy:ci:record --stream",
+    "test:e2e:ci:notify": "lerna run cy:ci:notify",
    "build:specs": "lerna run specs",
    "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
    "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy",
    "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy",
+    "build:docker:proxy:release": "node scripts/proxy/generateProxyConfig release && npm run build:docker:proxy",
    "build:docker:proxy:prod": "node scripts/proxy/generateProxyConfig prod && npm run build:docker:proxy",
    "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
    "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
    "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
+    "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
+    "build:docker:single": "lerna run build && lerna run predocker && npm run build:docker:single:image",
    "build:docs": "lerna run build:docs",
    "release:helm": "node scripts/releaseHelmChart",
    "env:multi:enable": "lerna run env:multi:enable",
@ -72,6 +78,8 @@
    "mode:cloud": "yarn env:selfhost:disable && yarn env:multi:enable && yarn env:account:disable",
    "mode:account": "yarn mode:cloud && yarn env:account:enable",
    "security:audit": "node scripts/audit.js",
-    "postinstall": "husky install"
+    "postinstall": "husky install",
+    "install:pro": "bash scripts/pro/install.sh",
+    "dep:clean": "yarn clean && yarn bootstrap"
  }
 }
--- a/packages/backend-core/.gitignore
+++ b/packages/backend-core/.gitignore
@ -44,9 +44,6 @@ jspm_packages/
 # Snowpack dependency directory (https://snowpack.dev/)
 web_modules/

-# TypeScript cache
-*.tsbuildinfo
-
 # Optional npm cache directory
 .npm

--- a/packages/backend-core/cache.js
+++ b/packages/backend-core/cache.js
@ -1,4 +1,7 @@
+const generic = require("./src/cache/generic")
+
 module.exports = {
  user: require("./src/cache/user"),
  app: require("./src/cache/appMetadata"),
+  ...generic,
 }
--- a/packages/backend-core/context.js
+++ b/packages/backend-core/context.js
@ -5,8 +5,11 @@ const {
  getAppId,
  updateAppId,
  doInAppContext,
+  doInTenant,
 } = require("./src/context")

+const identity = require("./src/context/identity")
+
 module.exports = {
  getAppDB,
  getDevAppDB,
@ -14,4 +17,6 @@ module.exports = {
  getAppId,
  updateAppId,
  doInAppContext,
+  doInTenant,
+  identity,
 }
--- a/packages/backend-core/db.js
+++ b/packages/backend-core/db.js
@ -3,4 +3,5 @@ module.exports = {
  ...require("./src/db/constants"),
  ...require("./src/db"),
  ...require("./src/db/views"),
+  ...require("./src/db/pouch"),
 }
--- a/packages/backend-core/logging.js
+++ b/packages/backend-core/logging.js
@ -0,0 +1 @@
+module.exports = require("./src/logging")
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,45 +1,80 @@
 {
  "name": "@budibase/backend-core",
-  "version": "1.0.98-alpha.1",
+  "version": "1.0.206",
  "description": "Budibase backend core libraries used in server and worker",
-  "main": "src/index.js",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "exports": {
+    ".": "./dist/src/index.js",
+    "./tests": "./dist/tests/index.js",
+    "./*": "./dist/*.js"
+  },
  "author": "Budibase",
  "license": "GPL-3.0",
  "scripts": {
+    "prebuild": "rimraf dist/",
+    "prepack": "cp package.json dist",
+    "build": "tsc -p tsconfig.build.json",
+    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
    "test": "jest",
    "test:watch": "jest --watchAll"
  },
  "dependencies": {
-    "@techpass/passport-openidconnect": "^0.3.0",
-    "aws-sdk": "^2.901.0",
-    "bcryptjs": "^2.4.3",
-    "cls-hooked": "^4.2.2",
-    "ioredis": "^4.27.1",
-    "jsonwebtoken": "^8.5.1",
-    "koa-passport": "^4.1.4",
-    "lodash": "^4.17.21",
-    "lodash.isarguments": "^3.1.0",
-    "node-fetch": "^2.6.1",
-    "passport-google-auth": "^1.0.2",
-    "passport-google-oauth": "^2.0.0",
-    "passport-jwt": "^4.0.0",
-    "passport-local": "^1.0.0",
-    "sanitize-s3-objectkey": "^0.0.1",
-    "tar-fs": "^2.1.1",
-    "uuid": "^8.3.2",
-    "zlib": "^1.0.5"
+    "@techpass/passport-openidconnect": "0.3.2",
+    "aws-sdk": "2.1030.0",
+    "bcrypt": "5.0.1",
+    "dotenv": "16.0.1",
+    "emitter-listener": "1.1.2",
+    "ioredis": "4.28.0",
+    "jsonwebtoken": "8.5.1",
+    "koa-passport": "4.1.4",
+    "lodash": "4.17.21",
+    "lodash.isarguments": "3.1.0",
+    "node-fetch": "2.6.7",
+    "passport-google-auth": "1.0.2",
+    "passport-google-oauth": "2.0.0",
+    "passport-jwt": "4.0.0",
+    "passport-local": "1.0.0",
+    "posthog-node": "1.3.0",
+    "pouchdb": "7.3.0",
+    "pouchdb-find": "7.2.2",
+    "pouchdb-replication-stream": "1.2.9",
+    "redlock": "4.2.0",
+    "sanitize-s3-objectkey": "0.0.1",
+    "semver": "7.3.7",
+    "tar-fs": "2.1.1",
+    "uuid": "8.3.2",
+    "zlib": "1.0.5"
  },
  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "moduleNameMapper": {
+      "@budibase/types": "<rootDir>/../types/src"
+    },
    "setupFiles": [
-      "./scripts/jestSetup.js"
+      "./scripts/jestSetup.ts"
    ]
  },
  "devDependencies": {
-    "ioredis-mock": "^5.5.5",
-    "jest": "^26.6.3",
-    "pouchdb": "^7.2.1",
-    "pouchdb-adapter-memory": "^7.2.2",
-    "pouchdb-all-dbs": "^1.0.2"
+    "@budibase/types": "^1.0.206",
+    "@shopify/jest-koa-mocks": "3.1.5",
+    "@types/jest": "27.5.1",
+    "@types/koa": "2.0.52",
+    "@types/node": "14.18.20",
+    "@types/node-fetch": "2.6.1",
+    "@types/redlock": "4.0.3",
+    "@types/semver": "7.3.7",
+    "@types/tar-fs": "2.0.1",
+    "@types/uuid": "8.3.4",
+    "ioredis-mock": "5.8.0",
+    "jest": "27.5.1",
+    "koa": "2.7.0",
+    "nodemon": "2.0.16",
+    "pouchdb-adapter-memory": "7.2.2",
+    "timekeeper": "2.2.0",
+    "ts-jest": "27.1.5",
+    "typescript": "4.7.3"
  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/backend-core/redis.js
+++ b/packages/backend-core/redis.js
@ -1,4 +1,5 @@
 module.exports = {
  Client: require("./src/redis"),
  utils: require("./src/redis/utils"),
+  clients: require("./src/redis/authRedis"),
 }
--- a/packages/backend-core/scripts/jestSetup.js
+++ b/packages/backend-core/scripts/jestSetup.js
@ -1,6 +0,0 @@
-const env = require("../src/environment")
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")
--- a/packages/backend-core/scripts/jestSetup.ts
+++ b/packages/backend-core/scripts/jestSetup.ts
@ -0,0 +1,12 @@
+import env from "../src/environment"
+import { mocks } from "../tests/utilities"
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")
--- a/packages/backend-core/src/auth.js
+++ b/packages/backend-core/src/auth.js
@ -29,7 +29,7 @@ passport.deserializeUser(async (user, done) => {
    const user = await db.get(user._id)
    return done(null, user)
  } catch (err) {
-    console.error("User not found", err)
+    console.error(`User not found`, err)
    return done(null, false, { message: "User not found" })
  }
 })
--- a/packages/backend-core/src/cache/appMetadata.js
+++ b/packages/backend-core/src/cache/appMetadata.js
@ -1,5 +1,5 @@
 const redis = require("../redis/authRedis")
-const { getCouch } = require("../db")
+const { doWithDB } = require("../db")
 const { DocumentTypes } = require("../db/constants")

 const AppState = {
@ -10,12 +10,14 @@ const EXPIRY_SECONDS = 3600
 /**
 * The default populate app metadata function
 */
-const populateFromDB = async (appId, CouchDB = null) => {
-  if (!CouchDB) {
-    CouchDB = getCouch()
-  }
-  const db = new CouchDB(appId, { skip_setup: true })
-  return db.get(DocumentTypes.APP_METADATA)
+const populateFromDB = async appId => {
+  return doWithDB(
+    appId,
+    db => {
+      return db.get(DocumentTypes.APP_METADATA)
+    },
+    { skip_setup: true }
+  )
 }

 const isInvalid = metadata => {
@ -27,17 +29,16 @@ const isInvalid = metadata => {
 * Use redis cache to first read the app metadata.
 * If not present fallback to loading the app metadata directly and re-caching.
 * @param {string} appId the id of the app to get metadata from.
- * @param {object} CouchDB the database being passed
 * @returns {object} the app metadata.
 */
-exports.getAppMetadata = async (appId, CouchDB = null) => {
+exports.getAppMetadata = async appId => {
  const client = await redis.getAppClient()
  // try cache
  let metadata = await client.get(appId)
  if (!metadata) {
    let expiry = EXPIRY_SECONDS
    try {
-      metadata = await populateFromDB(appId, CouchDB)
+      metadata = await populateFromDB(appId)
    } catch (err) {
      // app DB left around, but no metadata, it is invalid
      if (err && err.status === 404) {
--- a/packages/backend-core/src/cache/generic.js
+++ b/packages/backend-core/src/cache/generic.js
@ -0,0 +1,82 @@
+const redis = require("../redis/authRedis")
+const { getTenantId } = require("../context")
+
+exports.CacheKeys = {
+  CHECKLIST: "checklist",
+  INSTALLATION: "installation",
+  ANALYTICS_ENABLED: "analyticsEnabled",
+  UNIQUE_TENANT_ID: "uniqueTenantId",
+  EVENTS: "events",
+  BACKFILL_METADATA: "backfillMetadata",
+}
+
+exports.TTL = {
+  ONE_MINUTE: 600,
+  ONE_HOUR: 3600,
+  ONE_DAY: 86400,
+}
+
+function generateTenantKey(key) {
+  const tenantId = getTenantId()
+  return `${key}:${tenantId}`
+}
+
+exports.keys = async pattern => {
+  const client = await redis.getCacheClient()
+  return client.keys(pattern)
+}
+
+/**
+ * Read only from the cache.
+ */
+exports.get = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  const value = await client.get(key)
+  return value
+}
+
+/**
+ * Write to the cache.
+ */
+exports.store = async (key, value, ttl, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  await client.store(key, value, ttl)
+}
+
+exports.delete = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  return client.delete(key)
+}
+
+/**
+ * Read from the cache. Write to the cache if not exists.
+ */
+exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+  const cachedValue = await exports.get(key, opts)
+  if (cachedValue) {
+    return cachedValue
+  }
+
+  try {
+    const fetchedValue = await fetchFn()
+
+    await exports.store(key, fetchedValue, ttl, opts)
+    return fetchedValue
+  } catch (err) {
+    console.error("Error fetching before cache - ", err)
+    throw err
+  }
+}
+
+exports.bustCache = async key => {
+  const client = await redis.getCacheClient()
+  try {
+    await client.delete(generateTenantKey(key))
+  } catch (err) {
+    console.error("Error busting cache - ", err)
+    throw err
+  }
+}
--- a/packages/backend-core/src/cache/user.js
+++ b/packages/backend-core/src/cache/user.js
@ -1,5 +1,5 @@
 const redis = require("../redis/authRedis")
-const { getTenantId, lookupTenantId, getGlobalDB } = require("../tenancy")
+const { getTenantId, lookupTenantId, doWithGlobalDB } = require("../tenancy")
 const env = require("../environment")
 const accounts = require("../cloud/accounts")

@ -9,9 +9,8 @@ const EXPIRY_SECONDS = 3600
 * The default populate user function
 */
 const populateFromDB = async (userId, tenantId) => {
-  const user = await getGlobalDB(tenantId).get(userId)
+  const user = await doWithGlobalDB(tenantId, db => db.get(userId))
  user.budibaseAccess = true
-
  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
    const account = await accounts.getAccount(user.email)
    if (account) {
--- a/packages/backend-core/src/cloud/accounts.js
+++ b/packages/backend-core/src/cloud/accounts.js
@ -1,39 +0,0 @@
-const API = require("./api")
-const env = require("../environment")
-const { Headers } = require("../constants")
-
-const api = new API(env.ACCOUNT_PORTAL_URL)
-
-exports.getAccount = async email => {
-  const payload = {
-    email,
-  }
-  const response = await api.post(`/api/accounts/search`, {
-    body: payload,
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting account by email ${email}`, json)
-  }
-
-  return json[0]
-}
-
-exports.getStatus = async () => {
-  const response = await api.get(`/api/status`, {
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting status`)
-  }
-
-  return json
-}
--- a/packages/backend-core/src/cloud/accounts.ts
+++ b/packages/backend-core/src/cloud/accounts.ts
@ -0,0 +1,63 @@
+import API from "./api"
+import env from "../environment"
+import { Headers } from "../constants"
+import { CloudAccount } from "@budibase/types"
+
+const api = new API(env.ACCOUNT_PORTAL_URL)
+
+export const getAccount = async (
+  email: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    email,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by email ${email}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getAccountByTenantId = async (
+  tenantId: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    tenantId,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by tenantId ${tenantId}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getStatus = async () => {
+  const response = await api.get(`/api/status`, {
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+  const json = await response.json()
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting status`)
+  }
+
+  return json
+}
--- a/packages/backend-core/src/cloud/api.js
+++ b/packages/backend-core/src/cloud/api.js
@ -29,9 +29,7 @@ class API {
        credentials: "include",
      }

-      const resp = await fetch(`${this.host}${url}`, requestOptions)
-
-      return resp
+      return await fetch(`${this.host}${url}`, requestOptions)
    }

  post = this.apiCall("POST")
--- a/packages/backend-core/src/clshooked/index.js
+++ b/packages/backend-core/src/clshooked/index.js
@ -0,0 +1,650 @@
+const util = require("util")
+const assert = require("assert")
+const wrapEmitter = require("emitter-listener")
+const async_hooks = require("async_hooks")
+
+const CONTEXTS_SYMBOL = "cls@contexts"
+const ERROR_SYMBOL = "error@context"
+
+const DEBUG_CLS_HOOKED = process.env.DEBUG_CLS_HOOKED
+
+let currentUid = -1
+
+module.exports = {
+  getNamespace: getNamespace,
+  createNamespace: createNamespace,
+  destroyNamespace: destroyNamespace,
+  reset: reset,
+  ERROR_SYMBOL: ERROR_SYMBOL,
+}
+
+function Namespace(name) {
+  this.name = name
+  // changed in 2.7: no default context
+  this.active = null
+  this._set = []
+  this.id = null
+  this._contexts = new Map()
+  this._indent = 0
+  this._hook = null
+}
+
+Namespace.prototype.set = function set(key, value) {
+  if (!this.active) {
+    throw new Error(
+      "No context available. ns.run() or ns.bind() must be called first."
+    )
+  }
+
+  this.active[key] = value
+
+  if (DEBUG_CLS_HOOKED) {
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      indentStr +
+        "CONTEXT-SET KEY:" +
+        key +
+        "=" +
+        value +
+        " in ns:" +
+        this.name +
+        " currentUid:" +
+        currentUid +
+        " active:" +
+        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
+    )
+  }
+
+  return value
+}
+
+Namespace.prototype.get = function get(key) {
+  if (!this.active) {
+    if (DEBUG_CLS_HOOKED) {
+      const asyncHooksCurrentId = async_hooks.currentId()
+      const triggerId = async_hooks.triggerAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-GETTING KEY NO ACTIVE NS: (${this.name}) ${key}=undefined currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${this._set.length}`
+      )
+    }
+    return undefined
+  }
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      indentStr +
+        "CONTEXT-GETTING KEY:" +
+        key +
+        "=" +
+        this.active[key] +
+        " (" +
+        this.name +
+        ") currentUid:" +
+        currentUid +
+        " active:" +
+        util.inspect(this.active, { showHidden: true, depth: 2, colors: true })
+    )
+    debug2(
+      `${indentStr}CONTEXT-GETTING KEY: (${this.name}) ${key}=${
+        this.active[key]
+      } currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
+        this._set.length
+      } active:${util.inspect(this.active)}`
+    )
+  }
+  return this.active[key]
+}
+
+Namespace.prototype.createContext = function createContext() {
+  // Prototype inherit existing context if created a new child context within existing context.
+  let context = Object.create(this.active ? this.active : Object.prototype)
+  context._ns_name = this.name
+  context.id = currentUid
+
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-CREATED Context: (${
+        this.name
+      }) currentUid:${currentUid} asyncHooksCurrentId:${asyncHooksCurrentId} triggerId:${triggerId} len:${
+        this._set.length
+      } context:${util.inspect(context, {
+        showHidden: true,
+        depth: 2,
+        colors: true,
+      })}`
+    )
+  }
+
+  return context
+}
+
+Namespace.prototype.run = function run(fn) {
+  let context = this.createContext()
+  this.enter(context)
+
+  try {
+    if (DEBUG_CLS_HOOKED) {
+      const triggerId = async_hooks.triggerAsyncId()
+      const asyncHooksCurrentId = async_hooks.executionAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-RUN BEGIN: (${
+          this.name
+        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+          this._set.length
+        } context:${util.inspect(context)}`
+      )
+    }
+    fn(context)
+    return context
+  } catch (exception) {
+    if (exception) {
+      exception[ERROR_SYMBOL] = context
+    }
+    throw exception
+  } finally {
+    if (DEBUG_CLS_HOOKED) {
+      const triggerId = async_hooks.triggerAsyncId()
+      const asyncHooksCurrentId = async_hooks.executionAsyncId()
+      const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+      debug2(
+        `${indentStr}CONTEXT-RUN END: (${
+          this.name
+        }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+          this._set.length
+        } ${util.inspect(context)}`
+      )
+    }
+    this.exit(context)
+  }
+}
+
+Namespace.prototype.runAndReturn = function runAndReturn(fn) {
+  let value
+  this.run(function (context) {
+    value = fn(context)
+  })
+  return value
+}
+
+/**
+ * Uses global Promise and assumes Promise is cls friendly or wrapped already.
+ * @param {function} fn
+ * @returns {*}
+ */
+Namespace.prototype.runPromise = function runPromise(fn) {
+  let context = this.createContext()
+  this.enter(context)
+
+  let promise = fn(context)
+  if (!promise || !promise.then || !promise.catch) {
+    throw new Error("fn must return a promise.")
+  }
+
+  if (DEBUG_CLS_HOOKED) {
+    debug2(
+      "CONTEXT-runPromise BEFORE: (" +
+        this.name +
+        ") currentUid:" +
+        currentUid +
+        " len:" +
+        this._set.length +
+        " " +
+        util.inspect(context)
+    )
+  }
+
+  return promise
+    .then(result => {
+      if (DEBUG_CLS_HOOKED) {
+        debug2(
+          "CONTEXT-runPromise AFTER then: (" +
+            this.name +
+            ") currentUid:" +
+            currentUid +
+            " len:" +
+            this._set.length +
+            " " +
+            util.inspect(context)
+        )
+      }
+      this.exit(context)
+      return result
+    })
+    .catch(err => {
+      err[ERROR_SYMBOL] = context
+      if (DEBUG_CLS_HOOKED) {
+        debug2(
+          "CONTEXT-runPromise AFTER catch: (" +
+            this.name +
+            ") currentUid:" +
+            currentUid +
+            " len:" +
+            this._set.length +
+            " " +
+            util.inspect(context)
+        )
+      }
+      this.exit(context)
+      throw err
+    })
+}
+
+Namespace.prototype.bind = function bindFactory(fn, context) {
+  if (!context) {
+    if (!this.active) {
+      context = this.createContext()
+    } else {
+      context = this.active
+    }
+  }
+
+  let self = this
+  return function clsBind() {
+    self.enter(context)
+    try {
+      return fn.apply(this, arguments)
+    } catch (exception) {
+      if (exception) {
+        exception[ERROR_SYMBOL] = context
+      }
+      throw exception
+    } finally {
+      self.exit(context)
+    }
+  }
+}
+
+Namespace.prototype.enter = function enter(context) {
+  assert.ok(context, "context must be provided for entering")
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-ENTER: (${
+        this.name
+      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+        this._set.length
+      } ${util.inspect(context)}`
+    )
+  }
+
+  this._set.push(this.active)
+  this.active = context
+}
+
+Namespace.prototype.exit = function exit(context) {
+  assert.ok(context, "context must be provided for exiting")
+  if (DEBUG_CLS_HOOKED) {
+    const asyncHooksCurrentId = async_hooks.executionAsyncId()
+    const triggerId = async_hooks.triggerAsyncId()
+    const indentStr = " ".repeat(this._indent < 0 ? 0 : this._indent)
+    debug2(
+      `${indentStr}CONTEXT-EXIT: (${
+        this.name
+      }) currentUid:${currentUid} triggerId:${triggerId} asyncHooksCurrentId:${asyncHooksCurrentId} len:${
+        this._set.length
+      } ${util.inspect(context)}`
+    )
+  }
+
+  // Fast path for most exits that are at the top of the stack
+  if (this.active === context) {
+    assert.ok(this._set.length, "can't remove top context")
+    this.active = this._set.pop()
+    return
+  }
+
+  // Fast search in the stack using lastIndexOf
+  let index = this._set.lastIndexOf(context)
+
+  if (index < 0) {
+    if (DEBUG_CLS_HOOKED) {
+      debug2(
+        "??ERROR?? context exiting but not entered - ignoring: " +
+          util.inspect(context)
+      )
+    }
+    assert.ok(
+      index >= 0,
+      "context not currently entered; can't exit. \n" +
+        util.inspect(this) +
+        "\n" +
+        util.inspect(context)
+    )
+  } else {
+    assert.ok(index, "can't remove top context")
+    this._set.splice(index, 1)
+  }
+}
+
+Namespace.prototype.bindEmitter = function bindEmitter(emitter) {
+  assert.ok(
+    emitter.on && emitter.addListener && emitter.emit,
+    "can only bind real EEs"
+  )
+
+  let namespace = this
+  let thisSymbol = "context@" + this.name
+
+  // Capture the context active at the time the emitter is bound.
+  function attach(listener) {
+    if (!listener) {
+      return
+    }
+    if (!listener[CONTEXTS_SYMBOL]) {
+      listener[CONTEXTS_SYMBOL] = Object.create(null)
+    }
+
+    listener[CONTEXTS_SYMBOL][thisSymbol] = {
+      namespace: namespace,
+      context: namespace.active,
+    }
+  }
+
+  // At emit time, bind the listener within the correct context.
+  function bind(unwrapped) {
+    if (!(unwrapped && unwrapped[CONTEXTS_SYMBOL])) {
+      return unwrapped
+    }
+
+    let wrapped = unwrapped
+    let unwrappedContexts = unwrapped[CONTEXTS_SYMBOL]
+    Object.keys(unwrappedContexts).forEach(function (name) {
+      let thunk = unwrappedContexts[name]
+      wrapped = thunk.namespace.bind(wrapped, thunk.context)
+    })
+    return wrapped
+  }
+
+  wrapEmitter(emitter, attach, bind)
+}
+
+/**
+ * If an error comes out of a namespace, it will have a context attached to it.
+ * This function knows how to find it.
+ *
+ * @param {Error} exception Possibly annotated error.
+ */
+Namespace.prototype.fromException = function fromException(exception) {
+  return exception[ERROR_SYMBOL]
+}
+
+function getNamespace(name) {
+  return process.namespaces[name]
+}
+
+function createNamespace(name) {
+  assert.ok(name, "namespace must be given a name.")
+
+  if (DEBUG_CLS_HOOKED) {
+    debug2(`NS-CREATING NAMESPACE (${name})`)
+  }
+  let namespace = new Namespace(name)
+  namespace.id = currentUid
+
+  const hook = async_hooks.createHook({
+    init(asyncId, type, triggerId, resource) {
+      currentUid = async_hooks.executionAsyncId()
+
+      //CHAIN Parent's Context onto child if none exists. This is needed to pass net-events.spec
+      // let initContext = namespace.active;
+      // if(!initContext && triggerId) {
+      //   let parentContext = namespace._contexts.get(triggerId);
+      //   if (parentContext) {
+      //     namespace.active = parentContext;
+      //     namespace._contexts.set(currentUid, parentContext);
+      //     if (DEBUG_CLS_HOOKED) {
+      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //       debug2(`${indentStr}INIT [${type}] (${name}) WITH PARENT CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //     }
+      //   } else if (DEBUG_CLS_HOOKED) {
+      //       const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //       debug2(`${indentStr}INIT [${type}] (${name}) MISSING CONTEXT asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //     }
+      // }else {
+      //   namespace._contexts.set(currentUid, namespace.active);
+      //   if (DEBUG_CLS_HOOKED) {
+      //     const indentStr = ' '.repeat(namespace._indent < 0 ? 0 : namespace._indent);
+      //     debug2(`${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(namespace.active, true)} resource:${resource}`);
+      //   }
+      // }
+      if (namespace.active) {
+        namespace._contexts.set(asyncId, namespace.active)
+
+        if (DEBUG_CLS_HOOKED) {
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}INIT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} resource:${resource}`
+          )
+        }
+      } else if (currentUid === 0) {
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        const triggerId = async_hooks.triggerAsyncId()
+        const triggerIdContext = namespace._contexts.get(triggerId)
+        if (triggerIdContext) {
+          namespace._contexts.set(asyncId, triggerIdContext)
+          if (DEBUG_CLS_HOOKED) {
+            const indentStr = " ".repeat(
+              namespace._indent < 0 ? 0 : namespace._indent
+            )
+            debug2(
+              `${indentStr}INIT USING CONTEXT FROM TRIGGERID [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+                namespace.active,
+                { showHidden: true, depth: 2, colors: true }
+              )} resource:${resource}`
+            )
+          }
+        } else if (DEBUG_CLS_HOOKED) {
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}INIT MISSING CONTEXT [${type}] (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} resource:${resource}`
+          )
+        }
+      }
+
+      if (DEBUG_CLS_HOOKED && type === "PROMISE") {
+        debug2(util.inspect(resource, { showHidden: true }))
+        const parentId = resource.parentId
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}INIT RESOURCE-PROMISE [${type}] (${name}) parentId:${parentId} asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} resource:${resource}`
+        )
+      }
+    },
+    before(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      let context
+
+      /*
+      if(currentUid === 0){
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        //const triggerId = async_hooks.triggerAsyncId();
+        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
+      }else{
+        context = namespace._contexts.get(currentUid);
+      }
+      */
+
+      //HACK to work with promises until they are fixed in node > 8.1.1
+      context =
+        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
+
+      if (context) {
+        if (DEBUG_CLS_HOOKED) {
+          const triggerId = async_hooks.triggerAsyncId()
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}BEFORE (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} context:${util.inspect(context)}`
+          )
+          namespace._indent += 2
+        }
+
+        namespace.enter(context)
+      } else if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}BEFORE MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} namespace._contexts:${util.inspect(namespace._contexts, {
+            showHidden: true,
+            depth: 2,
+            colors: true,
+          })}`
+        )
+        namespace._indent += 2
+      }
+    },
+    after(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      let context // = namespace._contexts.get(currentUid);
+      /*
+      if(currentUid === 0){
+        // CurrentId will be 0 when triggered from C++. Promise events
+        // https://github.com/nodejs/node/blob/master/doc/api/async_hooks.md#triggerid
+        //const triggerId = async_hooks.triggerAsyncId();
+        context = namespace._contexts.get(asyncId); // || namespace._contexts.get(triggerId);
+      }else{
+        context = namespace._contexts.get(currentUid);
+      }
+      */
+      //HACK to work with promises until they are fixed in node > 8.1.1
+      context =
+        namespace._contexts.get(asyncId) || namespace._contexts.get(currentUid)
+
+      if (context) {
+        if (DEBUG_CLS_HOOKED) {
+          const triggerId = async_hooks.triggerAsyncId()
+          namespace._indent -= 2
+          const indentStr = " ".repeat(
+            namespace._indent < 0 ? 0 : namespace._indent
+          )
+          debug2(
+            `${indentStr}AFTER (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+              namespace.active,
+              { showHidden: true, depth: 2, colors: true }
+            )} context:${util.inspect(context)}`
+          )
+        }
+
+        namespace.exit(context)
+      } else if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        namespace._indent -= 2
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}AFTER MISSING CONTEXT (${name}) asyncId:${asyncId} currentUid:${currentUid} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} context:${util.inspect(context)}`
+        )
+      }
+    },
+    destroy(asyncId) {
+      currentUid = async_hooks.executionAsyncId()
+      if (DEBUG_CLS_HOOKED) {
+        const triggerId = async_hooks.triggerAsyncId()
+        const indentStr = " ".repeat(
+          namespace._indent < 0 ? 0 : namespace._indent
+        )
+        debug2(
+          `${indentStr}DESTROY (${name}) currentUid:${currentUid} asyncId:${asyncId} triggerId:${triggerId} active:${util.inspect(
+            namespace.active,
+            { showHidden: true, depth: 2, colors: true }
+          )} context:${util.inspect(namespace._contexts.get(currentUid))}`
+        )
+      }
+
+      namespace._contexts.delete(asyncId)
+    },
+  })
+
+  hook.enable()
+  namespace._hook = hook
+
+  process.namespaces[name] = namespace
+  return namespace
+}
+
+function destroyNamespace(name) {
+  let namespace = getNamespace(name)
+
+  assert.ok(namespace, "can't delete nonexistent namespace! \"" + name + '"')
+  assert.ok(
+    namespace.id,
+    "don't assign to process.namespaces directly! " + util.inspect(namespace)
+  )
+
+  namespace._hook.disable()
+  namespace._contexts = null
+  process.namespaces[name] = null
+}
+
+function reset() {
+  // must unregister async listeners
+  if (process.namespaces) {
+    Object.keys(process.namespaces).forEach(function (name) {
+      destroyNamespace(name)
+    })
+  }
+  process.namespaces = Object.create(null)
+}
+
+process.namespaces = process.namespaces || {}
+
+//const fs = require('fs');
+function debug2(...args) {
+  if (DEBUG_CLS_HOOKED) {
+    //fs.writeSync(1, `${util.format(...args)}\n`);
+    process._rawDebug(`${util.format(...args)}`)
+  }
+}
+
+/*function getFunctionName(fn) {
+  if (!fn) {
+    return fn;
+  }
+  if (typeof fn === 'function') {
+    if (fn.name) {
+      return fn.name;
+    }
+    return (fn.toString().trim().match(/^function\s*([^\s(]+)/) || [])[1];
+  } else if (fn.constructor && fn.constructor.name) {
+    return fn.constructor.name;
+  }
+}*/
--- a/packages/backend-core/src/constants.js
+++ b/packages/backend-core/src/constants.js
@ -13,9 +13,11 @@ exports.Cookies = {

 exports.Headers = {
  API_KEY: "x-budibase-api-key",
+  LICENSE_KEY: "x-budibase-license-key",
  API_VER: "x-budibase-api-version",
  APP_ID: "x-budibase-app-id",
  TYPE: "x-budibase-type",
+  PREVIEW_ROLE: "x-budibase-role",
  TENANT_ID: "x-budibase-tenant-id",
  TOKEN: "x-budibase-token",
  CSRF_TOKEN: "x-csrf-token",
--- a/packages/backend-core/src/context/FunctionContext.js
+++ b/packages/backend-core/src/context/FunctionContext.js
@ -1,73 +1,47 @@
-const cls = require("cls-hooked")
+const cls = require("../clshooked")
 const { newid } = require("../hashing")

 const REQUEST_ID_KEY = "requestId"
+const MAIN_CTX = cls.createNamespace("main")
+
+function getContextStorage(namespace) {
+  if (namespace && namespace.active) {
+    let contextData = namespace.active
+    delete contextData.id
+    delete contextData._ns_name
+    return contextData
+  }
+  return {}
+}

 class FunctionContext {
-  static getMiddleware(updateCtxFn = null, contextName = "session") {
-    const namespace = this.createNamespace(contextName)
-
-    return async function (ctx, next) {
-      await new Promise(
-        namespace.bind(function (resolve, reject) {
-          // store a contextual request ID that can be used anywhere (audit logs)
-          namespace.set(REQUEST_ID_KEY, newid())
-          namespace.bindEmitter(ctx.req)
-          namespace.bindEmitter(ctx.res)
-
-          if (updateCtxFn) {
-            updateCtxFn(ctx)
-          }
-          next().then(resolve).catch(reject)
-        })
-      )
-    }
+  static run(callback) {
+    return MAIN_CTX.runAndReturn(async () => {
+      const namespaceId = newid()
+      MAIN_CTX.set(REQUEST_ID_KEY, namespaceId)
+      const namespace = cls.createNamespace(namespaceId)
+      let response = await namespace.runAndReturn(callback)
+      cls.destroyNamespace(namespaceId)
+      return response
+    })
  }

-  static run(callback, contextName = "session") {
-    const namespace = this.createNamespace(contextName)
-
-    return namespace.runAndReturn(callback)
-  }
-
-  static setOnContext(key, value, contextName = "session") {
-    const namespace = this.createNamespace(contextName)
+  static setOnContext(key, value) {
+    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
+    const namespace = cls.getNamespace(namespaceId)
    namespace.set(key, value)
  }

-  static getContextStorage() {
-    if (this._namespace && this._namespace.active) {
-      let contextData = this._namespace.active
-      delete contextData.id
-      delete contextData._ns_name
-      return contextData
-    }
-
-    return {}
-  }
-
  static getFromContext(key) {
-    const context = this.getContextStorage()
+    const namespaceId = MAIN_CTX.get(REQUEST_ID_KEY)
+    const namespace = cls.getNamespace(namespaceId)
+    const context = getContextStorage(namespace)
    if (context) {
      return context[key]
    } else {
      return null
    }
  }
-
-  static destroyNamespace(name = "session") {
-    if (this._namespace) {
-      cls.destroyNamespace(name)
-      this._namespace = null
-    }
-  }
-
-  static createNamespace(name = "session") {
-    if (!this._namespace) {
-      this._namespace = cls.createNamespace(name)
-    }
-    return this._namespace
-  }
 }

 module.exports = FunctionContext
--- a/packages/backend-core/src/context/deprovision.js
+++ b/packages/backend-core/src/context/deprovision.js
@ -1,6 +1,6 @@
 const { getGlobalUserParams, getAllApps } = require("../db/utils")
-const { getDB } = require("../db")
-const { getGlobalDB } = require("../tenancy")
+const { doWithDB } = require("../db")
+const { doWithGlobalDB } = require("../tenancy")
 const { StaticDatabases } = require("../db/constants")

 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
@ -8,11 +8,12 @@ const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name

 const removeTenantFromInfoDB = async tenantId => {
  try {
-    const infoDb = getDB(PLATFORM_INFO_DB)
-    let tenants = await infoDb.get(TENANT_DOC)
-    tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)
+    await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+      let tenants = await infoDb.get(TENANT_DOC)
+      tenants.tenantIds = tenants.tenantIds.filter(id => id !== tenantId)

-    await infoDb.put(tenants)
+      await infoDb.put(tenants)
+    })
  } catch (err) {
    console.error(`Error removing tenant ${tenantId} from info db`, err)
    throw err
@ -20,36 +21,8 @@ const removeTenantFromInfoDB = async tenantId => {
 }

 exports.removeUserFromInfoDB = async dbUser => {
-  const infoDb = getDB(PLATFORM_INFO_DB)
-  const keys = [dbUser._id, dbUser.email]
-  const userDocs = await infoDb.allDocs({
-    keys,
-    include_docs: true,
-  })
-  const toDelete = userDocs.rows.map(row => {
-    return {
-      ...row.doc,
-      _deleted: true,
-    }
-  })
-  await infoDb.bulkDocs(toDelete)
-}
-
-const removeUsersFromInfoDB = async tenantId => {
-  try {
-    const globalDb = getGlobalDB(tenantId)
-    const infoDb = getDB(PLATFORM_INFO_DB)
-    const allUsers = await globalDb.allDocs(
-      getGlobalUserParams(null, {
-        include_docs: true,
-      })
-    )
-    const allEmails = allUsers.rows.map(row => row.doc.email)
-    // get the id docs
-    let keys = allUsers.rows.map(row => row.id)
-    // and the email docs
-    keys = keys.concat(allEmails)
-    // retrieve the docs and delete them
+  await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+    const keys = [dbUser._id, dbUser.email]
    const userDocs = await infoDb.allDocs({
      keys,
      include_docs: true,
@ -61,26 +34,60 @@ const removeUsersFromInfoDB = async tenantId => {
      }
    })
    await infoDb.bulkDocs(toDelete)
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} users from info db`, err)
-    throw err
-  }
+  })
+}
+
+const removeUsersFromInfoDB = async tenantId => {
+  return doWithGlobalDB(tenantId, async db => {
+    try {
+      const allUsers = await db.allDocs(
+        getGlobalUserParams(null, {
+          include_docs: true,
+        })
+      )
+      await doWithDB(PLATFORM_INFO_DB, async infoDb => {
+        const allEmails = allUsers.rows.map(row => row.doc.email)
+        // get the id docs
+        let keys = allUsers.rows.map(row => row.id)
+        // and the email docs
+        keys = keys.concat(allEmails)
+        // retrieve the docs and delete them
+        const userDocs = await infoDb.allDocs({
+          keys,
+          include_docs: true,
+        })
+        const toDelete = userDocs.rows.map(row => {
+          return {
+            ...row.doc,
+            _deleted: true,
+          }
+        })
+        await infoDb.bulkDocs(toDelete)
+      })
+    } catch (err) {
+      console.error(`Error removing tenant ${tenantId} users from info db`, err)
+      throw err
+    }
+  })
 }

 const removeGlobalDB = async tenantId => {
-  try {
-    const globalDb = getGlobalDB(tenantId)
-    await globalDb.destroy()
-  } catch (err) {
-    console.error(`Error removing tenant ${tenantId} users from info db`, err)
-    throw err
-  }
+  return doWithGlobalDB(tenantId, async db => {
+    try {
+      await db.destroy()
+    } catch (err) {
+      console.error(`Error removing tenant ${tenantId} users from info db`, err)
+      throw err
+    }
+  })
 }

 const removeTenantApps = async tenantId => {
  try {
    const apps = await getAllApps({ all: true })
-    const destroyPromises = apps.map(app => getDB(app.appId).destroy())
+    const destroyPromises = apps.map(app =>
+      doWithDB(app.appId, db => db.destroy())
+    )
    await Promise.allSettled(destroyPromises)
  } catch (err) {
    console.error(`Error removing tenant ${tenantId} apps`, err)
--- a/packages/backend-core/src/context/identity.ts
+++ b/packages/backend-core/src/context/identity.ts
@ -0,0 +1,50 @@
+import {
+  IdentityContext,
+  IdentityType,
+  User,
+  UserContext,
+  isCloudAccount,
+  Account,
+  AccountUserContext,
+} from "@budibase/types"
+import * as context from "."
+
+export const getIdentity = (): IdentityContext | undefined => {
+  return context.getIdentity()
+}
+
+export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+  return context.doInIdentityContext(identity, task)
+}
+
+export const doInUserContext = (user: User, task: any) => {
+  const userContext: UserContext = {
+    ...user,
+    _id: user._id as string,
+    type: IdentityType.USER,
+  }
+  return doInIdentityContext(userContext, task)
+}
+
+export const doInAccountContext = (account: Account, task: any) => {
+  const _id = getAccountUserId(account)
+  const tenantId = account.tenantId
+  const accountContext: AccountUserContext = {
+    _id,
+    type: IdentityType.USER,
+    tenantId,
+    account,
+  }
+  return doInIdentityContext(accountContext, task)
+}
+
+export const getAccountUserId = (account: Account) => {
+  let userId: string
+  if (isCloudAccount(account)) {
+    userId = account.budibaseUserId
+  } else {
+    // use account id as user id for self hosting
+    userId = account.accountId
+  }
+  return userId
+}
--- a/packages/backend-core/src/context/index.js
+++ b/packages/backend-core/src/context/index.js
@ -1,9 +1,10 @@
 const env = require("../environment")
-const { Headers } = require("../../constants")
 const { SEPARATOR, DocumentTypes } = require("../db/constants")
+const { DEFAULT_TENANT_ID } = require("../constants")
 const cls = require("./FunctionContext")
-const { getCouch } = require("../db")
+const { dangerousGetDB, closeDB } = require("../db")
 const { getProdAppID, getDevelopmentAppID } = require("../db/conversions")
+const { baseGlobalDBName } = require("../tenancy/utils")
 const { isEqual } = require("lodash")

 // some test cases call functions directly, need to
@ -12,7 +13,9 @@ let TEST_APP_ID = null

 const ContextKeys = {
  TENANT_ID: "tenantId",
+  GLOBAL_DB: "globalDb",
  APP_ID: "appId",
+  IDENTITY: "identity",
  // whatever the request app DB was
  CURRENT_DB: "currentDb",
  // get the prod app DB from the request
@ -20,9 +23,46 @@ const ContextKeys = {
  // get the dev app DB from the request
  DEV_DB: "devDb",
  DB_OPTS: "dbOpts",
+  // check if something else is using the context, don't close DB
+  IN_USE: "inUse",
 }

-exports.DEFAULT_TENANT_ID = "default"
+exports.DEFAULT_TENANT_ID = DEFAULT_TENANT_ID
+
+// this function makes sure the PouchDB objects are closed and
+// fully deleted when finished - this protects against memory leaks
+async function closeAppDBs() {
+  const dbKeys = [
+    ContextKeys.CURRENT_DB,
+    ContextKeys.PROD_DB,
+    ContextKeys.DEV_DB,
+  ]
+  for (let dbKey of dbKeys) {
+    const db = cls.getFromContext(dbKey)
+    if (!db) {
+      continue
+    }
+    await closeDB(db)
+    // clear the DB from context, incase someone tries to use it again
+    cls.setOnContext(dbKey, null)
+  }
+  // clear the app ID now that the databases are closed
+  if (cls.getFromContext(ContextKeys.APP_ID)) {
+    cls.setOnContext(ContextKeys.APP_ID, null)
+  }
+  if (cls.getFromContext(ContextKeys.DB_OPTS)) {
+    cls.setOnContext(ContextKeys.DB_OPTS, null)
+  }
+}
+
+exports.closeTenancy = async () => {
+  if (env.USE_COUCH) {
+    await closeDB(exports.getGlobalDB())
+  }
+  // clear from context now that database is closed/task is finished
+  cls.setOnContext(ContextKeys.TENANT_ID, null)
+  cls.setOnContext(ContextKeys.GLOBAL_DB, null)
+}

 exports.isDefaultTenant = () => {
  return exports.getTenantId() === exports.DEFAULT_TENANT_ID
@ -33,14 +73,42 @@ exports.isMultiTenant = () => {
 }

 // used for automations, API endpoints should always be in context already
-exports.doInTenant = (tenantId, task) => {
-  return cls.run(() => {
+exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
+  // the internal function is so that we can re-use an existing
+  // context - don't want to close DB on a parent context
+  async function internal(opts = { existing: false }) {
    // set the tenant id
-    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+    if (!opts.existing) {
+      exports.updateTenantId(tenantId)
+    }

-    // invoke the task
-    return task()
-  })
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        await exports.closeTenancy()
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (
+    !forceNew &&
+    using &&
+    cls.getFromContext(ContextKeys.TENANT_ID) === tenantId
+  ) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal()
+    })
+  }
 }

 /**
@ -64,37 +132,117 @@ exports.getTenantIDFromAppID = appId => {
 }

 const setAppTenantId = appId => {
-  const appTenantId = this.getTenantIDFromAppID(appId) || this.DEFAULT_TENANT_ID
-  this.updateTenantId(appTenantId)
+  const appTenantId =
+    exports.getTenantIDFromAppID(appId) || exports.DEFAULT_TENANT_ID
+  exports.updateTenantId(appTenantId)
 }

-exports.doInAppContext = (appId, task) => {
+exports.doInAppContext = (appId, task, { forceNew } = {}) => {
  if (!appId) {
    throw new Error("appId is required")
  }
-  return cls.run(() => {
-    // set the app tenant id
-    setAppTenantId(appId)

+  const identity = exports.getIdentity()
+
+  // the internal function is so that we can re-use an existing
+  // context - don't want to close DB on a parent context
+  async function internal(opts = { existing: false }) {
+    // set the app tenant id
+    if (!opts.existing) {
+      setAppTenantId(appId)
+    }
    // set the app ID
    cls.setOnContext(ContextKeys.APP_ID, appId)
+    // preserve the identity
+    exports.setIdentity(identity)
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        await closeAppDBs()
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (!forceNew && using && cls.getFromContext(ContextKeys.APP_ID) === appId) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal()
+    })
+  }
+}

-    // invoke the task
-    return task()
-  })
+exports.doInIdentityContext = (identity, task) => {
+  if (!identity) {
+    throw new Error("identity is required")
+  }
+
+  async function internal(opts = { existing: false }) {
+    if (!opts.existing) {
+      cls.setOnContext(ContextKeys.IDENTITY, identity)
+      // set the tenant so that doInTenant will preserve identity
+      if (identity.tenantId) {
+        exports.updateTenantId(identity.tenantId)
+      }
+    }
+
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        exports.setIdentity(null)
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+
+  const existing = cls.getFromContext(ContextKeys.IDENTITY)
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (using && existing && existing._id === identity._id) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal({ existing: false })
+    })
+  }
+}
+
+exports.setIdentity = identity => {
+  cls.setOnContext(ContextKeys.IDENTITY, identity)
+}
+
+exports.getIdentity = () => {
+  try {
+    return cls.getFromContext(ContextKeys.IDENTITY)
+  } catch (e) {
+    // do nothing - identity is not in context
+  }
 }

 exports.updateTenantId = tenantId => {
  cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+  if (env.USE_COUCH) {
+    exports.setGlobalDB(tenantId)
+  }
 }

-exports.updateAppId = appId => {
+exports.updateAppId = async appId => {
  try {
+    // have to close first, before removing the databases from context
+    await closeAppDBs()
    cls.setOnContext(ContextKeys.APP_ID, appId)
-    cls.setOnContext(ContextKeys.PROD_DB, null)
-    cls.setOnContext(ContextKeys.DEV_DB, null)
-    cls.setOnContext(ContextKeys.CURRENT_DB, null)
-    cls.setOnContext(ContextKeys.DB_OPTS, null)
  } catch (err) {
    if (env.isTest()) {
      TEST_APP_ID = appId
@ -104,42 +252,19 @@ exports.updateAppId = appId => {
  }
 }

-exports.setTenantId = (
-  ctx,
-  opts = { allowQs: false, allowNoTenant: false }
-) => {
-  let tenantId
-  // exit early if not multi-tenant
-  if (!exports.isMultiTenant()) {
-    cls.setOnContext(ContextKeys.TENANT_ID, this.DEFAULT_TENANT_ID)
-    return
-  }
+exports.setGlobalDB = tenantId => {
+  const dbName = baseGlobalDBName(tenantId)
+  const db = dangerousGetDB(dbName)
+  cls.setOnContext(ContextKeys.GLOBAL_DB, db)
+  return db
+}

-  const allowQs = opts && opts.allowQs
-  const allowNoTenant = opts && opts.allowNoTenant
-  const header = ctx.request.headers[Headers.TENANT_ID]
-  const user = ctx.user || {}
-  if (allowQs) {
-    const query = ctx.request.query || {}
-    tenantId = query.tenantId
-  }
-  // override query string (if allowed) by user, or header
-  // URL params cannot be used in a middleware, as they are
-  // processed later in the chain
-  tenantId = user.tenantId || header || tenantId
-
-  // Set the tenantId from the subdomain
-  if (!tenantId) {
-    tenantId = ctx.subdomains && ctx.subdomains[0]
-  }
-
-  if (!tenantId && !allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-  // check tenant ID just incase no tenant was allowed
-  if (tenantId) {
-    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+exports.getGlobalDB = () => {
+  const db = cls.getFromContext(ContextKeys.GLOBAL_DB)
+  if (!db) {
+    throw new Error("Global DB not found")
  }
+  return db
 }

 exports.isTenantIdSet = () => {
@ -167,16 +292,17 @@ exports.getAppId = () => {
  }
 }

-function getDB(key, opts) {
+function getContextDB(key, opts) {
  const dbOptsKey = `${key}${ContextKeys.DB_OPTS}`
  let storedOpts = cls.getFromContext(dbOptsKey)
  let db = cls.getFromContext(key)
  if (db && isEqual(opts, storedOpts)) {
    return db
  }
+
  const appId = exports.getAppId()
-  const CouchDB = getCouch()
  let toUseAppId
+
  switch (key) {
    case ContextKeys.CURRENT_DB:
      toUseAppId = appId
@ -188,7 +314,7 @@ function getDB(key, opts) {
      toUseAppId = getDevelopmentAppID(appId)
      break
  }
-  db = new CouchDB(toUseAppId, opts)
+  db = dangerousGetDB(toUseAppId, opts)
  try {
    cls.setOnContext(key, db)
    if (opts) {
@ -206,22 +332,22 @@ function getDB(key, opts) {
 * Opens the app database based on whatever the request
 * contained, dev or prod.
 */
-exports.getAppDB = opts => {
-  return getDB(ContextKeys.CURRENT_DB, opts)
+exports.getAppDB = (opts = null) => {
+  return getContextDB(ContextKeys.CURRENT_DB, opts)
 }

 /**
 * This specifically gets the prod app ID, if the request
 * contained a development app ID, this will open the prod one.
 */
-exports.getProdAppDB = opts => {
-  return getDB(ContextKeys.PROD_DB, opts)
+exports.getProdAppDB = (opts = null) => {
+  return getContextDB(ContextKeys.PROD_DB, opts)
 }

 /**
 * This specifically gets the dev app ID, if the request
 * contained a prod app ID, this will open the dev one.
 */
-exports.getDevAppDB = opts => {
-  return getDB(ContextKeys.DEV_DB, opts)
+exports.getDevAppDB = (opts = null) => {
+  return getContextDB(ContextKeys.DEV_DB, opts)
 }
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@ -1,27 +1,35 @@
-const { getDB } = require(".")
+import { dangerousGetDB, closeDB } from "."

 class Replication {
+  source: any
+  target: any
+  replication: any
+
  /**
   *
   * @param {String} source - the DB you want to replicate or rollback to
   * @param {String} target - the DB you want to replicate to, or rollback from
   */
-  constructor({ source, target }) {
-    this.source = getDB(source)
-    this.target = getDB(target)
+  constructor({ source, target }: any) {
+    this.source = dangerousGetDB(source)
+    this.target = dangerousGetDB(target)
  }

-  promisify(operation, opts = {}) {
+  close() {
+    return Promise.all([closeDB(this.source), closeDB(this.target)])
+  }
+
+  promisify(operation: any, opts = {}) {
    return new Promise(resolve => {
      operation(this.target, opts)
-        .on("denied", function (err) {
+        .on("denied", function (err: any) {
          // a document failed to replicate (e.g. due to permissions)
          throw new Error(`Denied: Document failed to replicate ${err}`)
        })
-        .on("complete", function (info) {
+        .on("complete", function (info: any) {
          return resolve(info)
        })
-        .on("error", function (err) {
+        .on("error", function (err: any) {
          throw new Error(`Replication Error: ${err}`)
        })
    })
@ -51,7 +59,7 @@ class Replication {
  async rollback() {
    await this.target.destroy()
    // Recreate the DB again
-    this.target = getDB(this.target.name)
+    this.target = dangerousGetDB(this.target.name)
    await this.replicate()
  }

@ -60,4 +68,4 @@ class Replication {
  }
 }

-module.exports = Replication
+export default Replication
--- a/packages/backend-core/src/db/constants.js
+++ b/packages/backend-core/src/db/constants.js
@ -23,6 +23,7 @@ exports.StaticDatabases = {
    docs: {
      apiKeys: "apikeys",
      usageQuota: "usage_quota",
+      licenseInfo: "license_info",
    },
  },
  // contains information about tenancy and so on
@ -30,6 +31,7 @@ exports.StaticDatabases = {
    name: "global-info",
    docs: {
      tenants: "tenants",
+      install: "install",
    },
  },
 }
--- a/packages/backend-core/src/db/conversions.js
+++ b/packages/backend-core/src/db/conversions.js
@ -23,24 +23,30 @@ exports.isDevApp = app => {
 }

 /**
- * Convert a development app ID to a deployed app ID.
+ * Generates a development app ID from a real app ID.
+ * @returns {string} the dev app ID which can be used for dev database.
 */
-exports.getProdAppID = appId => {
-  // if dev, convert it
-  if (appId.startsWith(APP_DEV_PREFIX)) {
-    const id = appId.split(APP_DEV_PREFIX)[1]
-    return `${APP_PREFIX}${id}`
+exports.getDevelopmentAppID = appId => {
+  if (!appId || appId.startsWith(APP_DEV_PREFIX)) {
+    return appId
  }
-  return appId
+  // split to take off the app_ element, then join it together incase any other app_ exist
+  const split = appId.split(APP_PREFIX)
+  split.shift()
+  const rest = split.join(APP_PREFIX)
+  return `${APP_DEV_PREFIX}${rest}`
 }

 /**
- * Convert a deployed app ID to a development app ID.
+ * Convert a development app ID to a deployed app ID.
 */
-exports.getDevelopmentAppID = appId => {
-  if (!appId.startsWith(APP_DEV_PREFIX)) {
-    const id = appId.split(APP_PREFIX)[1]
-    return `${APP_DEV_PREFIX}${id}`
+exports.getProdAppID = appId => {
+  if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
+    return appId
  }
-  return appId
+  // split to take off the app_dev element, then join it together incase any other app_ exist
+  const split = appId.split(APP_DEV_PREFIX)
+  split.shift()
+  const rest = split.join(APP_DEV_PREFIX)
+  return `${APP_PREFIX}${rest}`
 }
--- a/packages/backend-core/src/db/index.js
+++ b/packages/backend-core/src/db/index.js
@ -1,13 +1,77 @@
-let Pouch
+const pouch = require("./pouch")
+const env = require("../environment")

-module.exports.setDB = pouch => {
-  Pouch = pouch
+let PouchDB
+let initialised = false
+const dbList = new Set()
+
+const put =
+  dbPut =>
+  async (doc, options = {}) => {
+    if (!doc.createdAt) {
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
+  }
+
+const checkInitialised = () => {
+  if (!initialised) {
+    throw new Error("init has not been called")
+  }
 }

-module.exports.getDB = dbName => {
-  return new Pouch(dbName)
+exports.init = opts => {
+  PouchDB = pouch.getPouch(opts)
+  initialised = true
 }

-module.exports.getCouch = () => {
-  return Pouch
+// NOTE: THIS IS A DANGEROUS FUNCTION - USE WITH CAUTION
+// this function is prone to leaks, should only be used
+// in situations that using the function doWithDB does not work
+exports.dangerousGetDB = (dbName, opts) => {
+  checkInitialised()
+  if (env.isTest()) {
+    dbList.add(dbName)
+  }
+  const db = new PouchDB(dbName, opts)
+  const dbPut = db.put
+  db.put = put(dbPut)
+  return db
+}
+
+// use this function if you have called dangerousGetDB - close
+// the databases you've opened once finished
+exports.closeDB = async db => {
+  if (!db || env.isTest()) {
+    return
+  }
+  try {
+    // specifically await so that if there is an error, it can be ignored
+    return await db.close()
+  } catch (err) {
+    // ignore error, already closed
+  }
+}
+
+// we have to use a callback for this so that we can close
+// the DB when we're done, without this manual requests would
+// need to close the database when done with it to avoid memory leaks
+exports.doWithDB = async (dbName, cb, opts = {}) => {
+  const db = exports.dangerousGetDB(dbName, opts)
+  // need this to be async so that we can correctly close DB after all
+  // async operations have been completed
+  try {
+    return await cb(db)
+  } finally {
+    await exports.closeDB(db)
+  }
+}
+
+exports.allDbs = () => {
+  if (!env.isTest()) {
+    throw new Error("Cannot be used outside test environment.")
+  }
+  checkInitialised()
+  return [...dbList]
 }
--- a/packages/backend-core/src/db/pouch.js
+++ b/packages/backend-core/src/db/pouch.js
@ -0,0 +1,96 @@
+const PouchDB = require("pouchdb")
+const env = require("../environment")
+
+function getUrlInfo() {
+  let url = env.COUCH_DB_URL
+  let username, password, host
+  const [protocol, rest] = url.split("://")
+  if (url.includes("@")) {
+    const hostParts = rest.split("@")
+    host = hostParts[1]
+    const authParts = hostParts[0].split(":")
+    username = authParts[0]
+    password = authParts[1]
+  } else {
+    host = rest
+  }
+  return {
+    url: `${protocol}://${host}`,
+    auth: {
+      username,
+      password,
+    },
+  }
+}
+
+exports.getCouchInfo = () => {
+  const urlInfo = getUrlInfo()
+  let username
+  let password
+  if (env.COUCH_DB_USERNAME) {
+    // set from env
+    username = env.COUCH_DB_USERNAME
+  } else if (urlInfo.auth.username) {
+    // set from url
+    username = urlInfo.auth.username
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB username not set")
+  }
+  if (env.COUCH_DB_PASSWORD) {
+    // set from env
+    password = env.COUCH_DB_PASSWORD
+  } else if (urlInfo.auth.password) {
+    // set from url
+    password = urlInfo.auth.password
+  } else if (!env.isTest()) {
+    throw new Error("CouchDB password not set")
+  }
+  const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
+  return {
+    url: urlInfo.url,
+    auth: {
+      username: username,
+      password: password,
+    },
+    cookie: `Basic ${authCookie}`,
+  }
+}
+
+/**
+ * Return a constructor for PouchDB.
+ * This should be rarely used outside of the main application config.
+ * Exposed for exceptional cases such as in-memory views.
+ */
+exports.getPouch = (opts = {}) => {
+  let { url, cookie } = exports.getCouchInfo()
+  let POUCH_DB_DEFAULTS = {
+    prefix: url,
+    fetch: (url, opts) => {
+      // use a specific authorization cookie - be very explicit about how we authenticate
+      opts.headers.set("Authorization", cookie)
+      return PouchDB.fetch(url, opts)
+    },
+  }
+
+  if (opts.inMemory) {
+    const inMemory = require("pouchdb-adapter-memory")
+    PouchDB.plugin(inMemory)
+    POUCH_DB_DEFAULTS = {
+      prefix: undefined,
+      adapter: "memory",
+    }
+  }
+
+  if (opts.replication) {
+    const replicationStream = require("pouchdb-replication-stream")
+    PouchDB.plugin(replicationStream.plugin)
+    PouchDB.adapter("writableStream", replicationStream.adapters.writableStream)
+  }
+
+  if (opts.find) {
+    const find = require("pouchdb-find")
+    PouchDB.plugin(find)
+  }
+
+  return PouchDB.defaults(POUCH_DB_DEFAULTS)
+}
--- a/packages/backend-core/src/db/tests/index.spec.js
+++ b/packages/backend-core/src/db/tests/index.spec.js
@ -0,0 +1,26 @@
+require("../../../tests/utilities/TestConfiguration")
+const { dangerousGetDB } = require("../")
+
+describe("db", () => { 
+  
+  describe("getDB", () => {
+    it("returns a db", async () => {
+      const db = dangerousGetDB("test")
+      expect(db).toBeDefined()
+      expect(db._adapter).toBe("memory")
+      expect(db.prefix).toBe("_pouch_")
+      expect(db.name).toBe("test")
+    })
+
+    it("uses the custom put function", async () => {
+      const db = dangerousGetDB("test")
+      let doc = { _id: "test" }
+      await db.put(doc)
+      doc = await db.get(doc._id)
+      expect(doc.createdAt).toBe(new Date().toISOString())
+      expect(doc.updatedAt).toBe(new Date().toISOString())
+      await db.destroy()
+    })
+  })
+})
+
--- a/packages/backend-core/src/db/tests/utils.spec.js
+++ b/packages/backend-core/src/db/tests/utils.spec.js
@ -0,0 +1,194 @@
+require("../../../tests/utilities/TestConfiguration");
+const {
+  generateAppID,
+  getDevelopmentAppID,
+  getProdAppID,
+  isDevAppID,
+  isProdAppID,
+  getPlatformUrl,
+  getScopedConfig
+} = require("../utils")
+const tenancy = require("../../tenancy");
+const { Configs, DEFAULT_TENANT_ID } = require("../../constants");
+const env = require("../../environment")
+
+describe("utils", () => {
+  describe("app ID manipulation", () => {
+
+    function getID() {
+      const appId = generateAppID()
+      const split = appId.split("_")
+      const uuid = split[split.length - 1]
+      const devAppId = `app_dev_${uuid}`
+      return { appId, devAppId, split, uuid }
+    }
+
+    it("should be able to generate a new app ID", () => {
+      expect(generateAppID().startsWith("app_")).toEqual(true)
+    })
+  
+    it("should be able to convert a production app ID to development", () => {
+      const { appId, uuid } = getID()
+      expect(getDevelopmentAppID(appId)).toEqual(`app_dev_${uuid}`)
+    })
+  
+    it("should be able to convert a development app ID to development", () => {
+      const { devAppId, uuid } = getID()
+      expect(getDevelopmentAppID(devAppId)).toEqual(`app_dev_${uuid}`)
+    })
+  
+    it("should be able to convert a development ID to a production", () => {
+      const { devAppId, uuid } = getID()
+      expect(getProdAppID(devAppId)).toEqual(`app_${uuid}`)
+    })
+  
+    it("should be able to convert a production ID to production", () => {
+      const { appId, uuid } = getID()
+      expect(getProdAppID(appId)).toEqual(`app_${uuid}`)
+    })
+  
+    it("should be able to confirm dev app ID is development", () => {
+      const { devAppId } = getID()
+      expect(isDevAppID(devAppId)).toEqual(true)
+    })
+  
+    it("should be able to confirm prod app ID is not development", () => {
+      const { appId } = getID()
+      expect(isDevAppID(appId)).toEqual(false)
+    })
+  
+    it("should be able to confirm prod app ID is prod", () => {
+      const { appId } = getID()
+      expect(isProdAppID(appId)).toEqual(true)
+    })
+  
+    it("should be able to confirm dev app ID is not prod", () => {
+      const { devAppId } = getID()
+      expect(isProdAppID(devAppId)).toEqual(false)
+    })
+  })
+})
+
+const DB_URL = "http://dburl.com"
+const DEFAULT_URL = "http://localhost:10000"
+const ENV_URL = "http://env.com"
+
+const setDbPlatformUrl = async () => {
+  const db = tenancy.getGlobalDB()
+  db.put({
+    _id: "config_settings",
+    type: Configs.SETTINGS,
+    config: {
+      platformUrl: DB_URL
+    }
+  })
+}
+
+const clearSettingsConfig = async () => {
+  await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+    const db = tenancy.getGlobalDB()
+    try {
+      const config = await db.get("config_settings")
+      await db.remove("config_settings", config._rev)
+    } catch (e) {
+      if (e.status !== 404) {
+        throw e
+      }
+    }
+  })
+}
+ 
+describe("getPlatformUrl", () => {
+  describe("self host", () => {
+
+    beforeEach(async () => {
+      env._set("SELF_HOST", 1)
+      await clearSettingsConfig()
+    })
+
+    it("gets the default url", async () => {
+      await tenancy.doInTenant(null, async () => {
+        const url = await getPlatformUrl()
+        expect(url).toBe(DEFAULT_URL)
+      })
+    })
+
+    it("gets the platform url from the environment", async () => {
+      await tenancy.doInTenant(null, async () => {
+        env._set("PLATFORM_URL", ENV_URL)
+        const url = await getPlatformUrl()
+        expect(url).toBe(ENV_URL)
+      })
+    })
+
+    it("gets the platform url from the database", async () => {
+      await tenancy.doInTenant(null, async () => {
+        await setDbPlatformUrl()
+        const url = await getPlatformUrl()
+        expect(url).toBe(DB_URL)
+      })
+    })    
+  })
+
+
+  describe("cloud", () => {
+    const TENANT_AWARE_URL = "http://default.env.com"
+
+    beforeEach(async () => {
+      env._set("SELF_HOSTED", 0)
+      env._set("MULTI_TENANCY", 1)
+      env._set("PLATFORM_URL", ENV_URL)
+      await clearSettingsConfig()
+    })
+
+    it("gets the platform url from the environment without tenancy", async () => {
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        const url = await getPlatformUrl({ tenantAware: false })
+        expect(url).toBe(ENV_URL)
+      })
+    })
+
+    it("gets the platform url from the environment with tenancy", async () => {
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        const url = await getPlatformUrl()
+        expect(url).toBe(TENANT_AWARE_URL)
+      })
+    })
+
+    it("never gets the platform url from the database", async () => {
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        await setDbPlatformUrl()
+        const url = await getPlatformUrl()
+        expect(url).toBe(TENANT_AWARE_URL)
+      })
+    })    
+  })
+})
+
+describe("getScopedConfig", () => {
+  describe("settings config", () => {
+
+    beforeEach(async () => {
+      env._set("SELF_HOSTED", 1)
+      env._set("PLATFORM_URL", "")
+      await clearSettingsConfig()
+    })
+
+    it("returns the platform url with an existing config", async () => {
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        await setDbPlatformUrl()
+        const db = tenancy.getGlobalDB()
+        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        expect(config.platformUrl).toBe(DB_URL)
+      })
+    })
+
+    it("returns the platform url without an existing config", async () => {
+      await tenancy.doInTenant(DEFAULT_TENANT_ID, async () => {
+        const db = tenancy.getGlobalDB()
+        const config = await getScopedConfig(db, { type: Configs.SETTINGS })
+        expect(config.platformUrl).toBe(DEFAULT_URL)
+      })
+    })
+  })
+})
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@ -1,45 +1,40 @@
-const { newid } = require("../hashing")
-const Replication = require("./Replication")
-const { DEFAULT_TENANT_ID, Configs } = require("../constants")
-const env = require("../environment")
-const {
-  StaticDatabases,
-  SEPARATOR,
-  DocumentTypes,
-  APP_PREFIX,
-  APP_DEV,
-} = require("./constants")
-const { getTenantId, getGlobalDBName } = require("../tenancy")
-const fetch = require("node-fetch")
-const { getCouch } = require("./index")
-const { getAppMetadata } = require("../cache/appMetadata")
-const { checkSlashesInUrl } = require("../helpers")
-const {
-  isDevApp,
-  isProdAppID,
-  isDevAppID,
-  getDevelopmentAppID,
-  getProdAppID,
-} = require("./conversions")
+import { newid } from "../hashing"
+import { DEFAULT_TENANT_ID, Configs } from "../constants"
+import env from "../environment"
+import { SEPARATOR, DocumentTypes } from "./constants"
+import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
+import fetch from "node-fetch"
+import { doWithDB, allDbs } from "./index"
+import { getCouchInfo } from "./pouch"
+import { getAppMetadata } from "../cache/appMetadata"
+import { checkSlashesInUrl } from "../helpers"
+import { isDevApp, isDevAppID } from "./conversions"
+import { APP_PREFIX } from "./constants"
+import * as events from "../events"

 const UNICODE_MAX = "\ufff0"

-exports.ViewNames = {
+export const ViewNames = {
  USER_BY_EMAIL: "by_email",
  BY_API_KEY: "by_api_key",
+  USER_BY_BUILDERS: "by_builders",
 }

-exports.StaticDatabases = StaticDatabases
+export * from "./constants"
+export * from "./conversions"
+export { default as Replication } from "./Replication"

-exports.DocumentTypes = DocumentTypes
-exports.APP_PREFIX = APP_PREFIX
-exports.APP_DEV = exports.APP_DEV_PREFIX = APP_DEV
-exports.SEPARATOR = SEPARATOR
-exports.isDevApp = isDevApp
-exports.isProdAppID = isProdAppID
-exports.isDevAppID = isDevAppID
-exports.getDevelopmentAppID = getDevelopmentAppID
-exports.getProdAppID = getProdAppID
+/**
+ * Generates a new app ID.
+ * @returns {string} The new app ID which the app doc can be stored under.
+ */
+export const generateAppID = (tenantId = null) => {
+  let id = APP_PREFIX
+  if (tenantId) {
+    id += `${tenantId}${SEPARATOR}`
+  }
+  return `${id}${newid()}`
+}

 /**
 * If creating DB allDocs/query params with only a single top level ID this can be used, this
@ -53,7 +48,11 @@ exports.getProdAppID = getProdAppID
 * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
 * @returns {object} Parameters which can then be used with an allDocs request.
 */
-function getDocParams(docType, docId = null, otherProps = {}) {
+export function getDocParams(
+  docType: any,
+  docId: any = null,
+  otherProps: any = {}
+) {
  if (docId == null) {
    docId = ""
  }
@ -63,20 +62,19 @@ function getDocParams(docType, docId = null, otherProps = {}) {
    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
  }
 }
-exports.getDocParams = getDocParams

 /**
 * Generates a new workspace ID.
 * @returns {string} The new workspace ID which the workspace doc can be stored under.
 */
-exports.generateWorkspaceID = () => {
+export function generateWorkspaceID() {
  return `${DocumentTypes.WORKSPACE}${SEPARATOR}${newid()}`
 }

 /**
 * Gets parameters for retrieving workspaces.
 */
-exports.getWorkspaceParams = (id = "", otherProps = {}) => {
+export function getWorkspaceParams(id = "", otherProps = {}) {
  return {
    ...otherProps,
    startkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}`,
@ -88,14 +86,14 @@ exports.getWorkspaceParams = (id = "", otherProps = {}) => {
 * Generates a new global user ID.
 * @returns {string} The new user ID which the user doc can be stored under.
 */
-exports.generateGlobalUserID = id => {
+export function generateGlobalUserID(id?: any) {
  return `${DocumentTypes.USER}${SEPARATOR}${id || newid()}`
 }

 /**
 * Gets parameters for retrieving users.
 */
-exports.getGlobalUserParams = (globalId, otherProps = {}) => {
+export function getGlobalUserParams(globalId: any, otherProps = {}) {
  if (!globalId) {
    globalId = ""
  }
@ -110,14 +108,18 @@ exports.getGlobalUserParams = (globalId, otherProps = {}) => {
 * Generates a template ID.
 * @param ownerId The owner/user of the template, this could be global or a workspace level.
 */
-exports.generateTemplateID = ownerId => {
+export function generateTemplateID(ownerId: any) {
  return `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
 }

 /**
 * Gets parameters for retrieving templates. Owner ID must be specified, either global or a workspace level.
 */
-exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
+export function getTemplateParams(
+  ownerId: any,
+  templateId: any,
+  otherProps = {}
+) {
  if (!templateId) {
    templateId = ""
  }
@ -138,37 +140,18 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
 * Generates a new role ID.
 * @returns {string} The new role ID which the role doc can be stored under.
 */
-exports.generateRoleID = id => {
+export function generateRoleID(id: any) {
  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
 }

 /**
 * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
 */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
+export function getRoleParams(roleId = null, otherProps = {}) {
  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }

-exports.getCouchUrl = () => {
-  if (!env.COUCH_DB_URL) return
-
-  // username and password already exist in URL
-  if (env.COUCH_DB_URL.includes("@")) {
-    return env.COUCH_DB_URL
-  }
-
-  const [protocol, ...rest] = env.COUCH_DB_URL.split("://")
-
-  if (!env.COUCH_DB_USERNAME || !env.COUCH_DB_PASSWORD) {
-    throw new Error(
-      "CouchDB configuration invalid. You must provide a fully qualified CouchDB url, or the COUCH_DB_USER and COUCH_DB_PASSWORD environment variables."
-    )
-  }
-
-  return `${protocol}://${env.COUCH_DB_USERNAME}:${env.COUCH_DB_PASSWORD}@${rest}`
-}
-
-exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
+export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) {
  const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
  return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
@ -179,15 +162,21 @@ exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
 * opts.efficient can be provided to make sure this call is always quick in a multi-tenant environment,
 * but it may not be 100% accurate in full efficiency mode (some tenantless apps may be missed).
 */
-exports.getAllDbs = async (opts = { efficient: false }) => {
+export async function getAllDbs(opts = { efficient: false }) {
  const efficient = opts && opts.efficient
  // specifically for testing we use the pouch package for this
  if (env.isTest()) {
-    return getCouch().allDbs()
+    return allDbs()
  }
-  let dbs = []
-  async function addDbs(url) {
-    const response = await fetch(checkSlashesInUrl(encodeURI(url)))
+  let dbs: any[] = []
+  let { url, cookie } = getCouchInfo()
+  async function addDbs(couchUrl: string) {
+    const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
+      method: "GET",
+      headers: {
+        Authorization: cookie,
+      },
+    })
    if (response.status === 200) {
      let json = await response.json()
      dbs = dbs.concat(json)
@ -195,7 +184,7 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
      throw "Cannot connect to CouchDB instance"
    }
  }
-  let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
+  let couchUrl = `${url}/_all_dbs`
  let tenantId = getTenantId()
  if (!env.MULTI_TENANCY || (!efficient && tenantId === DEFAULT_TENANT_ID)) {
    // just get all DBs when:
@ -206,13 +195,9 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
    await addDbs(couchUrl)
  } else {
    // get prod apps
-    await addDbs(
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
-    )
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId))
    // get dev apps
-    await addDbs(
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId)
-    )
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId))
    // add global db name
    dbs.push(getGlobalDBName(tenantId))
  }
@ -225,14 +210,13 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
 *
 * @return {Promise<object[]>} returns the app information document stored in each app database.
 */
-exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
-  const CouchDB = getCouch()
+export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
  let tenantId = getTenantId()
  if (!env.MULTI_TENANCY && !tenantId) {
    tenantId = DEFAULT_TENANT_ID
  }
-  let dbs = await exports.getAllDbs({ efficient })
-  const appDbNames = dbs.filter(dbName => {
+  let dbs = await getAllDbs({ efficient })
+  const appDbNames = dbs.filter((dbName: any) => {
    const split = dbName.split(SEPARATOR)
    // it is an app, check the tenantId
    if (split[0] === DocumentTypes.APP) {
@ -252,26 +236,28 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
  if (idsOnly) {
    return appDbNames
  }
-  const appPromises = appDbNames.map(app =>
+  const appPromises = appDbNames.map((app: any) =>
    // skip setup otherwise databases could be re-created
-    getAppMetadata(app, CouchDB)
+    getAppMetadata(app)
  )
  if (appPromises.length === 0) {
    return []
  } else {
    const response = await Promise.allSettled(appPromises)
    const apps = response
-      .filter(result => result.status === "fulfilled" && result.value != null)
-      .map(({ value }) => value)
+      .filter(
+        (result: any) => result.status === "fulfilled" && result.value != null
+      )
+      .map(({ value }: any) => value)
    if (!all) {
-      return apps.filter(app => {
+      return apps.filter((app: any) => {
        if (dev) {
          return isDevApp(app)
        }
        return !isDevApp(app)
      })
    } else {
-      return apps.map(app => ({
+      return apps.map((app: any) => ({
        ...app,
        status: isDevApp(app) ? "development" : "published",
      }))
@ -282,42 +268,46 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
 /**
 * Utility function for getAllApps but filters to production apps only.
 */
-exports.getProdAppIDs = async () => {
-  return (await exports.getAllApps({ idsOnly: true })).filter(
-    id => !exports.isDevAppID(id)
+export async function getProdAppIDs() {
+  return (await getAllApps({ idsOnly: true })).filter(
+    (id: any) => !isDevAppID(id)
  )
 }

 /**
 * Utility function for the inverse of above.
 */
-exports.getDevAppIDs = async () => {
-  return (await exports.getAllApps({ idsOnly: true })).filter(id =>
-    exports.isDevAppID(id)
+export async function getDevAppIDs() {
+  return (await getAllApps({ idsOnly: true })).filter((id: any) =>
+    isDevAppID(id)
  )
 }

-exports.dbExists = async dbName => {
-  const CouchDB = getCouch()
+export async function dbExists(dbName: any) {
  let exists = false
-  try {
-    const db = CouchDB(dbName, { skip_setup: true })
-    // check if database exists
-    const info = await db.info()
-    if (info && !info.error) {
-      exists = true
-    }
-  } catch (err) {
-    exists = false
-  }
-  return exists
+  return doWithDB(
+    dbName,
+    async (db: any) => {
+      try {
+        // check if database exists
+        const info = await db.info()
+        if (info && !info.error) {
+          exists = true
+        }
+      } catch (err) {
+        exists = false
+      }
+      return exists
+    },
+    { skip_setup: true }
+  )
 }

 /**
 * Generates a new configuration ID.
 * @returns {string} The new configuration ID which the config doc can be stored under.
 */
-const generateConfigID = ({ type, workspace, user }) => {
+export const generateConfigID = ({ type, workspace, user }: any) => {
  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)

  return `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`
@ -326,7 +316,10 @@ const generateConfigID = ({ type, workspace, user }) => {
 /**
 * Gets parameters for retrieving configurations.
 */
-const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
+export const getConfigParams = (
+  { type, workspace, user }: any,
+  otherProps = {}
+) => {
  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)

  return {
@ -340,7 +333,7 @@ const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
 * Generates a new dev info document ID - this is scoped to a user.
 * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
 */
-const generateDevInfoID = userId => {
+export const generateDevInfoID = (userId: any) => {
  return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}`
 }

@ -350,7 +343,10 @@ const generateDevInfoID = userId => {
 * @param {Object} scopes - the type, workspace and userID scopes of the configuration.
 * @returns The most granular configuration document based on the scope.
 */
-const getScopedFullConfig = async function (db, { type, user, workspace }) {
+export const getScopedFullConfig = async function (
+  db: any,
+  { type, user, workspace }: any
+) {
  const response = await db.allDocs(
    getConfigParams(
      { type, user, workspace },
@ -360,7 +356,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
    )
  )

-  function determineScore(row) {
+  function determineScore(row: any) {
    const config = row.doc

    // Config is specific to a user and a workspace
@ -381,21 +377,24 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {

  // Find the config with the most granular scope based on context
  let scopedConfig = response.rows.sort(
-    (a, b) => determineScore(a) - determineScore(b)
+    (a: any, b: any) => determineScore(a) - determineScore(b)
  )[0]

  // custom logic for settings doc
-  // always provide the platform URL
  if (type === Configs.SETTINGS) {
    if (scopedConfig && scopedConfig.doc) {
-      scopedConfig.doc.config.platformUrl = await getPlatformUrl(
-        scopedConfig.doc.config
-      )
+      // overrides affected by environment variables
+      scopedConfig.doc.config.platformUrl = await getPlatformUrl()
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
    } else {
+      // defaults
      scopedConfig = {
        doc: {
+          _id: generateConfigID({ type, user, workspace }),
          config: {
            platformUrl: await getPlatformUrl(),
+            analyticsEnabled: await events.analytics.enabled(),
          },
        },
      }
@ -405,58 +404,37 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
  return scopedConfig && scopedConfig.doc
 }

-const getPlatformUrl = async settings => {
+export const getPlatformUrl = async (opts = { tenantAware: true }) => {
  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"

-  if (!env.SELF_HOSTED && env.MULTI_TENANCY) {
+  if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
    // cloud and multi tenant - add the tenant to the default platform url
    const tenantId = getTenantId()
    if (!platformUrl.includes("localhost:")) {
      platformUrl = platformUrl.replace("://", `://${tenantId}.`)
    }
-  } else {
+  } else if (env.SELF_HOSTED) {
+    const db = getGlobalDB()
+    // get the doc directly instead of with getScopedConfig to prevent loop
+    let settings
+    try {
+      settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
+    } catch (e: any) {
+      if (e.status !== 404) {
+        throw e
+      }
+    }
+
    // self hosted - check for platform url override
-    if (settings && settings.platformUrl) {
-      platformUrl = settings.platformUrl
+    if (settings && settings.config && settings.config.platformUrl) {
+      platformUrl = settings.config.platformUrl
    }
  }

  return platformUrl
 }

-async function getScopedConfig(db, params) {
+export async function getScopedConfig(db: any, params: any) {
  const configDoc = await getScopedFullConfig(db, params)
  return configDoc && configDoc.config ? configDoc.config : configDoc
 }
-
-function generateNewUsageQuotaDoc() {
-  return {
-    _id: StaticDatabases.GLOBAL.docs.usageQuota,
-    quotaReset: Date.now() + 2592000000,
-    usageQuota: {
-      automationRuns: 0,
-      rows: 0,
-      storage: 0,
-      apps: 0,
-      users: 0,
-      views: 0,
-      emails: 0,
-    },
-    usageLimits: {
-      automationRuns: 1000,
-      rows: 4000,
-      apps: 4,
-      storage: 1000,
-      users: 10,
-      emails: 50,
-    },
-  }
-}
-
-exports.Replication = Replication
-exports.getScopedConfig = getScopedConfig
-exports.generateConfigID = generateConfigID
-exports.getConfigParams = getConfigParams
-exports.getScopedFullConfig = getScopedFullConfig
-exports.generateNewUsageQuotaDoc = generateNewUsageQuotaDoc
-exports.generateDevInfoID = generateDevInfoID
--- a/packages/backend-core/src/db/views.js
+++ b/packages/backend-core/src/db/views.js
@ -56,10 +56,34 @@ exports.createApiKeyView = async () => {
  await db.put(designDoc)
 }

+exports.createUserBuildersView = async () => {
+  const db = getGlobalDB()
+  let designDoc
+  try {
+    designDoc = await db.get("_design/database")
+  } catch (err) {
+    // no design doc, make one
+    designDoc = DesignDoc()
+  }
+  const view = {
+    map: `function(doc) {
+      if (doc.builder && doc.builder.global === true) {
+        emit(doc._id, doc._id)
+      }
+    }`,
+  }
+  designDoc.views = {
+    ...designDoc.views,
+    [ViewNames.USER_BY_BUILDERS]: view,
+  }
+  await db.put(designDoc)
+}
+
 exports.queryGlobalView = async (viewName, params, db = null) => {
  const CreateFuncByName = {
    [ViewNames.USER_BY_EMAIL]: exports.createUserEmailView,
    [ViewNames.BY_API_KEY]: exports.createApiKeyView,
+    [ViewNames.USER_BY_BUILDERS]: exports.createUserBuildersView,
  }
  // can pass DB in if working with something specific
  if (!db) {
--- a/packages/backend-core/src/environment.js
+++ b/packages/backend-core/src/environment.js
@ -1,36 +0,0 @@
-function isTest() {
-  return (
-    process.env.NODE_ENV === "jest" ||
-    process.env.NODE_ENV === "cypress" ||
-    process.env.JEST_WORKER_ID != null
-  )
-}
-
-module.exports = {
-  JWT_SECRET: process.env.JWT_SECRET,
-  COUCH_DB_URL: process.env.COUCH_DB_URL,
-  COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
-  COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
-  GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
-  GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
-  SALT_ROUNDS: process.env.SALT_ROUNDS,
-  REDIS_URL: process.env.REDIS_URL,
-  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
-  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
-  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
-  AWS_REGION: process.env.AWS_REGION,
-  MINIO_URL: process.env.MINIO_URL,
-  INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
-  MULTI_TENANCY: process.env.MULTI_TENANCY,
-  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
-  ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
-  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
-  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
-  PLATFORM_URL: process.env.PLATFORM_URL,
-  isTest,
-  _set(key, value) {
-    process.env[key] = value
-    module.exports[key] = value
-  },
-}
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@ -0,0 +1,73 @@
+function isTest() {
+  return (
+    process.env.NODE_ENV === "jest" ||
+    process.env.NODE_ENV === "cypress" ||
+    process.env.JEST_WORKER_ID != null
+  )
+}
+
+function isDev() {
+  return process.env.NODE_ENV !== "production"
+}
+
+let LOADED = false
+if (!LOADED && isDev() && !isTest()) {
+  require("dotenv").config()
+  LOADED = true
+}
+
+const env: any = {
+  isTest,
+  isDev,
+  JWT_SECRET: process.env.JWT_SECRET,
+  COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
+  COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
+  COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
+  GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
+  GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
+  SALT_ROUNDS: process.env.SALT_ROUNDS,
+  REDIS_URL: process.env.REDIS_URL,
+  REDIS_PASSWORD: process.env.REDIS_PASSWORD,
+  MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
+  MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  AWS_REGION: process.env.AWS_REGION,
+  MINIO_URL: process.env.MINIO_URL,
+  INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
+  MULTI_TENANCY: process.env.MULTI_TENANCY,
+  ACCOUNT_PORTAL_URL:
+    process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
+  ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
+  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
+  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
+  PLATFORM_URL: process.env.PLATFORM_URL,
+  POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
+  BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
+  APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
+  TEMPLATES_BUCKET_NAME: process.env.TEMPLATES_BUCKET_NAME || "templates",
+  GLOBAL_BUCKET_NAME: process.env.GLOBAL_BUCKET_NAME || "global",
+  GLOBAL_CLOUD_BUCKET_NAME:
+    process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
+  USE_COUCH: process.env.USE_COUCH || true,
+  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
+  SERVICE: process.env.SERVICE || "budibase",
+  DEPLOYMENT_ENVIRONMENT:
+    process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  _set(key: any, value: any) {
+    process.env[key] = value
+    module.exports[key] = value
+  },
+}
+
+// clean up any environment variable edge cases
+for (let [key, value] of Object.entries(env)) {
+  // handle the edge case of "0" to disable an environment variable
+  if (value === "0") {
+    env[key] = 0
+  }
+}
+
+export = env
--- a/packages/backend-core/src/errors/base.js
+++ b/packages/backend-core/src/errors/base.js
@ -0,0 +1,11 @@
+class BudibaseError extends Error {
+  constructor(message, code, type) {
+    super(message)
+    this.code = code
+    this.type = type
+  }
+}
+
+module.exports = {
+  BudibaseError,
+}
--- a/packages/backend-core/src/errors/generic.js
+++ b/packages/backend-core/src/errors/generic.js
@ -0,0 +1,11 @@
+const { BudibaseError } = require("./base")
+
+class GenericError extends BudibaseError {
+  constructor(message, code, type) {
+    super(message, code, type ? type : "generic")
+  }
+}
+
+module.exports = {
+  GenericError,
+}
--- a/packages/backend-core/src/errors/http.js
+++ b/packages/backend-core/src/errors/http.js
@ -0,0 +1,12 @@
+const { GenericError } = require("./generic")
+
+class HTTPError extends GenericError {
+  constructor(message, httpStatus, code = "http", type = "generic") {
+    super(message, code, type)
+    this.status = httpStatus
+  }
+}
+
+module.exports = {
+  HTTPError,
+}
--- a/packages/backend-core/src/errors/index.js
+++ b/packages/backend-core/src/errors/index.js
@ -0,0 +1,43 @@
+const http = require("./http")
+const licensing = require("./licensing")
+
+const codes = {
+  ...licensing.codes,
+}
+
+const types = [licensing.type]
+
+const context = {
+  ...licensing.context,
+}
+
+const getPublicError = err => {
+  let error
+  if (err.code || err.type) {
+    // add generic error information
+    error = {
+      code: err.code,
+      type: err.type,
+    }
+
+    if (err.code && context[err.code]) {
+      error = {
+        ...error,
+        // get any additional context from this error
+        ...context[err.code](err),
+      }
+    }
+  }
+
+  return error
+}
+
+module.exports = {
+  codes,
+  types,
+  errors: {
+    UsageLimitError: licensing.UsageLimitError,
+    HTTPError: http.HTTPError,
+  },
+  getPublicError,
+}
--- a/packages/backend-core/src/errors/licensing.js
+++ b/packages/backend-core/src/errors/licensing.js
@ -0,0 +1,29 @@
+const { HTTPError } = require("./http")
+
+const type = "license_error"
+
+const codes = {
+  USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded",
+}
+
+const context = {
+  [codes.USAGE_LIMIT_EXCEEDED]: err => {
+    return {
+      limitName: err.limitName,
+    }
+  },
+}
+
+class UsageLimitError extends HTTPError {
+  constructor(message, limitName) {
+    super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type)
+    this.limitName = limitName
+  }
+}
+
+module.exports = {
+  type,
+  codes,
+  context,
+  UsageLimitError,
+}
--- a/packages/backend-core/src/events/analytics.ts
+++ b/packages/backend-core/src/events/analytics.ts
@ -0,0 +1,57 @@
+import env from "../environment"
+import tenancy from "../tenancy"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+export const enabled = async () => {
+  // cloud - always use the environment variable
+  if (!env.SELF_HOSTED) {
+    return !!env.ENABLE_ANALYTICS
+  }
+
+  // self host - prefer the settings doc
+  // use cache as events have high throughput
+  const enabledInDB = await withCache(
+    CacheKeys.ANALYTICS_ENABLED,
+    TTL.ONE_DAY,
+    async () => {
+      const settings = await getSettingsDoc()
+
+      // need to do explicit checks in case the field is not set
+      if (settings?.config?.analyticsEnabled === false) {
+        return false
+      } else if (settings?.config?.analyticsEnabled === true) {
+        return true
+      }
+    }
+  )
+
+  if (enabledInDB !== undefined) {
+    return enabledInDB
+  }
+
+  // fallback to the environment variable
+  // explicitly check for 0 or false here, undefined or otherwise is treated as true
+  const envEnabled: any = env.ENABLE_ANALYTICS
+  if (envEnabled === 0 || envEnabled === false) {
+    return false
+  } else {
+    return true
+  }
+}
+
+const getSettingsDoc = async () => {
+  const db = tenancy.getGlobalDB()
+  let settings
+  try {
+    settings = await db.get(
+      dbUtils.generateConfigID({ type: Configs.SETTINGS })
+    )
+  } catch (e: any) {
+    if (e.status !== 404) {
+      throw e
+    }
+  }
+  return settings
+}
--- a/packages/backend-core/src/events/backfill.ts
+++ b/packages/backend-core/src/events/backfill.ts
@ -0,0 +1,183 @@
+import {
+  Event,
+  BackfillMetadata,
+  CachedEvent,
+  SSOCreatedEvent,
+  AutomationCreatedEvent,
+  AutomationStepCreatedEvent,
+  DatasourceCreatedEvent,
+  LayoutCreatedEvent,
+  QueryCreatedEvent,
+  RoleCreatedEvent,
+  ScreenCreatedEvent,
+  TableCreatedEvent,
+  ViewCreatedEvent,
+  ViewCalculationCreatedEvent,
+  ViewFilterCreatedEvent,
+  AppPublishedEvent,
+  UserCreatedEvent,
+  RoleAssignedEvent,
+  UserPermissionAssignedEvent,
+  AppCreatedEvent,
+} from "@budibase/types"
+import * as context from "../context"
+import { CacheKeys } from "../cache/generic"
+import * as cache from "../cache/generic"
+
+// LIFECYCLE
+
+export const start = async (events: Event[]) => {
+  const metadata: BackfillMetadata = {
+    eventWhitelist: events,
+  }
+  return saveBackfillMetadata(metadata)
+}
+
+export const recordEvent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  // don't use a ttl - cleaned up by migration
+  // don't use tenancy - already in the key
+  await cache.store(eventKey, properties, undefined, { useTenancy: false })
+}
+
+export const end = async () => {
+  await deleteBackfillMetadata()
+  await clearEvents()
+}
+
+// CRUD
+
+const getBackfillMetadata = async (): Promise<BackfillMetadata | null> => {
+  return cache.get(CacheKeys.BACKFILL_METADATA)
+}
+
+const saveBackfillMetadata = async (
+  backfill: BackfillMetadata
+): Promise<void> => {
+  // no TTL - deleted by backfill
+  return cache.store(CacheKeys.BACKFILL_METADATA, backfill)
+}
+
+const deleteBackfillMetadata = async (): Promise<void> => {
+  await cache.delete(CacheKeys.BACKFILL_METADATA)
+}
+
+const clearEvents = async () => {
+  // wildcard
+  const pattern = getEventKey()
+  const keys = await cache.keys(pattern)
+
+  for (const key of keys) {
+    // delete each key
+    // don't  use tenancy, already in the key
+    await cache.delete(key, { useTenancy: false })
+  }
+}
+
+// HELPERS
+
+export const isBackfillingEvent = async (event: Event) => {
+  const backfill = await getBackfillMetadata()
+  const events = backfill?.eventWhitelist
+  if (events && events.includes(event)) {
+    return true
+  } else {
+    return false
+  }
+}
+
+export const isAlreadySent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  const cachedEvent: CachedEvent = await cache.get(eventKey, {
+    useTenancy: false,
+  })
+  return !!cachedEvent
+}
+
+const CUSTOM_PROPERTY_SUFFIX: any = {
+  // APP EVENTS
+  [Event.AUTOMATION_CREATED]: (properties: AutomationCreatedEvent) => {
+    return properties.automationId
+  },
+  [Event.AUTOMATION_STEP_CREATED]: (properties: AutomationStepCreatedEvent) => {
+    return properties.stepId
+  },
+  [Event.DATASOURCE_CREATED]: (properties: DatasourceCreatedEvent) => {
+    return properties.datasourceId
+  },
+  [Event.LAYOUT_CREATED]: (properties: LayoutCreatedEvent) => {
+    return properties.layoutId
+  },
+  [Event.QUERY_CREATED]: (properties: QueryCreatedEvent) => {
+    return properties.queryId
+  },
+  [Event.ROLE_CREATED]: (properties: RoleCreatedEvent) => {
+    return properties.roleId
+  },
+  [Event.SCREEN_CREATED]: (properties: ScreenCreatedEvent) => {
+    return properties.screenId
+  },
+  [Event.TABLE_CREATED]: (properties: TableCreatedEvent) => {
+    return properties.tableId
+  },
+  [Event.VIEW_CREATED]: (properties: ViewCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_CALCULATION_CREATED]: (
+    properties: ViewCalculationCreatedEvent
+  ) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_FILTER_CREATED]: (properties: ViewFilterCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.APP_CREATED]: (properties: AppCreatedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  // GLOBAL EVENTS
+  [Event.AUTH_SSO_CREATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.AUTH_SSO_ACTIVATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.USER_CREATED]: (properties: UserCreatedEvent) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.ROLE_ASSIGNED]: (properties: RoleAssignedEvent) => {
+    return `${properties.roleId}-${properties.userId}`
+  },
+}
+
+const getEventKey = (event?: Event, properties?: any) => {
+  let eventKey: string
+
+  const tenantId = context.getTenantId()
+  if (event) {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}`
+
+    // use some properties to make the key more unique
+    const custom = CUSTOM_PROPERTY_SUFFIX[event]
+    const suffix = custom ? custom(properties) : undefined
+    if (suffix) {
+      eventKey = `${eventKey}:${suffix}`
+    }
+  } else {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:*`
+  }
+
+  return eventKey
+}
--- a/packages/backend-core/src/events/events.ts
+++ b/packages/backend-core/src/events/events.ts
@ -0,0 +1,31 @@
+import { Event } from "@budibase/types"
+import { processors } from "./processors"
+import * as identification from "./identification"
+import * as backfill from "./backfill"
+
+export const publishEvent = async (
+  event: Event,
+  properties: any,
+  timestamp?: string | number
+) => {
+  // in future this should use async events via a distributed queue.
+  const identity = await identification.getCurrentIdentity()
+
+  const backfilling = await backfill.isBackfillingEvent(event)
+  // no backfill - send the event and exit
+  if (!backfilling) {
+    await processors.processEvent(event, identity, properties, timestamp)
+    return
+  }
+
+  // backfill active - check if the event has been sent already
+  const alreadySent = await backfill.isAlreadySent(event, properties)
+  if (alreadySent) {
+    // do nothing
+    return
+  } else {
+    // send and record the event
+    await processors.processEvent(event, identity, properties, timestamp)
+    await backfill.recordEvent(event, properties)
+  }
+}
--- a/packages/backend-core/src/events/identification.ts
+++ b/packages/backend-core/src/events/identification.ts
@ -0,0 +1,302 @@
+import * as context from "../context"
+import * as identityCtx from "../context/identity"
+import env from "../environment"
+import {
+  Hosting,
+  User,
+  Identity,
+  IdentityType,
+  Account,
+  isCloudAccount,
+  isSSOAccount,
+  TenantGroup,
+  SettingsConfig,
+  CloudAccount,
+  UserIdentity,
+  InstallationGroup,
+  UserContext,
+  Group,
+} from "@budibase/types"
+import { processors } from "./processors"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import * as hashing from "../hashing"
+import * as installation from "../installation"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+const pkg = require("../../package.json")
+
+/**
+ * An identity can be:
+ * - account user (Self host)
+ * - budibase user
+ * - tenant
+ * - installation
+ */
+export const getCurrentIdentity = async (): Promise<Identity> => {
+  let identityContext = identityCtx.getIdentity()
+  const environment = getDeploymentEnvironment()
+
+  let identityType
+
+  if (!identityContext) {
+    identityType = IdentityType.TENANT
+  } else {
+    identityType = identityContext.type
+  }
+
+  if (identityType === IdentityType.INSTALLATION) {
+    const installationId = await getInstallationId()
+    const hosting = getHostingFromEnv()
+    return {
+      id: formatDistinctId(installationId, identityType),
+      hosting,
+      type: identityType,
+      installationId,
+      environment,
+    }
+  } else if (identityType === IdentityType.TENANT) {
+    const installationId = await getInstallationId()
+    const tenantId = await getEventTenantId(context.getTenantId())
+    const hosting = getHostingFromEnv()
+
+    return {
+      id: formatDistinctId(tenantId, identityType),
+      type: identityType,
+      hosting,
+      installationId,
+      tenantId,
+      environment,
+    }
+  } else if (identityType === IdentityType.USER) {
+    const userContext = identityContext as UserContext
+    const tenantId = await getEventTenantId(context.getTenantId())
+    const installationId = await getInstallationId()
+
+    const account = userContext.account
+    let hosting
+    if (account) {
+      hosting = account.hosting
+    } else {
+      hosting = getHostingFromEnv()
+    }
+
+    return {
+      id: userContext._id,
+      type: identityType,
+      hosting,
+      installationId,
+      tenantId,
+      environment,
+    }
+  } else {
+    throw new Error("Unknown identity type")
+  }
+}
+
+export const identifyInstallationGroup = async (
+  installId: string,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = installId
+  const type = IdentityType.INSTALLATION
+  const hosting = getHostingFromEnv()
+  const version = pkg.version
+  const environment = getDeploymentEnvironment()
+
+  const group: InstallationGroup = {
+    id,
+    type,
+    hosting,
+    version,
+    environment,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the empty auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyTenantGroup = async (
+  tenantId: string,
+  account: Account | undefined,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = await getEventTenantId(tenantId)
+  const type = IdentityType.TENANT
+  const installationId = await getInstallationId()
+  const environment = getDeploymentEnvironment()
+
+  let hosting: Hosting
+  let profession: string | undefined
+  let companySize: string | undefined
+
+  if (account) {
+    profession = account.profession
+    companySize = account.size
+    hosting = account.hosting
+  } else {
+    hosting = getHostingFromEnv()
+  }
+
+  const group: TenantGroup = {
+    id,
+    type,
+    hosting,
+    environment,
+    installationId,
+    profession,
+    companySize,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyUser = async (
+  user: User,
+  account: CloudAccount | undefined,
+  timestamp?: string | number
+) => {
+  const id = user._id as string
+  const tenantId = await getEventTenantId(user.tenantId)
+  const type = IdentityType.USER
+  let builder = user.builder?.global || false
+  let admin = user.admin?.global || false
+  let providerType = user.providerType
+  const accountHolder = account?.budibaseUserId === user._id || false
+  const verified =
+    account && account?.budibaseUserId === user._id ? account.verified : false
+  const installationId = await getInstallationId()
+  const hosting = account ? account.hosting : getHostingFromEnv()
+  const environment = getDeploymentEnvironment()
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    hosting,
+    installationId,
+    tenantId,
+    verified,
+    accountHolder,
+    providerType,
+    builder,
+    admin,
+    environment,
+  }
+
+  await identify(identity, timestamp)
+}
+
+export const identifyAccount = async (account: Account) => {
+  let id = account.accountId
+  const tenantId = account.tenantId
+  let type = IdentityType.USER
+  let providerType = isSSOAccount(account) ? account.providerType : undefined
+  const verified = account.verified
+  const accountHolder = true
+  const hosting = account.hosting
+  const installationId = await getInstallationId()
+  const environment = getDeploymentEnvironment()
+
+  if (isCloudAccount(account)) {
+    if (account.budibaseUserId) {
+      // use the budibase user as the id if set
+      id = account.budibaseUserId
+    }
+  }
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    hosting,
+    installationId,
+    tenantId,
+    providerType,
+    verified,
+    accountHolder,
+    environment,
+  }
+
+  await identify(identity)
+}
+
+export const identify = async (
+  identity: Identity,
+  timestamp?: string | number
+) => {
+  await processors.identify(identity, timestamp)
+}
+
+export const identifyGroup = async (
+  group: Group,
+  timestamp?: string | number
+) => {
+  await processors.identifyGroup(group, timestamp)
+}
+
+const getDeploymentEnvironment = () => {
+  if (env.isDev()) {
+    return "development"
+  } else {
+    return env.DEPLOYMENT_ENVIRONMENT
+  }
+}
+
+const getHostingFromEnv = () => {
+  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
+}
+
+export const getInstallationId = async () => {
+  if (isAccountPortal()) {
+    return "account-portal"
+  }
+  const install = await installation.getInstall()
+  return install.installId
+}
+
+const getEventTenantId = async (tenantId: string): Promise<string> => {
+  if (env.SELF_HOSTED) {
+    return getUniqueTenantId(tenantId)
+  } else {
+    // tenant id's in the cloud are already unique
+    return tenantId
+  }
+}
+
+const getUniqueTenantId = async (tenantId: string): Promise<string> => {
+  // make sure this tenantId always matches the tenantId in context
+  return context.doInTenant(tenantId, () => {
+    return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
+      const db = context.getGlobalDB()
+      const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
+        type: Configs.SETTINGS,
+      })
+
+      let uniqueTenantId: string
+      if (config.config.uniqueTenantId) {
+        return config.config.uniqueTenantId
+      } else {
+        uniqueTenantId = `${hashing.newid()}_${tenantId}`
+        config.config.uniqueTenantId = uniqueTenantId
+        await db.put(config)
+        return uniqueTenantId
+      }
+    })
+  })
+}
+
+const isAccountPortal = () => {
+  return env.SERVICE === "account-portal"
+}
+
+const formatDistinctId = (id: string, type: IdentityType) => {
+  if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) {
+    return `$${type}_${id}`
+  } else {
+    return id
+  }
+}
--- a/packages/backend-core/src/events/index.ts
+++ b/packages/backend-core/src/events/index.ts
@ -0,0 +1,11 @@
+export * from "./publishers"
+export * as processors from "./processors"
+export * as analytics from "./analytics"
+export * as identification from "./identification"
+export * as backfillCache from "./backfill"
+
+import { processors } from "./processors"
+
+export const shutdown = () => {
+  processors.shutdown()
+}
--- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
+++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
@ -0,0 +1,64 @@
+import { Event, Identity, Group, IdentityType } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import * as analytics from "../analytics"
+import PosthogProcessor from "./PosthogProcessor"
+
+/**
+ * Events that are always captured.
+ */
+const EVENT_WHITELIST = [
+  Event.INSTALLATION_VERSION_UPGRADED,
+  Event.INSTALLATION_VERSION_DOWNGRADED,
+]
+const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT]
+
+export default class AnalyticsProcessor implements EventProcessor {
+  posthog: PosthogProcessor | undefined
+
+  constructor() {
+    if (env.POSTHOG_TOKEN && !env.isTest()) {
+      this.posthog = new PosthogProcessor(env.POSTHOG_TOKEN)
+    }
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (
+      !IDENTITY_WHITELIST.includes(identity.type) &&
+      !(await analytics.enabled())
+    ) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (this.posthog) {
+      this.posthog.identifyGroup(group, timestamp)
+    }
+  }
+
+  shutdown() {
+    if (this.posthog) {
+      this.posthog.shutdown()
+    }
+  }
+}
--- a/packages/backend-core/src/events/processors/LoggingProcessor.ts
+++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts
@ -0,0 +1,54 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+
+const getTimestampString = (timestamp?: string | number) => {
+  let timestampString = ""
+  if (timestamp) {
+    timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+  }
+  return timestampString
+}
+
+const skipLogging = env.SELF_HOSTED && !env.isDev()
+
+export default class LoggingProcessor implements EventProcessor {
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+    )
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(identity)}] ${timestampString} identified`
+    )
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(group)}] ${timestampString} group identified`
+    )
+  }
+
+  shutdown(): void {
+    // no-op
+  }
+}
--- a/packages/backend-core/src/events/processors/PosthogProcessor.ts
+++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts
@ -0,0 +1,81 @@
+import PostHog from "posthog-node"
+import { Event, Identity, Group, BaseEvent } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import context from "../../context"
+const pkg = require("../../../package.json")
+
+export default class PosthogProcessor implements EventProcessor {
+  posthog: PostHog
+
+  constructor(token: string | undefined) {
+    if (!token) {
+      throw new Error("Posthog token is not defined")
+    }
+    this.posthog = new PostHog(token)
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: BaseEvent,
+    timestamp?: string | number
+  ): Promise<void> {
+    properties.version = pkg.version
+    properties.service = env.SERVICE
+    properties.environment = identity.environment
+    properties.hosting = identity.hosting
+
+    const appId = context.getAppId()
+    if (appId) {
+      properties.appId = appId
+    }
+
+    const payload: any = { distinctId: identity.id, event, properties }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+
+    // add groups to the event
+    if (identity.installationId || identity.tenantId) {
+      payload.groups = {}
+      if (identity.installationId) {
+        payload.groups.installation = identity.installationId
+        payload.properties.installationId = identity.installationId
+      }
+      if (identity.tenantId) {
+        payload.groups.tenant = identity.tenantId
+        payload.properties.tenantId = identity.tenantId
+      }
+    }
+
+    this.posthog.capture(payload)
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    const payload: any = { distinctId: identity.id, properties: identity }
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.identify(payload)
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    const payload: any = {
+      distinctId: group.id,
+      groupType: group.type,
+      groupKey: group.id,
+      properties: group,
+    }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.groupIdentify(payload)
+  }
+
+  shutdown() {
+    this.posthog.shutdown()
+  }
+}
--- a/packages/backend-core/src/events/processors/Processors.ts
+++ b/packages/backend-core/src/events/processors/Processors.ts
@ -0,0 +1,46 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+
+export default class Processor implements EventProcessor {
+  initialised: boolean = false
+  processors: EventProcessor[] = []
+
+  constructor(processors: EventProcessor[]) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(
+    identity: Identity,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(
+    identity: Group,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identifyGroup(identity, timestamp)
+    }
+  }
+
+  shutdown() {
+    for (const eventProcessor of this.processors) {
+      eventProcessor.shutdown()
+    }
+  }
+}
--- a/packages/backend-core/src/events/processors/index.ts
+++ b/packages/backend-core/src/events/processors/index.ts
@ -0,0 +1,8 @@
+import AnalyticsProcessor from "./AnalyticsProcessor"
+import LoggingProcessor from "./LoggingProcessor"
+import Processors from "./Processors"
+
+export const analyticsProcessor = new AnalyticsProcessor()
+const loggingProcessor = new LoggingProcessor()
+
+export const processors = new Processors([analyticsProcessor, loggingProcessor])
--- a/packages/backend-core/src/events/processors/types.ts
+++ b/packages/backend-core/src/events/processors/types.ts
@ -0,0 +1,18 @@
+import { Event, Identity, Group } from "@budibase/types"
+
+export enum EventProcessorType {
+  POSTHOG = "posthog",
+  LOGGING = "logging",
+}
+
+export interface EventProcessor {
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
+  identify(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
+  shutdown(): void
+}
--- a/packages/backend-core/src/events/publishers/account.ts
+++ b/packages/backend-core/src/events/publishers/account.ts
@ -0,0 +1,29 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Account,
+  AccountCreatedEvent,
+  AccountDeletedEvent,
+  AccountVerifiedEvent,
+} from "@budibase/types"
+
+export async function created(account: Account) {
+  const properties: AccountCreatedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_CREATED, properties)
+}
+
+export async function deleted(account: Account) {
+  const properties: AccountDeletedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_DELETED, properties)
+}
+
+export async function verified(account: Account) {
+  const properties: AccountVerifiedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_VERIFIED, properties)
+}
--- a/packages/backend-core/src/events/publishers/app.ts
+++ b/packages/backend-core/src/events/publishers/app.ts
@ -0,0 +1,108 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  App,
+  AppCreatedEvent,
+  AppUpdatedEvent,
+  AppDeletedEvent,
+  AppPublishedEvent,
+  AppUnpublishedEvent,
+  AppFileImportedEvent,
+  AppTemplateImportedEvent,
+  AppVersionUpdatedEvent,
+  AppVersionRevertedEvent,
+  AppRevertedEvent,
+  AppExportedEvent,
+} from "@budibase/types"
+
+export const created = async (app: App, timestamp?: string | number) => {
+  const properties: AppCreatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_CREATED, properties, timestamp)
+}
+
+export async function updated(app: App) {
+  const properties: AppUpdatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_UPDATED, properties)
+}
+
+export async function deleted(app: App) {
+  const properties: AppDeletedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_DELETED, properties)
+}
+
+export async function published(app: App, timestamp?: string | number) {
+  const properties: AppPublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
+}
+
+export async function unpublished(app: App) {
+  const properties: AppUnpublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_UNPUBLISHED, properties)
+}
+
+export async function fileImported(app: App) {
+  const properties: AppFileImportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_FILE_IMPORTED, properties)
+}
+
+export async function templateImported(app: App, templateKey: string) {
+  const properties: AppTemplateImportedEvent = {
+    appId: app.appId,
+    templateKey,
+  }
+  await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
+}
+
+export async function versionUpdated(
+  app: App,
+  currentVersion: string,
+  updatedToVersion: string
+) {
+  const properties: AppVersionUpdatedEvent = {
+    appId: app.appId,
+    currentVersion,
+    updatedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_UPDATED, properties)
+}
+
+export async function versionReverted(
+  app: App,
+  currentVersion: string,
+  revertedToVersion: string
+) {
+  const properties: AppVersionRevertedEvent = {
+    appId: app.appId,
+    currentVersion,
+    revertedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_REVERTED, properties)
+}
+
+export async function reverted(app: App) {
+  const properties: AppRevertedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_REVERTED, properties)
+}
+
+export async function exported(app: App) {
+  const properties: AppExportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_EXPORTED, properties)
+}
--- a/packages/backend-core/src/events/publishers/auth.ts
+++ b/packages/backend-core/src/events/publishers/auth.ts
@ -0,0 +1,58 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  LoginEvent,
+  LoginSource,
+  LogoutEvent,
+  SSOActivatedEvent,
+  SSOCreatedEvent,
+  SSODeactivatedEvent,
+  SSOType,
+  SSOUpdatedEvent,
+} from "@budibase/types"
+import { identification } from ".."
+
+export async function login(source: LoginSource) {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LoginEvent = {
+    userId: identity.id,
+    source,
+  }
+  await publishEvent(Event.AUTH_LOGIN, properties)
+}
+
+export async function logout() {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LogoutEvent = {
+    userId: identity.id,
+  }
+  await publishEvent(Event.AUTH_LOGOUT, properties)
+}
+
+export async function SSOCreated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOCreatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp)
+}
+
+export async function SSOUpdated(type: SSOType) {
+  const properties: SSOUpdatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_UPDATED, properties)
+}
+
+export async function SSOActivated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOActivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp)
+}
+
+export async function SSODeactivated(type: SSOType) {
+  const properties: SSODeactivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_DEACTIVATED, properties)
+}
--- a/packages/backend-core/src/events/publishers/automation.ts
+++ b/packages/backend-core/src/events/publishers/automation.ts
@ -0,0 +1,94 @@
+import { publishEvent } from "../events"
+import {
+  Automation,
+  Event,
+  AutomationStep,
+  AutomationCreatedEvent,
+  AutomationDeletedEvent,
+  AutomationTestedEvent,
+  AutomationStepCreatedEvent,
+  AutomationStepDeletedEvent,
+  AutomationTriggerUpdatedEvent,
+  AutomationsRunEvent,
+} from "@budibase/types"
+
+export async function created(
+  automation: Automation,
+  timestamp?: string | number
+) {
+  const properties: AutomationCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
+}
+
+export async function triggerUpdated(automation: Automation) {
+  const properties: AutomationTriggerUpdatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TRIGGER_UPDATED, properties)
+}
+
+export async function deleted(automation: Automation) {
+  const properties: AutomationDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_DELETED, properties)
+}
+
+export async function tested(automation: Automation) {
+  const properties: AutomationTestedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TESTED, properties)
+}
+
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: AutomationsRunEvent = {
+    count,
+  }
+  await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
+}
+
+export async function stepCreated(
+  automation: Automation,
+  step: AutomationStep,
+  timestamp?: string | number
+) {
+  const properties: AutomationStepCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
+}
+
+export async function stepDeleted(
+  automation: Automation,
+  step: AutomationStep
+) {
+  const properties: AutomationStepDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/backfill.ts
+++ b/packages/backend-core/src/events/publishers/backfill.ts
@ -0,0 +1,67 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  AppBackfillSucceededEvent,
+  AppBackfillFailedEvent,
+  TenantBackfillSucceededEvent,
+  TenantBackfillFailedEvent,
+  InstallationBackfillSucceededEvent,
+  InstallationBackfillFailedEvent,
+} from "@budibase/types"
+const env = require("../../environment")
+
+const shouldSkip = !env.SELF_HOSTED && !env.isDev()
+
+export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function appFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: AppBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.APP_BACKFILL_FAILED, properties)
+}
+
+export async function tenantSucceeded(
+  properties: TenantBackfillSucceededEvent
+) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function tenantFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: TenantBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
+}
+
+export async function installationSucceeded() {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillSucceededEvent = {}
+  await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function installationFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
+}
--- a/packages/backend-core/src/events/publishers/datasource.ts
+++ b/packages/backend-core/src/events/publishers/datasource.ts
@ -0,0 +1,35 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Datasource,
+  DatasourceCreatedEvent,
+  DatasourceUpdatedEvent,
+  DatasourceDeletedEvent,
+} from "@budibase/types"
+
+export async function created(
+  datasource: Datasource,
+  timestamp?: string | number
+) {
+  const properties: DatasourceCreatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
+}
+
+export async function updated(datasource: Datasource) {
+  const properties: DatasourceUpdatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_UPDATED, properties)
+}
+
+export async function deleted(datasource: Datasource) {
+  const properties: DatasourceDeletedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/email.ts
+++ b/packages/backend-core/src/events/publishers/email.ts
@ -0,0 +1,12 @@
+import { publishEvent } from "../events"
+import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
+
+export async function SMTPCreated(timestamp?: string | number) {
+  const properties: SMTPCreatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
+}
+
+export async function SMTPUpdated() {
+  const properties: SMTPUpdatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
+}
--- a/packages/backend-core/src/events/publishers/index.ts
+++ b/packages/backend-core/src/events/publishers/index.ts
@ -0,0 +1,19 @@
+export * as account from "./account"
+export * as app from "./app"
+export * as auth from "./auth"
+export * as automation from "./automation"
+export * as datasource from "./datasource"
+export * as email from "./email"
+export * as license from "./license"
+export * as layout from "./layout"
+export * as org from "./org"
+export * as query from "./query"
+export * as role from "./role"
+export * as screen from "./screen"
+export * as rows from "./rows"
+export * as table from "./table"
+export * as serve from "./serve"
+export * as user from "./user"
+export * as view from "./view"
+export * as installation from "./installation"
+export * as backfill from "./backfill"
--- a/packages/backend-core/src/events/publishers/installation.ts
+++ b/packages/backend-core/src/events/publishers/installation.ts
@ -0,0 +1,31 @@
+import { publishEvent } from "../events"
+import { Event, VersionCheckedEvent, VersionChangeEvent } from "@budibase/types"
+
+export async function versionChecked(version: string) {
+  const properties: VersionCheckedEvent = {
+    currentVersion: version,
+  }
+  await publishEvent(Event.INSTALLATION_VERSION_CHECKED, properties)
+}
+
+export async function upgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+
+  await publishEvent(Event.INSTALLATION_VERSION_UPGRADED, properties)
+}
+
+export async function downgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+  await publishEvent(Event.INSTALLATION_VERSION_DOWNGRADED, properties)
+}
+
+export async function firstStartup() {
+  const properties = {}
+  await publishEvent(Event.INSTALLATION_FIRST_STARTUP, properties)
+}
--- a/packages/backend-core/src/events/publishers/layout.ts
+++ b/packages/backend-core/src/events/publishers/layout.ts
@ -0,0 +1,21 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Layout,
+  LayoutCreatedEvent,
+  LayoutDeletedEvent,
+} from "@budibase/types"
+
+export async function created(layout: Layout, timestamp?: string | number) {
+  const properties: LayoutCreatedEvent = {
+    layoutId: layout._id as string,
+  }
+  await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
+}
+
+export async function deleted(layoutId: string) {
+  const properties: LayoutDeletedEvent = {
+    layoutId,
+  }
+  await publishEvent(Event.LAYOUT_DELETED, properties)
+}
--- a/Show More
+++ b/Show More