Merge branch 'develop' of github.com:Budibase/budibase into cheeks-lab-day-spreadsheet

2023-02-08 15:10:48 +00:00 · 2023-02-08 15:10:48 +00:00 · 5eeea14de0
parent e40e82fd5f 568ac4da1b
commit 5eeea14de0
1239 changed files with 35363 additions and 29267 deletions
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@ -6,6 +6,8 @@ labels: bug
 assignees: ''

 ---
+**Checklist**
+- [ ] I have searched budibase discussions and github issues to check if my issue already exists

 **Hosting**
 <!-- Delete as appropriate -->
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@ -0,0 +1 @@
+blank_issues_enabled: false
--- a/.github/ISSUE_TEMPLATE/epic.md
+++ b/.github/ISSUE_TEMPLATE/epic.md
@ -1,24 +0,0 @@
---
-name: Epic
-about: Plan a new project
-title: ''
-labels: epic
-assignees: ''
-
---
-
-## Description
-Brief summary of what this Epic is, whether it's a larger project, goal, or user story. Describe the job to be done, which persona this Epic is mainly for, or if more multiple, break it down by user and job story.
-
-## Spec
-Link to confluence spec
-
-## Teams and Stakeholders
-Describe who needs to be kept up-to-date about this Epic, included in discussions, or updated along the way. Stakeholders can be both in Product/Engineering, as well as other teams like Customer Success who might want to keep customers updated on the Epic project.
-
-
-## Workflow
- [ ]  Spec Created and pasted above
- [ ]  Product Review
- [ ]  Designs created
- [ ]  Individual Tasks created and assigned to Epic
--- a/.github/workflows/deploy-cloud.yaml
+++ b/.github/workflows/deploy-cloud.yaml
@ -38,17 +38,6 @@ jobs:
          fi
          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
      
-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:prod
-          docker tag proxy-service budibase/proxy:$PROD_TAG
-          docker push budibase/proxy:$PROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PROD_TAG: k8s
-      
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
--- a/.github/workflows/deploy-preprod.yml
+++ b/.github/workflows/deploy-preprod.yml
@ -28,17 +28,6 @@ jobs:
          release_version=$(cat lerna.json | jq -r '.version')
          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV

-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:preprod
-          docker tag proxy-service budibase/proxy:$PREPROD_TAG
-          docker push budibase/proxy:$PREPROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PREPROD_TAG: k8s-preprod
-
      - name: Pull values.yaml from budibase-infra
        run: | 
          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
--- a/.github/workflows/deploy-release.yml
+++ b/.github/workflows/deploy-release.yml
@ -29,17 +29,6 @@ jobs:
          release_version=$(cat lerna.json | jq -r '.version')
          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV

-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:release
-          docker tag proxy-service budibase/proxy:$RELEASE_TAG
-          docker push budibase/proxy:$RELEASE_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          RELEASE_TAG: k8s-release
-
      - name: Pull values.yaml from budibase-infra
        run: | 
          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -26,7 +26,7 @@ env:
  FEATURE_PREVIEW_URL: https://budirelease.live
      
 jobs:
-  release:
+  release-images:
    runs-on: ubuntu-latest 

    steps:
@ -44,19 +44,12 @@ jobs:
        run: yarn install:pro develop

      - run: yarn 
-      - run: yarn bootstrap 
+      - run: yarn bootstrap
      - run: yarn lint 
      - run: yarn build
      - run: yarn build:sdk
      - run: yarn test

-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-1
-
      - name: Publish budibase packages to NPM
        env:
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
@ -76,22 +69,25 @@ jobs:
          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

-      - name: Get the latest budibase release version
+  deploy-to-release-env:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get the current budibase release version
        id: version
-        run: | 
+        run: |
          release_version=$(cat lerna.json | jq -r '.version')
          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV

-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:release
-          docker tag proxy-service budibase/proxy:$RELEASE_TAG
-          docker push budibase/proxy:$RELEASE_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          RELEASE_TAG: k8s-release
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1

      - name: Pull values.yaml from budibase-infra
        run: | 
@ -149,3 +145,54 @@ jobs:
          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
          content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
          embed-title: ${{ env.RELEASE_VERSION }}
+
+  release-helm-chart:
+    needs: [release-images]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        id: helm-install
+
+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
+      - name: Build and release helm chart
+        run: |
+          git config user.name "Budibase Helm Bot"
+          git config user.email "<>"
+          git reset --hard
+          git pull
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version 0.0.0-develop --app-version develop --destination sync
+          echo "Packaging successful"
+          git checkout gh-pages
+          echo "Indexing helm repo"         
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
+          git add -A
+          git commit -m "Helm Release: develop"
+          git push
+
+  trigger-deploy-to-qa-env:
+    needs: [release-helm-chart]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get the current budibase release version
+        id: version
+        run: |
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+
+      - uses: passeidireto/trigger-external-workflow-action@main
+        env:
+          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
+        with:
+          repository: budibase/budibase-deploys
+          event: deploy-budibase-develop-to-qa
+          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
--- a/.github/workflows/release-selfhost.yml
+++ b/.github/workflows/release-selfhost.yml
@ -67,16 +67,24 @@ jobs:
        uses: azure/setup-helm@v1
        id: helm-install

+      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
+      # we need to create new package in a different dir, merge the index and move the package back
      - name: Build and release helm chart
        run: | 
          git config user.name "Budibase Helm Bot"
          git config user.email "<>"
          git reset --hard
          git pull
-          helm package charts/budibase
+          mkdir sync
+          echo "Packaging chart to sync dir"
+          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
+          echo "Packaging successful"
          git checkout gh-pages
-          mv *.tgz docs
-          helm repo index docs
+          echo "Indexing helm repo"
+          helm repo index --merge docs/index.yaml sync
+          mv -f sync/* docs
+          rm -rf sync
+          echo "Pushing new helm release"
          git add -A
          git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
          git push
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -98,17 +98,6 @@ jobs:
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1

-      - name: Tag and release Proxy service docker image
-        run: | 
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build:docker:proxy:preprod
-          docker tag proxy-service budibase/proxy:$PREPROD_TAG
-          docker push budibase/proxy:$PREPROD_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          PREPROD_TAG: k8s-preprod
-
      - name: Pull values.yaml from budibase-infra
        run: | 
          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
--- a/.github/workflows/smoke_test.yaml
+++ b/.github/workflows/smoke_test.yaml
@ -18,30 +18,18 @@ jobs:
      - run: yarn
      - run: yarn bootstrap
      - run: yarn build
-      - name: Pull cypress.env.yaml from budibase-infra
+      - name: Pull  from budibase-infra
        run: | 
          curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
          -H 'Accept: application/vnd.github.v3.raw' \
-          -o packages/builder/cypress.env.json \
-          -L https://api.github.com/repos/budibase/budibase-infra/contents/test/cypress.env.json
-          wc -l packages/builder/cypress.env.json
-
-      - name: Cypress run
-        id: cypress
-        continue-on-error: true
-        uses: cypress-io/github-action@v2
-        with:
-          record: true
-          install: false
-          tag: nightly
-          command: yarn test:e2e:ci:record
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          -o 
+          -L 
+          wc -l 
      
      - uses: actions/upload-artifact@v3
        with:
          name: Test Reports
-          path: packages/builder/cypress/reports/testReport.html
+          path: 

      # TODO: enable once running in QA test env
      # - name: Configure AWS Credentials
@ -54,11 +42,3 @@ jobs:
      # - name: Upload test results HTML
      #   uses: aws-actions/configure-aws-credentials@v1
      #   run: aws s3 cp packages/builder/cypress/reports/testReport.html s3://{{ secrets.BUDI_QA_REPORTS_BUCKET_NAME }}/$GITHUB_RUN_ID/index.html
-
-      - name: Cypress Discord Notify
-        run: yarn test:e2e:ci:notify
-        env:
-          CYPRESS_WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }}
-          CYPRESS_OUTCOME: ${{ steps.cypress.outcome }}
-          CYPRESS_DASHBOARD_URL: ${{ steps.cypress.outputs.dashboardUrl }}
-          GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@ builder/*
 packages/server/runtime_apps/
 .idea/
 bb-airgapped.tar.gz
+*.iml

 # Logs
 logs
@ -65,8 +66,6 @@ typings/
 .env
 !qa-core/.env
 !hosting/.env
-hosting/.generated-nginx.dev.conf
-hosting/proxy/.generated-nginx.prod.conf

 # parcel-bundler cache (https://parceljs.org/)
 .cache
@ -104,5 +103,9 @@ stats.html

 # TypeScript cache
 *.tsbuildinfo
+
+# plugins
 budibase-component
 budibase-datasource
+
+*.iml
--- a/.nvmrc
+++ b/.nvmrc
@ -0,0 +1 @@
+v14.19.3
--- a/.python-version
+++ b/.python-version
@ -0,0 +1 @@
+3.11.1
--- a/.tool-versions
+++ b/.tool-versions
@ -0,0 +1,2 @@
+nodejs 14.19.3
+python 3.11.1
--- a/.vscode/extensions.json
+++ b/.vscode/extensions.json
@ -0,0 +1,6 @@
+{
+    "recommendations": [
+        "esbenp.prettier-vscode",
+        "svelte.svelte-vscode"
+    ]
+}
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@ -1,19 +1,28 @@
 {
-    "editor.formatOnSave": true,
-    "editor.codeActionsOnSave": {
-        "source.fixAll": true
-    },
-    "editor.defaultFormatter": "svelte.svelte-vscode",
-    "[json]": {
-        "editor.defaultFormatter": "vscode.json-language-features"
-    },
-    "[javascript]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
-    },
-    "debug.javascript.terminalOptions": {
-        "skipFiles": [
-            "${workspaceFolder}/packages/backend-core/node_modules/**",
-            "<node_internals>/**"
-        ]
-    },
+  "editor.formatOnSave": true,
+  "editor.codeActionsOnSave": {
+    "source.fixAll": true
+  },
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[javascript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "debug.javascript.terminalOptions": {
+    "skipFiles": [
+      "${workspaceFolder}/packages/backend-core/node_modules/**",
+      "<node_internals>/**"
+    ]
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[dockercompose]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[svelte]": {
+    "editor.defaultFormatter": "svelte.svelte-vscode"
+  }
 }
--- a/artifacthub-repo.yml
+++ b/artifacthub-repo.yml
@ -0,0 +1,10 @@
+# Artifact Hub repository metadata file 
+# This file is used to verify ownership of the budibase Helm chart repo 
+# so that we appear as a verified owner on artifacthub.io
+
+repositoryID: a7536764-e72e-4961-87d8-efe7c8dedfa3
+owners: # (optional, used to claim repository ownership)
+  - name: Martin
+    email: budimaster@budibase.com
+  - name: DevOps
+    email: devops@budibase.com
--- a/charts/budibase/Chart.yaml
+++ b/charts/budibase/Chart.yaml
@ -11,11 +11,13 @@ sources:
  - https://github.com/Budibase/budibase
  - https://budibase.com
 type: application
-version: 0.2.11
-appVersion: 1.0.214
+# populates on packaging
+version: 0.0.0
+# populates on packaging
+appVersion: 0.0.0
 dependencies:
  - name: couchdb
-    version: 3.6.1
+    version: 3.3.4
    repository: https://apache.github.io/couchdb-helm
    condition: services.couchdb.enabled
  - name: ingress-nginx
--- a/charts/budibase/templates/alb-ingress.yaml
+++ b/charts/budibase/templates/alb-ingress.yaml
@ -14,6 +14,9 @@ metadata:
    alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]'
    alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} 
    {{- end }}
+    {{- if .Values.ingress.securityGroups }}
+    alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }}
+    {{- end }}
 spec:
  rules:
    - http:
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@ -20,6 +20,9 @@ spec:
      annotations:
        kompose.cmd: kompose convert
        kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.apps.annotations }}
+{{- toYaml .Values.services.apps.annotations | indent 8 -}}
+{{ end }}
      creationTimestamp: null
      labels:
        io.kompose.service: app-service
@ -64,6 +67,8 @@ spec:
        - name: AWS_REGION
          value: {{ .Values.services.objectStore.region }}
        {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
        - name: MINIO_ACCESS_KEY
          valueFrom:
            secretKeyRef:
@ -74,13 +79,19 @@ spec:
            secretKeyRef:
              name: {{ template "budibase.fullname" . }}
              key: objectStoreSecret
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
        - name: MINIO_URL
          value: {{ .Values.services.objectStore.url }}
        - name: PLUGIN_BUCKET_NAME
          value: {{ .Values.services.objectStore.pluginBucketName | quote }}
        - name: APPS_BUCKET_NAME
          value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
          value: {{ .Values.services.objectStore.globalBucketName | quote }}
        - name: BACKUPS_BUCKET_NAME
          value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@ -128,6 +139,8 @@ spec:
          value: {{ .Values.globals.automationMaxIterations | quote }}
        - name: TENANT_FEATURE_FLAGS
          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
        {{ if .Values.globals.bbAdminUserEmail }}
        - name: BB_ADMIN_USER_EMAIL
          value: {{ .Values.globals.bbAdminUserEmail | quote }}
@ -156,8 +169,24 @@ spec:
        - name: ELASTIC_APM_SERVER_URL
          value: {{ .Values.globals.elasticApmServerUrl | quote }}
        {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
        - name: CDN_URL
          value: {{ .Values.globals.cdnUrl }}
+        {{ if .Values.services.tlsRejectUnauthorized }}
+        - name: NODE_TLS_REJECT_UNAUTHORIZED
+          value: {{ .Values.services.tlsRejectUnauthorized }}
+        {{ end }}

        image: budibase/apps:{{ .Values.globals.appVersion }}
        imagePullPolicy: Always
--- a/charts/budibase/templates/minio-service-deployment.yaml
+++ b/charts/budibase/templates/minio-service-deployment.yaml
@ -42,6 +42,7 @@ spec:
            secretKeyRef:
              name: {{ template "budibase.fullname" . }}
              key: objectStoreSecret
+
        image: minio/minio
        imagePullPolicy: ""
        livenessProbe:
--- a/charts/budibase/templates/proxy-service-deployment.yaml
+++ b/charts/budibase/templates/proxy-service-deployment.yaml
@ -20,16 +20,34 @@ spec:
      annotations:
        kompose.cmd: kompose convert
        kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.proxy.annotations }}
+{{- toYaml .Values.services.proxy.annotations | indent 8 -}}
+{{ end }}
      creationTimestamp: null
      labels:
        app.kubernetes.io/name: budibase-proxy
    spec:
      containers:
-      - image: budibase/proxy:{{ .Values.services.proxy.tag | default "k8s" }}
+      - image: budibase/proxy:{{ .Values.globals.appVersion }}
        imagePullPolicy: Always
        name: proxy-service
        ports:
        - containerPort: {{ .Values.services.proxy.port }}
+        env:
+        - name: APPS_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.apps . | quote }}
+        - name: WORKER_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.worker . | quote }}
+        - name: MINIO_UPSTREAM_URL
+          value: {{ tpl .Values.services.proxy.upstreams.minio . | quote }}
+        - name: COUCHDB_UPSTREAM_URL
+          value: {{ .Values.services.couchdb.url | default (tpl .Values.services.proxy.upstreams.couchdb .) | quote }}
+        - name: RESOLVER
+          {{ if .Values.services.proxy.resolver }}
+          value: {{ .Values.services.proxy.resolver }}
+          {{ else }}
+          value: kube-dns.kube-system.svc.{{ .Values.services.dns }}
+          {{ end }}
        {{ with .Values.services.proxy.resources }}
        resources:
        {{- toYaml . | nindent 10 }}
--- a/charts/budibase/templates/redis-service-deployment.yaml
+++ b/charts/budibase/templates/redis-service-deployment.yaml
@ -60,5 +60,6 @@ spec:
      - name: redis-data
        persistentVolumeClaim:
          claimName: redis-data
+
 status: {}
 {{- end }}
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@ -21,6 +21,9 @@ spec:
      annotations:
        kompose.cmd: kompose convert
        kompose.version: 1.21.0 (992df58d8)
+{{ if .Values.services.worker.annotations }}
+{{- toYaml .Values.services.worker.annotations | indent 8 -}}
+{{ end }}
      creationTimestamp: null
      labels:
        io.kompose.service: worker-service
@ -65,6 +68,8 @@ spec:
        - name: AWS_REGION
          value: {{ .Values.services.objectStore.region }}
        {{ end }}
+        - name: MINIO_ENABLED
+          value: {{ .Values.services.objectStore.minio | quote }}
        - name: MINIO_ACCESS_KEY
          valueFrom:
            secretKeyRef:
@ -77,11 +82,17 @@ spec:
              key: objectStoreSecret
        - name: MINIO_URL
          value: {{ .Values.services.objectStore.url }}
+        - name: CLOUDFRONT_CDN
+          value: {{ .Values.services.objectStore.cloudfront.cdn | quote }}
+        - name: CLOUDFRONT_PUBLIC_KEY_ID
+          value: {{ .Values.services.objectStore.cloudfront.publicKeyId | quote }}
+        - name: CLOUDFRONT_PRIVATE_KEY_64
+          value: {{ .Values.services.objectStore.cloudfront.privateKey64 | quote }}
        - name: PLUGIN_BUCKET_NAME
          value: {{ .Values.services.objectStore.pluginBucketName | quote }}
        - name: APPS_BUCKET_NAME
          value: {{ .Values.services.objectStore.appsBucketName | quote }}
-        - name: GLOBAL_CLOUD_BUCKET_NAME
+        - name: GLOBAL_BUCKET_NAME
          value: {{ .Values.services.objectStore.globalBucketName | quote }}
        - name: BACKUPS_BUCKET_NAME
          value: {{ .Values.services.objectStore.backupsBucketName | quote }}
@ -135,6 +146,8 @@ spec:
          value: {{ .Values.globals.google.secret | quote }}
        - name: TENANT_FEATURE_FLAGS
          value: {{ .Values.globals.tenantFeatureFlags | quote }}
+        - name: ENCRYPTION_KEY
+          value: {{ .Values.globals.bbEncryptionKey | quote }}
        {{ if .Values.globals.elasticApmEnabled }}
        - name: ELASTIC_APM_ENABLED
          value: {{ .Values.globals.elasticApmEnabled | quote }}
@ -147,8 +160,24 @@ spec:
        - name: ELASTIC_APM_SERVER_URL
          value: {{ .Values.globals.elasticApmServerUrl | quote }}
        {{ end }}
+        {{ if .Values.globals.globalAgentHttpProxy }}
+        - name: GLOBAL_AGENT_HTTP_PROXY
+          value: {{ .Values.globals.globalAgentHttpProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentHttpsProxy }}
+        - name: GLOBAL_AGENT_HTTPS_PROXY
+          value: {{ .Values.globals.globalAgentHttpsProxy | quote }}
+        {{ end }}
+        {{ if .Values.globals.globalAgentNoProxy }}
+        - name: GLOBAL_AGENT_NO_PROXY
+          value: {{ .Values.globals.globalAgentNoProxy | quote }}
+        {{ end }}
        - name: CDN_URL
          value: {{ .Values.globals.cdnUrl }}
+        {{ if .Values.services.tlsRejectUnauthorized }}
+        - name: NODE_TLS_REJECT_UNAUTHORIZED
+          value: {{ .Values.services.tlsRejectUnauthorized }}
+        {{ end }}

        image: budibase/worker:{{ .Values.globals.appVersion }}
        imagePullPolicy: Always
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@ -76,7 +76,7 @@ affinity: {}
 globals:
  appVersion: "latest"
  budibaseEnv: PRODUCTION
-  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS"
+  tenantFeatureFlags: "*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
  enableAnalytics: "1"
  sentryDSN: ""
  posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
@ -106,15 +106,28 @@ globals:
 #  elasticApmEnabled:
 #  elasticApmSecretToken:
 #  elasticApmServerUrl:
+#  globalAgentHttpProxy:
+#  globalAgentHttpsProxy:
+#  globalAgentNoProxy:

 services:
  budibaseVersion: latest
  dns: cluster.local
+  # tlsRejectUnauthorized: 0

  proxy:
    port: 10000
    replicaCount: 1
+    upstreams:
+      apps: 'http://app-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.apps.port }}'
+      worker: 'http://worker-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.worker.port }}'
+      minio: 'http://minio-service.{{ .Release.Namespace }}.svc.{{ .Values.services.dns }}:{{ .Values.services.objectStore.port }}'
+      couchdb: 'http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}'
    resources: {}
+#    annotations:
+#      co.elastic.logs/module: nginx
+#      co.elastic.logs/fileset.stdout: access
+#      co.elastic.logs/fileset.stderr: error

  apps:
    port: 4002
@ -122,11 +135,20 @@ services:
    logLevel: info
    resources: {}
 #    nodeDebug: "" # set the value of NODE_DEBUG
-
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after
  worker:
    port: 4003
    replicaCount: 1
    resources: {}
+#    annotations:
+#      co.elastic.logs/multiline.type: pattern
+#      co.elastic.logs/multiline.pattern: '^[[:space:]]'
+#      co.elastic.logs/multiline.negate: false
+#      co.elastic.logs/multiline.match: after

  couchdb:
    enabled: true
@ -157,6 +179,7 @@ services:
    resources: {}

  objectStore:
+    # Set to false if using another object store such as S3
    minio: true
    browser: true
    port: 9000
@ -172,6 +195,13 @@ services:
    ##   set, choosing the default provisioner.
    storageClass: ""
    resources: {}
+    cloudfront:
+      # Set the url of a distribution to enable cloudfront
+      cdn: ""
+      # ID of public key stored in cloudfront
+      publicKeyId: ""
+      # Base64 encoded private key for the above public key
+      privateKey64: ""

 # Override values in couchDB subchart
 couchdb:
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@ -9,7 +9,6 @@ From opening a bug report to creating a pull request: every contribution is appr
 - [Glossary of Terms](#glossary-of-terms)
 - [Contributing to Budibase](#contributing-to-budibase)

-
 ## Not Sure Where to Start?

 Budibase is a low-code web application builder that creates svelte-based web applications.
@ -22,7 +21,7 @@ Budibase is a monorepo managed by [lerna](https://github.com/lerna/lerna). Lerna

 - **packages/server** - The budibase server. This [Koa](https://koajs.com/) app is responsible for serving the JS for the builder and budibase apps, as well as providing the API for interaction with the database and file system.

- **packages/worker** - This [Koa](https://koajs.com/) app is responsible for providing global apis for managing your budibase installation. Authentication, Users, Email, Org and Auth configs are all provided by the worker. 
+- **packages/worker** - This [Koa](https://koajs.com/) app is responsible for providing global apis for managing your budibase installation. Authentication, Users, Email, Org and Auth configs are all provided by the worker.

 ## Contributor License Agreement (CLA)

@ -45,7 +44,7 @@ A client represents a single budibase customer. Each budibase client will have 1

 ### App

-A client can have one or more budibase applications. Budibase applications would be things like "Developer Inventory Management" or "Goat Herder CRM". Think of a budibase application as a tree. 
+A client can have one or more budibase applications. Budibase applications would be things like "Developer Inventory Management" or "Goat Herder CRM". Think of a budibase application as a tree.

 ### Database

@ -73,28 +72,55 @@ A component is the basic frontend building block of a budibase app.

 ### Component Library

-Component libraries are collections of components as well as the definition of their props contained in a file called `components.json`. 
+Component libraries are collections of components as well as the definition of their props contained in a file called `components.json`.

 ## Contributing to Budibase

-* Please maintain the existing code style. 
+- Please maintain the existing code style.

-* Please try to keep your commits small and focused.
+- Please try to keep your commits small and focused.

-* Please write tests.
+- Please write tests.

-* If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.
+- If the project diverges from your branch, please rebase instead of merging. This makes the commit graph easier to read.

-* Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.
+- Once your work is completed, please raise a PR against the `develop` branch with some information about what has changed and why.

 ### Getting Started For Contributors
-#### 1.  Prerequisites

-NodeJS Version `14.x.x`
+#### 1. Prerequisites

-*yarn -* `npm install -g yarn`
+- NodeJS version `14.x.x`
+- Python version `3.x`

-*jest* - `npm install -g jest`
+### Using asdf (recommended)
+
+Asdf is a package manager that allows managing multiple dependencies.
+
+You can install them following any of the steps described below:
+
+- Install using script (only for mac users):
+
+`./scripts/install-contributor-dependencies.sh`
+
+- Or, manually:
+
+  - Installation steps: https://asdf-vm.com/guide/getting-started.html
+  - asdf plugin add nodejs
+  - asdf plugin add python
+  - npm install -g yarn
+
+### Using NVM and pyenv
+
+- NVM:
+  - Install: https://github.com/nvm-sh/nvm#installing-and-updating
+  - Setup: `nvm use`
+- Pyenv:
+
+  - Install: https://github.com/pyenv/pyenv#installation
+  - Setup: `pyenv install -v 3.7.2`
+
+- _yarn -_ `npm install -g yarn`

 #### 2. Clone this repository

@ -102,7 +128,7 @@ NodeJS Version `14.x.x`

 then `cd ` into your local copy.

-#### 3.  Install and Build
+#### 3. Install and Build

 | **NOTE**: On Windows, all yarn commands must be executed on a bash shell (e.g. git bash)

@ -134,9 +160,9 @@ This will enable watch mode for both the builder app, server, client library and

 #### 5. Debugging using VS Code

-To debug the budibase server and worker a VS Code launch configuration has been provided. 
+To debug the budibase server and worker a VS Code launch configuration has been provided.

-Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component. 
+Visit the debug window and select `Budibase Server` or `Budibase Worker` to debug the respective component.
 Alternatively to start both components simultaneously select `Start Budibase`.

 In addition to the above, the remaining budibase components may be run in dev mode using: `yarn dev:noserver`.
@ -156,11 +182,11 @@ For the backend we run [Redis](https://redis.io/), [CouchDB](https://couchdb.apa

 When you are running locally, budibase stores data on disk using docker volumes. The volumes and the types of data associated with each are:

- `redis_data` 
+- `redis_data`
  - Sessions, email tokens
- `couchdb3_data` 
+- `couchdb3_data`
  - Global and app databases
- `minio_data` 
+- `minio_data`
  - App manifest, budibase client, static assets

 ### Development Modes
@ -172,34 +198,42 @@ A combination of environment variables controls the mode budibase runs in.
 Yarn commands can be used to mimic the different modes as described in the sections below:

 #### Self Hosted
-The default mode. A single tenant installation with no usage restrictions. 
+
+The default mode. A single tenant installation with no usage restrictions.

 To enable this mode, use:
+
 ```
 yarn mode:self
 ```

 #### Cloud
-The cloud mode, with account portal turned off. 
+
+The cloud mode, with account portal turned off.

 To enable this mode, use:
+
 ```
 yarn mode:cloud
 ```
-#### Cloud & Account
-The cloud mode, with account portal turned on. This is a replica of the mode that runs at https://budibase.app 

+#### Cloud & Account
+
+The cloud mode, with account portal turned on. This is a replica of the mode that runs at https://budibase.app

 To enable this mode, use:
+
 ```
 yarn mode:account
 ```
+
 ### CI
- An overview of the CI pipelines can be found [here](../.github/workflows/README.md)
+
+An overview of the CI pipelines can be found [here](../.github/workflows/README.md)

 ### Pro

-@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g. 
+@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g.

 ```
 .
@ -207,13 +241,14 @@ yarn mode:account
 |_ budibase-pro
 ```

-Note that only budibase maintainers will be able to access the pro repo. 
+Note that only budibase maintainers will be able to access the pro repo.

-The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev. 
+The `yarn bootstrap` command can be used to replace the NPM supplied dependency with the local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.

 ### Troubleshooting

 Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation.
+
 ### Running tests

 #### End-to-end Tests
@ -226,12 +261,11 @@ yarn test:e2e

 Or if you are in the builder you can run `yarn cy:test`.

-
 ### Other Useful Information

-* The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).
+- The contributors are listed in [AUTHORS.md](https://github.com/Budibase/budibase/blob/master/.github/AUTHORS.md) (add yourself).

-* This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).
+- This project uses a modified version of the MPLv2 license, see [LICENSE](https://github.com/budibase/server/blob/master/LICENSE).

-* We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
+- We use the [C4 (Collective Code Construction Contract)](https://rfc.zeromq.org/spec:42/C4/) process for contributions.
  Please read this if you are unfamiliar with it.
--- a/examples/nextjs-api-sales/yarn.lock
+++ b/examples/nextjs-api-sales/yarn.lock
--- a/hosting/.env
+++ b/hosting/.env
@ -19,10 +19,11 @@ COUCH_DB_PORT=4005
 REDIS_PORT=6379
 WATCHTOWER_PORT=6161
 BUDIBASE_ENVIRONMENT=PRODUCTION
+SQL_MAX_ROWS=

 # An admin user can be automatically created initially if these are set
 BB_ADMIN_USER_EMAIL=
 BB_ADMIN_USER_PASSWORD=

 # A path that is watched for plugin bundles. Any bundles found are imported automatically/
-PLUGINS_DIR=
+PLUGINS_DIR=
--- a/hosting/couchdb/Dockerfile
+++ b/hosting/couchdb/Dockerfile
@ -0,0 +1,32 @@
+FROM couchdb:3.2.1
+
+ENV COUCHDB_USER admin
+ENV COUCHDB_PASSWORD admin
+EXPOSE 5984
+
+RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
+    apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
+    apt-get update && apt-get install -y --no-install-recommends openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/
+
+# setup clouseau
+WORKDIR /
+RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
+  unzip clouseau-2.21.0-dist.zip && \
+  mv clouseau-2.21.0 /opt/clouseau && \
+  rm clouseau-2.21.0-dist.zip
+
+WORKDIR /opt/clouseau
+RUN mkdir ./bin
+ADD clouseau/clouseau ./bin/
+ADD clouseau/log4j.properties clouseau/clouseau.ini ./
+
+# setup CouchDB
+WORKDIR /opt/couchdb
+ADD couch/vm.args couch/local.ini ./etc/
+
+WORKDIR /
+ADD build-target-paths.sh .
+ADD runner.sh ./bbcouch-runner.sh
+RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau ./build-target-paths.sh
+CMD ["./bbcouch-runner.sh"]
--- a/hosting/couchdb/build-target-paths.sh
+++ b/hosting/couchdb/build-target-paths.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+echo ${TARGETBUILD} > /buildtarget.txt
+if [[ "${TARGETBUILD}" = "aas" ]]; then
+    # Azure AppService uses /home for persisent data & SSH on port 2222
+    DATA_DIR=/home
+    WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
+    mkdir -p $DATA_DIR/{search,minio,couch}
+    mkdir -p $DATA_DIR/couch/{dbs,views}
+    chown -R couchdb:couchdb $DATA_DIR/couch/
+    apt update
+    apt-get install -y openssh-server
+    echo "root:Docker!" | chpasswd
+    mkdir -p /tmp
+    chmod +x /tmp/ssh_setup.sh \
+        && (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
+    cp /etc/sshd_config /etc/ssh/sshd_config
+    /etc/init.d/ssh restart
+    sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
+else
+    sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
+    sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
+fi
--- a/hosting/couchdb/clouseau/clouseau
+++ b/hosting/couchdb/clouseau/clouseau
--- a/hosting/couchdb/clouseau/clouseau.ini
+++ b/hosting/couchdb/clouseau/clouseau.ini
--- a/hosting/couchdb/clouseau/log4j.properties
+++ b/hosting/couchdb/clouseau/log4j.properties
--- a/hosting/couchdb/couch/local.ini
+++ b/hosting/couchdb/couch/local.ini
--- a/hosting/couchdb/couch/vm.args
+++ b/hosting/couchdb/couch/vm.args
--- a/hosting/couchdb/runner.sh
+++ b/hosting/couchdb/runner.sh
@ -0,0 +1,14 @@
+#!/bin/bash
+
+DATA_DIR=${DATA_DIR:-/data}
+mkdir -p ${DATA_DIR}
+mkdir -p ${DATA_DIR}/couch/{dbs,views}
+mkdir -p ${DATA_DIR}/search
+chown -R couchdb:couchdb ${DATA_DIR}/couch
+/build-target-paths.sh
+/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
+/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+sleep 10
+curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_users
+curl -X PUT http://${COUCHDB_USER}:${COUCHDB_PASSWORD}@localhost:5984/_replicator
+sleep infinity
--- a/hosting/dependencies/Dockerfile
+++ b/hosting/dependencies/Dockerfile
@ -0,0 +1,23 @@
+FROM budibase/couchdb
+
+ENV DATA_DIR /data
+RUN mkdir /data
+
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends redis-server
+
+WORKDIR /minio
+ADD scripts/install-minio.sh ./install.sh
+RUN chmod +x install.sh && ./install.sh
+
+WORKDIR /
+
+ADD dependencies/runner.sh .
+RUN chmod +x ./runner.sh
+
+EXPOSE 5984
+EXPOSE 9000
+EXPOSE 9001
+EXPOSE 6379
+
+CMD ["./runner.sh"]
--- a/hosting/dependencies/README.md
+++ b/hosting/dependencies/README.md
@ -0,0 +1,57 @@
+# Docker Image for Running Budibase Tests
+
+## Overview
+This image contains the basic setup for running 
+
+## Usage
+
+- Build the Image
+- Run the Container
+
+
+### Build the Image
+The guidance below is based on building the Budibase single image on Debian 11 and AlmaLinux 8. If you use another distro or OS you will need to amend the commands to suit. 
+#### Install Node
+Budibase requires a more recent version of node (14+) than is available in the base Debian repos so:
+
+```
+curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -
+apt install -y nodejs
+node -v
+```
+Install yarn and lerna:
+```
+npm install -g yarn jest lerna
+```
+#### Install Docker
+
+```
+apt install -y docker.io
+```
+
+Check the versions of each installed version. This process was tested with the version numbers below so YMMV using anything else:
+
+- Docker: 20.10.5
+- node: 16.15.1
+- yarn: 1.22.19
+- lerna: 5.1.4
+
+#### Get the Code
+Clone the Budibase repo
+```
+git clone https://github.com/Budibase/budibase.git
+cd budibase
+```
+#### Setup Node
+Node setup:
+```
+node ./hosting/scripts/setup.js
+yarn
+yarn bootstrap
+yarn build
+```
+#### Build Image
+The following yarn command does some prep and then runs the docker build command:
+```
+yarn build:docker:dependencies
+```
--- a/hosting/dependencies/runner.sh
+++ b/hosting/dependencies/runner.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+
+redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+/bbcouch-runner.sh &
+/minio/minio server ${DATA_DIR}/minio --console-address ":9001" > /dev/stdout 2>&1 &
+
+echo "Budibase dependencies started..."
+sleep infinity
--- a/hosting/docker-compose.dev.yaml
+++ b/hosting/docker-compose.dev.yaml
@ -6,7 +6,8 @@ services:
  minio-service:
    container_name: budi-minio-dev
    restart: on-failure
-    image: minio/minio
+    # Last version that supports the "fs" backend
+    image: minio/minio:RELEASE.2022-10-24T18-35-07Z
    volumes:
      - minio_data:/data
    ports:
@ -25,9 +26,9 @@ services:
  proxy-service:
    container_name: budi-nginx-dev
    restart: on-failure
-    image: nginx:latest
+    image: budibase/proxy:latest
    volumes:
-      - ./.generated-nginx.dev.conf:/etc/nginx/nginx.conf
+      - ./nginx.dev.conf:/etc/nginx/templates/nginx.conf.template
      - ./proxy/error.html:/usr/share/nginx/html/error.html
    ports:
      - "${MAIN_PORT}:10000"
@ -36,28 +37,21 @@ services:
      - couchdb-service
    extra_hosts:
      - "host.docker.internal:host-gateway"
+    environment:
+      - PROXY_ADDRESS=host.docker.internal

  couchdb-service:
    # platform: linux/amd64
-    container_name: budi-couchdb-dev
+    container_name: budi-couchdb3-dev
    restart: on-failure
-    image: ibmcom/couchdb3
+    image: budibase/couchdb
    environment:
      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
      - COUCHDB_USER=${COUCH_DB_USER}
    ports:
      - "${COUCH_DB_PORT}:5984"
    volumes:
-      - couchdb3_data:/opt/couchdb/data
-  
-  couch-init:
-    container_name: budi-couchdb-init-dev
-    image: curlimages/curl
-    environment:
-      PUT_CALL: "curl -u ${COUCH_DB_USER}:${COUCH_DB_PASSWORD} -X PUT couchdb-service:5984"
-    depends_on:
-      - couchdb-service
-    command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"]
+      - couchdb_data:/data

  redis-service:
    container_name: budi-redis-dev
@ -70,7 +64,7 @@ services:
      - redis_data:/data

 volumes:
-  couchdb3_data:
+  couchdb_data:
    driver: local
  minio_data:
    driver: local
--- a/hosting/docker-compose.test.yaml
+++ b/hosting/docker-compose.test.yaml
@ -0,0 +1,47 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  minio-service:
+    restart: on-failure
+    # Last version that supports the "fs" backend
+    image: minio/minio:RELEASE.2022-10-24T18-35-07Z
+    ports:
+      - 9000
+      - 9001
+    environment:
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+    command: server /data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  couchdb-service:
+    # platform: linux/amd64
+    restart: on-failure
+    image: budibase/couchdb
+    environment:
+      - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
+      - COUCHDB_USER=${COUCH_DB_USER}
+    ports:
+      - 5984
+      - 4369
+      - 9100
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:5984/_up"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  redis-service:
+    restart: on-failure
+    image: redis
+    command: redis-server --requirepass ${REDIS_PASSWORD}
+    ports:
+      - 6379
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
--- a/hosting/docker-compose.yaml
+++ b/hosting/docker-compose.yaml
@ -82,6 +82,12 @@ services:
    environment:
      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
      - PROXY_RATE_LIMIT_API_PER_SECOND=20
+      - APPS_UPSTREAM_URL=http://app-service:4002
+      - WORKER_UPSTREAM_URL=http://worker-service:4003
+      - MINIO_UPSTREAM_URL=http://minio-service:9000
+      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+      - RESOLVER=127.0.0.11
    depends_on:
      - minio-service
      - worker-service
--- a/hosting/nginx.dev.conf.hbs
+++ b/hosting/nginx.dev.conf.hbs
@ -25,17 +25,17 @@ http {
  }

  upstream app-service {
-    server {{address}}:4001;
+    server ${PROXY_ADDRESS}:4001;
    keepalive 32;
  }

  upstream worker-service {
-    server {{address}}:4002;
+    server ${PROXY_ADDRESS}:4002;
    keepalive 32;
  }

  upstream builder {
-    server {{address}}:3000;
+    server ${PROXY_ADDRESS}:3000;
    keepalive 32;
  }

@ -186,6 +186,26 @@ http {
      proxy_pass      http://minio-service:9000;
    }

+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://minio-service:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
+    }
+
    client_header_timeout 60;
    client_body_timeout   60;
    keepalive_timeout     60;
--- a/hosting/proxy/Dockerfile
+++ b/hosting/proxy/Dockerfile
@ -4,7 +4,7 @@ FROM nginx:latest
 # use the default nginx behaviour for *.template files which are processed with envsubst
 # override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
 ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
-COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
+COPY nginx.prod.conf /etc/nginx/templates/nginx.conf.template

 # IPv6 removal needs to happen after envsubst
 RUN rm -rf /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
@ -16,4 +16,11 @@ COPY error.html /usr/share/nginx/html/error.html

 # Default environment
 ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
-ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
+ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
+# Use docker-compose values as defaults for backwards compatibility
+ENV APPS_UPSTREAM_URL=http://app-service:4002
+ENV WORKER_UPSTREAM_URL=http://worker-service:4003
+ENV MINIO_UPSTREAM_URL=http://minio-service:9000
+ENV COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+ENV WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+ENV RESOLVER=127.0.0.11
--- a/hosting/proxy/nginx.prod.conf
+++ b/hosting/proxy/nginx.prod.conf
@ -23,7 +23,7 @@ http {
  tcp_nodelay on;
  server_tokens off;
  types_hash_max_size 2048;
-  resolver {{ resolver }} valid=10s ipv6=off;
+  resolver ${RESOLVER} valid=10s ipv6=off;

  # buffering
  client_header_buffer_size 1k;
@ -76,27 +76,23 @@ http {
    add_header Content-Security-Policy "${csp_default}; ${csp_script}; ${csp_style}; ${csp_object}; ${csp_base_uri}; ${csp_connect}; ${csp_font}; ${csp_frame}; ${csp_img}; ${csp_manifest}; ${csp_media}; ${csp_worker};" always;

    # upstreams
-    set $apps {{ apps }};
-    set $worker {{ worker }};
-    set $minio {{ minio }};
-    set $couchdb {{ couchdb }};
-    {{#if watchtower}}
-    set $watchtower {{ watchtower }};
-    {{/if}}
+    set $apps ${APPS_UPSTREAM_URL};
+    set $worker ${WORKER_UPSTREAM_URL};
+    set $minio ${MINIO_UPSTREAM_URL};
+    set $couchdb ${COUCHDB_UPSTREAM_URL};
+    set $watchtower ${WATCHTOWER_UPSTREAM_URL};

    location /app {
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

    location = / {
-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

-    {{#if watchtower}}
    location = /v1/update {
-      proxy_pass      http://$watchtower:8080;
+      proxy_pass      $watchtower;
    }
-    {{/if}}

    location ~ ^/(builder|app_) {
      proxy_http_version  1.1;
@ -107,19 +103,17 @@ http {
      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
      proxy_set_header    Host                $host;

-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

    location ~ ^/api/(system|admin|global)/ {
      proxy_set_header    Host                $host;
-
-      proxy_pass      http://$worker:4003;
+      proxy_pass      $worker;
    }

    location /worker/ {
      proxy_set_header    Host                $host;
-
-      proxy_pass      http://$worker:4003;
+      proxy_pass      $worker;
      rewrite ^/worker/(.*)$ /$1 break;
    }

@ -138,7 +132,7 @@ http {
      proxy_set_header    X-Real-IP           $remote_addr;
      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;

-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

    location /api/ {
@ -157,7 +151,7 @@ http {
      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
      proxy_set_header    Host                $host;

-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

    location /api/webhooks/ {
@ -177,11 +171,11 @@ http {
      proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
      proxy_set_header    Host                $host;

-      proxy_pass      http://$apps:4002;
+      proxy_pass      $apps;
    }

    location /db/ {
-      proxy_pass      http://$couchdb:5984;
+      proxy_pass      $couchdb;
      rewrite ^/db/(.*)$ /$1 break;
    }

@ -191,7 +185,7 @@ http {
      proxy_set_header    Connection  'upgrade';
      proxy_set_header    Host        $host;
      proxy_cache_bypass  $http_upgrade;
-      proxy_pass          http://$apps:4002;
+      proxy_pass      $apps;
    }

    location / {
@ -205,7 +199,27 @@ http {
      proxy_set_header Connection "";
      chunked_transfer_encoding off;

-      proxy_pass      http://$minio:9000;
+      proxy_pass      $minio;
+    }
+
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      $minio;
+      rewrite ^/files/signed/(.*)$ /$1 break;
    }

    client_header_timeout 60;
--- a/hosting/scripts/install-minio.sh
+++ b/hosting/scripts/install-minio.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+if [[ $TARGETARCH == arm* ]] ;
+then
+  echo "INSTALLING ARM64 MINIO"
+  wget https://dl.min.io/server/minio/release/linux-arm64/minio
+else
+  echo "INSTALLING AMD64 MINIO"
+  wget https://dl.min.io/server/minio/release/linux-amd64/minio
+fi
+chmod +x minio
--- a/hosting/scripts/linux/release-couch.sh
+++ b/hosting/scripts/linux/release-couch.sh
@ -0,0 +1,15 @@
+#!/bin/bash
+
+tag=$1
+
+if [[ ! "$tag" ]]; then
+        echo "No tag present. You must pass a tag to this script"
+        exit 1
+fi
+
+echo "Tagging images with tag: $tag"
+
+docker tag budibase-couchdb budibase/couchdb:$tag
+
+docker push --all-tags budibase/couchdb
+
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@ -18,7 +18,7 @@ WORKDIR /worker
 ADD packages/worker .
 RUN node /pinVersions.js && yarn && yarn build && /cleanup.sh

-FROM couchdb:3.2.1
+FROM budibase/couchdb
 ARG TARGETARCH
 ENV TARGETARCH $TARGETARCH
 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service)
@ -29,23 +29,9 @@ ENV TARGETBUILD $TARGETBUILD
 COPY --from=build /app /app
 COPY --from=build /worker /worker

-# ENV CUSTOM_DOMAIN=budi001.custom.com \
-#  See runner.sh for Env Vars
-# These secret env variables are generated by the runner at startup
-# their values can be overriden by the user, they will be written
-# to the .env file in the /data directory for use later on
-#  REDIS_PASSWORD=budibase \
-#  COUCHDB_PASSWORD=budibase \
-#  COUCHDB_USER=budibase \
-#  COUCH_DB_URL=http://budibase:budibase@localhost:5984 \
-#  INTERNAL_API_KEY=budibase \
-#  JWT_SECRET=testsecret \
-#  MINIO_ACCESS_KEY=budibase \
-#  MINIO_SECRET_KEY=budibase \
-
 # install base dependencies
 RUN apt-get update && \
-  apt-get install -y software-properties-common wget nginx uuid-runtime && \
+  apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server && \
  apt-add-repository 'deb http://security.debian.org/debian-security stretch/updates main' && \
  apt-get update

@ -53,7 +39,7 @@ RUN apt-get update && \
 WORKDIR /nodejs
 RUN curl -sL https://deb.nodesource.com/setup_16.x -o /tmp/nodesource_setup.sh && \
  bash /tmp/nodesource_setup.sh && \
-  apt-get install -y libaio1 nodejs nginx openjdk-8-jdk redis-server unzip && \
+  apt-get install -y --no-install-recommends libaio1 nodejs && \
  npm install --global yarn pm2

 # setup nginx
@ -61,30 +47,14 @@ ADD hosting/single/nginx/nginx.conf /etc/nginx
 ADD hosting/single/nginx/nginx-default-site.conf /etc/nginx/sites-enabled/default
 RUN mkdir -p /var/log/nginx && \
  touch /var/log/nginx/error.log && \
-  touch /var/run/nginx.pid
+  touch /var/run/nginx.pid && \
+  usermod -a -G tty www-data

 WORKDIR /
 RUN mkdir -p scripts/integrations/oracle
 ADD packages/server/scripts/integrations/oracle scripts/integrations/oracle
 RUN /bin/bash -e ./scripts/integrations/oracle/instantclient/linux/install.sh

-# setup clouseau
-WORKDIR /
-RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
-  unzip clouseau-2.21.0-dist.zip && \
-  mv clouseau-2.21.0 /opt/clouseau && \
-  rm clouseau-2.21.0-dist.zip
-
-WORKDIR /opt/clouseau
-RUN mkdir ./bin
-ADD hosting/single/clouseau/clouseau ./bin/
-ADD hosting/single/clouseau/log4j.properties hosting/single/clouseau/clouseau.ini ./
-RUN chmod +x ./bin/clouseau
-
-# setup CouchDB
-WORKDIR /opt/couchdb
-ADD hosting/single/couch/vm.args hosting/single/couch/local.ini ./etc/
-
 # setup minio
 WORKDIR /minio
 ADD scripts/install-minio.sh ./install.sh
@ -97,9 +67,6 @@ RUN chmod +x ./runner.sh
 ADD hosting/single/healthcheck.sh .
 RUN chmod +x ./healthcheck.sh

-ADD hosting/scripts/build-target-paths.sh .
-RUN chmod +x ./build-target-paths.sh
-
 # Script below sets the path for storing data based on $DATA_DIR
 # For Azure App Service install SSH & point data locations to /home
 ADD hosting/single/ssh/sshd_config /etc/
--- a/hosting/single/nginx/nginx-default-site.conf
+++ b/hosting/single/nginx/nginx-default-site.conf
@ -2,7 +2,8 @@ server {
    listen       80 default_server;
    listen  [::]:80 default_server;
    server_name  _;
-
+    error_log            /dev/stderr warn;
+    access_log           /dev/stdout main;
    client_max_body_size 1000m;
    ignore_invalid_headers off;
    proxy_buffering off;
@ -94,15 +95,37 @@ server {
    }

    location / {
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;

-        proxy_connect_timeout 300;
-        proxy_http_version 1.1;
-        proxy_set_header Connection "";
-        chunked_transfer_encoding off;
-        proxy_pass      http://127.0.0.1:9000;
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://127.0.0.1:9000;
+    }
+
+    location /files/signed/ {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+
+      # IMPORTANT: Signed urls will inspect the host header of the request.
+      # Normally a signed url will need to be generated with a specified client host in mind.
+      # To support dynamic hosts, e.g. some unknown self-hosted installation url,
+      # use a predefined host header. The host 'minio-service' is also used at the time of url signing.
+      proxy_set_header Host minio-service;
+
+      proxy_connect_timeout 300;
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      proxy_pass      http://127.0.0.1:9000;
+      rewrite ^/files/signed/(.*)$ /$1 break;
    }

    client_header_timeout 60;
--- a/hosting/single/nginx/nginx.conf
+++ b/hosting/single/nginx/nginx.conf
@ -1,5 +1,5 @@
 user                 www-data www-data;
-error_log            /var/log/nginx/error.log;
+error_log            /dev/stderr warn;
 pid                  /var/run/nginx.pid;
 worker_processes     auto;
 worker_rlimit_nofile 8192;
--- a/hosting/single/runner.sh
+++ b/hosting/single/runner.sh
@ -10,7 +10,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 [[ -z "${MINIO_URL}" ]] && export MINIO_URL=http://localhost:9000
 [[ -z "${NODE_ENV}" ]] && export NODE_ENV=production
 [[ -z "${POSTHOG_TOKEN}" ]] && export POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS"
+[[ -z "${TENANT_FEATURE_FLAGS}" ]] && export TENANT_FEATURE_FLAGS="*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"
 [[ -z "${ACCOUNT_PORTAL_URL}" ]] && export ACCOUNT_PORTAL_URL=https://account.budibase.app
 [[ -z "${REDIS_URL}" ]] && export REDIS_URL=localhost:6379
 [[ -z "${SELF_HOSTED}" ]] && export SELF_HOSTED=1
@ -72,14 +72,11 @@ for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
 ln -s ${DATA_DIR}/.env /app/.env
 ln -s ${DATA_DIR}/.env /worker/.env
 # make these directories in runner, incase of mount
-mkdir -p ${DATA_DIR}/couch/{dbs,views}
 mkdir -p ${DATA_DIR}/minio
-mkdir -p ${DATA_DIR}/search
 chown -R couchdb:couchdb ${DATA_DIR}/couch
 redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
-/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
-/minio/minio server ${DATA_DIR}/minio > /dev/stdout 2>&1 &
-/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
+/bbcouch-runner.sh &
+/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
 /etc/init.d/nginx restart
 if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
    # Add monthly cron job to renew certbot certificate
@ -90,15 +87,14 @@ if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
    /etc/init.d/nginx restart
 fi

+# wait for backend services to start
+sleep 10
+
 pushd app
 pm2 start -l /dev/stdout --name app "yarn run:docker"
 popd
 pushd worker
 pm2 start -l /dev/stdout --name worker "yarn run:docker"
 popd
-sleep 10
-echo "curl to couchdb endpoints"
-curl -X PUT ${COUCH_DB_URL}/_users
-curl -X PUT ${COUCH_DB_URL}/_replicator
 echo "end of runner.sh, sleeping ..."
 sleep infinity
--- a/jestTestcontainersConfigGenerator.js
+++ b/jestTestcontainersConfigGenerator.js
@ -0,0 +1,9 @@
+module.exports = () => {
+  return {
+    dockerCompose: {
+      composeFilePath: "../../hosting",
+      composeFile: "docker-compose.test.yaml",
+      startupTimeout: 10000,
+    },
+  }
+}
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.1.32-alpha.3",
+  "version": "2.3.2-alpha.3",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/package.json
+++ b/package.json
@ -3,7 +3,8 @@
  "private": true,
  "devDependencies": {
    "@rollup/plugin-json": "^4.0.2",
-    "@typescript-eslint/parser": "4.28.0",
+    "@types/supertest": "^2.0.12",
+    "@typescript-eslint/parser": "5.45.0",
    "babel-eslint": "^10.0.3",
    "eslint": "^7.28.0",
    "eslint-plugin-cypress": "^2.11.3",
@ -25,6 +26,7 @@
    "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
    "build": "lerna run build",
    "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
+    "build:backend": "lerna run build --ignore @budibase/client --ignore @budibase/bbui --ignore @budibase/builder --ignore @budibase/cli",
    "build:sdk": "lerna run build:sdk",
    "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular",
    "release": "lerna publish ${RELEASE_VERSION_TYPE:-patch} --yes --force-publish && yarn release:pro",
@ -44,31 +46,26 @@
    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
    "test": "lerna run test && yarn test:pro",
    "test:pro": "bash scripts/pro/test.sh",
-    "lint:eslint": "eslint packages",
-    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
+    "lint:eslint": "eslint packages && eslint qa-core",
+    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
    "lint": "yarn run lint:eslint && yarn run lint:prettier",
    "lint:fix:eslint": "eslint --fix packages qa-core",
    "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"",
    "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint",
-    "test:e2e": "lerna run cy:test --stream",
-    "test:e2e:ci": "lerna run cy:ci --stream",
-    "test:e2e:ci:record": "lerna run cy:ci:record --stream",
-    "test:e2e:ci:notify": "lerna run cy:ci:notify",
    "build:specs": "lerna run specs",
-    "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
+    "build:docker": "lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -",
    "build:docker:pre": "lerna run build && lerna run predocker",
    "build:docker:proxy": "docker build hosting/proxy -t proxy-service",
-    "build:docker:proxy:compose": "node scripts/proxy/generateProxyConfig compose && npm run build:docker:proxy",
-    "build:docker:proxy:preprod": "node scripts/proxy/generateProxyConfig preprod && npm run build:docker:proxy",
-    "build:docker:proxy:release": "node scripts/proxy/generateProxyConfig release && npm run build:docker:proxy",
-    "build:docker:proxy:prod": "node scripts/proxy/generateProxyConfig prod && npm run build:docker:proxy",
    "build:docker:selfhost": "lerna run build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -",
-    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
+    "build:docker:develop": "node scripts/pinVersions && lerna run build:docker && npm run build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -",
    "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
    "build:digitalocean": "cd hosting/digitalocean && ./build.sh && cd -",
    "build:docker:single:multiarch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/single/Dockerfile -t budibase:latest .",
    "build:docker:single:image": "docker build -f hosting/single/Dockerfile -t budibase:latest .",
    "build:docker:single": "npm run build:docker:pre && npm run build:docker:single:image",
+    "build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
+    "publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
+    "publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
    "build:docs": "lerna run build:docs",
    "release:helm": "node scripts/releaseHelmChart",
    "env:multi:enable": "lerna run env:multi:enable",
--- a/packages/backend-core/mocks/aws-sdk.ts
+++ b/packages/backend-core/mocks/aws-sdk.ts
@ -3,7 +3,10 @@ const mockS3 = {
  deleteObject: jest.fn().mockReturnThis(),
  deleteObjects: jest.fn().mockReturnThis(),
  createBucket: jest.fn().mockReturnThis(),
-  listObjects: jest.fn().mockReturnThis(),
+  listObject: jest.fn().mockReturnThis(),
+  getSignedUrl: jest.fn((operation: string, params: any) => {
+    return `http://s3.example.com/${params.Bucket}/${params.Key}`
+  }),
  promise: jest.fn().mockReturnThis(),
  catch: jest.fn(),
 }
--- a/packages/backend-core/accounts.js
+++ b/packages/backend-core/accounts.js
@ -1 +0,0 @@
-module.exports = require("./src/cloud/accounts")
--- a/packages/backend-core/auth.js
+++ b/packages/backend-core/auth.js
@ -1 +0,0 @@
-module.exports = require("./src/auth")
--- a/packages/backend-core/cache.js
+++ b/packages/backend-core/cache.js
@ -1,9 +0,0 @@
-const generic = require("./src/cache/generic")
-
-module.exports = {
-  user: require("./src/cache/user"),
-  app: require("./src/cache/appMetadata"),
-  writethrough: require("./src/cache/writethrough"),
-  ...generic,
-  cache: generic,
-}
--- a/packages/backend-core/constants.js
+++ b/packages/backend-core/constants.js
@ -1 +0,0 @@
-module.exports = require("./src/constants")
--- a/packages/backend-core/context.js
+++ b/packages/backend-core/context.js
@ -1,24 +0,0 @@
-const {
-  getAppDB,
-  getDevAppDB,
-  getProdAppDB,
-  getAppId,
-  updateAppId,
-  doInAppContext,
-  doInTenant,
-  doInContext,
-} = require("./src/context")
-
-const identity = require("./src/context/identity")
-
-module.exports = {
-  getAppDB,
-  getDevAppDB,
-  getProdAppDB,
-  getAppId,
-  updateAppId,
-  doInAppContext,
-  doInTenant,
-  identity,
-  doInContext,
-}
--- a/packages/backend-core/db.js
+++ b/packages/backend-core/db.js
@ -1 +0,0 @@
-module.exports = require("./src/db")
--- a/packages/backend-core/deprovision.js
+++ b/packages/backend-core/deprovision.js
@ -1 +0,0 @@
-module.exports = require("./src/context/deprovision")
--- a/packages/backend-core/encryption.js
+++ b/packages/backend-core/encryption.js
@ -1 +0,0 @@
-module.exports = require("./src/security/encryption")
--- a/packages/backend-core/jest-testcontainers-config.js
+++ b/packages/backend-core/jest-testcontainers-config.js
@ -0,0 +1,8 @@
+const { join } = require("path")
+require("dotenv").config({
+  path: join(__dirname, "..", "..", "hosting", ".env"),
+})
+
+const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator")
+
+module.exports = jestTestcontainersConfigGenerator()
--- a/packages/backend-core/jest.config.ts
+++ b/packages/backend-core/jest.config.ts
@ -1,21 +1,40 @@
 import { Config } from "@jest/types"
+const preset = require("ts-jest/jest-preset")

-const config: Config.InitialOptions = {
-  preset: "ts-jest",
-  testEnvironment: "node",
-  setupFiles: ["./tests/jestSetup.ts"],
-  collectCoverageFrom: ["src/**/*.{js,ts}"],
-  coverageReporters: ["lcov", "json", "clover"],
+const baseConfig: Config.InitialProjectOptions = {
+  ...preset,
+  preset: "@trendyol/jest-testcontainers",
+  setupFiles: ["./tests/jestEnv.ts"],
+  setupFilesAfterEnv: ["./tests/jestSetup.ts"],
+  transform: {
+    "^.+\\.ts?$": "@swc/jest",
+  },
 }

 if (!process.env.CI) {
  // use sources when not in CI
-  config.moduleNameMapper = {
+  baseConfig.moduleNameMapper = {
    "@budibase/types": "<rootDir>/../types/src",
-    "^axios.*$": "<rootDir>/node_modules/axios/lib/axios.js",
  }
 } else {
  console.log("Running tests with compiled dependency sources")
 }

+const config: Config.InitialOptions = {
+  projects: [
+    {
+      ...baseConfig,
+      displayName: "sequential test",
+      testMatch: ["<rootDir>/**/*.seq.spec.[jt]s"],
+      runner: "jest-serial-runner",
+    },
+    {
+      ...baseConfig,
+      testMatch: ["<rootDir>/**/!(*.seq).spec.[jt]s"],
+    },
+  ],
+  collectCoverageFrom: ["src/**/*.{js,ts}"],
+  coverageReporters: ["lcov", "json", "clover"],
+}
+
 export default config
--- a/packages/backend-core/logging.js
+++ b/packages/backend-core/logging.js
@ -1 +0,0 @@
-module.exports = require("./src/logging")
--- a/packages/backend-core/middleware.js
+++ b/packages/backend-core/middleware.js
@ -1 +0,0 @@
-module.exports = require("./src/middleware")
--- a/packages/backend-core/migrations.js
+++ b/packages/backend-core/migrations.js
@ -1 +0,0 @@
-module.exports = require("./src/migrations")
--- a/packages/backend-core/objectStore.js
+++ b/packages/backend-core/objectStore.js
@ -1,4 +0,0 @@
-module.exports = {
-  ...require("./src/objectStore"),
-  ...require("./src/objectStore/utils"),
-}
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,6 +1,6 @@
 {
  "name": "@budibase/backend-core",
-  "version": "2.1.32-alpha.3",
+  "version": "2.3.2-alpha.3",
  "description": "Budibase backend core libraries used in server and worker",
  "main": "dist/src/index.js",
  "types": "dist/src/index.d.ts",
@ -15,29 +15,33 @@
    "prebuild": "rimraf dist/",
    "prepack": "cp package.json dist",
    "build": "tsc -p tsconfig.build.json",
+    "build:pro": "../../scripts/pro/build.sh",
+    "postbuild": "yarn run build:pro",
    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
-    "test": "jest --coverage",
+    "test": "jest --coverage --maxWorkers=2",
    "test:watch": "jest --watchAll"
  },
  "dependencies": {
-    "@budibase/types": "2.1.32-alpha.3",
+    "@budibase/nano": "10.1.1",
+    "@budibase/pouchdb-replication-stream": "1.2.10",
+    "@budibase/types": "2.3.2-alpha.3",
    "@shopify/jest-koa-mocks": "5.0.1",
    "@techpass/passport-openidconnect": "0.3.2",
+    "aws-cloudfront-sign": "2.2.0",
    "aws-sdk": "2.1030.0",
    "bcrypt": "5.0.1",
    "bcryptjs": "2.4.3",
    "bull": "4.10.1",
+    "correlation-id": "4.0.0",
    "dotenv": "16.0.1",
    "emitter-listener": "1.1.2",
    "ioredis": "4.28.0",
    "joi": "17.6.0",
-    "jsonwebtoken": "8.5.1",
+    "jsonwebtoken": "9.0.0",
    "koa-passport": "4.1.4",
    "lodash": "4.17.21",
    "lodash.isarguments": "3.1.0",
-    "nano": "^10.1.0",
    "node-fetch": "2.6.7",
-    "passport-google-auth": "1.0.2",
    "passport-google-oauth": "2.0.0",
    "passport-jwt": "4.0.0",
    "passport-local": "1.0.0",
@ -45,7 +49,6 @@
    "posthog-node": "1.3.0",
    "pouchdb": "7.3.0",
    "pouchdb-find": "7.2.2",
-    "pouchdb-replication-stream": "1.2.9",
    "redlock": "4.2.0",
    "sanitize-s3-objectkey": "0.0.1",
    "semver": "7.3.7",
@ -54,21 +57,27 @@
    "zlib": "1.0.5"
  },
  "devDependencies": {
+    "@swc/core": "^1.3.25",
+    "@swc/jest": "^0.2.24",
+    "@trendyol/jest-testcontainers": "^2.1.1",
    "@types/chance": "1.1.3",
    "@types/ioredis": "4.28.0",
    "@types/jest": "27.5.1",
    "@types/koa": "2.13.4",
+    "@types/koa-pino-logger": "3.0.0",
    "@types/lodash": "4.14.180",
    "@types/node": "14.18.20",
    "@types/node-fetch": "2.6.1",
+    "@types/pino-http": "5.8.1",
    "@types/pouchdb": "6.4.0",
    "@types/redlock": "4.0.3",
    "@types/semver": "7.3.7",
    "@types/tar-fs": "2.0.1",
    "@types/uuid": "8.3.4",
-    "chance": "1.1.3",
+    "chance": "1.1.8",
    "ioredis-mock": "5.8.0",
    "jest": "28.1.1",
+    "jest-serial-runner": "^1.2.1",
    "koa": "2.13.4",
    "nodemon": "2.0.16",
    "pouchdb-adapter-memory": "7.2.2",
--- a/packages/backend-core/permissions.js
+++ b/packages/backend-core/permissions.js
@ -1 +0,0 @@
-module.exports = require("./src/security/permissions")
--- a/packages/backend-core/plugins.js
+++ b/packages/backend-core/plugins.js
@ -1,3 +0,0 @@
-module.exports = {
-  ...require("./src/plugin"),
-}
--- a/packages/backend-core/plugins.ts
+++ b/packages/backend-core/plugins.ts
@ -0,0 +1 @@
+export * from "./src/plugin"
--- a/packages/backend-core/redis.js
+++ b/packages/backend-core/redis.js
@ -1,5 +0,0 @@
-module.exports = {
-  Client: require("./src/redis"),
-  utils: require("./src/redis/utils"),
-  clients: require("./src/redis/init"),
-}
--- a/packages/backend-core/roles.js
+++ b/packages/backend-core/roles.js
@ -1 +0,0 @@
-module.exports = require("./src/security/roles")
--- a/packages/backend-core/sessions.js
+++ b/packages/backend-core/sessions.js
@ -1 +0,0 @@
-module.exports = require("./src/security/sessions")
--- a/packages/backend-core/src/auth/auth.ts
+++ b/packages/backend-core/src/auth/auth.ts
@ -1,42 +1,51 @@
-const passport = require("koa-passport")
+const _passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-import { getGlobalDB } from "./tenancy"
+import { getGlobalDB } from "../tenancy"
 const refresh = require("passport-oauth2-refresh")
-import { Config } from "./constants"
-import { getScopedConfig } from "./db/utils"
+import { Config } from "../constants"
+import { getScopedConfig } from "../db"
 import {
-  jwt,
+  jwt as jwtPassport,
  local,
  authenticated,
-  google,
-  oidc,
-  auditLog,
  tenancy,
-  authError,
-  ssoCallbackUrl,
  csrf,
+  oidc,
+  google,
+} from "../middleware"
+import { invalidateUser } from "../cache/user"
+import { User } from "@budibase/types"
+import { logAlert } from "../logging"
+export {
+  auditLog,
+  authError,
  internalApi,
+  ssoCallbackUrl,
  adminOnly,
  builderOnly,
  builderOrAdmin,
  joiValidator,
-} from "./middleware"
-import { invalidateUser } from "./cache/user"
-import { User } from "@budibase/types"
-import { logAlert } from "./logging"
+  google,
+  oidc,
+} from "../middleware"
+export const buildAuthMiddleware = authenticated
+export const buildTenancyMiddleware = tenancy
+export const buildCsrfMiddleware = csrf
+export const passport = _passport
+export const jwt = require("jsonwebtoken")

 // Strategies
-passport.use(new LocalStrategy(local.options, local.authenticate))
-if (jwt.options.secretOrKey) {
-  passport.use(new JwtStrategy(jwt.options, jwt.authenticate))
+_passport.use(new LocalStrategy(local.options, local.authenticate))
+if (jwtPassport.options.secretOrKey) {
+  _passport.use(new JwtStrategy(jwtPassport.options, jwtPassport.authenticate))
 } else {
  logAlert("No JWT Secret supplied, cannot configure JWT strategy")
 }

-passport.serializeUser((user: User, done: any) => done(null, user))
+_passport.serializeUser((user: User, done: any) => done(null, user))

-passport.deserializeUser(async (user: User, done: any) => {
+_passport.deserializeUser(async (user: User, done: any) => {
  const db = getGlobalDB()

  try {
@ -115,7 +124,7 @@ async function refreshGoogleAccessToken(
  })
 }

-async function refreshOAuthToken(
+export async function refreshOAuthToken(
  refreshToken: string,
  configType: string,
  configId: string
@ -152,7 +161,7 @@ async function refreshOAuthToken(
  return refreshResponse
 }

-async function updateUserOAuth(userId: string, oAuthConfig: any) {
+export async function updateUserOAuth(userId: string, oAuthConfig: any) {
  const details = {
    accessToken: oAuthConfig.accessToken,
    refreshToken: oAuthConfig.refreshToken,
@ -179,23 +188,3 @@ async function updateUserOAuth(userId: string, oAuthConfig: any) {
    console.error("Could not update OAuth details for current user", e)
  }
 }
-
-export = {
-  buildAuthMiddleware: authenticated,
-  passport,
-  google,
-  oidc,
-  jwt: require("jsonwebtoken"),
-  buildTenancyMiddleware: tenancy,
-  auditLog,
-  authError,
-  buildCsrfMiddleware: csrf,
-  internalApi,
-  refreshOAuthToken,
-  updateUserOAuth,
-  ssoCallbackUrl,
-  adminOnly,
-  builderOnly,
-  builderOrAdmin,
-  joiValidator,
-}
--- a/packages/backend-core/src/auth/index.ts
+++ b/packages/backend-core/src/auth/index.ts
@ -0,0 +1 @@
+export * from "./auth"
--- a/packages/backend-core/src/cache/appMetadata.ts
+++ b/packages/backend-core/src/cache/appMetadata.ts
@ -1,6 +1,6 @@
-const redis = require("../redis/init")
-const { doWithDB } = require("../db")
-const { DocumentType } = require("../db/constants")
+import { getAppClient } from "../redis/init"
+import { doWithDB, DocumentType } from "../db"
+import { Database } from "@budibase/types"

 const AppState = {
  INVALID: "invalid",
@ -10,17 +10,17 @@ const EXPIRY_SECONDS = 3600
 /**
 * The default populate app metadata function
 */
-const populateFromDB = async appId => {
+async function populateFromDB(appId: string) {
  return doWithDB(
    appId,
-    db => {
+    (db: Database) => {
      return db.get(DocumentType.APP_METADATA)
    },
    { skip_setup: true }
  )
 }

-const isInvalid = metadata => {
+function isInvalid(metadata?: { state: string }) {
  return !metadata || metadata.state === AppState.INVALID
 }

@ -31,15 +31,15 @@ const isInvalid = metadata => {
 * @param {string} appId the id of the app to get metadata from.
 * @returns {object} the app metadata.
 */
-exports.getAppMetadata = async appId => {
-  const client = await redis.getAppClient()
+export async function getAppMetadata(appId: string) {
+  const client = await getAppClient()
  // try cache
  let metadata = await client.get(appId)
  if (!metadata) {
-    let expiry = EXPIRY_SECONDS
+    let expiry: number | undefined = EXPIRY_SECONDS
    try {
      metadata = await populateFromDB(appId)
-    } catch (err) {
+    } catch (err: any) {
      // app DB left around, but no metadata, it is invalid
      if (err && err.status === 404) {
        metadata = { state: AppState.INVALID }
@ -74,11 +74,11 @@ exports.getAppMetadata = async appId => {
 * @param newMetadata {object|undefined} optional - can simply provide the new metadata to update with.
 * @return {Promise<void>} will respond with success when cache is updated.
 */
-exports.invalidateAppMetadata = async (appId, newMetadata = null) => {
+export async function invalidateAppMetadata(appId: string, newMetadata?: any) {
  if (!appId) {
    throw "Cannot invalidate if no app ID provided."
  }
-  const client = await redis.getAppClient()
+  const client = await getAppClient()
  await client.delete(appId)
  if (newMetadata) {
    await client.store(appId, newMetadata, EXPIRY_SECONDS)
--- a/packages/backend-core/src/cache/base/index.ts
+++ b/packages/backend-core/src/cache/base/index.ts
@ -1,16 +1,16 @@
 import { getTenantId } from "../../context"
-import redis from "../../redis/init"
-import RedisWrapper from "../../redis"
+import * as redis from "../../redis/init"
+import { Client } from "../../redis"

 function generateTenantKey(key: string) {
  const tenantId = getTenantId()
  return `${key}:${tenantId}`
 }

-export = class BaseCache {
-  client: RedisWrapper | undefined
+export default class BaseCache {
+  client: Client | undefined

-  constructor(client: RedisWrapper | undefined = undefined) {
+  constructor(client: Client | undefined = undefined) {
    this.client = client
  }

--- a/packages/backend-core/src/cache/generic.js
+++ b/packages/backend-core/src/cache/generic.js
@ -1,30 +0,0 @@
-const BaseCache = require("./base")
-
-const GENERIC = new BaseCache()
-
-exports.CacheKeys = {
-  CHECKLIST: "checklist",
-  INSTALLATION: "installation",
-  ANALYTICS_ENABLED: "analyticsEnabled",
-  UNIQUE_TENANT_ID: "uniqueTenantId",
-  EVENTS: "events",
-  BACKFILL_METADATA: "backfillMetadata",
-  EVENTS_RATE_LIMIT: "eventsRateLimit",
-}
-
-exports.TTL = {
-  ONE_MINUTE: 600,
-  ONE_HOUR: 3600,
-  ONE_DAY: 86400,
-}
-
-function performExport(funcName) {
-  return (...args) => GENERIC[funcName](...args)
-}
-
-exports.keys = performExport("keys")
-exports.get = performExport("get")
-exports.store = performExport("store")
-exports.delete = performExport("delete")
-exports.withCache = performExport("withCache")
-exports.bustCache = performExport("bustCache")
--- a/packages/backend-core/src/cache/generic.ts
+++ b/packages/backend-core/src/cache/generic.ts
@ -0,0 +1,30 @@
+const BaseCache = require("./base")
+
+const GENERIC = new BaseCache.default()
+
+export enum CacheKey {
+  CHECKLIST = "checklist",
+  INSTALLATION = "installation",
+  ANALYTICS_ENABLED = "analyticsEnabled",
+  UNIQUE_TENANT_ID = "uniqueTenantId",
+  EVENTS = "events",
+  BACKFILL_METADATA = "backfillMetadata",
+  EVENTS_RATE_LIMIT = "eventsRateLimit",
+}
+
+export enum TTL {
+  ONE_MINUTE = 600,
+  ONE_HOUR = 3600,
+  ONE_DAY = 86400,
+}
+
+function performExport(funcName: string) {
+  return (...args: any) => GENERIC[funcName](...args)
+}
+
+export const keys = performExport("keys")
+export const get = performExport("get")
+export const store = performExport("store")
+export const destroy = performExport("delete")
+export const withCache = performExport("withCache")
+export const bustCache = performExport("bustCache")
--- a/packages/backend-core/src/cache/index.ts
+++ b/packages/backend-core/src/cache/index.ts
@ -0,0 +1,5 @@
+export * as generic from "./generic"
+export * as user from "./user"
+export * as app from "./appMetadata"
+export * as writethrough from "./writethrough"
+export * from "./generic"
--- a/packages/backend-core/src/cache/tests/writethrough.spec.js
+++ b/packages/backend-core/src/cache/tests/writethrough.spec.js
@ -2,14 +2,16 @@ require("../../../tests")
 const { Writethrough } = require("../writethrough")
 const { getDB } = require("../../db")
 const tk = require("timekeeper")
+const { structures } = require("../../../tests")

 const START_DATE = Date.now()
 tk.freeze(START_DATE)

+
 const DELAY = 5000

-const db = getDB("test")
-const db2 = getDB("test2")
+const db = getDB(structures.db.id())
+const db2 = getDB(structures.db.id())
 const writethrough = new Writethrough(db, DELAY), writethrough2 = new Writethrough(db2, DELAY)

 describe("writethrough", () => {
--- a/packages/backend-core/src/cache/user.ts
+++ b/packages/backend-core/src/cache/user.ts
@ -1,15 +1,16 @@
-const redis = require("../redis/init")
-const { getTenantId, lookupTenantId, doWithGlobalDB } = require("../tenancy")
-const env = require("../environment")
-const accounts = require("../cloud/accounts")
+import * as redis from "../redis/init"
+import { getTenantId, lookupTenantId, doWithGlobalDB } from "../tenancy"
+import env from "../environment"
+import * as accounts from "../cloud/accounts"
+import { Database } from "@budibase/types"

 const EXPIRY_SECONDS = 3600

 /**
 * The default populate user function
 */
-const populateFromDB = async (userId, tenantId) => {
-  const user = await doWithGlobalDB(tenantId, db => db.get(userId))
+async function populateFromDB(userId: string, tenantId: string) {
+  const user = await doWithGlobalDB(tenantId, (db: Database) => db.get(userId))
  user.budibaseAccess = true
  if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
    const account = await accounts.getAccount(user.email)
@ -31,7 +32,11 @@ const populateFromDB = async (userId, tenantId) => {
 * @param {*} populateUser function to provide the user for re-caching. default to couch db
 * @returns
 */
-exports.getUser = async (userId, tenantId = null, populateUser = null) => {
+export async function getUser(
+  userId: string,
+  tenantId?: string,
+  populateUser?: any
+) {
  if (!populateUser) {
    populateUser = populateFromDB
  }
@ -47,7 +52,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => {
  let user = await client.get(userId)
  if (!user) {
    user = await populateUser(userId, tenantId)
-    client.store(userId, user, EXPIRY_SECONDS)
+    await client.store(userId, user, EXPIRY_SECONDS)
  }
  if (user && !user.tenantId && tenantId) {
    // make sure the tenant ID is always correct/set
@ -56,7 +61,7 @@ exports.getUser = async (userId, tenantId = null, populateUser = null) => {
  return user
 }

-exports.invalidateUser = async userId => {
+export async function invalidateUser(userId: string) {
  const client = await redis.getUserClient()
  await client.delete(userId)
 }
--- a/packages/backend-core/src/cloud/api.js
+++ b/packages/backend-core/src/cloud/api.js
@ -1,42 +0,0 @@
-const fetch = require("node-fetch")
-class API {
-  constructor(host) {
-    this.host = host
-  }
-
-  apiCall =
-    method =>
-    async (url = "", options = {}) => {
-      if (!options.headers) {
-        options.headers = {}
-      }
-
-      if (!options.headers["Content-Type"]) {
-        options.headers = {
-          "Content-Type": "application/json",
-          Accept: "application/json",
-          ...options.headers,
-        }
-      }
-
-      let json = options.headers["Content-Type"] === "application/json"
-
-      const requestOptions = {
-        method: method,
-        body: json ? JSON.stringify(options.body) : options.body,
-        headers: options.headers,
-        // TODO: See if this is necessary
-        credentials: "include",
-      }
-
-      return await fetch(`${this.host}${url}`, requestOptions)
-    }
-
-  post = this.apiCall("POST")
-  get = this.apiCall("GET")
-  patch = this.apiCall("PATCH")
-  del = this.apiCall("DELETE")
-  put = this.apiCall("PUT")
-}
-
-module.exports = API
--- a/packages/backend-core/src/cloud/api.ts
+++ b/packages/backend-core/src/cloud/api.ts
@ -0,0 +1,59 @@
+import fetch from "node-fetch"
+import * as logging from "../logging"
+
+export default class API {
+  host: string
+
+  constructor(host: string) {
+    this.host = host
+  }
+
+  async apiCall(method: string, url: string, options?: any) {
+    if (!options.headers) {
+      options.headers = {}
+    }
+
+    if (!options.headers["Content-Type"]) {
+      options.headers = {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+        ...options.headers,
+      }
+    }
+
+    let json = options.headers["Content-Type"] === "application/json"
+
+    // add x-budibase-correlation-id header
+    logging.correlation.setHeader(options.headers)
+
+    const requestOptions = {
+      method: method,
+      body: json ? JSON.stringify(options.body) : options.body,
+      headers: options.headers,
+      // TODO: See if this is necessary
+      credentials: "include",
+    }
+
+    return await fetch(`${this.host}${url}`, requestOptions)
+  }
+
+  async post(url: string, options?: any) {
+    return this.apiCall("POST", url, options)
+  }
+
+  async get(url: string, options?: any) {
+    return this.apiCall("GET", url, options)
+  }
+
+  async patch(url: string, options?: any) {
+    return this.apiCall("PATCH", url, options)
+  }
+
+  async del(url: string, options?: any) {
+    return this.apiCall("DELETE", url, options)
+  }
+
+  async put(url: string, options?: any) {
+    return this.apiCall("PUT", url, options)
+  }
+}
--- a/packages/backend-core/src/constants.js
+++ b/packages/backend-core/src/constants.js
@ -1,44 +0,0 @@
-exports.UserStatus = {
-  ACTIVE: "active",
-  INACTIVE: "inactive",
-}
-
-exports.Cookie = {
-  CurrentApp: "budibase:currentapp",
-  Auth: "budibase:auth",
-  Init: "budibase:init",
-  ACCOUNT_RETURN_URL: "budibase:account:returnurl",
-  DatasourceAuth: "budibase:datasourceauth",
-  OIDC_CONFIG: "budibase:oidc:config",
-}
-
-exports.Header = {
-  API_KEY: "x-budibase-api-key",
-  LICENSE_KEY: "x-budibase-license-key",
-  API_VER: "x-budibase-api-version",
-  APP_ID: "x-budibase-app-id",
-  TYPE: "x-budibase-type",
-  PREVIEW_ROLE: "x-budibase-role",
-  TENANT_ID: "x-budibase-tenant-id",
-  TOKEN: "x-budibase-token",
-  CSRF_TOKEN: "x-csrf-token",
-}
-
-exports.GlobalRoles = {
-  OWNER: "owner",
-  ADMIN: "admin",
-  BUILDER: "builder",
-  WORKSPACE_MANAGER: "workspace_manager",
-}
-
-exports.Config = {
-  SETTINGS: "settings",
-  ACCOUNT: "account",
-  SMTP: "smtp",
-  GOOGLE: "google",
-  OIDC: "oidc",
-  OIDC_LOGOS: "logos_oidc",
-}
-
-exports.MAX_VALID_DATE = new Date(2147483647000)
-exports.DEFAULT_TENANT_ID = "default"
--- a/packages/backend-core/src/constants/db.ts
+++ b/packages/backend-core/src/constants/db.ts
@ -77,6 +77,7 @@ export const StaticDatabases = {
      apiKeys: "apikeys",
      usageQuota: "usage_quota",
      licenseInfo: "license_info",
+      environmentVariables: "environmentvariables",
    },
  },
  // contains information about tenancy and so on
--- a/packages/backend-core/src/constants/index.ts
+++ b/packages/backend-core/src/constants/index.ts
@ -0,0 +1,2 @@
+export * from "./db"
+export * from "./misc"
--- a/packages/backend-core/src/constants/misc.ts
+++ b/packages/backend-core/src/constants/misc.ts
@ -22,6 +22,7 @@ export enum Header {
  TENANT_ID = "x-budibase-tenant-id",
  TOKEN = "x-budibase-token",
  CSRF_TOKEN = "x-csrf-token",
+  CORRELATION_ID = "x-budibase-correlation-id",
 }

 export enum GlobalRole {
--- a/packages/backend-core/src/context/Context.ts
+++ b/packages/backend-core/src/context/Context.ts
@ -1,5 +1,5 @@
 import { AsyncLocalStorage } from "async_hooks"
-import { ContextMap } from "./constants"
+import { ContextMap } from "./mainContext"

 export default class Context {
  static storage = new AsyncLocalStorage<ContextMap>()
@ -11,8 +11,4 @@ export default class Context {
  static get(): ContextMap {
    return Context.storage.getStore() as ContextMap
  }
-
-  static set(context: ContextMap) {
-    Context.storage.enterWith(context)
-  }
 }
--- a/packages/backend-core/src/context/constants.ts
+++ b/packages/backend-core/src/context/constants.ts
@ -1,7 +0,0 @@
-import { IdentityContext } from "@budibase/types"
-
-export type ContextMap = {
-  tenantId?: string
-  appId?: string
-  identity?: IdentityContext
-}
--- a/packages/backend-core/src/context/identity.ts
+++ b/packages/backend-core/src/context/identity.ts
@ -2,23 +2,22 @@ import {
  IdentityContext,
  IdentityType,
  User,
-  UserContext,
  isCloudAccount,
  Account,
  AccountUserContext,
 } from "@budibase/types"
 import * as context from "."

-export const getIdentity = (): IdentityContext | undefined => {
+export function getIdentity(): IdentityContext | undefined {
  return context.getIdentity()
 }

-export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+export function doInIdentityContext(identity: IdentityContext, task: any) {
  return context.doInIdentityContext(identity, task)
 }

-export const doInUserContext = (user: User, task: any) => {
-  const userContext: UserContext = {
+export function doInUserContext(user: User, task: any) {
+  const userContext: any = {
    ...user,
    _id: user._id as string,
    type: IdentityType.USER,
@ -26,7 +25,7 @@ export const doInUserContext = (user: User, task: any) => {
  return doInIdentityContext(userContext, task)
 }

-export const doInAccountContext = (account: Account, task: any) => {
+export function doInAccountContext(account: Account, task: any) {
  const _id = getAccountUserId(account)
  const tenantId = account.tenantId
  const accountContext: AccountUserContext = {
@ -38,12 +37,12 @@ export const doInAccountContext = (account: Account, task: any) => {
  return doInIdentityContext(accountContext, task)
 }

-export const getAccountUserId = (account: Account) => {
+export function getAccountUserId(account: Account) {
  let userId: string
  if (isCloudAccount(account)) {
    userId = account.budibaseUserId
  } else {
-    // use account id as user id for self hosting
+    // use account id as user id for self-hosting
    userId = account.accountId
  }
  return userId
--- a/packages/backend-core/src/context/index.ts
+++ b/packages/backend-core/src/context/index.ts
@ -1,223 +1,3 @@
-import env from "../environment"
-import {
-  SEPARATOR,
-  DocumentType,
-  getDevelopmentAppID,
-  getProdAppID,
-  baseGlobalDBName,
-  getDB,
-} from "../db"
-import Context from "./Context"
-import { IdentityContext, Database } from "@budibase/types"
-import { DEFAULT_TENANT_ID as _DEFAULT_TENANT_ID } from "../constants"
-import { ContextMap } from "./constants"
-export const DEFAULT_TENANT_ID = _DEFAULT_TENANT_ID
-
-// some test cases call functions directly, need to
-// store an app ID to pretend there is a context
-let TEST_APP_ID: string | null = null
-
-export function isMultiTenant() {
-  return env.MULTI_TENANCY
-}
-
-export function isTenantIdSet() {
-  const context = Context.get()
-  return !!context?.tenantId
-}
-
-export function isTenancyEnabled() {
-  return env.MULTI_TENANCY
-}
-
-/**
- * Given an app ID this will attempt to retrieve the tenant ID from it.
- * @return {null|string} The tenant ID found within the app ID.
- */
-export function getTenantIDFromAppID(appId: string) {
-  if (!appId) {
-    return undefined
-  }
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-  const split = appId.split(SEPARATOR)
-  const hasDev = split[1] === DocumentType.DEV
-  if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
-    return undefined
-  }
-  if (hasDev) {
-    return split[2]
-  } else {
-    return split[1]
-  }
-}
-
-function updateContext(updates: ContextMap) {
-  let context: ContextMap
-  try {
-    context = Context.get()
-  } catch (err) {
-    // no context, start empty
-    context = {}
-  }
-  context = {
-    ...context,
-    ...updates,
-  }
-  return context
-}
-
-async function newContext(updates: ContextMap, task: any) {
-  // see if there already is a context setup
-  let context: ContextMap = updateContext(updates)
-  return Context.run(context, task)
-}
-
-export async function doInContext(appId: string, task: any): Promise<any> {
-  const tenantId = getTenantIDFromAppID(appId)
-  return newContext(
-    {
-      tenantId,
-      appId,
-    },
-    task
-  )
-}
-
-export async function doInTenant(
-  tenantId: string | null,
-  task: any
-): Promise<any> {
-  // make sure default always selected in single tenancy
-  if (!env.MULTI_TENANCY) {
-    tenantId = tenantId || DEFAULT_TENANT_ID
-  }
-
-  const updates = tenantId ? { tenantId } : {}
-  return newContext(updates, task)
-}
-
-export async function doInAppContext(appId: string, task: any): Promise<any> {
-  if (!appId) {
-    throw new Error("appId is required")
-  }
-
-  const tenantId = getTenantIDFromAppID(appId)
-  const updates: ContextMap = { appId }
-  if (tenantId) {
-    updates.tenantId = tenantId
-  }
-  return newContext(updates, task)
-}
-
-export async function doInIdentityContext(
-  identity: IdentityContext,
-  task: any
-): Promise<any> {
-  if (!identity) {
-    throw new Error("identity is required")
-  }
-
-  const context: ContextMap = {
-    identity,
-  }
-  if (identity.tenantId) {
-    context.tenantId = identity.tenantId
-  }
-  return newContext(context, task)
-}
-
-export function getIdentity(): IdentityContext | undefined {
-  try {
-    const context = Context.get()
-    return context?.identity
-  } catch (e) {
-    // do nothing - identity is not in context
-  }
-}
-
-export function getTenantId(): string {
-  if (!isMultiTenant()) {
-    return DEFAULT_TENANT_ID
-  }
-  const context = Context.get()
-  const tenantId = context?.tenantId
-  if (!tenantId) {
-    throw new Error("Tenant id not found")
-  }
-  return tenantId
-}
-
-export function getAppId(): string | undefined {
-  const context = Context.get()
-  const foundId = context?.appId
-  if (!foundId && env.isTest() && TEST_APP_ID) {
-    return TEST_APP_ID
-  } else {
-    return foundId
-  }
-}
-
-export function updateTenantId(tenantId?: string) {
-  let context: ContextMap = updateContext({
-    tenantId,
-  })
-  Context.set(context)
-}
-
-export function updateAppId(appId: string) {
-  let context: ContextMap = updateContext({
-    appId,
-  })
-  try {
-    Context.set(context)
-  } catch (err) {
-    if (env.isTest()) {
-      TEST_APP_ID = appId
-    } else {
-      throw err
-    }
-  }
-}
-
-export function getGlobalDB(): Database {
-  const context = Context.get()
-  if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
-    throw new Error("Global DB not found")
-  }
-  return getDB(baseGlobalDBName(context?.tenantId))
-}
-
-/**
- * Gets the app database based on whatever the request
- * contained, dev or prod.
- */
-export function getAppDB(opts?: any): Database {
-  const appId = getAppId()
-  return getDB(appId, opts)
-}
-
-/**
- * This specifically gets the prod app ID, if the request
- * contained a development app ID, this will get the prod one.
- */
-export function getProdAppDB(opts?: any): Database {
-  const appId = getAppId()
-  if (!appId) {
-    throw new Error("Unable to retrieve prod DB - no app ID.")
-  }
-  return getDB(getProdAppID(appId), opts)
-}
-
-/**
- * This specifically gets the dev app ID, if the request
- * contained a prod app ID, this will get the dev one.
- */
-export function getDevAppDB(opts?: any): Database {
-  const appId = getAppId()
-  if (!appId) {
-    throw new Error("Unable to retrieve dev DB - no app ID.")
-  }
-  return getDB(getDevelopmentAppID(appId), opts)
-}
+export { DEFAULT_TENANT_ID } from "../constants"
+export * as identity from "./identity"
+export * from "./mainContext"
--- a/packages/backend-core/src/context/mainContext.ts
+++ b/packages/backend-core/src/context/mainContext.ts
@ -0,0 +1,262 @@
+// some test cases call functions directly, need to
+// store an app ID to pretend there is a context
+import env from "../environment"
+import Context from "./Context"
+import * as conversions from "../db/conversions"
+import { getDB } from "../db/db"
+import {
+  DocumentType,
+  SEPARATOR,
+  StaticDatabases,
+  DEFAULT_TENANT_ID,
+} from "../constants"
+import { Database, IdentityContext } from "@budibase/types"
+
+export type ContextMap = {
+  tenantId?: string
+  appId?: string
+  identity?: IdentityContext
+  environmentVariables?: Record<string, string>
+}
+
+let TEST_APP_ID: string | null = null
+
+export function getGlobalDBName(tenantId?: string) {
+  // tenant ID can be set externally, for example user API where
+  // new tenants are being created, this may be the case
+  if (!tenantId) {
+    tenantId = getTenantId()
+  }
+  return baseGlobalDBName(tenantId)
+}
+
+export function baseGlobalDBName(tenantId: string | undefined | null) {
+  let dbName
+  if (!tenantId || tenantId === DEFAULT_TENANT_ID) {
+    dbName = StaticDatabases.GLOBAL.name
+  } else {
+    dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
+  }
+  return dbName
+}
+
+export function isMultiTenant() {
+  return env.MULTI_TENANCY
+}
+
+export function isTenantIdSet() {
+  const context = Context.get()
+  return !!context?.tenantId
+}
+
+export function isTenancyEnabled() {
+  return env.MULTI_TENANCY
+}
+
+/**
+ * Given an app ID this will attempt to retrieve the tenant ID from it.
+ * @return {null|string} The tenant ID found within the app ID.
+ */
+export function getTenantIDFromAppID(appId: string) {
+  if (!appId) {
+    return undefined
+  }
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
+  const split = appId.split(SEPARATOR)
+  const hasDev = split[1] === DocumentType.DEV
+  if ((hasDev && split.length === 3) || (!hasDev && split.length === 2)) {
+    return undefined
+  }
+  if (hasDev) {
+    return split[2]
+  } else {
+    return split[1]
+  }
+}
+
+function updateContext(updates: ContextMap): ContextMap {
+  let context: ContextMap
+  try {
+    context = Context.get()
+  } catch (err) {
+    // no context, start empty
+    context = {}
+  }
+  context = {
+    ...context,
+    ...updates,
+  }
+  return context
+}
+
+async function newContext(updates: ContextMap, task: any) {
+  // see if there already is a context setup
+  let context: ContextMap = updateContext(updates)
+  return Context.run(context, task)
+}
+
+export async function doInContext(appId: string, task: any): Promise<any> {
+  const tenantId = getTenantIDFromAppID(appId)
+  return newContext(
+    {
+      tenantId,
+      appId,
+    },
+    task
+  )
+}
+
+export async function doInTenant(
+  tenantId: string | null,
+  task: any
+): Promise<any> {
+  // make sure default always selected in single tenancy
+  if (!env.MULTI_TENANCY) {
+    tenantId = tenantId || DEFAULT_TENANT_ID
+  }
+
+  const updates = tenantId ? { tenantId } : {}
+  return newContext(updates, task)
+}
+
+export async function doInAppContext(
+  appId: string | null,
+  task: any
+): Promise<any> {
+  if (!appId && !env.isTest()) {
+    throw new Error("appId is required")
+  }
+
+  let updates: ContextMap
+  if (!appId) {
+    updates = { appId: "" }
+  } else {
+    const tenantId = getTenantIDFromAppID(appId)
+    updates = { appId }
+    if (tenantId) {
+      updates.tenantId = tenantId
+    }
+  }
+  return newContext(updates, task)
+}
+
+export async function doInIdentityContext(
+  identity: IdentityContext,
+  task: any
+): Promise<any> {
+  if (!identity) {
+    throw new Error("identity is required")
+  }
+
+  const context: ContextMap = {
+    identity,
+  }
+  if (identity.tenantId) {
+    context.tenantId = identity.tenantId
+  }
+  return newContext(context, task)
+}
+
+export function getIdentity(): IdentityContext | undefined {
+  try {
+    const context = Context.get()
+    return context?.identity
+  } catch (e) {
+    // do nothing - identity is not in context
+  }
+}
+
+export function getTenantId(): string {
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
+  const context = Context.get()
+  const tenantId = context?.tenantId
+  if (!tenantId) {
+    throw new Error("Tenant id not found")
+  }
+  return tenantId
+}
+
+export function getAppId(): string | undefined {
+  const context = Context.get()
+  const foundId = context?.appId
+  if (!foundId && env.isTest() && TEST_APP_ID) {
+    return TEST_APP_ID
+  } else {
+    return foundId
+  }
+}
+
+export const getProdAppId = () => {
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Could not get appId")
+  }
+  return conversions.getProdAppID(appId)
+}
+
+export function doInEnvironmentContext(
+  values: Record<string, string>,
+  task: any
+) {
+  if (!values) {
+    throw new Error("Must supply environment variables.")
+  }
+  const updates = {
+    environmentVariables: values,
+  }
+  return newContext(updates, task)
+}
+
+export function getEnvironmentVariables() {
+  const context = Context.get()
+  if (!context.environmentVariables) {
+    return null
+  } else {
+    return context.environmentVariables
+  }
+}
+
+export function getGlobalDB(): Database {
+  const context = Context.get()
+  if (!context || (env.MULTI_TENANCY && !context.tenantId)) {
+    throw new Error("Global DB not found")
+  }
+  return getDB(baseGlobalDBName(context?.tenantId))
+}
+
+/**
+ * Gets the app database based on whatever the request
+ * contained, dev or prod.
+ */
+export function getAppDB(opts?: any): Database {
+  const appId = getAppId()
+  return getDB(appId, opts)
+}
+
+/**
+ * This specifically gets the prod app ID, if the request
+ * contained a development app ID, this will get the prod one.
+ */
+export function getProdAppDB(opts?: any): Database {
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve prod DB - no app ID.")
+  }
+  return getDB(conversions.getProdAppID(appId), opts)
+}
+
+/**
+ * This specifically gets the dev app ID, if the request
+ * contained a prod app ID, this will get the dev one.
+ */
+export function getDevAppDB(opts?: any): Database {
+  const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve dev DB - no app ID.")
+  }
+  return getDB(conversions.getDevelopmentAppID(appId), opts)
+}
--- a/packages/backend-core/src/context/tests/index.spec.ts
+++ b/packages/backend-core/src/context/tests/index.spec.ts
@ -1,7 +1,7 @@
 require("../../../tests")
 const context = require("../")
 const { DEFAULT_TENANT_ID } = require("../../constants")
-const env = require("../../environment")
+import env from "../../environment"

 describe("context", () => {
  describe("doInTenant", () => {
@ -26,7 +26,7 @@ describe("context", () => {

      it("fails when no tenant id is set", () => {
        const test = () => {
-          let error
+          let error: any
          try {
            context.getTenantId()
          } catch (e) {
@ -45,7 +45,7 @@ describe("context", () => {

      it("fails when no tenant db is set", () => {
        const test = () => {
-          let error
+          let error: any
          try {
            context.getGlobalDB()
          } catch (e) {
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@ -1,5 +1,5 @@
-import { getPouchDB, closePouchDB } from "./couch/pouchDB"
-import { DocumentType } from "./constants"
+import { getPouchDB, closePouchDB } from "./couch"
+import { DocumentType } from "../constants"

 class Replication {
  source: any
--- a/packages/backend-core/src/db/conversions.ts
+++ b/packages/backend-core/src/db/conversions.ts
@ -1,4 +1,4 @@
-import { APP_DEV_PREFIX, APP_PREFIX } from "./constants"
+import { APP_DEV_PREFIX, APP_PREFIX } from "../constants"
 import { App } from "@budibase/types"
 const NO_APP_ERROR = "No app provided"

--- a/packages/backend-core/src/db/couch/DatabaseImpl.ts
+++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts
@ -1,4 +1,4 @@
-import Nano from "nano"
+import Nano from "@budibase/nano"
 import {
  AllDocsResponse,
  AnyDocument,
@ -15,18 +15,47 @@ import { getCouchInfo } from "./connections"
 import { directCouchCall } from "./utils"
 import { getPouchDB } from "./pouchDB"
 import { WriteStream, ReadStream } from "fs"
+import { newid } from "../../newid"
+
+function buildNano(couchInfo: { url: string; cookie: string }) {
+  return Nano({
+    url: couchInfo.url,
+    requestDefaults: {
+      headers: {
+        Authorization: couchInfo.cookie,
+      },
+    },
+    parseUrl: false,
+  })
+}
+
+export function DatabaseWithConnection(
+  dbName: string,
+  connection: string,
+  opts?: DatabaseOpts
+) {
+  if (!connection) {
+    throw new Error("Must provide connection details")
+  }
+  return new DatabaseImpl(dbName, opts, connection)
+}

 export class DatabaseImpl implements Database {
  public readonly name: string
  private static nano: Nano.ServerScope
+  private readonly instanceNano?: Nano.ServerScope
  private readonly pouchOpts: DatabaseOpts

-  constructor(dbName?: string, opts?: DatabaseOpts) {
+  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
    if (dbName == null) {
      throw new Error("Database name cannot be undefined.")
    }
    this.name = dbName
    this.pouchOpts = opts || {}
+    if (connection) {
+      const couchInfo = getCouchInfo(connection)
+      this.instanceNano = buildNano(couchInfo)
+    }
    if (!DatabaseImpl.nano) {
      DatabaseImpl.init()
    }
@ -34,15 +63,7 @@ export class DatabaseImpl implements Database {

  static init() {
    const couchInfo = getCouchInfo()
-    DatabaseImpl.nano = Nano({
-      url: couchInfo.url,
-      requestDefaults: {
-        headers: {
-          Authorization: couchInfo.cookie,
-        },
-      },
-      parseUrl: false,
-    })
+    DatabaseImpl.nano = buildNano(couchInfo)
  }

  async exists() {
@ -50,6 +71,10 @@ export class DatabaseImpl implements Database {
    return response.status === 200
  }

+  private nano() {
+    return this.instanceNano || DatabaseImpl.nano
+  }
+
  async checkSetup() {
    let shouldCreate = !this.pouchOpts?.skip_setup
    // check exists in a lightweight fashion
@ -58,9 +83,16 @@ export class DatabaseImpl implements Database {
      throw new Error("DB does not exist")
    }
    if (!exists) {
-      await DatabaseImpl.nano.db.create(this.name)
+      try {
+        await this.nano().db.create(this.name)
+      } catch (err: any) {
+        // Handling race conditions
+        if (err.statusCode !== 412) {
+          throw err
+        }
+      }
    }
-    return DatabaseImpl.nano.db.use(this.name)
+    return this.nano().db.use(this.name)
  }

  private async updateOutput(fnc: any) {
@ -101,6 +133,13 @@ export class DatabaseImpl implements Database {
    return this.updateOutput(() => db.destroy(_id, _rev))
  }

+  async post(document: AnyDocument, opts?: DatabasePutOpts) {
+    if (!document._id) {
+      document._id = newid()
+    }
+    return this.put(document, opts)
+  }
+
  async put(document: AnyDocument, opts?: DatabasePutOpts) {
    if (!document._id) {
      throw new Error("Cannot store document without _id field.")
@ -146,7 +185,7 @@ export class DatabaseImpl implements Database {

  async destroy() {
    try {
-      await DatabaseImpl.nano.db.destroy(this.name)
+      return await this.nano().db.destroy(this.name)
    } catch (err: any) {
      // didn't exist, don't worry
      if (err.statusCode === 404) {
--- a/Show More
+++ b/Show More
				`@ -1 +0,0 @@`
				`module.exports = require("./src/cloud/accounts")`
				`@ -1 +0,0 @@`
				`module.exports = require("./src/context/deprovision")`
				`@ -1 +0,0 @@`
				`module.exports = require("./src/security/encryption")`
				`@ -1 +0,0 @@`
				`module.exports = require("./src/middleware")`
				`@ -1 +0,0 @@`
				`module.exports = require("./src/migrations")`