Merge master

2022-06-16 09:44:37 +01:00 · 2022-06-16 09:44:37 +01:00 · 51642ffe3e
parent 6b833e7529 8ec04b1d6c
commit 51642ffe3e
329 changed files with 14188 additions and 7971 deletions
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -11,6 +11,7 @@ on:
    branches:
      - master
      - develop 
+      - release 
 workflow_dispatch:

 env:
--- a/.github/workflows/deploy-preprod.yml
+++ b/.github/workflows/deploy-preprod.yml
@ -4,9 +4,7 @@ on:
 workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}

 jobs:
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -4,7 +4,7 @@ concurrency: release-develop
 on: 
 push:
    branches:
-      - develop
+      - release
    paths:
      - '.aws/**'
      - '.github/**'
@ -18,9 +18,9 @@ on:
 workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      
 jobs:
@ -28,11 +28,11 @@ jobs:
    runs-on: ubuntu-latest 

    steps:
-      - name: Fail if branch is not develop
-        if: github.ref != 'refs/heads/develop' 
-        run: | 
-          echo "Ref is not develop, you must run this job from develop."
-          exit 1
+      # - name: Fail if branch is not develop
+      #   if: github.ref != 'refs/heads/develop' 
+      #   run: | 
+      #     echo "Ref is not develop, you must run this job from develop."
+      #     exit 1
      - uses: actions/checkout@v2
      - uses: actions/setup-node@v1
        with:
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -18,9 +18,9 @@ on:
 workflow_dispatch:

 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}

--- a/.gitignore
+++ b/.gitignore
@ -101,3 +101,5 @@ packages/builder/cypress.env.json
 packages/builder/cypress/reports
 stats.html

+# TypeScript cache
+*.tsbuildinfo
--- a/charts/budibase/templates/app-service-deployment.yaml
+++ b/charts/budibase/templates/app-service-deployment.yaml
@ -28,6 +28,8 @@ spec:
      - env:
        - name: BUDIBASE_ENVIRONMENT
          value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
        - name: COUCH_DB_URL
          {{ if .Values.services.couchdb.url }}
          value: {{ .Values.services.couchdb.url }}
--- a/charts/budibase/templates/worker-service-deployment.yaml
+++ b/charts/budibase/templates/worker-service-deployment.yaml
@ -27,6 +27,8 @@ spec:
    spec:
      containers:
      - env:
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
        - name: CLUSTER_PORT
          value: {{ .Values.services.worker.port | quote }}
        {{ if .Values.services.couchdb.enabled }}
@ -91,6 +93,10 @@ spec:
          value: {{ .Values.globals.selfHosted | quote }}
        - name: SENTRY_DSN
          value: {{ .Values.globals.sentryDSN }}
+        - name: ENABLE_ANALYTICS
+          value: {{ .Values.globals.enableAnalytics | quote }}
+        - name: POSTHOG_TOKEN
+          value: {{ .Values.globals.posthogToken }}
        - name: ACCOUNT_PORTAL_URL
          value: {{ .Values.globals.accountPortalUrl | quote }}
        - name: ACCOUNT_PORTAL_API_KEY
--- a/charts/budibase/values.yaml
+++ b/charts/budibase/values.yaml
@ -89,9 +89,9 @@ affinity: {}
 globals:
  appVersion: "latest"
  budibaseEnv: PRODUCTION
-  enableAnalytics: true
+  enableAnalytics: "1"
  sentryDSN: ""
-  posthogToken: ""
+  posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS"
  logLevel: info
  selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
  multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
--- a/hosting/single/Dockerfile
+++ b/hosting/single/Dockerfile
@ -1,5 +1,7 @@
 FROM couchdb

+ENV DEPLOYMENT_ENVIRONMENT=docker
+ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
 ENV COUCHDB_PASSWORD=budibase
 ENV COUCHDB_USER=budibase
 ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "1.0.200",
+  "version": "1.0.204",
  "npmClient": "yarn",
  "packages": [
    "packages/*"
--- a/package.json
+++ b/package.json
@ -22,8 +22,9 @@
  },
  "scripts": {
    "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "lerna link && lerna bootstrap && ./scripts/link-dependencies.sh",
+    "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
    "build": "lerna run build",
+    "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
    "release": "lerna publish patch --yes --force-publish && yarn release:pro",
    "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop",
    "release:pro": "bash scripts/pro/release.sh",
@ -37,8 +38,8 @@
    "kill-server": "kill-port 4001 4002",
    "kill-all": "yarn run kill-builder && yarn run kill-server",
    "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
+    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
    "test": "lerna run test",
    "lint:eslint": "eslint packages",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",
--- a/packages/backend-core/.gitignore
+++ b/packages/backend-core/.gitignore
@ -44,9 +44,6 @@ jspm_packages/
 # Snowpack dependency directory (https://snowpack.dev/)
 web_modules/

-# TypeScript cache
-*.tsbuildinfo
-
 # Optional npm cache directory
 .npm

--- a/packages/backend-core/context.js
+++ b/packages/backend-core/context.js
@ -5,8 +5,11 @@ const {
  getAppId,
  updateAppId,
  doInAppContext,
+  doInTenant,
 } = require("./src/context")

+const identity = require("./src/context/identity")
+
 module.exports = {
  getAppDB,
  getDevAppDB,
@ -14,4 +17,6 @@ module.exports = {
  getAppId,
  updateAppId,
  doInAppContext,
+  doInTenant,
+  identity,
 }
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -1,48 +1,80 @@
 {
  "name": "@budibase/backend-core",
-  "version": "1.0.200",
+  "version": "1.0.204",
  "description": "Budibase backend core libraries used in server and worker",
-  "main": "src/index.js",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "exports": {
+    ".": "./dist/src/index.js",
+    "./tests": "./dist/tests/index.js",
+    "./*": "./dist/*.js"
+  },
  "author": "Budibase",
  "license": "GPL-3.0",
  "scripts": {
+    "prebuild": "rimraf dist/",
+    "prepack": "cp package.json dist",
+    "build": "tsc -p tsconfig.build.json",
+    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
    "test": "jest",
    "test:watch": "jest --watchAll"
  },
  "dependencies": {
-    "@techpass/passport-openidconnect": "^0.3.0",
-    "aws-sdk": "^2.901.0",
-    "bcrypt": "^5.0.1",
-    "dotenv": "^16.0.1",
-    "emitter-listener": "^1.1.2",
-    "ioredis": "^4.27.1",
-    "jsonwebtoken": "^8.5.1",
-    "koa-passport": "^4.1.4",
-    "lodash": "^4.17.21",
-    "lodash.isarguments": "^3.1.0",
-    "node-fetch": "^2.6.1",
-    "passport-google-auth": "^1.0.2",
-    "passport-google-oauth": "^2.0.0",
-    "passport-jwt": "^4.0.0",
-    "passport-local": "^1.0.0",
-    "posthog-node": "^1.3.0",
+    "@techpass/passport-openidconnect": "0.3.2",
+    "aws-sdk": "2.1030.0",
+    "bcrypt": "5.0.1",
+    "dotenv": "16.0.1",
+    "emitter-listener": "1.1.2",
+    "ioredis": "4.28.0",
+    "jsonwebtoken": "8.5.1",
+    "koa-passport": "4.1.4",
+    "lodash": "4.17.21",
+    "lodash.isarguments": "3.1.0",
+    "node-fetch": "2.6.7",
+    "passport-google-auth": "1.0.2",
+    "passport-google-oauth": "2.0.0",
+    "passport-jwt": "4.0.0",
+    "passport-local": "1.0.0",
+    "posthog-node": "1.3.0",
    "pouchdb": "7.3.0",
-    "pouchdb-find": "^7.2.2",
-    "pouchdb-replication-stream": "^1.2.9",
-    "sanitize-s3-objectkey": "^0.0.1",
-    "tar-fs": "^2.1.1",
-    "uuid": "^8.3.2",
-    "zlib": "^1.0.5"
+    "pouchdb-find": "7.2.2",
+    "pouchdb-replication-stream": "1.2.9",
+    "redlock": "4.2.0",
+    "sanitize-s3-objectkey": "0.0.1",
+    "semver": "7.3.7",
+    "tar-fs": "2.1.1",
+    "uuid": "8.3.2",
+    "zlib": "1.0.5"
  },
  "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "moduleNameMapper": {
+      "@budibase/types": "<rootDir>/../types/src"
+    },
    "setupFiles": [
-      "./scripts/jestSetup.js"
+      "./scripts/jestSetup.ts"
    ]
  },
  "devDependencies": {
-    "ioredis-mock": "^5.5.5",
-    "jest": "^26.6.3",
-    "pouchdb-adapter-memory": "^7.2.2"
+    "@budibase/types": "^1.0.204",
+    "@shopify/jest-koa-mocks": "3.1.5",
+    "@types/jest": "27.5.1",
+    "@types/koa": "2.0.52",
+    "@types/node": "14.18.20",
+    "@types/node-fetch": "2.6.1",
+    "@types/redlock": "4.0.3",
+    "@types/semver": "7.3.7",
+    "@types/tar-fs": "2.0.1",
+    "@types/uuid": "8.3.4",
+    "ioredis-mock": "5.8.0",
+    "jest": "27.5.1",
+    "koa": "2.7.0",
+    "nodemon": "2.0.16",
+    "pouchdb-adapter-memory": "7.2.2",
+    "timekeeper": "2.2.0",
+    "ts-jest": "27.1.5",
+    "typescript": "4.7.3"
  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/backend-core/redis.js
+++ b/packages/backend-core/redis.js
@ -1,4 +1,5 @@
 module.exports = {
  Client: require("./src/redis"),
  utils: require("./src/redis/utils"),
+  clients: require("./src/redis/authRedis"),
 }
--- a/packages/backend-core/scripts/jestSetup.js
+++ b/packages/backend-core/scripts/jestSetup.js
@ -1,6 +0,0 @@
-const env = require("../src/environment")
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")
--- a/packages/backend-core/scripts/jestSetup.ts
+++ b/packages/backend-core/scripts/jestSetup.ts
@ -0,0 +1,12 @@
+import env from "../src/environment"
+import { mocks } from "../tests/utilities"
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")
--- a/packages/backend-core/src/cache/generic.js
+++ b/packages/backend-core/src/cache/generic.js
@ -1,9 +1,13 @@
 const redis = require("../redis/authRedis")
-const env = require("../environment")
 const { getTenantId } = require("../context")

 exports.CacheKeys = {
  CHECKLIST: "checklist",
+  INSTALLATION: "installation",
+  ANALYTICS_ENABLED: "analyticsEnabled",
+  UNIQUE_TENANT_ID: "uniqueTenantId",
+  EVENTS: "events",
+  BACKFILL_METADATA: "backfillMetadata",
 }

 exports.TTL = {
@ -17,10 +21,41 @@ function generateTenantKey(key) {
  return `${key}:${tenantId}`
 }

-exports.withCache = async (key, ttl, fetchFn) => {
-  key = generateTenantKey(key)
+exports.keys = async pattern => {
  const client = await redis.getCacheClient()
-  const cachedValue = await client.get(key)
+  return client.keys(pattern)
+}
+
+/**
+ * Read only from the cache.
+ */
+exports.get = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  const value = await client.get(key)
+  return value
+}
+
+/**
+ * Write to the cache.
+ */
+exports.store = async (key, value, ttl, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  await client.store(key, value, ttl)
+}
+
+exports.delete = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  return client.delete(key)
+}
+
+/**
+ * Read from the cache. Write to the cache if not exists.
+ */
+exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+  const cachedValue = await exports.get(key, opts)
  if (cachedValue) {
    return cachedValue
  }
@ -28,9 +63,7 @@ exports.withCache = async (key, ttl, fetchFn) => {
  try {
    const fetchedValue = await fetchFn()

-    if (!env.isTest()) {
-      await client.store(key, fetchedValue, ttl)
-    }
+    await exports.store(key, fetchedValue, ttl, opts)
    return fetchedValue
  } catch (err) {
    console.error("Error fetching before cache - ", err)
--- a/packages/backend-core/src/cloud/accounts.js
+++ b/packages/backend-core/src/cloud/accounts.js
@ -1,39 +0,0 @@
-const API = require("./api")
-const env = require("../environment")
-const { Headers } = require("../constants")
-
-const api = new API(env.ACCOUNT_PORTAL_URL)
-
-exports.getAccount = async email => {
-  const payload = {
-    email,
-  }
-  const response = await api.post(`/api/accounts/search`, {
-    body: payload,
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting account by email ${email}`, json)
-  }
-
-  return json[0]
-}
-
-exports.getStatus = async () => {
-  const response = await api.get(`/api/status`, {
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting status`)
-  }
-
-  return json
-}
--- a/packages/backend-core/src/cloud/accounts.ts
+++ b/packages/backend-core/src/cloud/accounts.ts
@ -0,0 +1,63 @@
+import API from "./api"
+import env from "../environment"
+import { Headers } from "../constants"
+import { CloudAccount } from "@budibase/types"
+
+const api = new API(env.ACCOUNT_PORTAL_URL)
+
+export const getAccount = async (
+  email: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    email,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by email ${email}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getAccountByTenantId = async (
+  tenantId: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    tenantId,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by tenantId ${tenantId}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getStatus = async () => {
+  const response = await api.get(`/api/status`, {
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+  const json = await response.json()
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting status`)
+  }
+
+  return json
+}
--- a/packages/backend-core/src/context/identity.ts
+++ b/packages/backend-core/src/context/identity.ts
@ -0,0 +1,50 @@
+import {
+  IdentityContext,
+  IdentityType,
+  User,
+  UserContext,
+  isCloudAccount,
+  Account,
+  AccountUserContext,
+} from "@budibase/types"
+import * as context from "."
+
+export const getIdentity = (): IdentityContext | undefined => {
+  return context.getIdentity()
+}
+
+export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+  return context.doInIdentityContext(identity, task)
+}
+
+export const doInUserContext = (user: User, task: any) => {
+  const userContext: UserContext = {
+    ...user,
+    _id: user._id as string,
+    type: IdentityType.USER,
+  }
+  return doInIdentityContext(userContext, task)
+}
+
+export const doInAccountContext = (account: Account, task: any) => {
+  const _id = getAccountUserId(account)
+  const tenantId = account.tenantId
+  const accountContext: AccountUserContext = {
+    _id,
+    type: IdentityType.USER,
+    tenantId,
+    account,
+  }
+  return doInIdentityContext(accountContext, task)
+}
+
+export const getAccountUserId = (account: Account) => {
+  let userId: string
+  if (isCloudAccount(account)) {
+    userId = account.budibaseUserId
+  } else {
+    // use account id as user id for self hosting
+    userId = account.accountId
+  }
+  return userId
+}
--- a/packages/backend-core/src/context/index.js
+++ b/packages/backend-core/src/context/index.js
@ -1,5 +1,4 @@
 const env = require("../environment")
-const { Headers } = require("../../constants")
 const { SEPARATOR, DocumentTypes } = require("../db/constants")
 const { DEFAULT_TENANT_ID } = require("../constants")
 const cls = require("./FunctionContext")
@ -16,6 +15,7 @@ const ContextKeys = {
  TENANT_ID: "tenantId",
  GLOBAL_DB: "globalDb",
  APP_ID: "appId",
+  IDENTITY: "identity",
  // whatever the request app DB was
  CURRENT_DB: "currentDb",
  // get the prod app DB from the request
@ -79,10 +79,7 @@ exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
  async function internal(opts = { existing: false }) {
    // set the tenant id
    if (!opts.existing) {
-      cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
-      if (env.USE_COUCH) {
-        exports.setGlobalDB(tenantId)
-      }
+      exports.updateTenantId(tenantId)
    }

    try {
@ -97,6 +94,7 @@ exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
      }
    }
  }
+
  const using = cls.getFromContext(ContextKeys.IN_USE)
  if (
    !forceNew &&
@ -144,6 +142,8 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
    throw new Error("appId is required")
  }

+  const identity = exports.getIdentity()
+
  // the internal function is so that we can re-use an existing
  // context - don't want to close DB on a parent context
  async function internal(opts = { existing: false }) {
@ -153,6 +153,8 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
    }
    // set the app ID
    cls.setOnContext(ContextKeys.APP_ID, appId)
+    // preserve the identity
+    exports.setIdentity(identity)
    try {
      // invoke the task
      return await task()
@ -177,9 +179,63 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
  }
 }

+exports.doInIdentityContext = (identity, task) => {
+  if (!identity) {
+    throw new Error("identity is required")
+  }
+
+  async function internal(opts = { existing: false }) {
+    if (!opts.existing) {
+      cls.setOnContext(ContextKeys.IDENTITY, identity)
+      // set the tenant so that doInTenant will preserve identity
+      if (identity.tenantId) {
+        exports.updateTenantId(identity.tenantId)
+      }
+    }
+
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        exports.setIdentity(null)
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+
+  const existing = cls.getFromContext(ContextKeys.IDENTITY)
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (using && existing && existing._id === identity._id) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal({ existing: false })
+    })
+  }
+}
+
+exports.setIdentity = identity => {
+  cls.setOnContext(ContextKeys.IDENTITY, identity)
+}
+
+exports.getIdentity = () => {
+  try {
+    return cls.getFromContext(ContextKeys.IDENTITY)
+  } catch (e) {
+    // do nothing - identity is not in context
+  }
+}
+
 exports.updateTenantId = tenantId => {
  cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+  if (env.USE_COUCH) {
    exports.setGlobalDB(tenantId)
+  }
 }

 exports.updateAppId = async appId => {
@ -196,45 +252,6 @@ exports.updateAppId = async appId => {
  }
 }

-exports.setTenantId = (
-  ctx,
-  opts = { allowQs: false, allowNoTenant: false }
-) => {
-  let tenantId
-  // exit early if not multi-tenant
-  if (!exports.isMultiTenant()) {
-    cls.setOnContext(ContextKeys.TENANT_ID, exports.DEFAULT_TENANT_ID)
-    return exports.DEFAULT_TENANT_ID
-  }
-
-  const allowQs = opts && opts.allowQs
-  const allowNoTenant = opts && opts.allowNoTenant
-  const header = ctx.request.headers[Headers.TENANT_ID]
-  const user = ctx.user || {}
-  if (allowQs) {
-    const query = ctx.request.query || {}
-    tenantId = query.tenantId
-  }
-  // override query string (if allowed) by user, or header
-  // URL params cannot be used in a middleware, as they are
-  // processed later in the chain
-  tenantId = user.tenantId || header || tenantId
-
-  // Set the tenantId from the subdomain
-  if (!tenantId) {
-    tenantId = ctx.subdomains && ctx.subdomains[0]
-  }
-
-  if (!tenantId && !allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-  // check tenant ID just incase no tenant was allowed
-  if (tenantId) {
-    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
-  }
-  return tenantId
-}
-
 exports.setGlobalDB = tenantId => {
  const dbName = baseGlobalDBName(tenantId)
  const db = dangerousGetDB(dbName)
@ -315,7 +332,7 @@ function getContextDB(key, opts) {
 * Opens the app database based on whatever the request
 * contained, dev or prod.
 */
-exports.getAppDB = opts => {
+exports.getAppDB = (opts = null) => {
  return getContextDB(ContextKeys.CURRENT_DB, opts)
 }

@ -323,7 +340,7 @@ exports.getAppDB = opts => {
 * This specifically gets the prod app ID, if the request
 * contained a development app ID, this will open the prod one.
 */
-exports.getProdAppDB = opts => {
+exports.getProdAppDB = (opts = null) => {
  return getContextDB(ContextKeys.PROD_DB, opts)
 }

@ -331,6 +348,6 @@ exports.getProdAppDB = opts => {
 * This specifically gets the dev app ID, if the request
 * contained a prod app ID, this will open the dev one.
 */
-exports.getDevAppDB = opts => {
+exports.getDevAppDB = (opts = null) => {
  return getContextDB(ContextKeys.DEV_DB, opts)
 }
--- a/packages/backend-core/src/db/Replication.ts
+++ b/packages/backend-core/src/db/Replication.ts
@ -1,12 +1,16 @@
-const { dangerousGetDB, closeDB } = require(".")
+import { dangerousGetDB, closeDB } from "."

 class Replication {
+  source: any
+  target: any
+  replication: any
+
  /**
   *
   * @param {String} source - the DB you want to replicate or rollback to
   * @param {String} target - the DB you want to replicate to, or rollback from
   */
-  constructor({ source, target }) {
+  constructor({ source, target }: any) {
    this.source = dangerousGetDB(source)
    this.target = dangerousGetDB(target)
  }
@ -15,17 +19,17 @@ class Replication {
    return Promise.all([closeDB(this.source), closeDB(this.target)])
  }

-  promisify(operation, opts = {}) {
+  promisify(operation: any, opts = {}) {
    return new Promise(resolve => {
      operation(this.target, opts)
-        .on("denied", function (err) {
+        .on("denied", function (err: any) {
          // a document failed to replicate (e.g. due to permissions)
          throw new Error(`Denied: Document failed to replicate ${err}`)
        })
-        .on("complete", function (info) {
+        .on("complete", function (info: any) {
          return resolve(info)
        })
-        .on("error", function (err) {
+        .on("error", function (err: any) {
          throw new Error(`Replication Error: ${err}`)
        })
    })
@ -64,4 +68,4 @@ class Replication {
  }
 }

-module.exports = Replication
+export default Replication
--- a/packages/backend-core/src/db/constants.js
+++ b/packages/backend-core/src/db/constants.js
@ -31,6 +31,7 @@ exports.StaticDatabases = {
    name: "global-info",
    docs: {
      tenants: "tenants",
+      install: "install",
    },
  },
 }
--- a/packages/backend-core/src/db/index.js
+++ b/packages/backend-core/src/db/index.js
@ -8,8 +8,11 @@ const dbList = new Set()
 const put =
  dbPut =>
  async (doc, options = {}) => {
-    // TODO: add created / updated
-    return await dbPut(doc, options)
+    if (!doc.createdAt) {
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
  }

 const checkInitialised = () => {
@ -54,7 +57,7 @@ exports.closeDB = async db => {
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
-exports.doWithDB = async (dbName, cb, opts) => {
+exports.doWithDB = async (dbName, cb, opts = {}) => {
  const db = exports.dangerousGetDB(dbName, opts)
  // need this to be async so that we can correctly close DB after all
  // async operations have been completed
--- a/packages/backend-core/src/db/tests/index.spec.js
+++ b/packages/backend-core/src/db/tests/index.spec.js
@ -0,0 +1,26 @@
+require("../../../tests/utilities/TestConfiguration")
+const { dangerousGetDB } = require("../")
+
+describe("db", () => { 
+  
+  describe("getDB", () => {
+    it("returns a db", async () => {
+      const db = dangerousGetDB("test")
+      expect(db).toBeDefined()
+      expect(db._adapter).toBe("memory")
+      expect(db.prefix).toBe("_pouch_")
+      expect(db.name).toBe("test")
+    })
+
+    it("uses the custom put function", async () => {
+      const db = dangerousGetDB("test")
+      let doc = { _id: "test" }
+      await db.put(doc)
+      doc = await db.get(doc._id)
+      expect(doc.createdAt).toBe(new Date().toISOString())
+      expect(doc.updatedAt).toBe(new Date().toISOString())
+      await db.destroy()
+    })
+  })
+})
+
--- a/packages/backend-core/src/db/tests/utils.spec.js
+++ b/packages/backend-core/src/db/tests/utils.spec.js
@ -1,4 +1,4 @@
-require("../../tests/utilities/dbConfig");
+require("../../../tests/utilities/TestConfiguration");
 const {
  generateAppID,
  getDevelopmentAppID,
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@ -1,53 +1,34 @@
-const { newid } = require("../hashing")
-const Replication = require("./Replication")
-const { DEFAULT_TENANT_ID, Configs } = require("../constants")
-const env = require("../environment")
-const {
-  StaticDatabases,
-  SEPARATOR,
-  DocumentTypes,
-  APP_PREFIX,
-  APP_DEV,
-} = require("./constants")
-const { getTenantId, getGlobalDBName, getGlobalDB } = require("../tenancy")
-const fetch = require("node-fetch")
-const { doWithDB, allDbs } = require("./index")
-const { getCouchInfo } = require("./pouch")
-const { getAppMetadata } = require("../cache/appMetadata")
-const { checkSlashesInUrl } = require("../helpers")
-const {
-  isDevApp,
-  isProdAppID,
-  isDevAppID,
-  getDevelopmentAppID,
-  getProdAppID,
-} = require("./conversions")
+import { newid } from "../hashing"
+import { DEFAULT_TENANT_ID, Configs } from "../constants"
+import env from "../environment"
+import { SEPARATOR, DocumentTypes } from "./constants"
+import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
+import fetch from "node-fetch"
+import { doWithDB, allDbs } from "./index"
+import { getCouchInfo } from "./pouch"
+import { getAppMetadata } from "../cache/appMetadata"
+import { checkSlashesInUrl } from "../helpers"
+import { isDevApp, isDevAppID } from "./conversions"
+import { APP_PREFIX } from "./constants"
+import * as events from "../events"

 const UNICODE_MAX = "\ufff0"

-exports.ViewNames = {
+export const ViewNames = {
  USER_BY_EMAIL: "by_email",
  BY_API_KEY: "by_api_key",
  USER_BY_BUILDERS: "by_builders",
 }

-exports.StaticDatabases = StaticDatabases
-
-exports.DocumentTypes = DocumentTypes
-exports.APP_PREFIX = APP_PREFIX
-exports.APP_DEV = exports.APP_DEV_PREFIX = APP_DEV
-exports.SEPARATOR = SEPARATOR
-exports.isDevApp = isDevApp
-exports.isProdAppID = isProdAppID
-exports.isDevAppID = isDevAppID
-exports.getDevelopmentAppID = getDevelopmentAppID
-exports.getProdAppID = getProdAppID
+export * from "./constants"
+export * from "./conversions"
+export { default as Replication } from "./Replication"

 /**
 * Generates a new app ID.
 * @returns {string} The new app ID which the app doc can be stored under.
 */
-exports.generateAppID = (tenantId = null) => {
+export const generateAppID = (tenantId = null) => {
  let id = APP_PREFIX
  if (tenantId) {
    id += `${tenantId}${SEPARATOR}`
@ -67,7 +48,11 @@ exports.generateAppID = (tenantId = null) => {
 * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
 * @returns {object} Parameters which can then be used with an allDocs request.
 */
-function getDocParams(docType, docId = null, otherProps = {}) {
+export function getDocParams(
+  docType: any,
+  docId: any = null,
+  otherProps: any = {}
+) {
  if (docId == null) {
    docId = ""
  }
@ -77,20 +62,19 @@ function getDocParams(docType, docId = null, otherProps = {}) {
    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
  }
 }
-exports.getDocParams = getDocParams

 /**
 * Generates a new workspace ID.
 * @returns {string} The new workspace ID which the workspace doc can be stored under.
 */
-exports.generateWorkspaceID = () => {
+export function generateWorkspaceID() {
  return `${DocumentTypes.WORKSPACE}${SEPARATOR}${newid()}`
 }

 /**
 * Gets parameters for retrieving workspaces.
 */
-exports.getWorkspaceParams = (id = "", otherProps = {}) => {
+export function getWorkspaceParams(id = "", otherProps = {}) {
  return {
    ...otherProps,
    startkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}`,
@ -102,14 +86,14 @@ exports.getWorkspaceParams = (id = "", otherProps = {}) => {
 * Generates a new global user ID.
 * @returns {string} The new user ID which the user doc can be stored under.
 */
-exports.generateGlobalUserID = id => {
+export function generateGlobalUserID(id?: any) {
  return `${DocumentTypes.USER}${SEPARATOR}${id || newid()}`
 }

 /**
 * Gets parameters for retrieving users.
 */
-exports.getGlobalUserParams = (globalId, otherProps = {}) => {
+export function getGlobalUserParams(globalId: any, otherProps = {}) {
  if (!globalId) {
    globalId = ""
  }
@ -124,14 +108,18 @@ exports.getGlobalUserParams = (globalId, otherProps = {}) => {
 * Generates a template ID.
 * @param ownerId The owner/user of the template, this could be global or a workspace level.
 */
-exports.generateTemplateID = ownerId => {
+export function generateTemplateID(ownerId: any) {
  return `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
 }

 /**
 * Gets parameters for retrieving templates. Owner ID must be specified, either global or a workspace level.
 */
-exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
+export function getTemplateParams(
+  ownerId: any,
+  templateId: any,
+  otherProps = {}
+) {
  if (!templateId) {
    templateId = ""
  }
@ -152,18 +140,18 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
 * Generates a new role ID.
 * @returns {string} The new role ID which the role doc can be stored under.
 */
-exports.generateRoleID = id => {
+export function generateRoleID(id: any) {
  return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
 }

 /**
 * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
 */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
+export function getRoleParams(roleId = null, otherProps = {}) {
  return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }

-exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
+export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) {
  const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
  return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
@ -174,15 +162,15 @@ exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
 * opts.efficient can be provided to make sure this call is always quick in a multi-tenant environment,
 * but it may not be 100% accurate in full efficiency mode (some tenantless apps may be missed).
 */
-exports.getAllDbs = async (opts = { efficient: false }) => {
+export async function getAllDbs(opts = { efficient: false }) {
  const efficient = opts && opts.efficient
  // specifically for testing we use the pouch package for this
  if (env.isTest()) {
    return allDbs()
  }
-  let dbs = []
+  let dbs: any[] = []
  let { url, cookie } = getCouchInfo()
-  async function addDbs(couchUrl) {
+  async function addDbs(couchUrl: string) {
    const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
      method: "GET",
      headers: {
@ -207,13 +195,9 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
    await addDbs(couchUrl)
  } else {
    // get prod apps
-    await addDbs(
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
-    )
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId))
    // get dev apps
-    await addDbs(
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId)
-    )
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId))
    // add global db name
    dbs.push(getGlobalDBName(tenantId))
  }
@ -226,13 +210,13 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
 *
 * @return {Promise<object[]>} returns the app information document stored in each app database.
 */
-exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
+export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
  let tenantId = getTenantId()
  if (!env.MULTI_TENANCY && !tenantId) {
    tenantId = DEFAULT_TENANT_ID
  }
-  let dbs = await exports.getAllDbs({ efficient })
-  const appDbNames = dbs.filter(dbName => {
+  let dbs = await getAllDbs({ efficient })
+  const appDbNames = dbs.filter((dbName: any) => {
    const split = dbName.split(SEPARATOR)
    // it is an app, check the tenantId
    if (split[0] === DocumentTypes.APP) {
@ -252,7 +236,7 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
  if (idsOnly) {
    return appDbNames
  }
-  const appPromises = appDbNames.map(app =>
+  const appPromises = appDbNames.map((app: any) =>
    // skip setup otherwise databases could be re-created
    getAppMetadata(app)
  )
@ -261,17 +245,19 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
  } else {
    const response = await Promise.allSettled(appPromises)
    const apps = response
-      .filter(result => result.status === "fulfilled" && result.value != null)
-      .map(({ value }) => value)
+      .filter(
+        (result: any) => result.status === "fulfilled" && result.value != null
+      )
+      .map(({ value }: any) => value)
    if (!all) {
-      return apps.filter(app => {
+      return apps.filter((app: any) => {
        if (dev) {
          return isDevApp(app)
        }
        return !isDevApp(app)
      })
    } else {
-      return apps.map(app => ({
+      return apps.map((app: any) => ({
        ...app,
        status: isDevApp(app) ? "development" : "published",
      }))
@ -282,26 +268,26 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
 /**
 * Utility function for getAllApps but filters to production apps only.
 */
-exports.getProdAppIDs = async () => {
-  return (await exports.getAllApps({ idsOnly: true })).filter(
-    id => !exports.isDevAppID(id)
+export async function getProdAppIDs() {
+  return (await getAllApps({ idsOnly: true })).filter(
+    (id: any) => !isDevAppID(id)
  )
 }

 /**
 * Utility function for the inverse of above.
 */
-exports.getDevAppIDs = async () => {
-  return (await exports.getAllApps({ idsOnly: true })).filter(id =>
-    exports.isDevAppID(id)
+export async function getDevAppIDs() {
+  return (await getAllApps({ idsOnly: true })).filter((id: any) =>
+    isDevAppID(id)
  )
 }

-exports.dbExists = async dbName => {
+export async function dbExists(dbName: any) {
  let exists = false
  return doWithDB(
    dbName,
-    async db => {
+    async (db: any) => {
      try {
        // check if database exists
        const info = await db.info()
@ -321,7 +307,7 @@ exports.dbExists = async dbName => {
 * Generates a new configuration ID.
 * @returns {string} The new configuration ID which the config doc can be stored under.
 */
-const generateConfigID = ({ type, workspace, user }) => {
+export const generateConfigID = ({ type, workspace, user }: any) => {
  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)

  return `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`
@ -330,7 +316,10 @@ const generateConfigID = ({ type, workspace, user }) => {
 /**
 * Gets parameters for retrieving configurations.
 */
-const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
+export const getConfigParams = (
+  { type, workspace, user }: any,
+  otherProps = {}
+) => {
  const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)

  return {
@ -344,7 +333,7 @@ const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
 * Generates a new dev info document ID - this is scoped to a user.
 * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
 */
-const generateDevInfoID = userId => {
+export const generateDevInfoID = (userId: any) => {
  return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}`
 }

@ -354,7 +343,10 @@ const generateDevInfoID = userId => {
 * @param {Object} scopes - the type, workspace and userID scopes of the configuration.
 * @returns The most granular configuration document based on the scope.
 */
-const getScopedFullConfig = async function (db, { type, user, workspace }) {
+export const getScopedFullConfig = async function (
+  db: any,
+  { type, user, workspace }: any
+) {
  const response = await db.allDocs(
    getConfigParams(
      { type, user, workspace },
@ -364,7 +356,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
    )
  )

-  function determineScore(row) {
+  function determineScore(row: any) {
    const config = row.doc

    // Config is specific to a user and a workspace
@ -385,19 +377,24 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {

  // Find the config with the most granular scope based on context
  let scopedConfig = response.rows.sort(
-    (a, b) => determineScore(a) - determineScore(b)
+    (a: any, b: any) => determineScore(a) - determineScore(b)
  )[0]

  // custom logic for settings doc
-  // always provide the platform URL
  if (type === Configs.SETTINGS) {
    if (scopedConfig && scopedConfig.doc) {
+      // overrides affected by environment variables
      scopedConfig.doc.config.platformUrl = await getPlatformUrl()
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
    } else {
+      // defaults
      scopedConfig = {
        doc: {
+          _id: generateConfigID({ type, user, workspace }),
          config: {
            platformUrl: await getPlatformUrl(),
+            analyticsEnabled: await events.analytics.enabled(),
          },
        },
      }
@ -407,7 +404,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
  return scopedConfig && scopedConfig.doc
 }

-const getPlatformUrl = async (opts = { tenantAware: true }) => {
+export const getPlatformUrl = async (opts = { tenantAware: true }) => {
  let platformUrl = env.PLATFORM_URL || "http://localhost:10000"

  if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
@ -422,7 +419,7 @@ const getPlatformUrl = async (opts = { tenantAware: true }) => {
    let settings
    try {
      settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
-    } catch (e) {
+    } catch (e: any) {
      if (e.status !== 404) {
        throw e
      }
@ -437,15 +434,7 @@ const getPlatformUrl = async (opts = { tenantAware: true }) => {
  return platformUrl
 }

-async function getScopedConfig(db, params) {
+export async function getScopedConfig(db: any, params: any) {
  const configDoc = await getScopedFullConfig(db, params)
  return configDoc && configDoc.config ? configDoc.config : configDoc
 }
-
-exports.Replication = Replication
-exports.getScopedConfig = getScopedConfig
-exports.generateConfigID = generateConfigID
-exports.getConfigParams = getConfigParams
-exports.getScopedFullConfig = getScopedFullConfig
-exports.generateDevInfoID = generateDevInfoID
-exports.getPlatformUrl = getPlatformUrl
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@ -16,7 +16,7 @@ if (!LOADED && isDev() && !isTest()) {
  LOADED = true
 }

-module.exports = {
+const env: any = {
  isTest,
  isDev,
  JWT_SECRET: process.env.JWT_SECRET,
@ -38,9 +38,11 @@ module.exports = {
    process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
  ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
  DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
  COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
  PLATFORM_URL: process.env.PLATFORM_URL,
+  POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
  TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
  BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
  APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
@ -51,16 +53,21 @@ module.exports = {
  USE_COUCH: process.env.USE_COUCH || true,
  DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
  DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
-  _set(key, value) {
+  SERVICE: process.env.SERVICE || "budibase",
+  DEPLOYMENT_ENVIRONMENT:
+    process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  _set(key: any, value: any) {
    process.env[key] = value
    module.exports[key] = value
  },
 }

 // clean up any environment variable edge cases
-for (let [key, value] of Object.entries(module.exports)) {
+for (let [key, value] of Object.entries(env)) {
  // handle the edge case of "0" to disable an environment variable
  if (value === "0") {
-    module.exports[key] = 0
+    env[key] = 0
  }
 }
+
+export = env
--- a/packages/backend-core/src/errors/base.js
+++ b/packages/backend-core/src/errors/base.js
@ -1,8 +1,8 @@
 class BudibaseError extends Error {
-  constructor(message, type, code) {
+  constructor(message, code, type) {
    super(message)
-    this.type = type
    this.code = code
+    this.type = type
  }
 }

--- a/packages/backend-core/src/errors/generic.js
+++ b/packages/backend-core/src/errors/generic.js
@ -0,0 +1,11 @@
+const { BudibaseError } = require("./base")
+
+class GenericError extends BudibaseError {
+  constructor(message, code, type) {
+    super(message, code, type ? type : "generic")
+  }
+}
+
+module.exports = {
+  GenericError,
+}
--- a/packages/backend-core/src/errors/http.js
+++ b/packages/backend-core/src/errors/http.js
@ -0,0 +1,12 @@
+const { GenericError } = require("./generic")
+
+class HTTPError extends GenericError {
+  constructor(message, httpStatus, code = "http", type = "generic") {
+    super(message, code, type)
+    this.status = httpStatus
+  }
+}
+
+module.exports = {
+  HTTPError,
+}
--- a/packages/backend-core/src/errors/index.js
+++ b/packages/backend-core/src/errors/index.js
@ -1,12 +1,11 @@
+const http = require("./http")
 const licensing = require("./licensing")

 const codes = {
  ...licensing.codes,
 }

-const types = {
-  ...licensing.types,
-}
+const types = [licensing.type]

 const context = {
  ...licensing.context,
@ -36,6 +35,9 @@ const getPublicError = err => {
 module.exports = {
  codes,
  types,
+  errors: {
    UsageLimitError: licensing.UsageLimitError,
+    HTTPError: http.HTTPError,
+  },
  getPublicError,
 }
--- a/packages/backend-core/src/errors/licensing.js
+++ b/packages/backend-core/src/errors/licensing.js
@ -1,8 +1,6 @@
-const { BudibaseError } = require("./base")
+const { HTTPError } = require("./http")

-const types = {
-  LICENSE_ERROR: "license_error",
-}
+const type = "license_error"

 const codes = {
  USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded",
@ -16,16 +14,15 @@ const context = {
  },
 }

-class UsageLimitError extends BudibaseError {
+class UsageLimitError extends HTTPError {
  constructor(message, limitName) {
-    super(message, types.LICENSE_ERROR, codes.USAGE_LIMIT_EXCEEDED)
+    super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type)
    this.limitName = limitName
-    this.status = 400
  }
 }

 module.exports = {
-  types,
+  type,
  codes,
  context,
  UsageLimitError,
--- a/packages/backend-core/src/events/analytics.ts
+++ b/packages/backend-core/src/events/analytics.ts
@ -0,0 +1,57 @@
+import env from "../environment"
+import tenancy from "../tenancy"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+export const enabled = async () => {
+  // cloud - always use the environment variable
+  if (!env.SELF_HOSTED) {
+    return !!env.ENABLE_ANALYTICS
+  }
+
+  // self host - prefer the settings doc
+  // use cache as events have high throughput
+  const enabledInDB = await withCache(
+    CacheKeys.ANALYTICS_ENABLED,
+    TTL.ONE_DAY,
+    async () => {
+      const settings = await getSettingsDoc()
+
+      // need to do explicit checks in case the field is not set
+      if (settings?.config?.analyticsEnabled === false) {
+        return false
+      } else if (settings?.config?.analyticsEnabled === true) {
+        return true
+      }
+    }
+  )
+
+  if (enabledInDB !== undefined) {
+    return enabledInDB
+  }
+
+  // fallback to the environment variable
+  // explicitly check for 0 or false here, undefined or otherwise is treated as true
+  const envEnabled: any = env.ENABLE_ANALYTICS
+  if (envEnabled === 0 || envEnabled === false) {
+    return false
+  } else {
+    return true
+  }
+}
+
+const getSettingsDoc = async () => {
+  const db = tenancy.getGlobalDB()
+  let settings
+  try {
+    settings = await db.get(
+      dbUtils.generateConfigID({ type: Configs.SETTINGS })
+    )
+  } catch (e: any) {
+    if (e.status !== 404) {
+      throw e
+    }
+  }
+  return settings
+}
--- a/packages/backend-core/src/events/backfill.ts
+++ b/packages/backend-core/src/events/backfill.ts
@ -0,0 +1,183 @@
+import {
+  Event,
+  BackfillMetadata,
+  CachedEvent,
+  SSOCreatedEvent,
+  AutomationCreatedEvent,
+  AutomationStepCreatedEvent,
+  DatasourceCreatedEvent,
+  LayoutCreatedEvent,
+  QueryCreatedEvent,
+  RoleCreatedEvent,
+  ScreenCreatedEvent,
+  TableCreatedEvent,
+  ViewCreatedEvent,
+  ViewCalculationCreatedEvent,
+  ViewFilterCreatedEvent,
+  AppPublishedEvent,
+  UserCreatedEvent,
+  RoleAssignedEvent,
+  UserPermissionAssignedEvent,
+  AppCreatedEvent,
+} from "@budibase/types"
+import * as context from "../context"
+import { CacheKeys } from "../cache/generic"
+import * as cache from "../cache/generic"
+
+// LIFECYCLE
+
+export const start = async (events: Event[]) => {
+  const metadata: BackfillMetadata = {
+    eventWhitelist: events,
+  }
+  return saveBackfillMetadata(metadata)
+}
+
+export const recordEvent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  // don't use a ttl - cleaned up by migration
+  // don't use tenancy - already in the key
+  await cache.store(eventKey, properties, undefined, { useTenancy: false })
+}
+
+export const end = async () => {
+  await deleteBackfillMetadata()
+  await clearEvents()
+}
+
+// CRUD
+
+const getBackfillMetadata = async (): Promise<BackfillMetadata | null> => {
+  return cache.get(CacheKeys.BACKFILL_METADATA)
+}
+
+const saveBackfillMetadata = async (
+  backfill: BackfillMetadata
+): Promise<void> => {
+  // no TTL - deleted by backfill
+  return cache.store(CacheKeys.BACKFILL_METADATA, backfill)
+}
+
+const deleteBackfillMetadata = async (): Promise<void> => {
+  await cache.delete(CacheKeys.BACKFILL_METADATA)
+}
+
+const clearEvents = async () => {
+  // wildcard
+  const pattern = getEventKey()
+  const keys = await cache.keys(pattern)
+
+  for (const key of keys) {
+    // delete each key
+    // don't  use tenancy, already in the key
+    await cache.delete(key, { useTenancy: false })
+  }
+}
+
+// HELPERS
+
+export const isBackfillingEvent = async (event: Event) => {
+  const backfill = await getBackfillMetadata()
+  const events = backfill?.eventWhitelist
+  if (events && events.includes(event)) {
+    return true
+  } else {
+    return false
+  }
+}
+
+export const isAlreadySent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  const cachedEvent: CachedEvent = await cache.get(eventKey, {
+    useTenancy: false,
+  })
+  return !!cachedEvent
+}
+
+const CUSTOM_PROPERTY_SUFFIX: any = {
+  // APP EVENTS
+  [Event.AUTOMATION_CREATED]: (properties: AutomationCreatedEvent) => {
+    return properties.automationId
+  },
+  [Event.AUTOMATION_STEP_CREATED]: (properties: AutomationStepCreatedEvent) => {
+    return properties.stepId
+  },
+  [Event.DATASOURCE_CREATED]: (properties: DatasourceCreatedEvent) => {
+    return properties.datasourceId
+  },
+  [Event.LAYOUT_CREATED]: (properties: LayoutCreatedEvent) => {
+    return properties.layoutId
+  },
+  [Event.QUERY_CREATED]: (properties: QueryCreatedEvent) => {
+    return properties.queryId
+  },
+  [Event.ROLE_CREATED]: (properties: RoleCreatedEvent) => {
+    return properties.roleId
+  },
+  [Event.SCREEN_CREATED]: (properties: ScreenCreatedEvent) => {
+    return properties.screenId
+  },
+  [Event.TABLE_CREATED]: (properties: TableCreatedEvent) => {
+    return properties.tableId
+  },
+  [Event.VIEW_CREATED]: (properties: ViewCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_CALCULATION_CREATED]: (
+    properties: ViewCalculationCreatedEvent
+  ) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_FILTER_CREATED]: (properties: ViewFilterCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.APP_CREATED]: (properties: AppCreatedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  // GLOBAL EVENTS
+  [Event.AUTH_SSO_CREATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.AUTH_SSO_ACTIVATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.USER_CREATED]: (properties: UserCreatedEvent) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.ROLE_ASSIGNED]: (properties: RoleAssignedEvent) => {
+    return `${properties.roleId}-${properties.userId}`
+  },
+}
+
+const getEventKey = (event?: Event, properties?: any) => {
+  let eventKey: string
+
+  const tenantId = context.getTenantId()
+  if (event) {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}`
+
+    // use some properties to make the key more unique
+    const custom = CUSTOM_PROPERTY_SUFFIX[event]
+    const suffix = custom ? custom(properties) : undefined
+    if (suffix) {
+      eventKey = `${eventKey}:${suffix}`
+    }
+  } else {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:*`
+  }
+
+  return eventKey
+}
--- a/packages/backend-core/src/events/events.ts
+++ b/packages/backend-core/src/events/events.ts
@ -0,0 +1,31 @@
+import { Event } from "@budibase/types"
+import { processors } from "./processors"
+import * as identification from "./identification"
+import * as backfill from "./backfill"
+
+export const publishEvent = async (
+  event: Event,
+  properties: any,
+  timestamp?: string | number
+) => {
+  // in future this should use async events via a distributed queue.
+  const identity = await identification.getCurrentIdentity()
+
+  const backfilling = await backfill.isBackfillingEvent(event)
+  // no backfill - send the event and exit
+  if (!backfilling) {
+    await processors.processEvent(event, identity, properties, timestamp)
+    return
+  }
+
+  // backfill active - check if the event has been sent already
+  const alreadySent = await backfill.isAlreadySent(event, properties)
+  if (alreadySent) {
+    // do nothing
+    return
+  } else {
+    // send and record the event
+    await processors.processEvent(event, identity, properties, timestamp)
+    await backfill.recordEvent(event, properties)
+  }
+}
--- a/packages/backend-core/src/events/identification.ts
+++ b/packages/backend-core/src/events/identification.ts
@ -0,0 +1,302 @@
+import * as context from "../context"
+import * as identityCtx from "../context/identity"
+import env from "../environment"
+import {
+  Hosting,
+  User,
+  Identity,
+  IdentityType,
+  Account,
+  isCloudAccount,
+  isSSOAccount,
+  TenantGroup,
+  SettingsConfig,
+  CloudAccount,
+  UserIdentity,
+  InstallationGroup,
+  UserContext,
+  Group,
+} from "@budibase/types"
+import { processors } from "./processors"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import * as hashing from "../hashing"
+import * as installation from "../installation"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+const pkg = require("../../package.json")
+
+/**
+ * An identity can be:
+ * - account user (Self host)
+ * - budibase user
+ * - tenant
+ * - installation
+ */
+export const getCurrentIdentity = async (): Promise<Identity> => {
+  let identityContext = identityCtx.getIdentity()
+  const environment = getDeploymentEnvironment()
+
+  let identityType
+
+  if (!identityContext) {
+    identityType = IdentityType.TENANT
+  } else {
+    identityType = identityContext.type
+  }
+
+  if (identityType === IdentityType.INSTALLATION) {
+    const installationId = await getInstallationId()
+    const hosting = getHostingFromEnv()
+    return {
+      id: formatDistinctId(installationId, identityType),
+      hosting,
+      type: identityType,
+      installationId,
+      environment,
+    }
+  } else if (identityType === IdentityType.TENANT) {
+    const installationId = await getInstallationId()
+    const tenantId = await getEventTenantId(context.getTenantId())
+    const hosting = getHostingFromEnv()
+
+    return {
+      id: formatDistinctId(tenantId, identityType),
+      type: identityType,
+      hosting,
+      installationId,
+      tenantId,
+      environment,
+    }
+  } else if (identityType === IdentityType.USER) {
+    const userContext = identityContext as UserContext
+    const tenantId = await getEventTenantId(context.getTenantId())
+    const installationId = await getInstallationId()
+
+    const account = userContext.account
+    let hosting
+    if (account) {
+      hosting = account.hosting
+    } else {
+      hosting = getHostingFromEnv()
+    }
+
+    return {
+      id: userContext._id,
+      type: identityType,
+      hosting,
+      installationId,
+      tenantId,
+      environment,
+    }
+  } else {
+    throw new Error("Unknown identity type")
+  }
+}
+
+export const identifyInstallationGroup = async (
+  installId: string,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = installId
+  const type = IdentityType.INSTALLATION
+  const hosting = getHostingFromEnv()
+  const version = pkg.version
+  const environment = getDeploymentEnvironment()
+
+  const group: InstallationGroup = {
+    id,
+    type,
+    hosting,
+    version,
+    environment,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the empty auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyTenantGroup = async (
+  tenantId: string,
+  account: Account | undefined,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = await getEventTenantId(tenantId)
+  const type = IdentityType.TENANT
+  const installationId = await getInstallationId()
+  const environment = getDeploymentEnvironment()
+
+  let hosting: Hosting
+  let profession: string | undefined
+  let companySize: string | undefined
+
+  if (account) {
+    profession = account.profession
+    companySize = account.size
+    hosting = account.hosting
+  } else {
+    hosting = getHostingFromEnv()
+  }
+
+  const group: TenantGroup = {
+    id,
+    type,
+    hosting,
+    environment,
+    installationId,
+    profession,
+    companySize,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyUser = async (
+  user: User,
+  account: CloudAccount | undefined,
+  timestamp?: string | number
+) => {
+  const id = user._id as string
+  const tenantId = await getEventTenantId(user.tenantId)
+  const type = IdentityType.USER
+  let builder = user.builder?.global || false
+  let admin = user.admin?.global || false
+  let providerType = user.providerType
+  const accountHolder = account?.budibaseUserId === user._id || false
+  const verified =
+    account && account?.budibaseUserId === user._id ? account.verified : false
+  const installationId = await getInstallationId()
+  const hosting = account ? account.hosting : getHostingFromEnv()
+  const environment = getDeploymentEnvironment()
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    hosting,
+    installationId,
+    tenantId,
+    verified,
+    accountHolder,
+    providerType,
+    builder,
+    admin,
+    environment,
+  }
+
+  await identify(identity, timestamp)
+}
+
+export const identifyAccount = async (account: Account) => {
+  let id = account.accountId
+  const tenantId = account.tenantId
+  let type = IdentityType.USER
+  let providerType = isSSOAccount(account) ? account.providerType : undefined
+  const verified = account.verified
+  const accountHolder = true
+  const hosting = account.hosting
+  const installationId = await getInstallationId()
+  const environment = getDeploymentEnvironment()
+
+  if (isCloudAccount(account)) {
+    if (account.budibaseUserId) {
+      // use the budibase user as the id if set
+      id = account.budibaseUserId
+    }
+  }
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    hosting,
+    installationId,
+    tenantId,
+    providerType,
+    verified,
+    accountHolder,
+    environment,
+  }
+
+  await identify(identity)
+}
+
+export const identify = async (
+  identity: Identity,
+  timestamp?: string | number
+) => {
+  await processors.identify(identity, timestamp)
+}
+
+export const identifyGroup = async (
+  group: Group,
+  timestamp?: string | number
+) => {
+  await processors.identifyGroup(group, timestamp)
+}
+
+const getDeploymentEnvironment = () => {
+  if (env.isDev()) {
+    return "development"
+  } else {
+    return env.DEPLOYMENT_ENVIRONMENT
+  }
+}
+
+const getHostingFromEnv = () => {
+  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
+}
+
+export const getInstallationId = async () => {
+  if (isAccountPortal()) {
+    return "account-portal"
+  }
+  const install = await installation.getInstall()
+  return install.installId
+}
+
+const getEventTenantId = async (tenantId: string): Promise<string> => {
+  if (env.SELF_HOSTED) {
+    return getUniqueTenantId(tenantId)
+  } else {
+    // tenant id's in the cloud are already unique
+    return tenantId
+  }
+}
+
+const getUniqueTenantId = async (tenantId: string): Promise<string> => {
+  // make sure this tenantId always matches the tenantId in context
+  return context.doInTenant(tenantId, () => {
+    return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
+      const db = context.getGlobalDB()
+      const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
+        type: Configs.SETTINGS,
+      })
+
+      let uniqueTenantId: string
+      if (config.config.uniqueTenantId) {
+        return config.config.uniqueTenantId
+      } else {
+        uniqueTenantId = `${hashing.newid()}_${tenantId}`
+        config.config.uniqueTenantId = uniqueTenantId
+        await db.put(config)
+        return uniqueTenantId
+      }
+    })
+  })
+}
+
+const isAccountPortal = () => {
+  return env.SERVICE === "account-portal"
+}
+
+const formatDistinctId = (id: string, type: IdentityType) => {
+  if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) {
+    return `$${type}_${id}`
+  } else {
+    return id
+  }
+}
--- a/packages/backend-core/src/events/index.ts
+++ b/packages/backend-core/src/events/index.ts
@ -0,0 +1,11 @@
+export * from "./publishers"
+export * as processors from "./processors"
+export * as analytics from "./analytics"
+export * as identification from "./identification"
+export * as backfillCache from "./backfill"
+
+import { processors } from "./processors"
+
+export const shutdown = () => {
+  processors.shutdown()
+}
--- a/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
+++ b/packages/backend-core/src/events/processors/AnalyticsProcessor.ts
@ -0,0 +1,64 @@
+import { Event, Identity, Group, IdentityType } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import * as analytics from "../analytics"
+import PosthogProcessor from "./PosthogProcessor"
+
+/**
+ * Events that are always captured.
+ */
+const EVENT_WHITELIST = [
+  Event.INSTALLATION_VERSION_UPGRADED,
+  Event.INSTALLATION_VERSION_DOWNGRADED,
+]
+const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT]
+
+export default class AnalyticsProcessor implements EventProcessor {
+  posthog: PosthogProcessor | undefined
+
+  constructor() {
+    if (env.POSTHOG_TOKEN && !env.isTest()) {
+      this.posthog = new PosthogProcessor(env.POSTHOG_TOKEN)
+    }
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (
+      !IDENTITY_WHITELIST.includes(identity.type) &&
+      !(await analytics.enabled())
+    ) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (this.posthog) {
+      this.posthog.identifyGroup(group, timestamp)
+    }
+  }
+
+  shutdown() {
+    if (this.posthog) {
+      this.posthog.shutdown()
+    }
+  }
+}
--- a/packages/backend-core/src/events/processors/LoggingProcessor.ts
+++ b/packages/backend-core/src/events/processors/LoggingProcessor.ts
@ -0,0 +1,54 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+
+const getTimestampString = (timestamp?: string | number) => {
+  let timestampString = ""
+  if (timestamp) {
+    timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+  }
+  return timestampString
+}
+
+const skipLogging = env.SELF_HOSTED && !env.isDev()
+
+export default class LoggingProcessor implements EventProcessor {
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+    )
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(identity)}] ${timestampString} identified`
+    )
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(group)}] ${timestampString} group identified`
+    )
+  }
+
+  shutdown(): void {
+    // no-op
+  }
+}
--- a/packages/backend-core/src/events/processors/PosthogProcessor.ts
+++ b/packages/backend-core/src/events/processors/PosthogProcessor.ts
@ -0,0 +1,81 @@
+import PostHog from "posthog-node"
+import { Event, Identity, Group, BaseEvent } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import context from "../../context"
+const pkg = require("../../../package.json")
+
+export default class PosthogProcessor implements EventProcessor {
+  posthog: PostHog
+
+  constructor(token: string | undefined) {
+    if (!token) {
+      throw new Error("Posthog token is not defined")
+    }
+    this.posthog = new PostHog(token)
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: BaseEvent,
+    timestamp?: string | number
+  ): Promise<void> {
+    properties.version = pkg.version
+    properties.service = env.SERVICE
+    properties.environment = identity.environment
+    properties.hosting = identity.hosting
+
+    const appId = context.getAppId()
+    if (appId) {
+      properties.appId = appId
+    }
+
+    const payload: any = { distinctId: identity.id, event, properties }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+
+    // add groups to the event
+    if (identity.installationId || identity.tenantId) {
+      payload.groups = {}
+      if (identity.installationId) {
+        payload.groups.installation = identity.installationId
+        payload.properties.installationId = identity.installationId
+      }
+      if (identity.tenantId) {
+        payload.groups.tenant = identity.tenantId
+        payload.properties.tenantId = identity.tenantId
+      }
+    }
+
+    this.posthog.capture(payload)
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    const payload: any = { distinctId: identity.id, properties: identity }
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.identify(payload)
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    const payload: any = {
+      distinctId: group.id,
+      groupType: group.type,
+      groupKey: group.id,
+      properties: group,
+    }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.groupIdentify(payload)
+  }
+
+  shutdown() {
+    this.posthog.shutdown()
+  }
+}
--- a/packages/backend-core/src/events/processors/Processors.ts
+++ b/packages/backend-core/src/events/processors/Processors.ts
@ -0,0 +1,46 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+
+export default class Processor implements EventProcessor {
+  initialised: boolean = false
+  processors: EventProcessor[] = []
+
+  constructor(processors: EventProcessor[]) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(
+    identity: Identity,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(
+    identity: Group,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identifyGroup(identity, timestamp)
+    }
+  }
+
+  shutdown() {
+    for (const eventProcessor of this.processors) {
+      eventProcessor.shutdown()
+    }
+  }
+}
--- a/packages/backend-core/src/events/processors/index.ts
+++ b/packages/backend-core/src/events/processors/index.ts
@ -0,0 +1,8 @@
+import AnalyticsProcessor from "./AnalyticsProcessor"
+import LoggingProcessor from "./LoggingProcessor"
+import Processors from "./Processors"
+
+export const analyticsProcessor = new AnalyticsProcessor()
+const loggingProcessor = new LoggingProcessor()
+
+export const processors = new Processors([analyticsProcessor, loggingProcessor])
--- a/packages/backend-core/src/events/processors/types.ts
+++ b/packages/backend-core/src/events/processors/types.ts
@ -0,0 +1,18 @@
+import { Event, Identity, Group } from "@budibase/types"
+
+export enum EventProcessorType {
+  POSTHOG = "posthog",
+  LOGGING = "logging",
+}
+
+export interface EventProcessor {
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
+  identify(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
+  shutdown(): void
+}
--- a/packages/backend-core/src/events/publishers/account.ts
+++ b/packages/backend-core/src/events/publishers/account.ts
@ -0,0 +1,29 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Account,
+  AccountCreatedEvent,
+  AccountDeletedEvent,
+  AccountVerifiedEvent,
+} from "@budibase/types"
+
+export async function created(account: Account) {
+  const properties: AccountCreatedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_CREATED, properties)
+}
+
+export async function deleted(account: Account) {
+  const properties: AccountDeletedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_DELETED, properties)
+}
+
+export async function verified(account: Account) {
+  const properties: AccountVerifiedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_VERIFIED, properties)
+}
--- a/packages/backend-core/src/events/publishers/app.ts
+++ b/packages/backend-core/src/events/publishers/app.ts
@ -0,0 +1,108 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  App,
+  AppCreatedEvent,
+  AppUpdatedEvent,
+  AppDeletedEvent,
+  AppPublishedEvent,
+  AppUnpublishedEvent,
+  AppFileImportedEvent,
+  AppTemplateImportedEvent,
+  AppVersionUpdatedEvent,
+  AppVersionRevertedEvent,
+  AppRevertedEvent,
+  AppExportedEvent,
+} from "@budibase/types"
+
+export const created = async (app: App, timestamp?: string | number) => {
+  const properties: AppCreatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_CREATED, properties, timestamp)
+}
+
+export async function updated(app: App) {
+  const properties: AppUpdatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_UPDATED, properties)
+}
+
+export async function deleted(app: App) {
+  const properties: AppDeletedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_DELETED, properties)
+}
+
+export async function published(app: App, timestamp?: string | number) {
+  const properties: AppPublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
+}
+
+export async function unpublished(app: App) {
+  const properties: AppUnpublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_UNPUBLISHED, properties)
+}
+
+export async function fileImported(app: App) {
+  const properties: AppFileImportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_FILE_IMPORTED, properties)
+}
+
+export async function templateImported(app: App, templateKey: string) {
+  const properties: AppTemplateImportedEvent = {
+    appId: app.appId,
+    templateKey,
+  }
+  await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
+}
+
+export async function versionUpdated(
+  app: App,
+  currentVersion: string,
+  updatedToVersion: string
+) {
+  const properties: AppVersionUpdatedEvent = {
+    appId: app.appId,
+    currentVersion,
+    updatedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_UPDATED, properties)
+}
+
+export async function versionReverted(
+  app: App,
+  currentVersion: string,
+  revertedToVersion: string
+) {
+  const properties: AppVersionRevertedEvent = {
+    appId: app.appId,
+    currentVersion,
+    revertedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_REVERTED, properties)
+}
+
+export async function reverted(app: App) {
+  const properties: AppRevertedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_REVERTED, properties)
+}
+
+export async function exported(app: App) {
+  const properties: AppExportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_EXPORTED, properties)
+}
--- a/packages/backend-core/src/events/publishers/auth.ts
+++ b/packages/backend-core/src/events/publishers/auth.ts
@ -0,0 +1,58 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  LoginEvent,
+  LoginSource,
+  LogoutEvent,
+  SSOActivatedEvent,
+  SSOCreatedEvent,
+  SSODeactivatedEvent,
+  SSOType,
+  SSOUpdatedEvent,
+} from "@budibase/types"
+import { identification } from ".."
+
+export async function login(source: LoginSource) {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LoginEvent = {
+    userId: identity.id,
+    source,
+  }
+  await publishEvent(Event.AUTH_LOGIN, properties)
+}
+
+export async function logout() {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LogoutEvent = {
+    userId: identity.id,
+  }
+  await publishEvent(Event.AUTH_LOGOUT, properties)
+}
+
+export async function SSOCreated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOCreatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp)
+}
+
+export async function SSOUpdated(type: SSOType) {
+  const properties: SSOUpdatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_UPDATED, properties)
+}
+
+export async function SSOActivated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOActivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp)
+}
+
+export async function SSODeactivated(type: SSOType) {
+  const properties: SSODeactivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_DEACTIVATED, properties)
+}
--- a/packages/backend-core/src/events/publishers/automation.ts
+++ b/packages/backend-core/src/events/publishers/automation.ts
@ -0,0 +1,94 @@
+import { publishEvent } from "../events"
+import {
+  Automation,
+  Event,
+  AutomationStep,
+  AutomationCreatedEvent,
+  AutomationDeletedEvent,
+  AutomationTestedEvent,
+  AutomationStepCreatedEvent,
+  AutomationStepDeletedEvent,
+  AutomationTriggerUpdatedEvent,
+  AutomationsRunEvent,
+} from "@budibase/types"
+
+export async function created(
+  automation: Automation,
+  timestamp?: string | number
+) {
+  const properties: AutomationCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
+}
+
+export async function triggerUpdated(automation: Automation) {
+  const properties: AutomationTriggerUpdatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TRIGGER_UPDATED, properties)
+}
+
+export async function deleted(automation: Automation) {
+  const properties: AutomationDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_DELETED, properties)
+}
+
+export async function tested(automation: Automation) {
+  const properties: AutomationTestedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TESTED, properties)
+}
+
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: AutomationsRunEvent = {
+    count,
+  }
+  await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
+}
+
+export async function stepCreated(
+  automation: Automation,
+  step: AutomationStep,
+  timestamp?: string | number
+) {
+  const properties: AutomationStepCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
+}
+
+export async function stepDeleted(
+  automation: Automation,
+  step: AutomationStep
+) {
+  const properties: AutomationStepDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/backfill.ts
+++ b/packages/backend-core/src/events/publishers/backfill.ts
@ -0,0 +1,67 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  AppBackfillSucceededEvent,
+  AppBackfillFailedEvent,
+  TenantBackfillSucceededEvent,
+  TenantBackfillFailedEvent,
+  InstallationBackfillSucceededEvent,
+  InstallationBackfillFailedEvent,
+} from "@budibase/types"
+const env = require("../../environment")
+
+const shouldSkip = !env.SELF_HOSTED && !env.isDev()
+
+export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function appFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: AppBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.APP_BACKFILL_FAILED, properties)
+}
+
+export async function tenantSucceeded(
+  properties: TenantBackfillSucceededEvent
+) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function tenantFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: TenantBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
+}
+
+export async function installationSucceeded() {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillSucceededEvent = {}
+  await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function installationFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
+}
--- a/packages/backend-core/src/events/publishers/datasource.ts
+++ b/packages/backend-core/src/events/publishers/datasource.ts
@ -0,0 +1,35 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Datasource,
+  DatasourceCreatedEvent,
+  DatasourceUpdatedEvent,
+  DatasourceDeletedEvent,
+} from "@budibase/types"
+
+export async function created(
+  datasource: Datasource,
+  timestamp?: string | number
+) {
+  const properties: DatasourceCreatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
+}
+
+export async function updated(datasource: Datasource) {
+  const properties: DatasourceUpdatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_UPDATED, properties)
+}
+
+export async function deleted(datasource: Datasource) {
+  const properties: DatasourceDeletedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/email.ts
+++ b/packages/backend-core/src/events/publishers/email.ts
@ -0,0 +1,12 @@
+import { publishEvent } from "../events"
+import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
+
+export async function SMTPCreated(timestamp?: string | number) {
+  const properties: SMTPCreatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
+}
+
+export async function SMTPUpdated() {
+  const properties: SMTPUpdatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
+}
--- a/packages/backend-core/src/events/publishers/index.ts
+++ b/packages/backend-core/src/events/publishers/index.ts
@ -0,0 +1,19 @@
+export * as account from "./account"
+export * as app from "./app"
+export * as auth from "./auth"
+export * as automation from "./automation"
+export * as datasource from "./datasource"
+export * as email from "./email"
+export * as license from "./license"
+export * as layout from "./layout"
+export * as org from "./org"
+export * as query from "./query"
+export * as role from "./role"
+export * as screen from "./screen"
+export * as rows from "./rows"
+export * as table from "./table"
+export * as serve from "./serve"
+export * as user from "./user"
+export * as view from "./view"
+export * as installation from "./installation"
+export * as backfill from "./backfill"
--- a/packages/backend-core/src/events/publishers/installation.ts
+++ b/packages/backend-core/src/events/publishers/installation.ts
@ -0,0 +1,31 @@
+import { publishEvent } from "../events"
+import { Event, VersionCheckedEvent, VersionChangeEvent } from "@budibase/types"
+
+export async function versionChecked(version: string) {
+  const properties: VersionCheckedEvent = {
+    currentVersion: version,
+  }
+  await publishEvent(Event.INSTALLATION_VERSION_CHECKED, properties)
+}
+
+export async function upgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+
+  await publishEvent(Event.INSTALLATION_VERSION_UPGRADED, properties)
+}
+
+export async function downgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+  await publishEvent(Event.INSTALLATION_VERSION_DOWNGRADED, properties)
+}
+
+export async function firstStartup() {
+  const properties = {}
+  await publishEvent(Event.INSTALLATION_FIRST_STARTUP, properties)
+}
--- a/packages/backend-core/src/events/publishers/layout.ts
+++ b/packages/backend-core/src/events/publishers/layout.ts
@ -0,0 +1,21 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Layout,
+  LayoutCreatedEvent,
+  LayoutDeletedEvent,
+} from "@budibase/types"
+
+export async function created(layout: Layout, timestamp?: string | number) {
+  const properties: LayoutCreatedEvent = {
+    layoutId: layout._id as string,
+  }
+  await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
+}
+
+export async function deleted(layoutId: string) {
+  const properties: LayoutDeletedEvent = {
+    layoutId,
+  }
+  await publishEvent(Event.LAYOUT_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/license.ts
+++ b/packages/backend-core/src/events/publishers/license.ts
@ -0,0 +1,33 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  License,
+  LicenseActivatedEvent,
+  LicenseDowngradedEvent,
+  LicenseUpdatedEvent,
+  LicenseUpgradedEvent,
+} from "@budibase/types"
+
+// TODO
+export async function updgraded(license: License) {
+  const properties: LicenseUpgradedEvent = {}
+  await publishEvent(Event.LICENSE_UPGRADED, properties)
+}
+
+// TODO
+export async function downgraded(license: License) {
+  const properties: LicenseDowngradedEvent = {}
+  await publishEvent(Event.LICENSE_DOWNGRADED, properties)
+}
+
+// TODO
+export async function updated(license: License) {
+  const properties: LicenseUpdatedEvent = {}
+  await publishEvent(Event.LICENSE_UPDATED, properties)
+}
+
+// TODO
+export async function activated(license: License) {
+  const properties: LicenseActivatedEvent = {}
+  await publishEvent(Event.LICENSE_ACTIVATED, properties)
+}
--- a/packages/backend-core/src/events/publishers/org.ts
+++ b/packages/backend-core/src/events/publishers/org.ts
@ -0,0 +1,29 @@
+import { publishEvent } from "../events"
+import { Event } from "@budibase/types"
+
+export async function nameUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_NAME_UPDATED, properties, timestamp)
+}
+
+export async function logoUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_LOGO_UPDATED, properties, timestamp)
+}
+
+export async function platformURLUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties, timestamp)
+}
+
+// TODO
+
+export async function analyticsOptOut() {
+  const properties = {}
+  await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
+}
+
+export async function analyticsOptIn() {
+  const properties = {}
+  await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
+}
--- a/packages/backend-core/src/events/publishers/query.ts
+++ b/packages/backend-core/src/events/publishers/query.ts
@ -0,0 +1,79 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Datasource,
+  Query,
+  QueryCreatedEvent,
+  QueryUpdatedEvent,
+  QueryDeletedEvent,
+  QueryImportedEvent,
+  QueryPreviewedEvent,
+  QueriesRunEvent,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export const created = async (
+  datasource: Datasource,
+  query: Query,
+  timestamp?: string | number
+) => {
+  const properties: QueryCreatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_CREATED, properties, timestamp)
+}
+
+export const updated = async (datasource: Datasource, query: Query) => {
+  const properties: QueryUpdatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_UPDATED, properties)
+}
+
+export const deleted = async (datasource: Datasource, query: Query) => {
+  const properties: QueryDeletedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_DELETED, properties)
+}
+
+export const imported = async (
+  datasource: Datasource,
+  importSource: any,
+  count: any
+) => {
+  const properties: QueryImportedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    count,
+    importSource,
+  }
+  await publishEvent(Event.QUERY_IMPORT, properties)
+}
+
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: QueriesRunEvent = {
+    count,
+  }
+  await publishEvent(Event.QUERIES_RUN, properties, timestamp)
+}
+
+export const previewed = async (datasource: Datasource, query: Query) => {
+  const properties: QueryPreviewedEvent = {
+    queryId: query._id,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_PREVIEWED, properties)
+}
--- a/packages/backend-core/src/events/publishers/role.ts
+++ b/packages/backend-core/src/events/publishers/role.ts
@ -0,0 +1,54 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Role,
+  RoleAssignedEvent,
+  RoleCreatedEvent,
+  RoleDeletedEvent,
+  RoleUnassignedEvent,
+  RoleUpdatedEvent,
+  User,
+} from "@budibase/types"
+
+export async function created(role: Role, timestamp?: string | number) {
+  const properties: RoleCreatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_CREATED, properties, timestamp)
+}
+
+export async function updated(role: Role) {
+  const properties: RoleUpdatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_UPDATED, properties)
+}
+
+export async function deleted(role: Role) {
+  const properties: RoleDeletedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_DELETED, properties)
+}
+
+export async function assigned(user: User, roleId: string, timestamp?: number) {
+  const properties: RoleAssignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
+  await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp)
+}
+
+export async function unassigned(user: User, roleId: string) {
+  const properties: RoleUnassignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
+  await publishEvent(Event.ROLE_UNASSIGNED, properties)
+}
--- a/packages/backend-core/src/events/publishers/rows.ts
+++ b/packages/backend-core/src/events/publishers/rows.ts
@ -0,0 +1,30 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  RowsImportedEvent,
+  RowsCreatedEvent,
+  RowImportFormat,
+  Table,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export const created = async (count: number, timestamp?: string | number) => {
+  const properties: RowsCreatedEvent = {
+    count,
+  }
+  await publishEvent(Event.ROWS_CREATED, properties, timestamp)
+}
+
+export const imported = async (
+  table: Table,
+  format: RowImportFormat,
+  count: number
+) => {
+  const properties: RowsImportedEvent = {
+    tableId: table._id as string,
+    format,
+    count,
+  }
+  await publishEvent(Event.ROWS_IMPORTED, properties)
+}
--- a/packages/backend-core/src/events/publishers/screen.ts
+++ b/packages/backend-core/src/events/publishers/screen.ts
@ -0,0 +1,25 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Screen,
+  ScreenCreatedEvent,
+  ScreenDeletedEvent,
+} from "@budibase/types"
+
+export async function created(screen: Screen, timestamp?: string | number) {
+  const properties: ScreenCreatedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
+  await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
+}
+
+export async function deleted(screen: Screen) {
+  const properties: ScreenDeletedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
+  await publishEvent(Event.SCREEN_DELETED, properties)
+}
--- a/packages/backend-core/src/events/publishers/serve.ts
+++ b/packages/backend-core/src/events/publishers/serve.ts
@ -0,0 +1,28 @@
+import { publishEvent } from "../events"
+import {
+  App,
+  BuilderServedEvent,
+  Event,
+  AppPreviewServedEvent,
+  AppServedEvent,
+} from "@budibase/types"
+
+export async function servedBuilder() {
+  const properties: BuilderServedEvent = {}
+  await publishEvent(Event.SERVED_BUILDER, properties)
+}
+
+export async function servedApp(app: App) {
+  const properties: AppServedEvent = {
+    appVersion: app.version,
+  }
+  await publishEvent(Event.SERVED_APP, properties)
+}
+
+export async function servedAppPreview(app: App) {
+  const properties: AppPreviewServedEvent = {
+    appId: app.appId,
+    appVersion: app.version,
+  }
+  await publishEvent(Event.SERVED_APP_PREVIEW, properties)
+}
--- a/packages/backend-core/src/events/publishers/table.ts
+++ b/packages/backend-core/src/events/publishers/table.ts
@ -0,0 +1,49 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  TableExportFormat,
+  TableImportFormat,
+  Table,
+  TableCreatedEvent,
+  TableUpdatedEvent,
+  TableDeletedEvent,
+  TableExportedEvent,
+  TableImportedEvent,
+} from "@budibase/types"
+
+export async function created(table: Table, timestamp?: string | number) {
+  const properties: TableCreatedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_CREATED, properties, timestamp)
+}
+
+export async function updated(table: Table) {
+  const properties: TableUpdatedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_UPDATED, properties)
+}
+
+export async function deleted(table: Table) {
+  const properties: TableDeletedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_DELETED, properties)
+}
+
+export async function exported(table: Table, format: TableExportFormat) {
+  const properties: TableExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.TABLE_EXPORTED, properties)
+}
+
+export async function imported(table: Table, format: TableImportFormat) {
+  const properties: TableImportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.TABLE_IMPORTED, properties)
+}
--- a/packages/backend-core/src/events/publishers/user.ts
+++ b/packages/backend-core/src/events/publishers/user.ts
@ -0,0 +1,122 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  User,
+  UserCreatedEvent,
+  UserDeletedEvent,
+  UserInviteAcceptedEvent,
+  UserInvitedEvent,
+  UserPasswordForceResetEvent,
+  UserPasswordResetEvent,
+  UserPasswordResetRequestedEvent,
+  UserPasswordUpdatedEvent,
+  UserPermissionAssignedEvent,
+  UserPermissionRemovedEvent,
+  UserUpdatedEvent,
+} from "@budibase/types"
+
+export async function created(user: User, timestamp?: number) {
+  const properties: UserCreatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_CREATED, properties, timestamp)
+}
+
+export async function updated(user: User) {
+  const properties: UserUpdatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_UPDATED, properties)
+}
+
+export async function deleted(user: User) {
+  const properties: UserDeletedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_DELETED, properties)
+}
+
+// PERMISSIONS
+
+export async function permissionAdminAssigned(user: User, timestamp?: number) {
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(
+    Event.USER_PERMISSION_ADMIN_ASSIGNED,
+    properties,
+    timestamp
+  )
+}
+
+export async function permissionAdminRemoved(user: User) {
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
+}
+
+export async function permissionBuilderAssigned(
+  user: User,
+  timestamp?: number
+) {
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(
+    Event.USER_PERMISSION_BUILDER_ASSIGNED,
+    properties,
+    timestamp
+  )
+}
+
+export async function permissionBuilderRemoved(user: User) {
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties)
+}
+
+// INVITE
+
+export async function invited() {
+  const properties: UserInvitedEvent = {}
+  await publishEvent(Event.USER_INVITED, properties)
+}
+
+export async function inviteAccepted(user: User) {
+  const properties: UserInviteAcceptedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_INVITED_ACCEPTED, properties)
+}
+
+// PASSWORD
+
+export async function passwordForceReset(user: User) {
+  const properties: UserPasswordForceResetEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
+}
+
+export async function passwordUpdated(user: User) {
+  const properties: UserPasswordUpdatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
+}
+
+export async function passwordResetRequested(user: User) {
+  const properties: UserPasswordResetRequestedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
+}
+
+export async function passwordReset(user: User) {
+  const properties: UserPasswordResetEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_RESET, properties)
+}
--- a/packages/backend-core/src/events/publishers/view.ts
+++ b/packages/backend-core/src/events/publishers/view.ts
@ -0,0 +1,97 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  ViewCalculationCreatedEvent,
+  ViewCalculationDeletedEvent,
+  ViewCalculationUpdatedEvent,
+  ViewCreatedEvent,
+  ViewDeletedEvent,
+  ViewExportedEvent,
+  ViewFilterCreatedEvent,
+  ViewFilterDeletedEvent,
+  ViewFilterUpdatedEvent,
+  ViewUpdatedEvent,
+  View,
+  ViewCalculation,
+  Table,
+  TableExportFormat,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export async function created(view: View, timestamp?: string | number) {
+  const properties: ViewCreatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_CREATED, properties, timestamp)
+}
+
+export async function updated(view: View) {
+  const properties: ViewUpdatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_UPDATED, properties)
+}
+
+export async function deleted(view: View) {
+  const properties: ViewDeletedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_DELETED, properties)
+}
+
+export async function exported(table: Table, format: TableExportFormat) {
+  const properties: ViewExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.VIEW_EXPORTED, properties)
+}
+
+export async function filterCreated(view: View, timestamp?: string | number) {
+  const properties: ViewFilterCreatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp)
+}
+
+export async function filterUpdated(view: View) {
+  const properties: ViewFilterUpdatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_UPDATED, properties)
+}
+
+export async function filterDeleted(view: View) {
+  const properties: ViewFilterDeletedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_DELETED, properties)
+}
+
+export async function calculationCreated(
+  view: View,
+  timestamp?: string | number
+) {
+  const properties: ViewCalculationCreatedEvent = {
+    tableId: view.tableId,
+    calculation: view.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp)
+}
+
+export async function calculationUpdated(view: View) {
+  const properties: ViewCalculationUpdatedEvent = {
+    tableId: view.tableId,
+    calculation: view.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_UPDATED, properties)
+}
+
+export async function calculationDeleted(existingView: View) {
+  const properties: ViewCalculationDeletedEvent = {
+    tableId: existingView.tableId,
+    calculation: existingView.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_DELETED, properties)
+}
--- a/packages/backend-core/src/index.js
+++ b/packages/backend-core/src/index.js
@ -1,24 +0,0 @@
-const db = require("./db")
-
-module.exports = {
-  init(opts = {}) {
-    db.init(opts.db)
-  },
-  // some default exports from the library, however these ideally shouldn't
-  // be used, instead the syntax require("@budibase/backend-core/db") should be used
-  StaticDatabases: require("./db/utils").StaticDatabases,
-  db: require("../db"),
-  redis: require("../redis"),
-  objectStore: require("../objectStore"),
-  utils: require("../utils"),
-  cache: require("../cache"),
-  auth: require("../auth"),
-  constants: require("../constants"),
-  migrations: require("../migrations"),
-  errors: require("./errors"),
-  env: require("./environment"),
-  accounts: require("./cloud/accounts"),
-  tenancy: require("./tenancy"),
-  context: require("../context"),
-  featureFlags: require("./featureFlags"),
-}
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@ -0,0 +1,55 @@
+import errors from "./errors"
+const errorClasses = errors.errors
+import * as events from "./events"
+import * as migrations from "./migrations"
+import * as users from "./users"
+import * as accounts from "./cloud/accounts"
+import * as installation from "./installation"
+import env from "./environment"
+import tenancy from "./tenancy"
+import featureFlags from "./featureFlags"
+import sessions from "./security/sessions"
+import deprovisioning from "./context/deprovision"
+import auth from "./auth"
+import constants from "./constants"
+import * as dbConstants from "./db/constants"
+
+// mimic the outer package exports
+import * as db from "./pkg/db"
+import * as objectStore from "./pkg/objectStore"
+import * as utils from "./pkg/utils"
+import redis from "./pkg/redis"
+import cache from "./pkg/cache"
+import context from "./pkg/context"
+
+const init = (opts: any = {}) => {
+  db.init(opts.db)
+}
+
+const core = {
+  init,
+  db,
+  ...dbConstants,
+  redis,
+  objectStore,
+  utils,
+  users,
+  cache,
+  auth,
+  constants,
+  ...constants,
+  migrations,
+  env,
+  accounts,
+  tenancy,
+  context,
+  featureFlags,
+  events,
+  sessions,
+  deprovisioning,
+  installation,
+  errors,
+  ...errorClasses,
+}
+
+export = core
--- a/packages/backend-core/src/installation.ts
+++ b/packages/backend-core/src/installation.ts
@ -0,0 +1,96 @@
+import * as hashing from "./hashing"
+import * as events from "./events"
+import { StaticDatabases } from "./db/constants"
+import { doWithDB } from "./db"
+import { Installation, IdentityType } from "@budibase/types"
+import * as context from "./context"
+import semver from "semver"
+import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic"
+
+const pkg = require("../package.json")
+
+export const getInstall = async (): Promise<Installation> => {
+  return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
+    useTenancy: false,
+  })
+}
+
+const getInstallFromDB = async (): Promise<Installation> => {
+  return doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (platformDb: any) => {
+      let install: Installation
+      try {
+        install = await platformDb.get(
+          StaticDatabases.PLATFORM_INFO.docs.install
+        )
+      } catch (e: any) {
+        if (e.status === 404) {
+          install = {
+            _id: StaticDatabases.PLATFORM_INFO.docs.install,
+            installId: hashing.newid(),
+            version: pkg.version,
+          }
+          const resp = await platformDb.put(install)
+          install._rev = resp.rev
+        } else {
+          throw e
+        }
+      }
+      return install
+    }
+  )
+}
+
+const updateVersion = async (version: string): Promise<boolean> => {
+  try {
+    await doWithDB(
+      StaticDatabases.PLATFORM_INFO.name,
+      async (platformDb: any) => {
+        const install = await getInstall()
+        install.version = version
+        await platformDb.put(install)
+        await bustCache(CacheKeys.INSTALLATION)
+      }
+    )
+  } catch (e: any) {
+    if (e.status === 409) {
+      // do nothing - version has already been updated
+      // likely in clustered environment
+      return false
+    }
+    throw e
+  }
+  return true
+}
+
+export const checkInstallVersion = async (): Promise<void> => {
+  const install = await getInstall()
+
+  const currentVersion = install.version
+  const newVersion = pkg.version
+
+  if (currentVersion !== newVersion) {
+    const isUpgrade = semver.gt(newVersion, currentVersion)
+    const isDowngrade = semver.lt(newVersion, currentVersion)
+
+    const success = await updateVersion(newVersion)
+
+    if (success) {
+      await context.doInIdentityContext(
+        {
+          _id: install.installId,
+          type: IdentityType.INSTALLATION,
+        },
+        async () => {
+          if (isUpgrade) {
+            await events.installation.upgraded(currentVersion, newVersion)
+          } else if (isDowngrade) {
+            await events.installation.downgraded(currentVersion, newVersion)
+          }
+        }
+      )
+      await events.identification.identifyInstallationGroup(install.installId)
+    }
+  }
+}
--- a/packages/backend-core/src/logging.js
+++ b/packages/backend-core/src/logging.js
@ -4,7 +4,7 @@ function isSuppressed(e) {
  return e && e["suppressAlert"]
 }

-module.exports.logAlert = (message, e = null) => {
+module.exports.logAlert = (message, e) => {
  if (e && NonErrors.includes(e.name) && isSuppressed(e)) {
    return
  }
--- a/packages/backend-core/src/middleware/authenticated.js
+++ b/packages/backend-core/src/middleware/authenticated.js
@ -4,9 +4,12 @@ const { getUser } = require("../cache/user")
 const { getSession, updateSessionTTL } = require("../security/sessions")
 const { buildMatcherRegex, matches } = require("./matchers")
 const env = require("../environment")
-const { SEPARATOR, ViewNames, queryGlobalView } = require("../../db")
+const { SEPARATOR } = require("../db/constants")
+const { ViewNames } = require("../db/utils")
+const { queryGlobalView } = require("../db/views")
 const { getGlobalDB, doInTenant } = require("../tenancy")
 const { decrypt } = require("../security/encryption")
+const identity = require("../context/identity")

 function finalise(
  ctx,
@ -132,7 +135,12 @@ module.exports = (
      }
      // isAuthenticated is a function, so use a variable to be able to check authed state
      finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
+
+      if (user && user.email) {
+        return identity.doInUserContext(user, next)
+      } else {
        return next()
+      }
    } catch (err) {
      // invalid token, clear the cookie
      if (err && err.name === "JsonWebTokenError") {
--- a/packages/backend-core/src/middleware/passport/local.js
+++ b/packages/backend-core/src/middleware/passport/local.js
@ -2,7 +2,7 @@ const jwt = require("jsonwebtoken")
 const { UserStatus } = require("../../constants")
 const { compare } = require("../../hashing")
 const env = require("../../environment")
-const { getGlobalUserByEmail } = require("../../utils")
+const users = require("../../users")
 const { authError } = require("./utils")
 const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
@ -28,7 +28,7 @@ exports.authenticate = async function (ctx, email, password, done) {
  if (!email) return authError(done, "Email Required")
  if (!password) return authError(done, "Password Required")

-  const dbUser = await getGlobalUserByEmail(email)
+  const dbUser = await users.getGlobalUserByEmail(email)
  if (dbUser == null) {
    return authError(done, `User not found: [${email}]`)
  }
--- a/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
+++ b/packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js
@ -1,7 +1,4 @@
-// Mock data
-
-require("../../../tests/utilities/dbConfig")
-
+require("../../../../tests/utilities/TestConfiguration")
 const { authenticateThirdParty } = require("../third-party-common")
 const { data } = require("./utilities/mock-data")
 const { DEFAULT_TENANT_ID } = require("../../../constants")
--- a/packages/backend-core/src/middleware/passport/third-party-common.js
+++ b/packages/backend-core/src/middleware/passport/third-party-common.js
@ -4,7 +4,7 @@ const { generateGlobalUserID } = require("../../db/utils")
 const { authError } = require("./utils")
 const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
-const { getGlobalUserByEmail } = require("../../utils")
+const users = require("../../users")
 const { getGlobalDB, getTenantId } = require("../../tenancy")
 const fetch = require("node-fetch")

@ -52,7 +52,7 @@ exports.authenticateThirdParty = async function (

  // fallback to loading by email
  if (!dbUser) {
-    dbUser = await getGlobalUserByEmail(thirdPartyUser.email)
+    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
  }

  // exit early if there is still no user and auto creation is disabled
@ -79,14 +79,14 @@ exports.authenticateThirdParty = async function (
  dbUser.forceResetPassword = false

  // create or sync the user
-  let response
  try {
-    response = await saveUserFn(dbUser, getTenantId(), false, false)
+    await saveUserFn(dbUser, false, false)
  } catch (err) {
    return authError(done, err)
  }

-  dbUser._rev = response.rev
+  // now that we're sure user exists, load them from the db
+  dbUser = await db.get(dbUser._id)

  // authenticate
  const sessionId = newid()
--- a/packages/backend-core/src/middleware/tenancy.js
+++ b/packages/backend-core/src/middleware/tenancy.js
@ -1,6 +1,38 @@
-const { setTenantId, setGlobalDB, closeTenancy } = require("../tenancy")
-const cls = require("../context/FunctionContext")
+const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy")
 const { buildMatcherRegex, matches } = require("./matchers")
+const { Headers } = require("../constants")
+
+const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => {
+  // exit early if not multi-tenant
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
+
+  let tenantId
+  const allowQs = opts && opts.allowQs
+  const allowNoTenant = opts && opts.allowNoTenant
+  const header = ctx.request.headers[Headers.TENANT_ID]
+  const user = ctx.user || {}
+  if (allowQs) {
+    const query = ctx.request.query || {}
+    tenantId = query.tenantId
+  }
+  // override query string (if allowed) by user, or header
+  // URL params cannot be used in a middleware, as they are
+  // processed later in the chain
+  tenantId = user.tenantId || header || tenantId
+
+  // Set the tenantId from the subdomain
+  if (!tenantId) {
+    tenantId = ctx.subdomains && ctx.subdomains[0]
+  }
+
+  if (!tenantId && !allowNoTenant) {
+    ctx.throw(403, "Tenant id not set")
+  }
+
+  return tenantId
+}

 module.exports = (
  allowQueryStringPatterns,
@ -11,15 +43,10 @@ module.exports = (
  const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)

  return async function (ctx, next) {
-    return cls.run(async () => {
    const allowNoTenant =
      opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
    const allowQs = !!matches(ctx, allowQsOptions)
-      const tenantId = setTenantId(ctx, { allowQs, allowNoTenant })
-      setGlobalDB(tenantId)
-      const res = await next()
-      await closeTenancy()
-      return res
-    })
+    const tenantId = getTenantID(ctx, { allowQs, allowNoTenant })
+    return doInTenant(tenantId, next)
  }
 }
--- a/packages/backend-core/src/migrations/definitions.ts
+++ b/packages/backend-core/src/migrations/definitions.ts
@ -0,0 +1,40 @@
+import {
+  MigrationType,
+  MigrationName,
+  MigrationDefinition,
+} from "@budibase/types"
+
+export const DEFINITIONS: MigrationDefinition[] = [
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.USER_EMAIL_VIEW_CASING,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.QUOTAS_1,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.APP_URLS,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.DEVELOPER_QUOTA,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.PUBLISHED_APP_QUOTA,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.EVENT_APP_BACKFILL,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.EVENT_GLOBAL_BACKFILL,
+  },
+  {
+    type: MigrationType.INSTALLATION,
+    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
+  },
+]
--- a/packages/backend-core/src/migrations/index.js
+++ b/packages/backend-core/src/migrations/index.js
@ -1,117 +0,0 @@
-const { DEFAULT_TENANT_ID } = require("../constants")
-const { doWithDB } = require("../db")
-const { DocumentTypes } = require("../db/constants")
-const { getAllApps } = require("../db/utils")
-const environment = require("../environment")
-const {
-  doInTenant,
-  getTenantIds,
-  getGlobalDBName,
-  getTenantId,
-} = require("../tenancy")
-
-exports.MIGRATION_TYPES = {
-  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
-  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
-}
-
-exports.getMigrationsDoc = async db => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentTypes.MIGRATIONS)
-  } catch (err) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentTypes.MIGRATIONS }
-    }
-    console.error(err)
-  }
-}
-
-const runMigration = async (migration, options = {}) => {
-  const tenantId = getTenantId()
-  const migrationType = migration.type
-  const migrationName = migration.name
-
-  // get the db to store the migration in
-  let dbNames
-  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    const apps = await getAllApps(migration.opts)
-    dbNames = apps.map(app => app.appId)
-  } else {
-    throw new Error(
-      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
-    )
-  }
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    await doWithDB(dbName, async db => {
-      try {
-        const doc = await exports.getMigrationsDoc(db)
-
-        // exit if the migration has been performed already
-        if (doc[migrationName]) {
-          if (
-            options.force &&
-            options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
-          ) {
-            console.log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // the migration has already been performed
-            return
-          }
-        }
-
-        console.log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
-        )
-        // run the migration with tenant context
-        await migration.fn(db)
-        console.log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
-        )
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
-      }
-    })
-  }
-}
-
-exports.runMigrations = async (migrations, options = {}) => {
-  console.log("Running migrations")
-  let tenantIds
-  if (environment.MULTI_TENANCY) {
-    if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
-    }
-  }
-  console.log("Migrations complete")
-}
--- a/packages/backend-core/src/migrations/index.ts
+++ b/packages/backend-core/src/migrations/index.ts
@ -0,0 +1,2 @@
+export * from "./migrations"
+export * from "./definitions"
--- a/packages/backend-core/src/migrations/migrations.ts
+++ b/packages/backend-core/src/migrations/migrations.ts
@ -0,0 +1,189 @@
+import { DEFAULT_TENANT_ID } from "../constants"
+import { doWithDB } from "../db"
+import { DocumentTypes, StaticDatabases } from "../db/constants"
+import { getAllApps } from "../db/utils"
+import environment from "../environment"
+import {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} from "../tenancy"
+import context from "../context"
+import { DEFINITIONS } from "."
+import {
+  Migration,
+  MigrationOptions,
+  MigrationType,
+  MigrationNoOpOptions,
+} from "@budibase/types"
+
+export const getMigrationsDoc = async (db: any) => {
+  // get the migrations doc
+  try {
+    return await db.get(DocumentTypes.MIGRATIONS)
+  } catch (err: any) {
+    if (err.status && err.status === 404) {
+      return { _id: DocumentTypes.MIGRATIONS }
+    } else {
+      console.error(err)
+      throw err
+    }
+  }
+}
+
+export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
+  // filter migrations to the type and populate a no-op migration
+  const migrations: Migration[] = DEFINITIONS.filter(
+    def => def.type === opts.type
+  ).map(d => ({ ...d, fn: () => {} }))
+  await runMigrations(migrations, { noOp: opts })
+}
+
+export const runMigration = async (
+  migration: Migration,
+  options: MigrationOptions = {}
+) => {
+  const migrationType = migration.type
+  let tenantId: string
+  if (migrationType !== MigrationType.INSTALLATION) {
+    tenantId = getTenantId()
+  }
+  const migrationName = migration.name
+  const silent = migration.silent
+
+  const log = (message: string) => {
+    if (!silent) {
+      console.log(message)
+    }
+  }
+
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === MigrationType.GLOBAL) {
+    dbNames = [getGlobalDBName()]
+  } else if (migrationType === MigrationType.APP) {
+    if (options.noOp) {
+      dbNames = [options.noOp.appId]
+    } else {
+      const apps = await getAllApps(migration.appOpts)
+      dbNames = apps.map(app => app.appId)
+    }
+  } else if (migrationType === MigrationType.INSTALLATION) {
+    dbNames = [StaticDatabases.PLATFORM_INFO.name]
+  } else {
+    throw new Error(`Unrecognised migration type [${migrationType}]`)
+  }
+
+  const length = dbNames.length
+  let count = 0
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    count++
+    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
+
+    await doWithDB(dbName, async (db: any) => {
+      try {
+        const doc = await exports.getMigrationsDoc(db)
+
+        // the migration has already been run
+        if (doc[migrationName]) {
+          // check for force
+          if (
+            options.force &&
+            options.force[migrationType] &&
+            options.force[migrationType].includes(migrationName)
+          ) {
+            log(
+              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+            )
+          } else {
+            // no force, exit
+            return
+          }
+        }
+
+        // check if the migration is not a no-op
+        if (!options.noOp) {
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+          )
+
+          if (migration.preventRetry) {
+            // eagerly set the completion date
+            // so that we never run this migration twice even upon failure
+            doc[migrationName] = Date.now()
+            const response = await db.put(doc)
+            doc._rev = response.rev
+          }
+
+          // run the migration
+          if (migrationType === MigrationType.APP) {
+            await context.doInAppContext(db.name, async () => {
+              await migration.fn(db)
+            })
+          } else {
+            await migration.fn(db)
+          }
+
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+          )
+        }
+
+        // mark as complete
+        doc[migrationName] = Date.now()
+        await db.put(doc)
+      } catch (err) {
+        console.error(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+          err
+        )
+        throw err
+      }
+    })
+  }
+}
+
+export const runMigrations = async (
+  migrations: Migration[],
+  options: MigrationOptions = {}
+) => {
+  let tenantIds
+
+  if (environment.MULTI_TENANCY) {
+    if (options.noOp) {
+      tenantIds = [options.noOp.tenantId]
+    } else if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    } else {
+      tenantIds = options.tenantIds
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  if (tenantIds.length > 1) {
+    console.log(`Checking migrations for ${tenantIds.length} tenants`)
+  } else {
+    console.log("Checking migrations")
+  }
+
+  let count = 0
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    count++
+    if (tenantIds.length > 1) {
+      console.log(`Progress [${count}/${tenantIds.length}]`)
+    }
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await doInTenant(tenantId, () => runMigration(migration, options))
+    }
+  }
+  console.log("Migrations complete")
+}
--- a/packages/backend-core/src/migrations/tests/snapshots/index.spec.js.snap
+++ b/packages/backend-core/src/migrations/tests/snapshots/index.spec.js.snap
@ -3,7 +3,9 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
  "_id": "migrations",
-  "_rev": "1-6277abc4e3db950221768e5a2618a059",
-  "test": 1487076708000,
+  "_rev": "1-a32b0b708e59eeb006ed5e063cfeb36a",
+  "createdAt": "2020-01-01T00:00:00.000Z",
+  "test": 1577836800000,
+  "updatedAt": "2020-01-01T00:00:00.000Z",
 }
 `;
--- a/packages/backend-core/src/migrations/tests/index.spec.js
+++ b/packages/backend-core/src/migrations/tests/index.spec.js
@ -1,12 +1,10 @@
-require("../../tests/utilities/dbConfig")
-
+require("../../../tests/utilities/TestConfiguration")
 const { runMigrations, getMigrationsDoc } = require("../index")
 const { dangerousGetDB } = require("../../db")
 const {
  StaticDatabases,
 } = require("../../db/utils")

-Date.now = jest.fn(() => 1487076708000)
 let db

 describe("migrations", () => {
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@ -1,16 +1,16 @@
 const sanitize = require("sanitize-s3-objectkey")
-const AWS = require("aws-sdk")
-const stream = require("stream")
-const fetch = require("node-fetch")
-const tar = require("tar-fs")
+import AWS from "aws-sdk"
+import stream from "stream"
+import fetch from "node-fetch"
+import tar from "tar-fs"
 const zlib = require("zlib")
-const { promisify } = require("util")
-const { join } = require("path")
-const fs = require("fs")
-const env = require("../environment")
-const { budibaseTempDir, ObjectStoreBuckets } = require("./utils")
-const { v4 } = require("uuid")
-const { APP_PREFIX, APP_DEV_PREFIX } = require("../db/utils")
+import { promisify } from "util"
+import { join } from "path"
+import fs from "fs"
+import env from "../environment"
+import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
+import { v4 } from "uuid"
+import { APP_PREFIX, APP_DEV_PREFIX } from "../db/utils"

 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@ -18,7 +18,7 @@ const STATE = {
  bucketCreationPromises: {},
 }

-const CONTENT_TYPE_MAP = {
+const CONTENT_TYPE_MAP: any = {
  html: "text/html",
  css: "text/css",
  js: "application/javascript",
@ -32,20 +32,16 @@ const STRING_CONTENT_TYPES = [
 ]

 // does normal sanitization and then swaps dev apps to apps
-function sanitizeKey(input) {
+export function sanitizeKey(input: any) {
  return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
 }

-exports.sanitizeKey = sanitizeKey
-
 // simply handles the dev app to app conversion
-function sanitizeBucket(input) {
+export function sanitizeBucket(input: any) {
  return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }

-exports.sanitizeBucket = sanitizeBucket
-
-function publicPolicy(bucketName) {
+function publicPolicy(bucketName: any) {
  return {
    Version: "2012-10-17",
    Statement: [
@ -69,13 +65,13 @@ const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS, ObjectStoreBuckets.GLOBAL]
 * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
 * @constructor
 */
-exports.ObjectStore = bucket => {
+export const ObjectStore = (bucket: any) => {
  AWS.config.update({
    accessKeyId: env.MINIO_ACCESS_KEY,
    secretAccessKey: env.MINIO_SECRET_KEY,
    region: env.AWS_REGION,
  })
-  const config = {
+  const config: any = {
    s3ForcePathStyle: true,
    signatureVersion: "v4",
    apiVersion: "2006-03-01",
@ -93,7 +89,7 @@ exports.ObjectStore = bucket => {
 * Given an object store and a bucket name this will make sure the bucket exists,
 * if it does not exist then it will create it.
 */
-exports.makeSureBucketExists = async (client, bucketName) => {
+export const makeSureBucketExists = async (client: any, bucketName: any) => {
  bucketName = sanitizeBucket(bucketName)
  try {
    await client
@ -101,8 +97,8 @@ exports.makeSureBucketExists = async (client, bucketName) => {
        Bucket: bucketName,
      })
      .promise()
-  } catch (err) {
-    const promises = STATE.bucketCreationPromises
+  } catch (err: any) {
+    const promises: any = STATE.bucketCreationPromises
    const doesntExist = err.statusCode === 404,
      noAccess = err.statusCode === 403
    if (promises[bucketName]) {
@ -138,20 +134,20 @@ exports.makeSureBucketExists = async (client, bucketName) => {
 * Uploads the contents of a file given the required parameters, useful when
 * temp files in use (for example file uploaded as an attachment).
 */
-exports.upload = async ({
+export const upload = async ({
  bucket: bucketName,
  filename,
  path,
  type,
  metadata,
-}) => {
+}: any) => {
  const extension = [...filename.split(".")].pop()
  const fileBytes = fs.readFileSync(path)

-  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+  const objectStore = ObjectStore(bucketName)
+  await makeSureBucketExists(objectStore, bucketName)

-  const config = {
+  const config: any = {
    // windows file paths need to be converted to forward slashes for s3
    Key: sanitizeKey(filename),
    Body: fileBytes,
@ -167,9 +163,14 @@ exports.upload = async ({
 * Similar to the upload function but can be used to send a file stream
 * through to the object store.
 */
-exports.streamUpload = async (bucketName, filename, stream, extra = {}) => {
-  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+export const streamUpload = async (
+  bucketName: any,
+  filename: any,
+  stream: any,
+  extra = {}
+) => {
+  const objectStore = ObjectStore(bucketName)
+  await makeSureBucketExists(objectStore, bucketName)

  const params = {
    Bucket: sanitizeBucket(bucketName),
@ -184,13 +185,13 @@ exports.streamUpload = async (bucketName, filename, stream, extra = {}) => {
 * retrieves the contents of a file from the object store, if it is a known content type it
 * will be converted, otherwise it will be returned as a buffer stream.
 */
-exports.retrieve = async (bucketName, filepath) => {
-  const objectStore = exports.ObjectStore(bucketName)
+export const retrieve = async (bucketName: any, filepath: any) => {
+  const objectStore = ObjectStore(bucketName)
  const params = {
    Bucket: sanitizeBucket(bucketName),
    Key: sanitizeKey(filepath),
  }
-  const response = await objectStore.getObject(params).promise()
+  const response: any = await objectStore.getObject(params).promise()
  // currently these are all strings
  if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
    return response.Body.toString("utf8")
@ -202,10 +203,10 @@ exports.retrieve = async (bucketName, filepath) => {
 /**
 * Same as retrieval function but puts to a temporary file.
 */
-exports.retrieveToTmp = async (bucketName, filepath) => {
+export const retrieveToTmp = async (bucketName: any, filepath: any) => {
  bucketName = sanitizeBucket(bucketName)
  filepath = sanitizeKey(filepath)
-  const data = await exports.retrieve(bucketName, filepath)
+  const data = await retrieve(bucketName, filepath)
  const outputPath = join(budibaseTempDir(), v4())
  fs.writeFileSync(outputPath, data)
  return outputPath
@ -214,9 +215,9 @@ exports.retrieveToTmp = async (bucketName, filepath) => {
 /**
 * Delete a single file.
 */
-exports.deleteFile = async (bucketName, filepath) => {
-  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+export const deleteFile = async (bucketName: any, filepath: any) => {
+  const objectStore = ObjectStore(bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
  const params = {
    Bucket: bucketName,
    Key: filepath,
@ -224,13 +225,13 @@ exports.deleteFile = async (bucketName, filepath) => {
  return objectStore.deleteObject(params)
 }

-exports.deleteFiles = async (bucketName, filepaths) => {
-  const objectStore = exports.ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+export const deleteFiles = async (bucketName: any, filepaths: any) => {
+  const objectStore = ObjectStore(bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
  const params = {
    Bucket: bucketName,
    Delete: {
-      Objects: filepaths.map(path => ({ Key: path })),
+      Objects: filepaths.map((path: any) => ({ Key: path })),
    },
  }
  return objectStore.deleteObjects(params).promise()
@ -239,38 +240,45 @@ exports.deleteFiles = async (bucketName, filepaths) => {
 /**
 * Delete a path, including everything within.
 */
-exports.deleteFolder = async (bucketName, folder) => {
+export const deleteFolder = async (
+  bucketName: any,
+  folder: any
+): Promise<any> => {
  bucketName = sanitizeBucket(bucketName)
  folder = sanitizeKey(folder)
-  const client = exports.ObjectStore(bucketName)
+  const client = ObjectStore(bucketName)
  const listParams = {
    Bucket: bucketName,
    Prefix: folder,
  }

-  let response = await client.listObjects(listParams).promise()
+  let response: any = await client.listObjects(listParams).promise()
  if (response.Contents.length === 0) {
    return
  }
-  const deleteParams = {
+  const deleteParams: any = {
    Bucket: bucketName,
    Delete: {
      Objects: [],
    },
  }

-  response.Contents.forEach(content => {
+  response.Contents.forEach((content: any) => {
    deleteParams.Delete.Objects.push({ Key: content.Key })
  })

  response = await client.deleteObjects(deleteParams).promise()
  // can only empty 1000 items at once
  if (response.Deleted.length === 1000) {
-    return exports.deleteFolder(bucketName, folder)
+    return deleteFolder(bucketName, folder)
  }
 }

-exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
+export const uploadDirectory = async (
+  bucketName: any,
+  localPath: any,
+  bucketPath: any
+) => {
  bucketName = sanitizeBucket(bucketName)
  let uploads = []
  const files = fs.readdirSync(localPath, { withFileTypes: true })
@ -278,17 +286,15 @@ exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
    const path = sanitizeKey(join(bucketPath, file.name))
    const local = join(localPath, file.name)
    if (file.isDirectory()) {
-      uploads.push(exports.uploadDirectory(bucketName, local, path))
+      uploads.push(uploadDirectory(bucketName, local, path))
    } else {
-      uploads.push(
-        exports.streamUpload(bucketName, path, fs.createReadStream(local))
-      )
+      uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
    }
  }
  await Promise.all(uploads)
 }

-exports.downloadTarball = async (url, bucketName, path) => {
+export const downloadTarball = async (url: any, bucketName: any, path: any) => {
  bucketName = sanitizeBucket(bucketName)
  path = sanitizeKey(path)
  const response = await fetch(url)
@ -299,7 +305,7 @@ exports.downloadTarball = async (url, bucketName, path) => {
  const tmpPath = join(budibaseTempDir(), path)
  await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
  if (!env.isTest() && env.SELF_HOSTED) {
-    await exports.uploadDirectory(bucketName, tmpPath, path)
+    await uploadDirectory(bucketName, tmpPath, path)
  }
  // return the temporary path incase there is a use for it
  return tmpPath
--- a/packages/backend-core/src/pkg/cache.ts
+++ b/packages/backend-core/src/pkg/cache.ts
@ -0,0 +1,11 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import * as generic from "../cache/generic"
+import * as user from "../cache/user"
+import * as app from "../cache/appMetadata"
+
+export = {
+  app,
+  user,
+  ...generic,
+}
--- a/packages/backend-core/src/pkg/context.ts
+++ b/packages/backend-core/src/pkg/context.ts
@ -0,0 +1,24 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+  doInTenant,
+} from "../context"
+
+import * as identity from "../context/identity"
+
+export = {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+  doInTenant,
+  identity,
+}
--- a/packages/backend-core/src/pkg/db.ts
+++ b/packages/backend-core/src/pkg/db.ts
@ -0,0 +1,7 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../db"
+export * from "../db/utils"
+export * from "../db/views"
+export * from "../db/pouch"
+export * from "../db/constants"
--- a/packages/backend-core/src/pkg/objectStore.ts
+++ b/packages/backend-core/src/pkg/objectStore.ts
@ -0,0 +1,4 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../objectStore"
+export * from "../objectStore/utils"
--- a/packages/backend-core/src/pkg/redis.ts
+++ b/packages/backend-core/src/pkg/redis.ts
@ -0,0 +1,11 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import Client from "../redis"
+import utils from "../redis/utils"
+import clients from "../redis/authRedis"
+
+export = {
+  Client,
+  utils,
+  clients,
+}
--- a/packages/backend-core/src/pkg/utils.ts
+++ b/packages/backend-core/src/pkg/utils.ts
@ -0,0 +1,4 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../utils"
+export * from "../hashing"
--- a/packages/backend-core/src/redis/authRedis.js
+++ b/packages/backend-core/src/redis/authRedis.js
@ -1,13 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
+const { getRedlock } = require("./redlock")

 let userClient, sessionClient, appClient, cacheClient
+let migrationsRedlock
+
+// turn retry off so that only one instance can ever hold the lock
+const migrationsRedlockConfig = { retryCount: 0 }

 async function init() {
  userClient = await new Client(utils.Databases.USER_CACHE).init()
  sessionClient = await new Client(utils.Databases.SESSIONS).init()
  appClient = await new Client(utils.Databases.APP_METADATA).init()
  cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  // pass the underlying ioredis client to redlock
+  migrationsRedlock = getRedlock(
+    cacheClient.getClient(),
+    migrationsRedlockConfig
+  )
 }

 process.on("exit", async () => {
@ -42,4 +52,10 @@ module.exports = {
    }
    return cacheClient
  },
+  getMigrationsRedlock: async () => {
+    if (!migrationsRedlock) {
+      await init()
+    }
+    return migrationsRedlock
+  },
 }
--- a/packages/backend-core/src/redis/index.js
+++ b/packages/backend-core/src/redis/index.js
@ -139,6 +139,10 @@ class RedisWrapper {
    this._db = db
  }

+  getClient() {
+    return CLIENT
+  }
+
  async init() {
    CLOSED = false
    init()
@ -164,6 +168,11 @@ class RedisWrapper {
    return promisifyStream(stream)
  }

+  async keys(pattern) {
+    const db = this._db
+    return CLIENT.keys(addDbPrefix(db, pattern))
+  }
+
  async get(key) {
    const db = this._db
    let response = await CLIENT.get(addDbPrefix(db, key))
--- a/packages/backend-core/src/redis/redlock.ts
+++ b/packages/backend-core/src/redis/redlock.ts
@ -0,0 +1,21 @@
+import Redlock from "redlock"
+
+export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+  return new Redlock([redisClient], {
+    // the expected clock drift; for more details
+    // see http://redis.io/topics/distlock
+    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
+
+    // the max number of times Redlock will attempt
+    // to lock a resource before erroring
+    retryCount: opts.retryCount,
+
+    // the time in ms between attempts
+    retryDelay: 200, // time in ms
+
+    // the max time in ms randomly added to retries
+    // to improve performance under high contention
+    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
+    retryJitter: 200, // time in ms
+  })
+}
--- a/packages/backend-core/src/tenancy/index.js
+++ b/packages/backend-core/src/tenancy/index.js
@ -1,4 +0,0 @@
-module.exports = {
-  ...require("../context"),
-  ...require("./tenancy"),
-}
--- a/packages/backend-core/src/tenancy/index.ts
+++ b/packages/backend-core/src/tenancy/index.ts
@ -0,0 +1,9 @@
+import * as context from "../context"
+import * as tenancy from "./tenancy"
+
+const pkg = {
+  ...context,
+  ...tenancy,
+}
+
+export = pkg
--- a/packages/backend-core/src/tenancy/tenancy.ts
+++ b/packages/backend-core/src/tenancy/tenancy.ts
@ -1,18 +1,18 @@
-const { doWithDB } = require("../db")
-const { StaticDatabases } = require("../db/constants")
-const { baseGlobalDBName } = require("./utils")
-const {
+import { doWithDB } from "../db"
+import { StaticDatabases } from "../db/constants"
+import { baseGlobalDBName } from "./utils"
+import {
  getTenantId,
  DEFAULT_TENANT_ID,
  isMultiTenant,
  getTenantIDFromAppID,
-} = require("../context")
-const env = require("../environment")
+} from "../context"
+import env from "../environment"

 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name

-exports.addTenantToUrl = url => {
+export const addTenantToUrl = (url: string) => {
  const tenantId = getTenantId()

  if (isMultiTenant()) {
@ -23,8 +23,8 @@ exports.addTenantToUrl = url => {
  return url
 }

-exports.doesTenantExist = async tenantId => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const doesTenantExist = async (tenantId: string) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
    let tenants
    try {
      tenants = await db.get(TENANT_DOC)
@ -40,9 +40,14 @@ exports.doesTenantExist = async tenantId => {
  })
 }

-exports.tryAddTenant = async (tenantId, userId, email) => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
-    const getDoc = async id => {
+export const tryAddTenant = async (
+  tenantId: string,
+  userId: string,
+  email: string,
+  afterCreateTenant: () => Promise<void>
+) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
+    const getDoc = async (id: string) => {
      if (!id) {
        return null
      }
@ -76,12 +81,13 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
    if (tenants.tenantIds.indexOf(tenantId) === -1) {
      tenants.tenantIds.push(tenantId)
      promises.push(db.put(tenants))
+      await afterCreateTenant()
    }
    await Promise.all(promises)
  })
 }

-exports.getGlobalDBName = (tenantId = null) => {
+export const getGlobalDBName = (tenantId?: string) => {
  // tenant ID can be set externally, for example user API where
  // new tenants are being created, this may be the case
  if (!tenantId) {
@ -90,12 +96,12 @@ exports.getGlobalDBName = (tenantId = null) => {
  return baseGlobalDBName(tenantId)
 }

-exports.doWithGlobalDB = (tenantId, cb) => {
-  return doWithDB(exports.getGlobalDBName(tenantId), cb)
+export const doWithGlobalDB = (tenantId: string, cb: any) => {
+  return doWithDB(getGlobalDBName(tenantId), cb)
 }

-exports.lookupTenantId = async userId => {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
+export const lookupTenantId = async (userId: string) => {
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
    let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
    try {
      const doc = await db.get(userId)
@ -110,8 +116,8 @@ exports.lookupTenantId = async userId => {
 }

 // lookup, could be email or userId, either will return a doc
-exports.getTenantUser = async identifier => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const getTenantUser = async (identifier: string) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
    try {
      return await db.get(identifier)
    } catch (err) {
@ -120,7 +126,7 @@ exports.getTenantUser = async identifier => {
  })
 }

-exports.isUserInAppTenant = (appId, user = null) => {
+export const isUserInAppTenant = (appId: string, user: any) => {
  let userTenantId
  if (user) {
    userTenantId = user.tenantId || DEFAULT_TENANT_ID
@ -131,8 +137,8 @@ exports.isUserInAppTenant = (appId, user = null) => {
  return tenantId === userTenantId
 }

-exports.getTenantIds = async () => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+export const getTenantIds = async () => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
    let tenants
    try {
      tenants = await db.get(TENANT_DOC)
--- a/packages/backend-core/src/tests/utils.spec.js
+++ b/packages/backend-core/src/tests/utils.spec.js
@ -0,0 +1,17 @@
+require("../../tests/utilities/TestConfiguration")
+const { structures } = require("../../tests/utilities")
+const utils = require("../utils")
+const events = require("../events")
+const { doInTenant, DEFAULT_TENANT_ID }= require("../context")
+
+describe("utils", () => {
+  describe("platformLogout", () => {
+    it("should call platform logout", async () => {
+      await doInTenant(DEFAULT_TENANT_ID, async () => {
+        const ctx = structures.koa.newContext()
+        await utils.platformLogout({ ctx, userId: "test" })
+        expect(events.auth.logout).toBeCalledTimes(1)
+      })
+    })
+  })
+})
--- a/packages/backend-core/src/users.js
+++ b/packages/backend-core/src/users.js
@ -0,0 +1,21 @@
+const { ViewNames } = require("./db/utils")
+const { queryGlobalView } = require("./db/views")
+
+/**
+ * Given an email address this will use a view to search through
+ * all the users to find one with this email address.
+ * @param {string} email the email to lookup the user by.
+ * @return {Promise<object|null>}
+ */
+exports.getGlobalUserByEmail = async email => {
+  if (email == null) {
+    throw "Must supply an email address to view"
+  }
+
+  const response = await queryGlobalView(ViewNames.USER_BY_EMAIL, {
+    key: email.toLowerCase(),
+    include_docs: true,
+  })
+
+  return response
+}
--- a/packages/backend-core/src/utils.js
+++ b/packages/backend-core/src/utils.js
@ -2,25 +2,16 @@ const {
  DocumentTypes,
  SEPARATOR,
  ViewNames,
-  generateGlobalUserID,
  getAllApps,
 } = require("./db/utils")
 const jwt = require("jsonwebtoken")
 const { options } = require("./middleware/passport/jwt")
 const { queryGlobalView } = require("./db/views")
-const { Headers, UserStatus, Cookies, MAX_VALID_DATE } = require("./constants")
-const {
-  doWithGlobalDB,
-  updateTenantId,
-  getTenantUser,
-  tryAddTenant,
-} = require("./tenancy")
-const environment = require("./environment")
-const accounts = require("./cloud/accounts")
-const { hash } = require("./hashing")
-const userCache = require("./cache/user")
+const { Headers, Cookies, MAX_VALID_DATE } = require("./constants")
 const env = require("./environment")
+const userCache = require("./cache/user")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
+const events = require("./events")
 const tenancy = require("./tenancy")

 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@ -135,8 +126,8 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
    overwrite: true,
  }

-  if (environment.COOKIE_DOMAIN) {
-    config.domain = environment.COOKIE_DOMAIN
+  if (env.COOKIE_DOMAIN) {
+    config.domain = env.COOKIE_DOMAIN
  }

  ctx.cookies.set(name, value, config)
@ -159,23 +150,6 @@ exports.isClient = ctx => {
  return ctx.headers[Headers.TYPE] === "client"
 }

-/**
- * Given an email address this will use a view to search through
- * all the users to find one with this email address.
- * @param {string} email the email to lookup the user by.
- * @return {Promise<object|null>}
- */
-exports.getGlobalUserByEmail = async email => {
-  if (email == null) {
-    throw "Must supply an email address to view"
-  }
-
-  return queryGlobalView(ViewNames.USER_BY_EMAIL, {
-    key: email.toLowerCase(),
-    include_docs: true,
-  })
-}
-
 const getBuilders = async () => {
  const builders = await queryGlobalView(ViewNames.USER_BY_BUILDERS, {
    include_docs: false,
@ -197,124 +171,6 @@ exports.getBuildersCount = async () => {
  return builders.length
 }

-const DEFAULT_SAVE_USER = {
-  hashPassword: true,
-  requirePassword: true,
-  bulkCreate: false,
-}
-
-exports.internalSaveUser = async (
-  user,
-  tenantId,
-  { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
-) => {
-  if (!tenantId) {
-    throw "No tenancy specified."
-  }
-  // need to set the context for this request, as specified
-  updateTenantId(tenantId)
-  // specify the tenancy incase we're making a new admin user (public)
-  return doWithGlobalDB(tenantId, async db => {
-    let { email, password, _id } = user
-    // make sure another user isn't using the same email
-    let dbUser
-    // user can't exist in bulk creation
-    if (bulkCreate) {
-      dbUser = null
-    } else if (email) {
-      // check budibase users inside the tenant
-      dbUser = await exports.getGlobalUserByEmail(email)
-      if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
-        throw `Email address ${email} already in use.`
-      }
-
-      // check budibase users in other tenants
-      if (env.MULTI_TENANCY) {
-        const tenantUser = await getTenantUser(email)
-        if (tenantUser != null && tenantUser.tenantId !== tenantId) {
-          throw `Email address ${email} already in use.`
-        }
-      }
-
-      // check root account users in account portal
-      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-        const account = await accounts.getAccount(email)
-        if (account && account.verified && account.tenantId !== tenantId) {
-          throw `Email address ${email} already in use.`
-        }
-      }
-    } else {
-      dbUser = await db.get(_id)
-    }
-
-    // get the password, make sure one is defined
-    let hashedPassword
-    if (password) {
-      hashedPassword = hashPassword ? await hash(password) : password
-    } else if (dbUser) {
-      hashedPassword = dbUser.password
-    } else if (requirePassword) {
-      throw "Password must be specified."
-    }
-
-    _id = _id || generateGlobalUserID()
-    user = {
-      createdAt: Date.now(),
-      ...dbUser,
-      ...user,
-      _id,
-      password: hashedPassword,
-      tenantId,
-    }
-    // make sure the roles object is always present
-    if (!user.roles) {
-      user.roles = {}
-    }
-    // add the active status to a user if its not provided
-    if (user.status == null) {
-      user.status = UserStatus.ACTIVE
-    }
-    try {
-      const putOpts = {
-        password: hashedPassword,
-        ...user,
-      }
-      if (bulkCreate) {
-        return putOpts
-      }
-      const response = await db.put(putOpts)
-      if (env.MULTI_TENANCY) {
-        await tryAddTenant(tenantId, _id, email)
-      }
-      await userCache.invalidateUser(response.id)
-      return {
-        _id: response.id,
-        _rev: response.rev,
-        email,
-      }
-    } catch (err) {
-      if (err.status === 409) {
-        throw "User exists already"
-      } else {
-        throw err
-      }
-    }
-  })
-}
-
-// maintained for api compat, don't want to change function signature
-exports.saveUser = async (
-  user,
-  tenantId,
-  hashPassword = true,
-  requirePassword = true
-) => {
-  return exports.internalSaveUser(user, tenantId, {
-    hashPassword,
-    requirePassword,
-  })
-}
-
 /**
 * Logs a user out from budibase. Re-used across account portal and builder.
 */
@ -338,5 +194,6 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
    userId,
    sessions.map(({ sessionId }) => sessionId)
  )
+  await events.auth.logout()
  await userCache.invalidateUser(userId)
 }
--- a/packages/backend-core/tests/index.ts
+++ b/packages/backend-core/tests/index.ts
@ -0,0 +1 @@
+export * from "./utilities"
--- a/packages/backend-core/tests/utilities/TestConfiguration.js
+++ b/packages/backend-core/tests/utilities/TestConfiguration.js
@ -0,0 +1 @@
+require("./db")
--- a/packages/backend-core/src/tests/utilities/dbConfig.js
+++ b/packages/backend-core/src/tests/utilities/dbConfig.js
@ -1,5 +1,6 @@
-const core = require("../../index")
+const core = require("../../src/index")
 const dbConfig = {
  inMemory: true,
+  allDbs: true,
 }
 core.init({ db: dbConfig })
--- a/packages/backend-core/tests/utilities/index.ts
+++ b/packages/backend-core/tests/utilities/index.ts
@ -0,0 +1,2 @@
+export * as mocks from "./mocks"
+export * as structures from "./structures"
--- a/Show More
+++ b/Show More