Merge branch 'develop' of github.com:Budibase/budibase into new-design-ui

.github/workflows/budibase_ci.yml

@@ -11,6 +11,7 @@ on:
     branches:
       - master
       - develop 
+      - release 
  workflow_dispatch:
 
 env:

.github/workflows/deploy-preprod.yml

@@ -4,9 +4,7 @@ on:
  workflow_dispatch:
 
 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
 
 jobs:

.github/workflows/release-develop.yml

@@ -4,7 +4,7 @@ concurrency: release-develop
 on: 
  push:
     branches:
-      - develop
+      - release
     paths:
       - '.aws/**'
       - '.github/**'
@@ -18,9 +18,9 @@ on:
  workflow_dispatch:
 
 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
       
 jobs:
@@ -28,11 +28,11 @@ jobs:
     runs-on: ubuntu-latest 
 
     steps:
-      - name: Fail if branch is not develop
+      # - name: Fail if branch is not develop
-        if: github.ref != 'refs/heads/develop' 
+      #   if: github.ref != 'refs/heads/develop' 
-        run: | 
+      #   run: | 
-          echo "Ref is not develop, you must run this job from develop."
+      #     echo "Ref is not develop, you must run this job from develop."
-          exit 1
+      #     exit 1
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:

.github/workflows/release.yml

@@ -18,9 +18,9 @@ on:
  workflow_dispatch:
 
 env:
-  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  # Posthog token used by ui at build time 
+  POSTHOG_TOKEN: phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
   SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
   PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
 

.gitignore

@@ -101,3 +101,5 @@ packages/builder/cypress.env.json
 packages/builder/cypress/reports
 stats.html
 
+# TypeScript cache
+*.tsbuildinfo

charts/budibase/templates/app-service-deployment.yaml

@@ -28,6 +28,8 @@ spec:
       - env:
         - name: BUDIBASE_ENVIRONMENT
           value: {{ .Values.globals.budibaseEnv }}
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
         - name: COUCH_DB_URL
           {{ if .Values.services.couchdb.url }}
           value: {{ .Values.services.couchdb.url }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -27,6 +27,8 @@ spec:
     spec:
       containers:
       - env:
+        - name: DEPLOYMENT_ENVIRONMENT
+          value: "kubernetes"
         - name: CLUSTER_PORT
           value: {{ .Values.services.worker.port | quote }}
         {{ if .Values.services.couchdb.enabled }}

charts/budibase/values.yaml

@@ -91,7 +91,7 @@ globals:
   budibaseEnv: PRODUCTION
   enableAnalytics: true
   sentryDSN: ""
-  posthogToken: ""
+  posthogToken: "phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS"
   logLevel: info
   selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
   multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs

hosting/nginx.prod.conf.hbs

@@ -48,7 +48,7 @@ http {
     set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
     set $csp_object "object-src 'none'";
     set $csp_base_uri "base-uri 'self'";
-    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
+    set $csp_connect "connect-src 'self' https://api-iam.intercom.io https://api-iam.intercom.io  https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io  wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io  https://uploads.intercomcdn.com  https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com";
     set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com  https://js.intercomcdn.com  https://fonts.intercomcdn.com";
     set $csp_frame "frame-src 'self' https:";
     set $csp_img "img-src http: https: data: blob:";

hosting/single/Dockerfile

@@ -1,5 +1,7 @@
 FROM couchdb
 
+ENV DEPLOYMENT_ENVIRONMENT=docker
+ENV POSTHOG_TOKEN=phc_fg5I3nDOf6oJVMHSaycEhpPdlgS8rzXG2r6F2IpxCHS
 ENV COUCHDB_PASSWORD=budibase
 ENV COUCHDB_USER=budibase
 ENV COUCH_DB_URL=http://budibase:budibase@localhost:5984

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.192-alpha.6",
+  "version": "1.0.200-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -22,8 +22,9 @@
   },
   "scripts": {
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "lerna link && lerna bootstrap && ./scripts/link-dependencies.sh",
+    "bootstrap": "lerna bootstrap && lerna link && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
+    "build:dev": "lerna run prebuild && tsc --build --watch --preserveWatchOutput",
     "release": "lerna publish patch --yes --force-publish && yarn release:pro",
     "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop",
     "release:pro": "bash scripts/pro/release.sh",
@@ -37,8 +38,8 @@
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
     "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
+    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
+    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test",
     "lint:eslint": "eslint packages",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",

packages/backend-core/.gitignore

@@ -44,9 +44,6 @@ jspm_packages/
 # Snowpack dependency directory (https://snowpack.dev/)
 web_modules/
 
-# TypeScript cache
-*.tsbuildinfo
-
 # Optional npm cache directory
 .npm
 

packages/backend-core/context.js

@@ -5,8 +5,11 @@ const {
   getAppId,
   updateAppId,
   doInAppContext,
+  doInTenant,
 } = require("./src/context")
 
+const identity = require("./src/context/identity")
+
 module.exports = {
   getAppDB,
   getDevAppDB,
@@ -14,4 +17,6 @@ module.exports = {
   getAppId,
   updateAppId,
   doInAppContext,
+  doInTenant,
+  identity,
 }

packages/backend-core/package.json

@@ -1,48 +1,79 @@
 {
   "name": "@budibase/backend-core",
-  "version": "1.0.192-alpha.6",
+  "version": "1.0.200-alpha.3",
   "description": "Budibase backend core libraries used in server and worker",
-  "main": "src/index.js",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
+  "exports": {
+    ".": "./dist/src/index.js",
+    "./tests": "./dist/tests/index.js",
+    "./*": "./dist/*.js"
+  },
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {
+    "prebuild": "rimraf dist/",
+    "build": "tsc -p tsconfig.build.json",
+    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "test": "jest",
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
-    "@techpass/passport-openidconnect": "^0.3.0",
+    "@techpass/passport-openidconnect": "0.3.2",
-    "aws-sdk": "^2.901.0",
+    "aws-sdk": "2.1030.0",
-    "bcrypt": "^5.0.1",
+    "bcrypt": "5.0.1",
-    "dotenv": "^16.0.1",
+    "dotenv": "16.0.1",
-    "emitter-listener": "^1.1.2",
+    "emitter-listener": "1.1.2",
-    "ioredis": "^4.27.1",
+    "ioredis": "4.28.0",
-    "jsonwebtoken": "^8.5.1",
+    "jsonwebtoken": "8.5.1",
-    "koa-passport": "^4.1.4",
+    "koa-passport": "4.1.4",
-    "lodash": "^4.17.21",
+    "lodash": "4.17.21",
-    "lodash.isarguments": "^3.1.0",
+    "lodash.isarguments": "3.1.0",
-    "node-fetch": "^2.6.1",
+    "node-fetch": "2.6.7",
-    "passport-google-auth": "^1.0.2",
+    "passport-google-auth": "1.0.2",
-    "passport-google-oauth": "^2.0.0",
+    "passport-google-oauth": "2.0.0",
-    "passport-jwt": "^4.0.0",
+    "passport-jwt": "4.0.0",
-    "passport-local": "^1.0.0",
+    "passport-local": "1.0.0",
-    "posthog-node": "^1.3.0",
+    "posthog-node": "1.3.0",
     "pouchdb": "7.3.0",
-    "pouchdb-find": "^7.2.2",
+    "pouchdb-find": "7.2.2",
-    "pouchdb-replication-stream": "^1.2.9",
+    "pouchdb-replication-stream": "1.2.9",
-    "sanitize-s3-objectkey": "^0.0.1",
+    "redlock": "4.2.0",
-    "tar-fs": "^2.1.1",
+    "sanitize-s3-objectkey": "0.0.1",
-    "uuid": "^8.3.2",
+    "semver": "7.3.7",
-    "zlib": "^1.0.5"
+    "tar-fs": "2.1.1",
+    "uuid": "8.3.2",
+    "zlib": "1.0.5"
   },
   "jest": {
+    "preset": "ts-jest",
+    "testEnvironment": "node",
+    "moduleNameMapper": {
+      "@budibase/types": "<rootDir>/../types/src"
+    },
     "setupFiles": [
-      "./scripts/jestSetup.js"
+      "./scripts/jestSetup.ts"
     ]
   },
   "devDependencies": {
-    "ioredis-mock": "^5.5.5",
+    "@budibase/types": "^1.0.200-alpha.3",
-    "jest": "^26.6.3",
+    "@shopify/jest-koa-mocks": "3.1.5",
-    "pouchdb-adapter-memory": "^7.2.2"
+    "@types/jest": "27.5.1",
+    "@types/koa": "2.0.52",
+    "@types/node": "14.18.20",
+    "@types/node-fetch": "2.6.1",
+    "@types/redlock": "4.0.3",
+    "@types/semver": "7.3.7",
+    "@types/tar-fs": "2.0.1",
+    "@types/uuid": "8.3.4",
+    "ioredis-mock": "5.8.0",
+    "jest": "27.5.1",
+    "koa": "2.7.0",
+    "nodemon": "2.0.16",
+    "pouchdb-adapter-memory": "7.2.2",
+    "timekeeper": "2.2.0",
+    "ts-jest": "27.1.5",
+    "typescript": "4.7.3"
   },
   "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }

packages/backend-core/redis.js

@@ -1,4 +1,5 @@
 module.exports = {
   Client: require("./src/redis"),
   utils: require("./src/redis/utils"),
+  clients: require("./src/redis/authRedis"),
 }

packages/backend-core/scripts/jestSetup.js

@@ -1,6 +0,0 @@
-const env = require("../src/environment")
-
-env._set("SELF_HOSTED", "1")
-env._set("NODE_ENV", "jest")
-env._set("JWT_SECRET", "test-jwtsecret")
-env._set("LOG_LEVEL", "silent")

packages/backend-core/scripts/jestSetup.ts

@@ -0,0 +1,12 @@
+import env from "../src/environment"
+import { mocks } from "../tests/utilities"
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+import tk from "timekeeper"
+tk.freeze(mocks.date.MOCK_DATE)
+
+env._set("SELF_HOSTED", "1")
+env._set("NODE_ENV", "jest")
+env._set("JWT_SECRET", "test-jwtsecret")
+env._set("LOG_LEVEL", "silent")

packages/backend-core/src/cache/generic.js

@@ -1,9 +1,13 @@
 const redis = require("../redis/authRedis")
-const env = require("../environment")
 const { getTenantId } = require("../context")
 
 exports.CacheKeys = {
   CHECKLIST: "checklist",
+  INSTALLATION: "installation",
+  ANALYTICS_ENABLED: "analyticsEnabled",
+  UNIQUE_TENANT_ID: "uniqueTenantId",
+  EVENTS: "events",
+  BACKFILL_METADATA: "backfillMetadata",
 }
 
 exports.TTL = {
@@ -17,10 +21,41 @@ function generateTenantKey(key) {
   return `${key}:${tenantId}`
 }
 
-exports.withCache = async (key, ttl, fetchFn) => {
+exports.keys = async pattern => {
-  key = generateTenantKey(key)
   const client = await redis.getCacheClient()
-  const cachedValue = await client.get(key)
+  return client.keys(pattern)
+}
+
+/**
+ * Read only from the cache.
+ */
+exports.get = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  const value = await client.get(key)
+  return value
+}
+
+/**
+ * Write to the cache.
+ */
+exports.store = async (key, value, ttl, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  await client.store(key, value, ttl)
+}
+
+exports.delete = async (key, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
+  const client = await redis.getCacheClient()
+  return client.delete(key)
+}
+
+/**
+ * Read from the cache. Write to the cache if not exists.
+ */
+exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+  const cachedValue = await exports.get(key, opts)
   if (cachedValue) {
     return cachedValue
   }
@@ -28,9 +63,7 @@ exports.withCache = async (key, ttl, fetchFn) => {
   try {
     const fetchedValue = await fetchFn()
 
-    if (!env.isTest()) {
+    await exports.store(key, fetchedValue, ttl, opts)
-      await client.store(key, fetchedValue, ttl)
-    }
     return fetchedValue
   } catch (err) {
     console.error("Error fetching before cache - ", err)

packages/backend-core/src/cloud/accounts.js

@@ -1,39 +0,0 @@
-const API = require("./api")
-const env = require("../environment")
-const { Headers } = require("../constants")
-
-const api = new API(env.ACCOUNT_PORTAL_URL)
-
-exports.getAccount = async email => {
-  const payload = {
-    email,
-  }
-  const response = await api.post(`/api/accounts/search`, {
-    body: payload,
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting account by email ${email}`, json)
-  }
-
-  return json[0]
-}
-
-exports.getStatus = async () => {
-  const response = await api.get(`/api/status`, {
-    headers: {
-      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
-    },
-  })
-  const json = await response.json()
-
-  if (response.status !== 200) {
-    throw new Error(`Error getting status`)
-  }
-
-  return json
-}

packages/backend-core/src/cloud/accounts.ts

@@ -0,0 +1,63 @@
+import API from "./api"
+import env from "../environment"
+import { Headers } from "../constants"
+import { CloudAccount } from "@budibase/types"
+
+const api = new API(env.ACCOUNT_PORTAL_URL)
+
+export const getAccount = async (
+  email: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    email,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by email ${email}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getAccountByTenantId = async (
+  tenantId: string
+): Promise<CloudAccount | undefined> => {
+  const payload = {
+    tenantId,
+  }
+  const response = await api.post(`/api/accounts/search`, {
+    body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting account by tenantId ${tenantId}`)
+  }
+
+  const json: CloudAccount[] = await response.json()
+  return json[0]
+}
+
+export const getStatus = async () => {
+  const response = await api.get(`/api/status`, {
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
+  })
+  const json = await response.json()
+
+  if (response.status !== 200) {
+    throw new Error(`Error getting status`)
+  }
+
+  return json
+}

packages/backend-core/src/context/identity.ts

@@ -0,0 +1,50 @@
+import {
+  IdentityContext,
+  IdentityType,
+  User,
+  UserContext,
+  isCloudAccount,
+  Account,
+  AccountUserContext,
+} from "@budibase/types"
+import * as context from "."
+
+export const getIdentity = (): IdentityContext | undefined => {
+  return context.getIdentity()
+}
+
+export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+  return context.doInIdentityContext(identity, task)
+}
+
+export const doInUserContext = (user: User, task: any) => {
+  const userContext: UserContext = {
+    ...user,
+    _id: user._id as string,
+    type: IdentityType.USER,
+  }
+  return doInIdentityContext(userContext, task)
+}
+
+export const doInAccountContext = (account: Account, task: any) => {
+  const _id = getAccountUserId(account)
+  const tenantId = account.tenantId
+  const accountContext: AccountUserContext = {
+    _id,
+    type: IdentityType.USER,
+    tenantId,
+    account,
+  }
+  return doInIdentityContext(accountContext, task)
+}
+
+export const getAccountUserId = (account: Account) => {
+  let userId: string
+  if (isCloudAccount(account)) {
+    userId = account.budibaseUserId
+  } else {
+    // use account id as user id for self hosting
+    userId = account.accountId
+  }
+  return userId
+}

packages/backend-core/src/context/index.js

@@ -1,5 +1,4 @@
 const env = require("../environment")
-const { Headers } = require("../../constants")
 const { SEPARATOR, DocumentTypes } = require("../db/constants")
 const { DEFAULT_TENANT_ID } = require("../constants")
 const cls = require("./FunctionContext")
@@ -16,6 +15,7 @@ const ContextKeys = {
   TENANT_ID: "tenantId",
   GLOBAL_DB: "globalDb",
   APP_ID: "appId",
+  IDENTITY: "identity",
   // whatever the request app DB was
   CURRENT_DB: "currentDb",
   // get the prod app DB from the request
@@ -79,10 +79,7 @@ exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
   async function internal(opts = { existing: false }) {
     // set the tenant id
     if (!opts.existing) {
-      cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
+      exports.updateTenantId(tenantId)
-      if (env.USE_COUCH) {
-        exports.setGlobalDB(tenantId)
-      }
     }
 
     try {
@@ -97,6 +94,7 @@ exports.doInTenant = (tenantId, task, { forceNew } = {}) => {
       }
     }
   }
+
   const using = cls.getFromContext(ContextKeys.IN_USE)
   if (
     !forceNew &&
@@ -144,6 +142,8 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
     throw new Error("appId is required")
   }
 
+  const identity = exports.getIdentity()
+
   // the internal function is so that we can re-use an existing
   // context - don't want to close DB on a parent context
   async function internal(opts = { existing: false }) {
@@ -153,6 +153,8 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
     }
     // set the app ID
     cls.setOnContext(ContextKeys.APP_ID, appId)
+    // preserve the identity
+    exports.setIdentity(identity)
     try {
       // invoke the task
       return await task()
@@ -177,9 +179,63 @@ exports.doInAppContext = (appId, task, { forceNew } = {}) => {
   }
 }
 
+exports.doInIdentityContext = (identity, task) => {
+  if (!identity) {
+    throw new Error("identity is required")
+  }
+
+  async function internal(opts = { existing: false }) {
+    if (!opts.existing) {
+      cls.setOnContext(ContextKeys.IDENTITY, identity)
+      // set the tenant so that doInTenant will preserve identity
+      if (identity.tenantId) {
+        exports.updateTenantId(identity.tenantId)
+      }
+    }
+
+    try {
+      // invoke the task
+      return await task()
+    } finally {
+      const using = cls.getFromContext(ContextKeys.IN_USE)
+      if (!using || using <= 1) {
+        exports.setIdentity(null)
+      } else {
+        cls.setOnContext(using - 1)
+      }
+    }
+  }
+
+  const existing = cls.getFromContext(ContextKeys.IDENTITY)
+  const using = cls.getFromContext(ContextKeys.IN_USE)
+  if (using && existing && existing._id === identity._id) {
+    cls.setOnContext(ContextKeys.IN_USE, using + 1)
+    return internal({ existing: true })
+  } else {
+    return cls.run(async () => {
+      cls.setOnContext(ContextKeys.IN_USE, 1)
+      return internal({ existing: false })
+    })
+  }
+}
+
+exports.setIdentity = identity => {
+  cls.setOnContext(ContextKeys.IDENTITY, identity)
+}
+
+exports.getIdentity = () => {
+  try {
+    return cls.getFromContext(ContextKeys.IDENTITY)
+  } catch (e) {
+    // do nothing - identity is not in context
+  }
+}
+
 exports.updateTenantId = tenantId => {
   cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
-  exports.setGlobalDB(tenantId)
+  if (env.USE_COUCH) {
+    exports.setGlobalDB(tenantId)
+  }
 }
 
 exports.updateAppId = async appId => {
@@ -196,45 +252,6 @@ exports.updateAppId = async appId => {
   }
 }
 
-exports.setTenantId = (
-  ctx,
-  opts = { allowQs: false, allowNoTenant: false }
-) => {
-  let tenantId
-  // exit early if not multi-tenant
-  if (!exports.isMultiTenant()) {
-    cls.setOnContext(ContextKeys.TENANT_ID, exports.DEFAULT_TENANT_ID)
-    return exports.DEFAULT_TENANT_ID
-  }
-
-  const allowQs = opts && opts.allowQs
-  const allowNoTenant = opts && opts.allowNoTenant
-  const header = ctx.request.headers[Headers.TENANT_ID]
-  const user = ctx.user || {}
-  if (allowQs) {
-    const query = ctx.request.query || {}
-    tenantId = query.tenantId
-  }
-  // override query string (if allowed) by user, or header
-  // URL params cannot be used in a middleware, as they are
-  // processed later in the chain
-  tenantId = user.tenantId || header || tenantId
-
-  // Set the tenantId from the subdomain
-  if (!tenantId) {
-    tenantId = ctx.subdomains && ctx.subdomains[0]
-  }
-
-  if (!tenantId && !allowNoTenant) {
-    ctx.throw(403, "Tenant id not set")
-  }
-  // check tenant ID just incase no tenant was allowed
-  if (tenantId) {
-    cls.setOnContext(ContextKeys.TENANT_ID, tenantId)
-  }
-  return tenantId
-}
-
 exports.setGlobalDB = tenantId => {
   const dbName = baseGlobalDBName(tenantId)
   const db = dangerousGetDB(dbName)
@@ -315,7 +332,7 @@ function getContextDB(key, opts) {
  * Opens the app database based on whatever the request
  * contained, dev or prod.
  */
-exports.getAppDB = opts => {
+exports.getAppDB = (opts = null) => {
   return getContextDB(ContextKeys.CURRENT_DB, opts)
 }
 
@@ -323,7 +340,7 @@ exports.getAppDB = opts => {
  * This specifically gets the prod app ID, if the request
  * contained a development app ID, this will open the prod one.
  */
-exports.getProdAppDB = opts => {
+exports.getProdAppDB = (opts = null) => {
   return getContextDB(ContextKeys.PROD_DB, opts)
 }
 
@@ -331,6 +348,6 @@ exports.getProdAppDB = opts => {
  * This specifically gets the dev app ID, if the request
  * contained a prod app ID, this will open the dev one.
  */
-exports.getDevAppDB = opts => {
+exports.getDevAppDB = (opts = null) => {
   return getContextDB(ContextKeys.DEV_DB, opts)
 }

packages/backend-core/src/db/Replication.ts

@@ -1,12 +1,16 @@
-const { dangerousGetDB, closeDB } = require(".")
+import { dangerousGetDB, closeDB } from "."
 
 class Replication {
+  source: any
+  target: any
+  replication: any
+
   /**
    *
    * @param {String} source - the DB you want to replicate or rollback to
    * @param {String} target - the DB you want to replicate to, or rollback from
    */
-  constructor({ source, target }) {
+  constructor({ source, target }: any) {
     this.source = dangerousGetDB(source)
     this.target = dangerousGetDB(target)
   }
@@ -15,17 +19,17 @@ class Replication {
     return Promise.all([closeDB(this.source), closeDB(this.target)])
   }
 
-  promisify(operation, opts = {}) {
+  promisify(operation: any, opts = {}) {
     return new Promise(resolve => {
       operation(this.target, opts)
-        .on("denied", function (err) {
+        .on("denied", function (err: any) {
           // a document failed to replicate (e.g. due to permissions)
           throw new Error(`Denied: Document failed to replicate ${err}`)
         })
-        .on("complete", function (info) {
+        .on("complete", function (info: any) {
           return resolve(info)
         })
-        .on("error", function (err) {
+        .on("error", function (err: any) {
           throw new Error(`Replication Error: ${err}`)
         })
     })
@@ -64,4 +68,4 @@ class Replication {
   }
 }
 
-module.exports = Replication
+export default Replication

packages/backend-core/src/db/constants.js

@@ -31,6 +31,7 @@ exports.StaticDatabases = {
     name: "global-info",
     docs: {
       tenants: "tenants",
+      install: "install",
     },
   },
 }

packages/backend-core/src/db/index.js

@@ -8,8 +8,11 @@ const dbList = new Set()
 const put =
   dbPut =>
   async (doc, options = {}) => {
-    // TODO: add created / updated
+    if (!doc.createdAt) {
-    return await dbPut(doc, options)
+      doc.createdAt = new Date().toISOString()
+    }
+    doc.updatedAt = new Date().toISOString()
+    return dbPut(doc, options)
   }
 
 const checkInitialised = () => {
@@ -54,7 +57,7 @@ exports.closeDB = async db => {
 // we have to use a callback for this so that we can close
 // the DB when we're done, without this manual requests would
 // need to close the database when done with it to avoid memory leaks
-exports.doWithDB = async (dbName, cb, opts) => {
+exports.doWithDB = async (dbName, cb, opts = {}) => {
   const db = exports.dangerousGetDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
   // async operations have been completed

packages/backend-core/src/db/tests/index.spec.js

@@ -0,0 +1,26 @@
+require("../../../tests/utilities/TestConfiguration")
+const { dangerousGetDB } = require("../")
+
+describe("db", () => { 
+  
+  describe("getDB", () => {
+    it("returns a db", async () => {
+      const db = dangerousGetDB("test")
+      expect(db).toBeDefined()
+      expect(db._adapter).toBe("memory")
+      expect(db.prefix).toBe("_pouch_")
+      expect(db.name).toBe("test")
+    })
+
+    it("uses the custom put function", async () => {
+      const db = dangerousGetDB("test")
+      let doc = { _id: "test" }
+      await db.put(doc)
+      doc = await db.get(doc._id)
+      expect(doc.createdAt).toBe(new Date().toISOString())
+      expect(doc.updatedAt).toBe(new Date().toISOString())
+      await db.destroy()
+    })
+  })
+})
+

packages/backend-core/src/db/tests/utils.spec.js

@@ -1,4 +1,4 @@
-require("../../tests/utilities/dbConfig");
+require("../../../tests/utilities/TestConfiguration");
 const {
   generateAppID,
   getDevelopmentAppID,

packages/backend-core/src/db/utils.ts

@@ -1,53 +1,34 @@
-const { newid } = require("../hashing")
+import { newid } from "../hashing"
-const Replication = require("./Replication")
+import { DEFAULT_TENANT_ID, Configs } from "../constants"
-const { DEFAULT_TENANT_ID, Configs } = require("../constants")
+import env from "../environment"
-const env = require("../environment")
+import { SEPARATOR, DocumentTypes } from "./constants"
-const {
+import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
-  StaticDatabases,
+import fetch from "node-fetch"
-  SEPARATOR,
+import { doWithDB, allDbs } from "./index"
-  DocumentTypes,
+import { getCouchInfo } from "./pouch"
-  APP_PREFIX,
+import { getAppMetadata } from "../cache/appMetadata"
-  APP_DEV,
+import { checkSlashesInUrl } from "../helpers"
-} = require("./constants")
+import { isDevApp, isDevAppID } from "./conversions"
-const { getTenantId, getGlobalDBName, getGlobalDB } = require("../tenancy")
+import { APP_PREFIX } from "./constants"
-const fetch = require("node-fetch")
+import * as events from "../events"
-const { doWithDB, allDbs } = require("./index")
-const { getCouchInfo } = require("./pouch")
-const { getAppMetadata } = require("../cache/appMetadata")
-const { checkSlashesInUrl } = require("../helpers")
-const {
-  isDevApp,
-  isProdAppID,
-  isDevAppID,
-  getDevelopmentAppID,
-  getProdAppID,
-} = require("./conversions")
 
 const UNICODE_MAX = "\ufff0"
 
-exports.ViewNames = {
+export const ViewNames = {
   USER_BY_EMAIL: "by_email",
   BY_API_KEY: "by_api_key",
   USER_BY_BUILDERS: "by_builders",
 }
 
-exports.StaticDatabases = StaticDatabases
+export * from "./constants"
-
+export * from "./conversions"
-exports.DocumentTypes = DocumentTypes
+export { default as Replication } from "./Replication"
-exports.APP_PREFIX = APP_PREFIX
-exports.APP_DEV = exports.APP_DEV_PREFIX = APP_DEV
-exports.SEPARATOR = SEPARATOR
-exports.isDevApp = isDevApp
-exports.isProdAppID = isProdAppID
-exports.isDevAppID = isDevAppID
-exports.getDevelopmentAppID = getDevelopmentAppID
-exports.getProdAppID = getProdAppID
 
 /**
  * Generates a new app ID.
  * @returns {string} The new app ID which the app doc can be stored under.
  */
-exports.generateAppID = (tenantId = null) => {
+export const generateAppID = (tenantId = null) => {
   let id = APP_PREFIX
   if (tenantId) {
     id += `${tenantId}${SEPARATOR}`
@@ -67,7 +48,11 @@ exports.generateAppID = (tenantId = null) => {
  * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
  * @returns {object} Parameters which can then be used with an allDocs request.
  */
-function getDocParams(docType, docId = null, otherProps = {}) {
+export function getDocParams(
+  docType: any,
+  docId: any = null,
+  otherProps: any = {}
+) {
   if (docId == null) {
     docId = ""
   }
@@ -77,20 +62,19 @@ function getDocParams(docType, docId = null, otherProps = {}) {
     endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
   }
 }
-exports.getDocParams = getDocParams
 
 /**
  * Generates a new workspace ID.
  * @returns {string} The new workspace ID which the workspace doc can be stored under.
  */
-exports.generateWorkspaceID = () => {
+export function generateWorkspaceID() {
   return `${DocumentTypes.WORKSPACE}${SEPARATOR}${newid()}`
 }
 
 /**
  * Gets parameters for retrieving workspaces.
  */
-exports.getWorkspaceParams = (id = "", otherProps = {}) => {
+export function getWorkspaceParams(id = "", otherProps = {}) {
   return {
     ...otherProps,
     startkey: `${DocumentTypes.WORKSPACE}${SEPARATOR}${id}`,
@@ -102,14 +86,14 @@ exports.getWorkspaceParams = (id = "", otherProps = {}) => {
  * Generates a new global user ID.
  * @returns {string} The new user ID which the user doc can be stored under.
  */
-exports.generateGlobalUserID = id => {
+export function generateGlobalUserID(id?: any) {
   return `${DocumentTypes.USER}${SEPARATOR}${id || newid()}`
 }
 
 /**
  * Gets parameters for retrieving users.
  */
-exports.getGlobalUserParams = (globalId, otherProps = {}) => {
+export function getGlobalUserParams(globalId: any, otherProps = {}) {
   if (!globalId) {
     globalId = ""
   }
@@ -124,14 +108,18 @@ exports.getGlobalUserParams = (globalId, otherProps = {}) => {
  * Generates a template ID.
  * @param ownerId The owner/user of the template, this could be global or a workspace level.
  */
-exports.generateTemplateID = ownerId => {
+export function generateTemplateID(ownerId: any) {
   return `${DocumentTypes.TEMPLATE}${SEPARATOR}${ownerId}${SEPARATOR}${newid()}`
 }
 
 /**
  * Gets parameters for retrieving templates. Owner ID must be specified, either global or a workspace level.
  */
-exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
+export function getTemplateParams(
+  ownerId: any,
+  templateId: any,
+  otherProps = {}
+) {
   if (!templateId) {
     templateId = ""
   }
@@ -152,18 +140,18 @@ exports.getTemplateParams = (ownerId, templateId, otherProps = {}) => {
  * Generates a new role ID.
  * @returns {string} The new role ID which the role doc can be stored under.
  */
-exports.generateRoleID = id => {
+export function generateRoleID(id: any) {
   return `${DocumentTypes.ROLE}${SEPARATOR}${id || newid()}`
 }
 
 /**
  * Gets parameters for retrieving a role, this is a utility function for the getDocParams function.
  */
-exports.getRoleParams = (roleId = null, otherProps = {}) => {
+export function getRoleParams(roleId = null, otherProps = {}) {
   return getDocParams(DocumentTypes.ROLE, roleId, otherProps)
 }
 
-exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
+export function getStartEndKeyURL(base: any, baseKey: any, tenantId = null) {
   const tenancy = tenantId ? `${SEPARATOR}${tenantId}` : ""
   return `${base}?startkey="${baseKey}${tenancy}"&endkey="${baseKey}${tenancy}${UNICODE_MAX}"`
 }
@@ -174,15 +162,15 @@ exports.getStartEndKeyURL = (base, baseKey, tenantId = null) => {
  * opts.efficient can be provided to make sure this call is always quick in a multi-tenant environment,
  * but it may not be 100% accurate in full efficiency mode (some tenantless apps may be missed).
  */
-exports.getAllDbs = async (opts = { efficient: false }) => {
+export async function getAllDbs(opts = { efficient: false }) {
   const efficient = opts && opts.efficient
   // specifically for testing we use the pouch package for this
   if (env.isTest()) {
     return allDbs()
   }
-  let dbs = []
+  let dbs: any[] = []
   let { url, cookie } = getCouchInfo()
-  async function addDbs(couchUrl) {
+  async function addDbs(couchUrl: string) {
     const response = await fetch(checkSlashesInUrl(encodeURI(couchUrl)), {
       method: "GET",
       headers: {
@@ -207,13 +195,9 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
     await addDbs(couchUrl)
   } else {
     // get prod apps
-    await addDbs(
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId))
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
-    )
     // get dev apps
-    await addDbs(
+    await addDbs(getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId))
-      exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP_DEV, tenantId)
-    )
     // add global db name
     dbs.push(getGlobalDBName(tenantId))
   }
@@ -226,13 +210,13 @@ exports.getAllDbs = async (opts = { efficient: false }) => {
  *
  * @return {Promise<object[]>} returns the app information document stored in each app database.
  */
-exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
+export async function getAllApps({ dev, all, idsOnly, efficient }: any = {}) {
   let tenantId = getTenantId()
   if (!env.MULTI_TENANCY && !tenantId) {
     tenantId = DEFAULT_TENANT_ID
   }
-  let dbs = await exports.getAllDbs({ efficient })
+  let dbs = await getAllDbs({ efficient })
-  const appDbNames = dbs.filter(dbName => {
+  const appDbNames = dbs.filter((dbName: any) => {
     const split = dbName.split(SEPARATOR)
     // it is an app, check the tenantId
     if (split[0] === DocumentTypes.APP) {
@@ -252,7 +236,7 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
   if (idsOnly) {
     return appDbNames
   }
-  const appPromises = appDbNames.map(app =>
+  const appPromises = appDbNames.map((app: any) =>
     // skip setup otherwise databases could be re-created
     getAppMetadata(app)
   )
@@ -261,17 +245,19 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
   } else {
     const response = await Promise.allSettled(appPromises)
     const apps = response
-      .filter(result => result.status === "fulfilled" && result.value != null)
+      .filter(
-      .map(({ value }) => value)
+        (result: any) => result.status === "fulfilled" && result.value != null
+      )
+      .map(({ value }: any) => value)
     if (!all) {
-      return apps.filter(app => {
+      return apps.filter((app: any) => {
         if (dev) {
           return isDevApp(app)
         }
         return !isDevApp(app)
       })
     } else {
-      return apps.map(app => ({
+      return apps.map((app: any) => ({
         ...app,
         status: isDevApp(app) ? "development" : "published",
       }))
@@ -282,26 +268,26 @@ exports.getAllApps = async ({ dev, all, idsOnly, efficient } = {}) => {
 /**
  * Utility function for getAllApps but filters to production apps only.
  */
-exports.getProdAppIDs = async () => {
+export async function getProdAppIDs() {
-  return (await exports.getAllApps({ idsOnly: true })).filter(
+  return (await getAllApps({ idsOnly: true })).filter(
-    id => !exports.isDevAppID(id)
+    (id: any) => !isDevAppID(id)
   )
 }
 
 /**
  * Utility function for the inverse of above.
  */
-exports.getDevAppIDs = async () => {
+export async function getDevAppIDs() {
-  return (await exports.getAllApps({ idsOnly: true })).filter(id =>
+  return (await getAllApps({ idsOnly: true })).filter((id: any) =>
-    exports.isDevAppID(id)
+    isDevAppID(id)
   )
 }
 
-exports.dbExists = async dbName => {
+export async function dbExists(dbName: any) {
   let exists = false
   return doWithDB(
     dbName,
-    async db => {
+    async (db: any) => {
       try {
         // check if database exists
         const info = await db.info()
@@ -321,7 +307,7 @@ exports.dbExists = async dbName => {
  * Generates a new configuration ID.
  * @returns {string} The new configuration ID which the config doc can be stored under.
  */
-const generateConfigID = ({ type, workspace, user }) => {
+export const generateConfigID = ({ type, workspace, user }: any) => {
   const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)
 
   return `${DocumentTypes.CONFIG}${SEPARATOR}${scope}`
@@ -330,7 +316,10 @@ const generateConfigID = ({ type, workspace, user }) => {
 /**
  * Gets parameters for retrieving configurations.
  */
-const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
+export const getConfigParams = (
+  { type, workspace, user }: any,
+  otherProps = {}
+) => {
   const scope = [type, workspace, user].filter(Boolean).join(SEPARATOR)
 
   return {
@@ -344,7 +333,7 @@ const getConfigParams = ({ type, workspace, user }, otherProps = {}) => {
  * Generates a new dev info document ID - this is scoped to a user.
  * @returns {string} The new dev info ID which info for dev (like api key) can be stored under.
  */
-const generateDevInfoID = userId => {
+export const generateDevInfoID = (userId: any) => {
   return `${DocumentTypes.DEV_INFO}${SEPARATOR}${userId}`
 }
 
@@ -354,7 +343,10 @@ const generateDevInfoID = userId => {
  * @param {Object} scopes - the type, workspace and userID scopes of the configuration.
  * @returns The most granular configuration document based on the scope.
  */
-const getScopedFullConfig = async function (db, { type, user, workspace }) {
+export const getScopedFullConfig = async function (
+  db: any,
+  { type, user, workspace }: any
+) {
   const response = await db.allDocs(
     getConfigParams(
       { type, user, workspace },
@@ -364,7 +356,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
     )
   )
 
-  function determineScore(row) {
+  function determineScore(row: any) {
     const config = row.doc
 
     // Config is specific to a user and a workspace
@@ -385,19 +377,24 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
 
   // Find the config with the most granular scope based on context
   let scopedConfig = response.rows.sort(
-    (a, b) => determineScore(a) - determineScore(b)
+    (a: any, b: any) => determineScore(a) - determineScore(b)
   )[0]
 
   // custom logic for settings doc
-  // always provide the platform URL
   if (type === Configs.SETTINGS) {
     if (scopedConfig && scopedConfig.doc) {
+      // overrides affected by environment variables
       scopedConfig.doc.config.platformUrl = await getPlatformUrl()
+      scopedConfig.doc.config.analyticsEnabled =
+        await events.analytics.enabled()
     } else {
+      // defaults
       scopedConfig = {
         doc: {
+          _id: generateConfigID({ type, user, workspace }),
           config: {
             platformUrl: await getPlatformUrl(),
+            analyticsEnabled: await events.analytics.enabled(),
           },
         },
       }
@@ -407,7 +404,7 @@ const getScopedFullConfig = async function (db, { type, user, workspace }) {
   return scopedConfig && scopedConfig.doc
 }
 
-const getPlatformUrl = async (opts = { tenantAware: true }) => {
+export const getPlatformUrl = async (opts = { tenantAware: true }) => {
   let platformUrl = env.PLATFORM_URL || "http://localhost:10000"
 
   if (!env.SELF_HOSTED && env.MULTI_TENANCY && opts.tenantAware) {
@@ -422,7 +419,7 @@ const getPlatformUrl = async (opts = { tenantAware: true }) => {
     let settings
     try {
       settings = await db.get(generateConfigID({ type: Configs.SETTINGS }))
-    } catch (e) {
+    } catch (e: any) {
       if (e.status !== 404) {
         throw e
       }
@@ -437,15 +434,7 @@ const getPlatformUrl = async (opts = { tenantAware: true }) => {
   return platformUrl
 }
 
-async function getScopedConfig(db, params) {
+export async function getScopedConfig(db: any, params: any) {
   const configDoc = await getScopedFullConfig(db, params)
   return configDoc && configDoc.config ? configDoc.config : configDoc
 }
-
-exports.Replication = Replication
-exports.getScopedConfig = getScopedConfig
-exports.generateConfigID = generateConfigID
-exports.getConfigParams = getConfigParams
-exports.getScopedFullConfig = getScopedFullConfig
-exports.generateDevInfoID = generateDevInfoID
-exports.getPlatformUrl = getPlatformUrl

packages/backend-core/src/environment.ts

@@ -16,7 +16,7 @@ if (!LOADED && isDev() && !isTest()) {
   LOADED = true
 }
 
-module.exports = {
+const env: any = {
   isTest,
   isDev,
   JWT_SECRET: process.env.JWT_SECRET,
@@ -38,9 +38,11 @@ module.exports = {
     process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
   ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
-  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
+  SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED || ""),
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
   PLATFORM_URL: process.env.PLATFORM_URL,
+  POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
+  ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
   BACKUPS_BUCKET_NAME: process.env.BACKUPS_BUCKET_NAME || "backups",
   APPS_BUCKET_NAME: process.env.APPS_BUCKET_NAME || "prod-budi-app-assets",
@@ -51,16 +53,21 @@ module.exports = {
   USE_COUCH: process.env.USE_COUCH || true,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
   DEFAULT_LICENSE: process.env.DEFAULT_LICENSE,
-  _set(key, value) {
+  SERVICE: process.env.SERVICE || "budibase",
+  DEPLOYMENT_ENVIRONMENT:
+    process.env.DEPLOYMENT_ENVIRONMENT || "docker-compose",
+  _set(key: any, value: any) {
     process.env[key] = value
     module.exports[key] = value
   },
 }
 
 // clean up any environment variable edge cases
-for (let [key, value] of Object.entries(module.exports)) {
+for (let [key, value] of Object.entries(env)) {
   // handle the edge case of "0" to disable an environment variable
   if (value === "0") {
-    module.exports[key] = 0
+    env[key] = 0
   }
 }
+
+export = env

packages/backend-core/src/errors/base.js

@@ -1,8 +1,8 @@
 class BudibaseError extends Error {
-  constructor(message, type, code) {
+  constructor(message, code, type) {
     super(message)
-    this.type = type
     this.code = code
+    this.type = type
   }
 }
 

packages/backend-core/src/errors/generic.js

@@ -0,0 +1,11 @@
+const { BudibaseError } = require("./base")
+
+class GenericError extends BudibaseError {
+  constructor(message, code, type) {
+    super(message, code, type ? type : "generic")
+  }
+}
+
+module.exports = {
+  GenericError,
+}

packages/backend-core/src/errors/http.js

@@ -0,0 +1,12 @@
+const { GenericError } = require("./generic")
+
+class HTTPError extends GenericError {
+  constructor(message, httpStatus, code = "http", type = "generic") {
+    super(message, code, type)
+    this.status = httpStatus
+  }
+}
+
+module.exports = {
+  HTTPError,
+}

packages/backend-core/src/errors/index.js

@@ -1,12 +1,11 @@
+const http = require("./http")
 const licensing = require("./licensing")
 
 const codes = {
   ...licensing.codes,
 }
 
-const types = {
+const types = [licensing.type]
-  ...licensing.types,
-}
 
 const context = {
   ...licensing.context,
@@ -36,6 +35,9 @@ const getPublicError = err => {
 module.exports = {
   codes,
   types,
-  UsageLimitError: licensing.UsageLimitError,
+  errors: {
+    UsageLimitError: licensing.UsageLimitError,
+    HTTPError: http.HTTPError,
+  },
   getPublicError,
 }

packages/backend-core/src/errors/licensing.js

@@ -1,8 +1,6 @@
-const { BudibaseError } = require("./base")
+const { HTTPError } = require("./http")
 
-const types = {
+const type = "license_error"
-  LICENSE_ERROR: "license_error",
-}
 
 const codes = {
   USAGE_LIMIT_EXCEEDED: "usage_limit_exceeded",
@@ -16,16 +14,15 @@ const context = {
   },
 }
 
-class UsageLimitError extends BudibaseError {
+class UsageLimitError extends HTTPError {
   constructor(message, limitName) {
-    super(message, types.LICENSE_ERROR, codes.USAGE_LIMIT_EXCEEDED)
+    super(message, 400, codes.USAGE_LIMIT_EXCEEDED, type)
     this.limitName = limitName
-    this.status = 400
   }
 }
 
 module.exports = {
-  types,
+  type,
   codes,
   context,
   UsageLimitError,

packages/backend-core/src/events/analytics.ts

@@ -0,0 +1,57 @@
+import env from "../environment"
+import tenancy from "../tenancy"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+export const enabled = async () => {
+  // cloud - always use the environment variable
+  if (!env.SELF_HOSTED) {
+    return !!env.ENABLE_ANALYTICS
+  }
+
+  // self host - prefer the settings doc
+  // use cache as events have high throughput
+  const enabledInDB = await withCache(
+    CacheKeys.ANALYTICS_ENABLED,
+    TTL.ONE_DAY,
+    async () => {
+      const settings = await getSettingsDoc()
+
+      // need to do explicit checks in case the field is not set
+      if (settings?.config?.analyticsEnabled === false) {
+        return false
+      } else if (settings?.config?.analyticsEnabled === true) {
+        return true
+      }
+    }
+  )
+
+  if (enabledInDB !== undefined) {
+    return enabledInDB
+  }
+
+  // fallback to the environment variable
+  // explicitly check for 0 or false here, undefined or otherwise is treated as true
+  const envEnabled: any = env.ENABLE_ANALYTICS
+  if (envEnabled === 0 || envEnabled === false) {
+    return false
+  } else {
+    return true
+  }
+}
+
+const getSettingsDoc = async () => {
+  const db = tenancy.getGlobalDB()
+  let settings
+  try {
+    settings = await db.get(
+      dbUtils.generateConfigID({ type: Configs.SETTINGS })
+    )
+  } catch (e: any) {
+    if (e.status !== 404) {
+      throw e
+    }
+  }
+  return settings
+}

packages/backend-core/src/events/backfill.ts

@@ -0,0 +1,183 @@
+import {
+  Event,
+  BackfillMetadata,
+  CachedEvent,
+  SSOCreatedEvent,
+  AutomationCreatedEvent,
+  AutomationStepCreatedEvent,
+  DatasourceCreatedEvent,
+  LayoutCreatedEvent,
+  QueryCreatedEvent,
+  RoleCreatedEvent,
+  ScreenCreatedEvent,
+  TableCreatedEvent,
+  ViewCreatedEvent,
+  ViewCalculationCreatedEvent,
+  ViewFilterCreatedEvent,
+  AppPublishedEvent,
+  UserCreatedEvent,
+  RoleAssignedEvent,
+  UserPermissionAssignedEvent,
+  AppCreatedEvent,
+} from "@budibase/types"
+import * as context from "../context"
+import { CacheKeys } from "../cache/generic"
+import * as cache from "../cache/generic"
+
+// LIFECYCLE
+
+export const start = async (events: Event[]) => {
+  const metadata: BackfillMetadata = {
+    eventWhitelist: events,
+  }
+  return saveBackfillMetadata(metadata)
+}
+
+export const recordEvent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  // don't use a ttl - cleaned up by migration
+  // don't use tenancy - already in the key
+  await cache.store(eventKey, properties, undefined, { useTenancy: false })
+}
+
+export const end = async () => {
+  await deleteBackfillMetadata()
+  await clearEvents()
+}
+
+// CRUD
+
+const getBackfillMetadata = async (): Promise<BackfillMetadata | null> => {
+  return cache.get(CacheKeys.BACKFILL_METADATA)
+}
+
+const saveBackfillMetadata = async (
+  backfill: BackfillMetadata
+): Promise<void> => {
+  // no TTL - deleted by backfill
+  return cache.store(CacheKeys.BACKFILL_METADATA, backfill)
+}
+
+const deleteBackfillMetadata = async (): Promise<void> => {
+  await cache.delete(CacheKeys.BACKFILL_METADATA)
+}
+
+const clearEvents = async () => {
+  // wildcard
+  const pattern = getEventKey()
+  const keys = await cache.keys(pattern)
+
+  for (const key of keys) {
+    // delete each key
+    // don't  use tenancy, already in the key
+    await cache.delete(key, { useTenancy: false })
+  }
+}
+
+// HELPERS
+
+export const isBackfillingEvent = async (event: Event) => {
+  const backfill = await getBackfillMetadata()
+  const events = backfill?.eventWhitelist
+  if (events && events.includes(event)) {
+    return true
+  } else {
+    return false
+  }
+}
+
+export const isAlreadySent = async (event: Event, properties: any) => {
+  const eventKey = getEventKey(event, properties)
+  const cachedEvent: CachedEvent = await cache.get(eventKey, {
+    useTenancy: false,
+  })
+  return !!cachedEvent
+}
+
+const CUSTOM_PROPERTY_SUFFIX: any = {
+  // APP EVENTS
+  [Event.AUTOMATION_CREATED]: (properties: AutomationCreatedEvent) => {
+    return properties.automationId
+  },
+  [Event.AUTOMATION_STEP_CREATED]: (properties: AutomationStepCreatedEvent) => {
+    return properties.stepId
+  },
+  [Event.DATASOURCE_CREATED]: (properties: DatasourceCreatedEvent) => {
+    return properties.datasourceId
+  },
+  [Event.LAYOUT_CREATED]: (properties: LayoutCreatedEvent) => {
+    return properties.layoutId
+  },
+  [Event.QUERY_CREATED]: (properties: QueryCreatedEvent) => {
+    return properties.queryId
+  },
+  [Event.ROLE_CREATED]: (properties: RoleCreatedEvent) => {
+    return properties.roleId
+  },
+  [Event.SCREEN_CREATED]: (properties: ScreenCreatedEvent) => {
+    return properties.screenId
+  },
+  [Event.TABLE_CREATED]: (properties: TableCreatedEvent) => {
+    return properties.tableId
+  },
+  [Event.VIEW_CREATED]: (properties: ViewCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_CALCULATION_CREATED]: (
+    properties: ViewCalculationCreatedEvent
+  ) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.VIEW_FILTER_CREATED]: (properties: ViewFilterCreatedEvent) => {
+    return properties.tableId // best uniqueness
+  },
+  [Event.APP_CREATED]: (properties: AppCreatedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  [Event.APP_PUBLISHED]: (properties: AppPublishedEvent) => {
+    return properties.appId // best uniqueness
+  },
+  // GLOBAL EVENTS
+  [Event.AUTH_SSO_CREATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.AUTH_SSO_ACTIVATED]: (properties: SSOCreatedEvent) => {
+    return properties.type
+  },
+  [Event.USER_CREATED]: (properties: UserCreatedEvent) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_ADMIN_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.USER_PERMISSION_BUILDER_ASSIGNED]: (
+    properties: UserPermissionAssignedEvent
+  ) => {
+    return properties.userId
+  },
+  [Event.ROLE_ASSIGNED]: (properties: RoleAssignedEvent) => {
+    return `${properties.roleId}-${properties.userId}`
+  },
+}
+
+const getEventKey = (event?: Event, properties?: any) => {
+  let eventKey: string
+
+  const tenantId = context.getTenantId()
+  if (event) {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:${event}`
+
+    // use some properties to make the key more unique
+    const custom = CUSTOM_PROPERTY_SUFFIX[event]
+    const suffix = custom ? custom(properties) : undefined
+    if (suffix) {
+      eventKey = `${eventKey}:${suffix}`
+    }
+  } else {
+    eventKey = `${CacheKeys.EVENTS}:${tenantId}:*`
+  }
+
+  return eventKey
+}

packages/backend-core/src/events/events.ts

@@ -0,0 +1,31 @@
+import { Event } from "@budibase/types"
+import { processors } from "./processors"
+import * as identification from "./identification"
+import * as backfill from "./backfill"
+
+export const publishEvent = async (
+  event: Event,
+  properties: any,
+  timestamp?: string | number
+) => {
+  // in future this should use async events via a distributed queue.
+  const identity = await identification.getCurrentIdentity()
+
+  const backfilling = await backfill.isBackfillingEvent(event)
+  // no backfill - send the event and exit
+  if (!backfilling) {
+    await processors.processEvent(event, identity, properties, timestamp)
+    return
+  }
+
+  // backfill active - check if the event has been sent already
+  const alreadySent = await backfill.isAlreadySent(event, properties)
+  if (alreadySent) {
+    // do nothing
+    return
+  } else {
+    // send and record the event
+    await processors.processEvent(event, identity, properties, timestamp)
+    await backfill.recordEvent(event, properties)
+  }
+}

packages/backend-core/src/events/identification.ts

@@ -0,0 +1,267 @@
+import * as context from "../context"
+import * as identityCtx from "../context/identity"
+import env from "../environment"
+import {
+  Hosting,
+  User,
+  Identity,
+  IdentityType,
+  Account,
+  isCloudAccount,
+  isSSOAccount,
+  TenantGroup,
+  SettingsConfig,
+  CloudAccount,
+  UserIdentity,
+  InstallationGroup,
+  isSelfHostAccount,
+  UserContext,
+  Group,
+} from "@budibase/types"
+import { processors } from "./processors"
+import * as dbUtils from "../db/utils"
+import { Configs } from "../constants"
+import * as hashing from "../hashing"
+import * as installation from "../installation"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
+
+const pkg = require("../../package.json")
+
+/**
+ * An identity can be:
+ * - account user (Self host)
+ * - budibase user
+ * - tenant
+ * - installation
+ */
+export const getCurrentIdentity = async (): Promise<Identity> => {
+  let identityContext = identityCtx.getIdentity()
+
+  let identityType
+
+  if (!identityContext) {
+    identityType = IdentityType.TENANT
+  } else {
+    identityType = identityContext.type
+  }
+
+  if (identityType === IdentityType.INSTALLATION) {
+    const installationId = await getInstallationId()
+    return {
+      id: formatDistinctId(installationId, identityType),
+      type: identityType,
+      installationId,
+    }
+  } else if (identityType === IdentityType.TENANT) {
+    const installationId = await getInstallationId()
+    const tenantId = await getEventTenantId(context.getTenantId())
+
+    return {
+      id: formatDistinctId(tenantId, identityType),
+      type: identityType,
+      installationId,
+      tenantId,
+    }
+  } else if (identityType === IdentityType.USER) {
+    const userContext = identityContext as UserContext
+    const tenantId = await getEventTenantId(context.getTenantId())
+    let installationId: string | undefined
+
+    // self host account users won't have installation
+    if (!userContext.account || !isSelfHostAccount(userContext.account)) {
+      installationId = await getInstallationId()
+    }
+
+    return {
+      id: userContext._id,
+      type: identityType,
+      installationId,
+      tenantId,
+    }
+  } else {
+    throw new Error("Unknown identity type")
+  }
+}
+
+export const identifyInstallationGroup = async (
+  installId: string,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = installId
+  const type = IdentityType.INSTALLATION
+  const hosting = getHostingFromEnv()
+  const version = pkg.version
+
+  const group: InstallationGroup = {
+    id,
+    type,
+    hosting,
+    version,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the empty auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyTenantGroup = async (
+  tenantId: string,
+  account: Account | undefined,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = await getEventTenantId(tenantId)
+  const type = IdentityType.TENANT
+
+  let hosting: Hosting
+  let profession: string | undefined
+  let companySize: string | undefined
+
+  if (account) {
+    profession = account.profession
+    companySize = account.size
+    hosting = account.hosting
+  } else {
+    hosting = getHostingFromEnv()
+  }
+
+  const group: TenantGroup = {
+    id,
+    type,
+    hosting,
+    profession,
+    companySize,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyUser = async (
+  user: User,
+  account: CloudAccount | undefined,
+  timestamp?: string | number
+) => {
+  const id = user._id as string
+  const tenantId = await getEventTenantId(user.tenantId)
+  const type = IdentityType.USER
+  let builder = user.builder?.global || false
+  let admin = user.admin?.global || false
+  let providerType = user.providerType
+  const accountHolder = account?.budibaseUserId === user._id || false
+  const verified =
+    account && account?.budibaseUserId === user._id ? account.verified : false
+  const installationId = await getInstallationId()
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    installationId,
+    tenantId,
+    verified,
+    accountHolder,
+    providerType,
+    builder,
+    admin,
+  }
+
+  await identify(identity, timestamp)
+}
+
+export const identifyAccount = async (account: Account) => {
+  let id = account.accountId
+  const tenantId = account.tenantId
+  let type = IdentityType.USER
+  let providerType = isSSOAccount(account) ? account.providerType : undefined
+  const verified = account.verified
+  const accountHolder = true
+
+  if (isCloudAccount(account)) {
+    if (account.budibaseUserId) {
+      // use the budibase user as the id if set
+      id = account.budibaseUserId
+    }
+  }
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    tenantId,
+    providerType,
+    verified,
+    accountHolder,
+  }
+
+  await identify(identity)
+}
+
+export const identify = async (
+  identity: Identity,
+  timestamp?: string | number
+) => {
+  await processors.identify(identity, timestamp)
+}
+
+export const identifyGroup = async (
+  group: Group,
+  timestamp?: string | number
+) => {
+  await processors.identifyGroup(group, timestamp)
+}
+
+const getHostingFromEnv = () => {
+  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
+}
+
+export const getInstallationId = async () => {
+  if (isAccountPortal()) {
+    return "account-portal"
+  }
+  const install = await installation.getInstall()
+  return install.installId
+}
+
+const getEventTenantId = async (tenantId: string): Promise<string> => {
+  if (env.SELF_HOSTED) {
+    return getUniqueTenantId(tenantId)
+  } else {
+    // tenant id's in the cloud are already unique
+    return tenantId
+  }
+}
+
+const getUniqueTenantId = async (tenantId: string): Promise<string> => {
+  // make sure this tenantId always matches the tenantId in context
+  return context.doInTenant(tenantId, () => {
+    return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
+      const db = context.getGlobalDB()
+      const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
+        type: Configs.SETTINGS,
+      })
+
+      let uniqueTenantId: string
+      if (config.config.uniqueTenantId) {
+        return config.config.uniqueTenantId
+      } else {
+        uniqueTenantId = `${hashing.newid()}_${tenantId}`
+        config.config.uniqueTenantId = uniqueTenantId
+        await db.put(config)
+        return uniqueTenantId
+      }
+    })
+  })
+}
+
+const isAccountPortal = () => {
+  return env.SERVICE === "account-portal"
+}
+
+const formatDistinctId = (id: string, type: IdentityType) => {
+  if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) {
+    return `$${type}_${id}`
+  } else {
+    return id
+  }
+}

packages/backend-core/src/events/index.ts

@@ -0,0 +1,11 @@
+export * from "./publishers"
+export * as processors from "./processors"
+export * as analytics from "./analytics"
+export * as identification from "./identification"
+export * as backfillCache from "./backfill"
+
+import { processors } from "./processors"
+
+export const shutdown = () => {
+  processors.shutdown()
+}

packages/backend-core/src/events/processors/AnalyticsProcessor.ts

@@ -0,0 +1,61 @@
+import { Event, Identity, Group, IdentityType } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import * as analytics from "../analytics"
+import PosthogProcessor from "./PosthogProcessor"
+
+/**
+ * Events that are always captured.
+ */
+const EVENT_WHITELIST = [Event.VERSION_UPGRADED, Event.VERSION_DOWNGRADED]
+const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT]
+
+export default class AnalyticsProcessor implements EventProcessor {
+  posthog: PosthogProcessor | undefined
+
+  constructor() {
+    if (env.POSTHOG_TOKEN) {
+      this.posthog = new PosthogProcessor(env.POSTHOG_TOKEN)
+    }
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (
+      !IDENTITY_WHITELIST.includes(identity.type) &&
+      !(await analytics.enabled())
+    ) {
+      return
+    }
+    if (this.posthog) {
+      this.posthog.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (this.posthog) {
+      this.posthog.identifyGroup(group, timestamp)
+    }
+  }
+
+  shutdown() {
+    if (this.posthog) {
+      this.posthog.shutdown()
+    }
+  }
+}

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -0,0 +1,54 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+
+const getTimestampString = (timestamp?: string | number) => {
+  let timestampString = ""
+  if (timestamp) {
+    timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+  }
+  return timestampString
+}
+
+const skipLogging = env.SELF_HOSTED && !env.isDev()
+
+export default class LoggingProcessor implements EventProcessor {
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string
+  ): Promise<void> {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
+    )
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(identity)}] ${timestampString} identified`
+    )
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(group)}] ${timestampString} group identified`
+    )
+  }
+
+  shutdown(): void {
+    // no-op
+  }
+}

packages/backend-core/src/events/processors/PosthogProcessor.ts

@@ -0,0 +1,80 @@
+import PostHog from "posthog-node"
+import { Event, Identity, Group, BaseEvent } from "@budibase/types"
+import { EventProcessor } from "./types"
+import env from "../../environment"
+import context from "../../context"
+const pkg = require("../../../package.json")
+
+export default class PosthogProcessor implements EventProcessor {
+  posthog: PostHog
+
+  constructor(token: string | undefined) {
+    if (!token) {
+      throw new Error("Posthog token is not defined")
+    }
+    this.posthog = new PostHog(token)
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: BaseEvent,
+    timestamp?: string | number
+  ): Promise<void> {
+    properties.version = pkg.version
+    properties.service = env.SERVICE
+    properties.environment = env.DEPLOYMENT_ENVIRONMENT
+
+    const appId = context.getAppId()
+    if (appId) {
+      properties.appId = appId
+    }
+
+    const payload: any = { distinctId: identity.id, event, properties }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+
+    // add groups to the event
+    if (identity.installationId || identity.tenantId) {
+      payload.groups = {}
+      if (identity.installationId) {
+        payload.groups.installation = identity.installationId
+        payload.properties.installationId = identity.installationId
+      }
+      if (identity.tenantId) {
+        payload.groups.tenant = identity.tenantId
+        payload.properties.tenantId = identity.tenantId
+      }
+    }
+
+    this.posthog.capture(payload)
+  }
+
+  async identify(identity: Identity, timestamp?: string | number) {
+    const payload: any = { distinctId: identity.id, properties: identity }
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.identify(payload)
+  }
+
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    const payload: any = {
+      distinctId: group.id,
+      groupType: group.type,
+      groupKey: group.id,
+      properties: group,
+    }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.groupIdentify(payload)
+  }
+
+  shutdown() {
+    this.posthog.shutdown()
+  }
+}

packages/backend-core/src/events/processors/Processors.ts

@@ -0,0 +1,46 @@
+import { Event, Identity, Group } from "@budibase/types"
+import { EventProcessor } from "./types"
+
+export default class Processor implements EventProcessor {
+  initialised: boolean = false
+  processors: EventProcessor[] = []
+
+  constructor(processors: EventProcessor[]) {
+    this.processors = processors
+  }
+
+  async processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.processEvent(event, identity, properties, timestamp)
+    }
+  }
+
+  async identify(
+    identity: Identity,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identify(identity, timestamp)
+    }
+  }
+
+  async identifyGroup(
+    identity: Group,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identifyGroup(identity, timestamp)
+    }
+  }
+
+  shutdown() {
+    for (const eventProcessor of this.processors) {
+      eventProcessor.shutdown()
+    }
+  }
+}

packages/backend-core/src/events/processors/index.ts

@@ -0,0 +1,8 @@
+import AnalyticsProcessor from "./AnalyticsProcessor"
+import LoggingProcessor from "./LoggingProcessor"
+import Processors from "./Processors"
+
+export const analyticsProcessor = new AnalyticsProcessor()
+const loggingProcessor = new LoggingProcessor()
+
+export const processors = new Processors([analyticsProcessor, loggingProcessor])

packages/backend-core/src/events/processors/types.ts

@@ -0,0 +1,18 @@
+import { Event, Identity, Group } from "@budibase/types"
+
+export enum EventProcessorType {
+  POSTHOG = "posthog",
+  LOGGING = "logging",
+}
+
+export interface EventProcessor {
+  processEvent(
+    event: Event,
+    identity: Identity,
+    properties: any,
+    timestamp?: string | number
+  ): Promise<void>
+  identify(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
+  shutdown(): void
+}

packages/backend-core/src/events/publishers/account.ts

@@ -0,0 +1,29 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Account,
+  AccountCreatedEvent,
+  AccountDeletedEvent,
+  AccountVerifiedEvent,
+} from "@budibase/types"
+
+export async function created(account: Account) {
+  const properties: AccountCreatedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_CREATED, properties)
+}
+
+export async function deleted(account: Account) {
+  const properties: AccountDeletedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_DELETED, properties)
+}
+
+export async function verified(account: Account) {
+  const properties: AccountVerifiedEvent = {
+    tenantId: account.tenantId,
+  }
+  await publishEvent(Event.ACCOUNT_VERIFIED, properties)
+}

packages/backend-core/src/events/publishers/app.ts

@@ -0,0 +1,108 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  App,
+  AppCreatedEvent,
+  AppUpdatedEvent,
+  AppDeletedEvent,
+  AppPublishedEvent,
+  AppUnpublishedEvent,
+  AppFileImportedEvent,
+  AppTemplateImportedEvent,
+  AppVersionUpdatedEvent,
+  AppVersionRevertedEvent,
+  AppRevertedEvent,
+  AppExportedEvent,
+} from "@budibase/types"
+
+export const created = async (app: App, timestamp?: string | number) => {
+  const properties: AppCreatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_CREATED, properties, timestamp)
+}
+
+export async function updated(app: App) {
+  const properties: AppUpdatedEvent = {
+    appId: app.appId,
+    version: app.version,
+  }
+  await publishEvent(Event.APP_UPDATED, properties)
+}
+
+export async function deleted(app: App) {
+  const properties: AppDeletedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_DELETED, properties)
+}
+
+export async function published(app: App, timestamp?: string | number) {
+  const properties: AppPublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_PUBLISHED, properties, timestamp)
+}
+
+export async function unpublished(app: App) {
+  const properties: AppUnpublishedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_UNPUBLISHED, properties)
+}
+
+export async function fileImported(app: App) {
+  const properties: AppFileImportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_FILE_IMPORTED, properties)
+}
+
+export async function templateImported(app: App, templateKey: string) {
+  const properties: AppTemplateImportedEvent = {
+    appId: app.appId,
+    templateKey,
+  }
+  await publishEvent(Event.APP_TEMPLATE_IMPORTED, properties)
+}
+
+export async function versionUpdated(
+  app: App,
+  currentVersion: string,
+  updatedToVersion: string
+) {
+  const properties: AppVersionUpdatedEvent = {
+    appId: app.appId,
+    currentVersion,
+    updatedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_UPDATED, properties)
+}
+
+export async function versionReverted(
+  app: App,
+  currentVersion: string,
+  revertedToVersion: string
+) {
+  const properties: AppVersionRevertedEvent = {
+    appId: app.appId,
+    currentVersion,
+    revertedToVersion,
+  }
+  await publishEvent(Event.APP_VERSION_REVERTED, properties)
+}
+
+export async function reverted(app: App) {
+  const properties: AppRevertedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_REVERTED, properties)
+}
+
+export async function exported(app: App) {
+  const properties: AppExportedEvent = {
+    appId: app.appId,
+  }
+  await publishEvent(Event.APP_EXPORTED, properties)
+}

packages/backend-core/src/events/publishers/auth.ts

@@ -0,0 +1,58 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  LoginEvent,
+  LoginSource,
+  LogoutEvent,
+  SSOActivatedEvent,
+  SSOCreatedEvent,
+  SSODeactivatedEvent,
+  SSOType,
+  SSOUpdatedEvent,
+} from "@budibase/types"
+import { identification } from ".."
+
+export async function login(source: LoginSource) {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LoginEvent = {
+    userId: identity.id,
+    source,
+  }
+  await publishEvent(Event.AUTH_LOGIN, properties)
+}
+
+export async function logout() {
+  const identity = await identification.getCurrentIdentity()
+  const properties: LogoutEvent = {
+    userId: identity.id,
+  }
+  await publishEvent(Event.AUTH_LOGOUT, properties)
+}
+
+export async function SSOCreated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOCreatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_CREATED, properties, timestamp)
+}
+
+export async function SSOUpdated(type: SSOType) {
+  const properties: SSOUpdatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_UPDATED, properties)
+}
+
+export async function SSOActivated(type: SSOType, timestamp?: string | number) {
+  const properties: SSOActivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_ACTIVATED, properties, timestamp)
+}
+
+export async function SSODeactivated(type: SSOType) {
+  const properties: SSODeactivatedEvent = {
+    type,
+  }
+  await publishEvent(Event.AUTH_SSO_DEACTIVATED, properties)
+}

packages/backend-core/src/events/publishers/automation.ts

@@ -0,0 +1,94 @@
+import { publishEvent } from "../events"
+import {
+  Automation,
+  Event,
+  AutomationStep,
+  AutomationCreatedEvent,
+  AutomationDeletedEvent,
+  AutomationTestedEvent,
+  AutomationStepCreatedEvent,
+  AutomationStepDeletedEvent,
+  AutomationTriggerUpdatedEvent,
+  AutomationsRunEvent,
+} from "@budibase/types"
+
+export async function created(
+  automation: Automation,
+  timestamp?: string | number
+) {
+  const properties: AutomationCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_CREATED, properties, timestamp)
+}
+
+export async function triggerUpdated(automation: Automation) {
+  const properties: AutomationTriggerUpdatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TRIGGER_UPDATED, properties)
+}
+
+export async function deleted(automation: Automation) {
+  const properties: AutomationDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_DELETED, properties)
+}
+
+export async function tested(automation: Automation) {
+  const properties: AutomationTestedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_TESTED, properties)
+}
+
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: AutomationsRunEvent = {
+    count,
+  }
+  await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
+}
+
+export async function stepCreated(
+  automation: Automation,
+  step: AutomationStep,
+  timestamp?: string | number
+) {
+  const properties: AutomationStepCreatedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_CREATED, properties, timestamp)
+}
+
+export async function stepDeleted(
+  automation: Automation,
+  step: AutomationStep
+) {
+  const properties: AutomationStepDeletedEvent = {
+    appId: automation.appId,
+    automationId: automation._id as string,
+    triggerId: automation.definition?.trigger?.id,
+    triggerType: automation.definition?.trigger?.stepId,
+    stepId: step.id,
+    stepType: step.stepId,
+  }
+  await publishEvent(Event.AUTOMATION_STEP_DELETED, properties)
+}

packages/backend-core/src/events/publishers/backfill.ts

@@ -0,0 +1,67 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  AppBackfillSucceededEvent,
+  AppBackfillFailedEvent,
+  TenantBackfillSucceededEvent,
+  TenantBackfillFailedEvent,
+  InstallationBackfillSucceededEvent,
+  InstallationBackfillFailedEvent,
+} from "@budibase/types"
+const env = require("../../environment")
+
+const shouldSkip = !env.SELF_HOSTED && !env.isDev()
+
+export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function appFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: AppBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.APP_BACKFILL_FAILED, properties)
+}
+
+export async function tenantSucceeded(
+  properties: TenantBackfillSucceededEvent
+) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function tenantFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: TenantBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
+}
+
+export async function installationSucceeded() {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillSucceededEvent = {}
+  await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function installationFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
+}

packages/backend-core/src/events/publishers/datasource.ts

@@ -0,0 +1,35 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Datasource,
+  DatasourceCreatedEvent,
+  DatasourceUpdatedEvent,
+  DatasourceDeletedEvent,
+} from "@budibase/types"
+
+export async function created(
+  datasource: Datasource,
+  timestamp?: string | number
+) {
+  const properties: DatasourceCreatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_CREATED, properties, timestamp)
+}
+
+export async function updated(datasource: Datasource) {
+  const properties: DatasourceUpdatedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_UPDATED, properties)
+}
+
+export async function deleted(datasource: Datasource) {
+  const properties: DatasourceDeletedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+  }
+  await publishEvent(Event.DATASOURCE_DELETED, properties)
+}

packages/backend-core/src/events/publishers/email.ts

@@ -0,0 +1,12 @@
+import { publishEvent } from "../events"
+import { Event, SMTPCreatedEvent, SMTPUpdatedEvent } from "@budibase/types"
+
+export async function SMTPCreated(timestamp?: string | number) {
+  const properties: SMTPCreatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_CREATED, properties, timestamp)
+}
+
+export async function SMTPUpdated() {
+  const properties: SMTPUpdatedEvent = {}
+  await publishEvent(Event.EMAIL_SMTP_UPDATED, properties)
+}

packages/backend-core/src/events/publishers/index.ts

@@ -0,0 +1,19 @@
+export * as account from "./account"
+export * as app from "./app"
+export * as auth from "./auth"
+export * as automation from "./automation"
+export * as datasource from "./datasource"
+export * as email from "./email"
+export * as license from "./license"
+export * as layout from "./layout"
+export * as org from "./org"
+export * as query from "./query"
+export * as role from "./role"
+export * as screen from "./screen"
+export * as rows from "./rows"
+export * as table from "./table"
+export * as serve from "./serve"
+export * as user from "./user"
+export * as view from "./view"
+export * as version from "./version"
+export * as backfill from "./backfill"

packages/backend-core/src/events/publishers/layout.ts

@@ -0,0 +1,21 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Layout,
+  LayoutCreatedEvent,
+  LayoutDeletedEvent,
+} from "@budibase/types"
+
+export async function created(layout: Layout, timestamp?: string | number) {
+  const properties: LayoutCreatedEvent = {
+    layoutId: layout._id as string,
+  }
+  await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
+}
+
+export async function deleted(layoutId: string) {
+  const properties: LayoutDeletedEvent = {
+    layoutId,
+  }
+  await publishEvent(Event.LAYOUT_DELETED, properties)
+}

packages/backend-core/src/events/publishers/license.ts

@@ -0,0 +1,33 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  License,
+  LicenseActivatedEvent,
+  LicenseDowngradedEvent,
+  LicenseUpdatedEvent,
+  LicenseUpgradedEvent,
+} from "@budibase/types"
+
+// TODO
+export async function updgraded(license: License) {
+  const properties: LicenseUpgradedEvent = {}
+  await publishEvent(Event.LICENSE_UPGRADED, properties)
+}
+
+// TODO
+export async function downgraded(license: License) {
+  const properties: LicenseDowngradedEvent = {}
+  await publishEvent(Event.LICENSE_DOWNGRADED, properties)
+}
+
+// TODO
+export async function updated(license: License) {
+  const properties: LicenseUpdatedEvent = {}
+  await publishEvent(Event.LICENSE_UPDATED, properties)
+}
+
+// TODO
+export async function activated(license: License) {
+  const properties: LicenseActivatedEvent = {}
+  await publishEvent(Event.LICENSE_ACTIVATED, properties)
+}

packages/backend-core/src/events/publishers/org.ts

@@ -0,0 +1,29 @@
+import { publishEvent } from "../events"
+import { Event } from "@budibase/types"
+
+export async function nameUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_NAME_UPDATED, properties, timestamp)
+}
+
+export async function logoUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_LOGO_UPDATED, properties, timestamp)
+}
+
+export async function platformURLUpdated(timestamp?: string | number) {
+  const properties = {}
+  await publishEvent(Event.ORG_PLATFORM_URL_UPDATED, properties, timestamp)
+}
+
+// TODO
+
+export async function analyticsOptOut() {
+  const properties = {}
+  await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
+}
+
+export async function analyticsOptIn() {
+  const properties = {}
+  await publishEvent(Event.ANALYTICS_OPT_OUT, properties)
+}

packages/backend-core/src/events/publishers/query.ts

@@ -0,0 +1,79 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Datasource,
+  Query,
+  QueryCreatedEvent,
+  QueryUpdatedEvent,
+  QueryDeletedEvent,
+  QueryImportedEvent,
+  QueryPreviewedEvent,
+  QueriesRunEvent,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export const created = async (
+  datasource: Datasource,
+  query: Query,
+  timestamp?: string | number
+) => {
+  const properties: QueryCreatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_CREATED, properties, timestamp)
+}
+
+export const updated = async (datasource: Datasource, query: Query) => {
+  const properties: QueryUpdatedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_UPDATED, properties)
+}
+
+export const deleted = async (datasource: Datasource, query: Query) => {
+  const properties: QueryDeletedEvent = {
+    queryId: query._id as string,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_DELETED, properties)
+}
+
+export const imported = async (
+  datasource: Datasource,
+  importSource: any,
+  count: any
+) => {
+  const properties: QueryImportedEvent = {
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    count,
+    importSource,
+  }
+  await publishEvent(Event.QUERY_IMPORT, properties)
+}
+
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: QueriesRunEvent = {
+    count,
+  }
+  await publishEvent(Event.QUERIES_RUN, properties, timestamp)
+}
+
+export const previewed = async (datasource: Datasource, query: Query) => {
+  const properties: QueryPreviewedEvent = {
+    queryId: query._id,
+    datasourceId: datasource._id as string,
+    source: datasource.source,
+    queryVerb: query.queryVerb,
+  }
+  await publishEvent(Event.QUERY_PREVIEWED, properties)
+}

packages/backend-core/src/events/publishers/role.ts

@@ -0,0 +1,54 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Role,
+  RoleAssignedEvent,
+  RoleCreatedEvent,
+  RoleDeletedEvent,
+  RoleUnassignedEvent,
+  RoleUpdatedEvent,
+  User,
+} from "@budibase/types"
+
+export async function created(role: Role, timestamp?: string | number) {
+  const properties: RoleCreatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_CREATED, properties, timestamp)
+}
+
+export async function updated(role: Role) {
+  const properties: RoleUpdatedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_UPDATED, properties)
+}
+
+export async function deleted(role: Role) {
+  const properties: RoleDeletedEvent = {
+    roleId: role._id as string,
+    permissionId: role.permissionId,
+    inherits: role.inherits,
+  }
+  await publishEvent(Event.ROLE_DELETED, properties)
+}
+
+export async function assigned(user: User, roleId: string, timestamp?: number) {
+  const properties: RoleAssignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
+  await publishEvent(Event.ROLE_ASSIGNED, properties, timestamp)
+}
+
+export async function unassigned(user: User, roleId: string) {
+  const properties: RoleUnassignedEvent = {
+    userId: user._id as string,
+    roleId,
+  }
+  await publishEvent(Event.ROLE_UNASSIGNED, properties)
+}

packages/backend-core/src/events/publishers/rows.ts

@@ -0,0 +1,30 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  RowsImportedEvent,
+  RowsCreatedEvent,
+  RowImportFormat,
+  Table,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export const created = async (count: number, timestamp?: string | number) => {
+  const properties: RowsCreatedEvent = {
+    count,
+  }
+  await publishEvent(Event.ROWS_CREATED, properties, timestamp)
+}
+
+export const imported = async (
+  table: Table,
+  format: RowImportFormat,
+  count: number
+) => {
+  const properties: RowsImportedEvent = {
+    tableId: table._id as string,
+    format,
+    count,
+  }
+  await publishEvent(Event.ROWS_IMPORTED, properties)
+}

packages/backend-core/src/events/publishers/screen.ts

@@ -0,0 +1,25 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  Screen,
+  ScreenCreatedEvent,
+  ScreenDeletedEvent,
+} from "@budibase/types"
+
+export async function created(screen: Screen, timestamp?: string | number) {
+  const properties: ScreenCreatedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
+  await publishEvent(Event.SCREEN_CREATED, properties, timestamp)
+}
+
+export async function deleted(screen: Screen) {
+  const properties: ScreenDeletedEvent = {
+    layoutId: screen.layoutId,
+    screenId: screen._id as string,
+    roleId: screen.routing.roleId,
+  }
+  await publishEvent(Event.SCREEN_DELETED, properties)
+}

packages/backend-core/src/events/publishers/serve.ts

@@ -0,0 +1,28 @@
+import { publishEvent } from "../events"
+import {
+  App,
+  BuilderServedEvent,
+  Event,
+  AppPreviewServedEvent,
+  AppServedEvent,
+} from "@budibase/types"
+
+export async function servedBuilder() {
+  const properties: BuilderServedEvent = {}
+  await publishEvent(Event.SERVED_BUILDER, properties)
+}
+
+export async function servedApp(app: App) {
+  const properties: AppServedEvent = {
+    appVersion: app.version,
+  }
+  await publishEvent(Event.SERVED_APP, properties)
+}
+
+export async function servedAppPreview(app: App) {
+  const properties: AppPreviewServedEvent = {
+    appId: app.appId,
+    appVersion: app.version,
+  }
+  await publishEvent(Event.SERVED_APP_PREVIEW, properties)
+}

packages/backend-core/src/events/publishers/table.ts

@@ -0,0 +1,49 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  TableExportFormat,
+  TableImportFormat,
+  Table,
+  TableCreatedEvent,
+  TableUpdatedEvent,
+  TableDeletedEvent,
+  TableExportedEvent,
+  TableImportedEvent,
+} from "@budibase/types"
+
+export async function created(table: Table, timestamp?: string | number) {
+  const properties: TableCreatedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_CREATED, properties, timestamp)
+}
+
+export async function updated(table: Table) {
+  const properties: TableUpdatedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_UPDATED, properties)
+}
+
+export async function deleted(table: Table) {
+  const properties: TableDeletedEvent = {
+    tableId: table._id as string,
+  }
+  await publishEvent(Event.TABLE_DELETED, properties)
+}
+
+export async function exported(table: Table, format: TableExportFormat) {
+  const properties: TableExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.TABLE_EXPORTED, properties)
+}
+
+export async function imported(table: Table, format: TableImportFormat) {
+  const properties: TableImportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.TABLE_IMPORTED, properties)
+}

packages/backend-core/src/events/publishers/user.ts

@@ -0,0 +1,122 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  User,
+  UserCreatedEvent,
+  UserDeletedEvent,
+  UserInviteAcceptedEvent,
+  UserInvitedEvent,
+  UserPasswordForceResetEvent,
+  UserPasswordResetEvent,
+  UserPasswordResetRequestedEvent,
+  UserPasswordUpdatedEvent,
+  UserPermissionAssignedEvent,
+  UserPermissionRemovedEvent,
+  UserUpdatedEvent,
+} from "@budibase/types"
+
+export async function created(user: User, timestamp?: number) {
+  const properties: UserCreatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_CREATED, properties, timestamp)
+}
+
+export async function updated(user: User) {
+  const properties: UserUpdatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_UPDATED, properties)
+}
+
+export async function deleted(user: User) {
+  const properties: UserDeletedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_DELETED, properties)
+}
+
+// PERMISSIONS
+
+export async function permissionAdminAssigned(user: User, timestamp?: number) {
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(
+    Event.USER_PERMISSION_ADMIN_ASSIGNED,
+    properties,
+    timestamp
+  )
+}
+
+export async function permissionAdminRemoved(user: User) {
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PERMISSION_ADMIN_REMOVED, properties)
+}
+
+export async function permissionBuilderAssigned(
+  user: User,
+  timestamp?: number
+) {
+  const properties: UserPermissionAssignedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(
+    Event.USER_PERMISSION_BUILDER_ASSIGNED,
+    properties,
+    timestamp
+  )
+}
+
+export async function permissionBuilderRemoved(user: User) {
+  const properties: UserPermissionRemovedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PERMISSION_BUILDER_REMOVED, properties)
+}
+
+// INVITE
+
+export async function invited() {
+  const properties: UserInvitedEvent = {}
+  await publishEvent(Event.USER_INVITED, properties)
+}
+
+export async function inviteAccepted(user: User) {
+  const properties: UserInviteAcceptedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_INVITED_ACCEPTED, properties)
+}
+
+// PASSWORD
+
+export async function passwordForceReset(user: User) {
+  const properties: UserPasswordForceResetEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_FORCE_RESET, properties)
+}
+
+export async function passwordUpdated(user: User) {
+  const properties: UserPasswordUpdatedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_UPDATED, properties)
+}
+
+export async function passwordResetRequested(user: User) {
+  const properties: UserPasswordResetRequestedEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_RESET_REQUESTED, properties)
+}
+
+export async function passwordReset(user: User) {
+  const properties: UserPasswordResetEvent = {
+    userId: user._id as string,
+  }
+  await publishEvent(Event.USER_PASSWORD_RESET, properties)
+}

packages/backend-core/src/events/publishers/version.ts

@@ -0,0 +1,26 @@
+import { publishEvent } from "../events"
+import { Event, VersionCheckedEvent, VersionChangeEvent } from "@budibase/types"
+
+export async function checked(version: string) {
+  const properties: VersionCheckedEvent = {
+    currentVersion: version,
+  }
+  await publishEvent(Event.VERSION_CHECKED, properties)
+}
+
+export async function upgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+
+  await publishEvent(Event.VERSION_UPGRADED, properties)
+}
+
+export async function downgraded(from: string, to: string) {
+  const properties: VersionChangeEvent = {
+    from,
+    to,
+  }
+  await publishEvent(Event.VERSION_DOWNGRADED, properties)
+}

packages/backend-core/src/events/publishers/view.ts

@@ -0,0 +1,97 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  ViewCalculationCreatedEvent,
+  ViewCalculationDeletedEvent,
+  ViewCalculationUpdatedEvent,
+  ViewCreatedEvent,
+  ViewDeletedEvent,
+  ViewExportedEvent,
+  ViewFilterCreatedEvent,
+  ViewFilterDeletedEvent,
+  ViewFilterUpdatedEvent,
+  ViewUpdatedEvent,
+  View,
+  ViewCalculation,
+  Table,
+  TableExportFormat,
+} from "@budibase/types"
+
+/* eslint-disable */
+
+export async function created(view: View, timestamp?: string | number) {
+  const properties: ViewCreatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_CREATED, properties, timestamp)
+}
+
+export async function updated(view: View) {
+  const properties: ViewUpdatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_UPDATED, properties)
+}
+
+export async function deleted(view: View) {
+  const properties: ViewDeletedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_DELETED, properties)
+}
+
+export async function exported(table: Table, format: TableExportFormat) {
+  const properties: ViewExportedEvent = {
+    tableId: table._id as string,
+    format,
+  }
+  await publishEvent(Event.VIEW_EXPORTED, properties)
+}
+
+export async function filterCreated(view: View, timestamp?: string | number) {
+  const properties: ViewFilterCreatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_CREATED, properties, timestamp)
+}
+
+export async function filterUpdated(view: View) {
+  const properties: ViewFilterUpdatedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_UPDATED, properties)
+}
+
+export async function filterDeleted(view: View) {
+  const properties: ViewFilterDeletedEvent = {
+    tableId: view.tableId,
+  }
+  await publishEvent(Event.VIEW_FILTER_DELETED, properties)
+}
+
+export async function calculationCreated(
+  view: View,
+  timestamp?: string | number
+) {
+  const properties: ViewCalculationCreatedEvent = {
+    tableId: view.tableId,
+    calculation: view.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_CREATED, properties, timestamp)
+}
+
+export async function calculationUpdated(view: View) {
+  const properties: ViewCalculationUpdatedEvent = {
+    tableId: view.tableId,
+    calculation: view.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_UPDATED, properties)
+}
+
+export async function calculationDeleted(existingView: View) {
+  const properties: ViewCalculationDeletedEvent = {
+    tableId: existingView.tableId,
+    calculation: existingView.calculation as ViewCalculation,
+  }
+  await publishEvent(Event.VIEW_CALCULATION_DELETED, properties)
+}

packages/backend-core/src/index.js

@@ -1,24 +0,0 @@
-const db = require("./db")
-
-module.exports = {
-  init(opts = {}) {
-    db.init(opts.db)
-  },
-  // some default exports from the library, however these ideally shouldn't
-  // be used, instead the syntax require("@budibase/backend-core/db") should be used
-  StaticDatabases: require("./db/utils").StaticDatabases,
-  db: require("../db"),
-  redis: require("../redis"),
-  objectStore: require("../objectStore"),
-  utils: require("../utils"),
-  cache: require("../cache"),
-  auth: require("../auth"),
-  constants: require("../constants"),
-  migrations: require("../migrations"),
-  errors: require("./errors"),
-  env: require("./environment"),
-  accounts: require("./cloud/accounts"),
-  tenancy: require("./tenancy"),
-  context: require("../context"),
-  featureFlags: require("./featureFlags"),
-}

packages/backend-core/src/index.ts

@@ -0,0 +1,55 @@
+import errors from "./errors"
+const errorClasses = errors.errors
+import * as events from "./events"
+import * as migrations from "./migrations"
+import * as users from "./users"
+import * as accounts from "./cloud/accounts"
+import * as installation from "./installation"
+import env from "./environment"
+import tenancy from "./tenancy"
+import featureFlags from "./featureFlags"
+import sessions from "./security/sessions"
+import deprovisioning from "./context/deprovision"
+import auth from "./auth"
+import constants from "./constants"
+import * as dbConstants from "./db/constants"
+
+// mimic the outer package exports
+import * as db from "./pkg/db"
+import * as objectStore from "./pkg/objectStore"
+import * as utils from "./pkg/utils"
+import redis from "./pkg/redis"
+import cache from "./pkg/cache"
+import context from "./pkg/context"
+
+const init = (opts: any = {}) => {
+  db.init(opts.db)
+}
+
+const core = {
+  init,
+  db,
+  ...dbConstants,
+  redis,
+  objectStore,
+  utils,
+  users,
+  cache,
+  auth,
+  constants,
+  ...constants,
+  migrations,
+  env,
+  accounts,
+  tenancy,
+  context,
+  featureFlags,
+  events,
+  sessions,
+  deprovisioning,
+  installation,
+  errors,
+  ...errorClasses,
+}
+
+export = core

packages/backend-core/src/installation.ts

@@ -0,0 +1,96 @@
+import * as hashing from "./hashing"
+import * as events from "./events"
+import { StaticDatabases } from "./db/constants"
+import { doWithDB } from "./db"
+import { Installation, IdentityType } from "@budibase/types"
+import * as context from "./context"
+import semver from "semver"
+import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic"
+
+const pkg = require("../package.json")
+
+export const getInstall = async (): Promise<Installation> => {
+  return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
+    useTenancy: false,
+  })
+}
+
+const getInstallFromDB = async (): Promise<Installation> => {
+  return doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
+    async (platformDb: any) => {
+      let install: Installation
+      try {
+        install = await platformDb.get(
+          StaticDatabases.PLATFORM_INFO.docs.install
+        )
+      } catch (e: any) {
+        if (e.status === 404) {
+          install = {
+            _id: StaticDatabases.PLATFORM_INFO.docs.install,
+            installId: hashing.newid(),
+            version: pkg.version,
+          }
+          const resp = await platformDb.put(install)
+          install._rev = resp.rev
+        } else {
+          throw e
+        }
+      }
+      return install
+    }
+  )
+}
+
+const updateVersion = async (version: string): Promise<boolean> => {
+  try {
+    await doWithDB(
+      StaticDatabases.PLATFORM_INFO.name,
+      async (platformDb: any) => {
+        const install = await getInstall()
+        install.version = version
+        await platformDb.put(install)
+        await bustCache(CacheKeys.INSTALLATION)
+      }
+    )
+  } catch (e: any) {
+    if (e.status === 409) {
+      // do nothing - version has already been updated
+      // likely in clustered environment
+      return false
+    }
+    throw e
+  }
+  return true
+}
+
+export const checkInstallVersion = async (): Promise<void> => {
+  const install = await getInstall()
+
+  const currentVersion = install.version
+  const newVersion = pkg.version
+
+  if (currentVersion !== newVersion) {
+    const isUpgrade = semver.gt(newVersion, currentVersion)
+    const isDowngrade = semver.lt(newVersion, currentVersion)
+
+    const success = await updateVersion(newVersion)
+
+    if (success) {
+      await context.doInIdentityContext(
+        {
+          _id: install.installId,
+          type: IdentityType.INSTALLATION,
+        },
+        async () => {
+          if (isUpgrade) {
+            await events.version.upgraded(currentVersion, newVersion)
+          } else if (isDowngrade) {
+            await events.version.downgraded(currentVersion, newVersion)
+          }
+        }
+      )
+      await events.identification.identifyInstallationGroup(install.installId)
+    }
+  }
+}

packages/backend-core/src/logging.js

@@ -4,7 +4,7 @@ function isSuppressed(e) {
   return e && e["suppressAlert"]
 }
 
-module.exports.logAlert = (message, e = null) => {
+module.exports.logAlert = (message, e) => {
   if (e && NonErrors.includes(e.name) && isSuppressed(e)) {
     return
   }

packages/backend-core/src/middleware/authenticated.js

@@ -4,9 +4,12 @@ const { getUser } = require("../cache/user")
 const { getSession, updateSessionTTL } = require("../security/sessions")
 const { buildMatcherRegex, matches } = require("./matchers")
 const env = require("../environment")
-const { SEPARATOR, ViewNames, queryGlobalView } = require("../../db")
+const { SEPARATOR } = require("../db/constants")
+const { ViewNames } = require("../db/utils")
+const { queryGlobalView } = require("../db/views")
 const { getGlobalDB, doInTenant } = require("../tenancy")
 const { decrypt } = require("../security/encryption")
+const identity = require("../context/identity")
 
 function finalise(
   ctx,
@@ -132,7 +135,12 @@ module.exports = (
       }
       // isAuthenticated is a function, so use a variable to be able to check authed state
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
-      return next()
+
+      if (user && user.email) {
+        return identity.doInUserContext(user, next)
+      } else {
+        return next()
+      }
     } catch (err) {
       // invalid token, clear the cookie
       if (err && err.name === "JsonWebTokenError") {

packages/backend-core/src/middleware/passport/datasource/google.js

@@ -1,4 +1,5 @@
 const google = require("../google")
+const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy
 const { Cookies, Configs } = require("../../../constants")
 const { clearCookie, getCookie } = require("../../../utils")
 const { getScopedConfig, getPlatformUrl } = require("../../../db/utils")
@@ -46,19 +47,20 @@ async function postAuth(passport, ctx, next) {
   const platformUrl = await getPlatformUrl({ tenantAware: false })
 
   let callbackUrl = `${platformUrl}/api/global/auth/datasource/google/callback`
-  const strategy = await google.strategyFactory(
-    config,
-    callbackUrl,
-    (accessToken, refreshToken, profile, done) => {
-      clearCookie(ctx, Cookies.DatasourceAuth)
-      done(null, { refreshToken })
-    }
-  )
-
   const authStateCookie = getCookie(ctx, Cookies.DatasourceAuth)
 
   return passport.authenticate(
-    strategy,
+    new GoogleStrategy(
+      {
+        clientID: config.clientID,
+        clientSecret: config.clientSecret,
+        callbackURL: callbackUrl,
+      },
+      (accessToken, refreshToken, profile, done) => {
+        clearCookie(ctx, Cookies.DatasourceAuth)
+        done(null, { accessToken, refreshToken })
+      }
+    ),
     { successRedirect: "/", failureRedirect: "/error" },
     async (err, tokens) => {
       // update the DB for the datasource with all the user info

packages/backend-core/src/middleware/passport/google.js

@@ -11,8 +11,8 @@ const buildVerifyFn = saveUserFn => {
       profile: profile,
       email: profile._json.email,
       oauth2: {
-        accessToken: accessToken,
+        accessToken,
-        refreshToken: refreshToken,
+        refreshToken,
       },
     }
 

packages/backend-core/src/middleware/passport/local.js

@@ -2,7 +2,7 @@ const jwt = require("jsonwebtoken")
 const { UserStatus } = require("../../constants")
 const { compare } = require("../../hashing")
 const env = require("../../environment")
-const { getGlobalUserByEmail } = require("../../utils")
+const users = require("../../users")
 const { authError } = require("./utils")
 const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
@@ -28,7 +28,7 @@ exports.authenticate = async function (ctx, email, password, done) {
   if (!email) return authError(done, "Email Required")
   if (!password) return authError(done, "Password Required")
 
-  const dbUser = await getGlobalUserByEmail(email)
+  const dbUser = await users.getGlobalUserByEmail(email)
   if (dbUser == null) {
     return authError(done, `User not found: [${email}]`)
   }

packages/backend-core/src/middleware/passport/tests/third-party-common.spec.js

@@ -1,7 +1,4 @@
-// Mock data
+require("../../../../tests/utilities/TestConfiguration")
-
-require("../../../tests/utilities/dbConfig")
-
 const { authenticateThirdParty } = require("../third-party-common")
 const { data } = require("./utilities/mock-data")
 const { DEFAULT_TENANT_ID } = require("../../../constants")

packages/backend-core/src/middleware/passport/third-party-common.js

@@ -4,7 +4,7 @@ const { generateGlobalUserID } = require("../../db/utils")
 const { authError } = require("./utils")
 const { newid } = require("../../hashing")
 const { createASession } = require("../../security/sessions")
-const { getGlobalUserByEmail } = require("../../utils")
+const users = require("../../users")
 const { getGlobalDB, getTenantId } = require("../../tenancy")
 const fetch = require("node-fetch")
 
@@ -52,7 +52,7 @@ exports.authenticateThirdParty = async function (
 
   // fallback to loading by email
   if (!dbUser) {
-    dbUser = await getGlobalUserByEmail(thirdPartyUser.email)
+    dbUser = await users.getGlobalUserByEmail(thirdPartyUser.email)
   }
 
   // exit early if there is still no user and auto creation is disabled
@@ -79,14 +79,14 @@ exports.authenticateThirdParty = async function (
   dbUser.forceResetPassword = false
 
   // create or sync the user
-  let response
   try {
-    response = await saveUserFn(dbUser, getTenantId(), false, false)
+    await saveUserFn(dbUser, false, false)
   } catch (err) {
     return authError(done, err)
   }
 
-  dbUser._rev = response.rev
+  // now that we're sure user exists, load them from the db
+  dbUser = await db.get(dbUser._id)
 
   // authenticate
   const sessionId = newid()

packages/backend-core/src/middleware/tenancy.js

@@ -1,6 +1,38 @@
-const { setTenantId, setGlobalDB, closeTenancy } = require("../tenancy")
+const { doInTenant, isMultiTenant, DEFAULT_TENANT_ID } = require("../tenancy")
-const cls = require("../context/FunctionContext")
 const { buildMatcherRegex, matches } = require("./matchers")
+const { Headers } = require("../constants")
+
+const getTenantID = (ctx, opts = { allowQs: false, allowNoTenant: false }) => {
+  // exit early if not multi-tenant
+  if (!isMultiTenant()) {
+    return DEFAULT_TENANT_ID
+  }
+
+  let tenantId
+  const allowQs = opts && opts.allowQs
+  const allowNoTenant = opts && opts.allowNoTenant
+  const header = ctx.request.headers[Headers.TENANT_ID]
+  const user = ctx.user || {}
+  if (allowQs) {
+    const query = ctx.request.query || {}
+    tenantId = query.tenantId
+  }
+  // override query string (if allowed) by user, or header
+  // URL params cannot be used in a middleware, as they are
+  // processed later in the chain
+  tenantId = user.tenantId || header || tenantId
+
+  // Set the tenantId from the subdomain
+  if (!tenantId) {
+    tenantId = ctx.subdomains && ctx.subdomains[0]
+  }
+
+  if (!tenantId && !allowNoTenant) {
+    ctx.throw(403, "Tenant id not set")
+  }
+
+  return tenantId
+}
 
 module.exports = (
   allowQueryStringPatterns,
@@ -11,15 +43,10 @@ module.exports = (
   const noTenancyOptions = buildMatcherRegex(noTenancyPatterns)
 
   return async function (ctx, next) {
-    return cls.run(async () => {
+    const allowNoTenant =
-      const allowNoTenant =
+      opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
-        opts.noTenancyRequired || !!matches(ctx, noTenancyOptions)
+    const allowQs = !!matches(ctx, allowQsOptions)
-      const allowQs = !!matches(ctx, allowQsOptions)
+    const tenantId = getTenantID(ctx, { allowQs, allowNoTenant })
-      const tenantId = setTenantId(ctx, { allowQs, allowNoTenant })
+    return doInTenant(tenantId, next)
-      setGlobalDB(tenantId)
-      const res = await next()
-      await closeTenancy()
-      return res
-    })
   }
 }

packages/backend-core/src/migrations/definitions.ts

@@ -0,0 +1,40 @@
+import {
+  MigrationType,
+  MigrationName,
+  MigrationDefinition,
+} from "@budibase/types"
+
+export const DEFINITIONS: MigrationDefinition[] = [
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.USER_EMAIL_VIEW_CASING,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.QUOTAS_1,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.APP_URLS,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.DEVELOPER_QUOTA,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.PUBLISHED_APP_QUOTA,
+  },
+  {
+    type: MigrationType.APP,
+    name: MigrationName.EVENT_APP_BACKFILL,
+  },
+  {
+    type: MigrationType.GLOBAL,
+    name: MigrationName.EVENT_GLOBAL_BACKFILL,
+  },
+  {
+    type: MigrationType.INSTALLATION,
+    name: MigrationName.EVENT_INSTALLATION_BACKFILL,
+  },
+]

packages/backend-core/src/migrations/index.js

@@ -1,117 +0,0 @@
-const { DEFAULT_TENANT_ID } = require("../constants")
-const { doWithDB } = require("../db")
-const { DocumentTypes } = require("../db/constants")
-const { getAllApps } = require("../db/utils")
-const environment = require("../environment")
-const {
-  doInTenant,
-  getTenantIds,
-  getGlobalDBName,
-  getTenantId,
-} = require("../tenancy")
-
-exports.MIGRATION_TYPES = {
-  GLOBAL: "global", // run once, recorded in global db, global db is provided as an argument
-  APP: "app", // run per app, recorded in each app db, app db is provided as an argument
-}
-
-exports.getMigrationsDoc = async db => {
-  // get the migrations doc
-  try {
-    return await db.get(DocumentTypes.MIGRATIONS)
-  } catch (err) {
-    if (err.status && err.status === 404) {
-      return { _id: DocumentTypes.MIGRATIONS }
-    }
-    console.error(err)
-  }
-}
-
-const runMigration = async (migration, options = {}) => {
-  const tenantId = getTenantId()
-  const migrationType = migration.type
-  const migrationName = migration.name
-
-  // get the db to store the migration in
-  let dbNames
-  if (migrationType === exports.MIGRATION_TYPES.GLOBAL) {
-    dbNames = [getGlobalDBName()]
-  } else if (migrationType === exports.MIGRATION_TYPES.APP) {
-    const apps = await getAllApps(migration.opts)
-    dbNames = apps.map(app => app.appId)
-  } else {
-    throw new Error(
-      `[Tenant: ${tenantId}] Unrecognised migration type [${migrationType}]`
-    )
-  }
-
-  // run the migration against each db
-  for (const dbName of dbNames) {
-    await doWithDB(dbName, async db => {
-      try {
-        const doc = await exports.getMigrationsDoc(db)
-
-        // exit if the migration has been performed already
-        if (doc[migrationName]) {
-          if (
-            options.force &&
-            options.force[migrationType] &&
-            options.force[migrationType].includes(migrationName)
-          ) {
-            console.log(
-              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
-            )
-          } else {
-            // the migration has already been performed
-            return
-          }
-        }
-
-        console.log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running`
-        )
-        // run the migration with tenant context
-        await migration.fn(db)
-        console.log(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
-        )
-
-        // mark as complete
-        doc[migrationName] = Date.now()
-        await db.put(doc)
-      } catch (err) {
-        console.error(
-          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
-          err
-        )
-        throw err
-      }
-    })
-  }
-}
-
-exports.runMigrations = async (migrations, options = {}) => {
-  console.log("Running migrations")
-  let tenantIds
-  if (environment.MULTI_TENANCY) {
-    if (!options.tenantIds || !options.tenantIds.length) {
-      // run for all tenants
-      tenantIds = await getTenantIds()
-    } else {
-      tenantIds = options.tenantIds
-    }
-  } else {
-    // single tenancy
-    tenantIds = [DEFAULT_TENANT_ID]
-  }
-
-  // for all tenants
-  for (const tenantId of tenantIds) {
-    // for all migrations
-    for (const migration of migrations) {
-      // run the migration
-      await doInTenant(tenantId, () => runMigration(migration, options))
-    }
-  }
-  console.log("Migrations complete")
-}

packages/backend-core/src/migrations/index.ts

@@ -0,0 +1,2 @@
+export * from "./migrations"
+export * from "./definitions"

packages/backend-core/src/migrations/migrations.ts

@@ -0,0 +1,189 @@
+import { DEFAULT_TENANT_ID } from "../constants"
+import { doWithDB } from "../db"
+import { DocumentTypes, StaticDatabases } from "../db/constants"
+import { getAllApps } from "../db/utils"
+import environment from "../environment"
+import {
+  doInTenant,
+  getTenantIds,
+  getGlobalDBName,
+  getTenantId,
+} from "../tenancy"
+import context from "../context"
+import { DEFINITIONS } from "."
+import {
+  Migration,
+  MigrationOptions,
+  MigrationType,
+  MigrationNoOpOptions,
+} from "@budibase/types"
+
+export const getMigrationsDoc = async (db: any) => {
+  // get the migrations doc
+  try {
+    return await db.get(DocumentTypes.MIGRATIONS)
+  } catch (err: any) {
+    if (err.status && err.status === 404) {
+      return { _id: DocumentTypes.MIGRATIONS }
+    } else {
+      console.error(err)
+      throw err
+    }
+  }
+}
+
+export const backPopulateMigrations = async (opts: MigrationNoOpOptions) => {
+  // filter migrations to the type and populate a no-op migration
+  const migrations: Migration[] = DEFINITIONS.filter(
+    def => def.type === opts.type
+  ).map(d => ({ ...d, fn: () => {} }))
+  await runMigrations(migrations, { noOp: opts })
+}
+
+export const runMigration = async (
+  migration: Migration,
+  options: MigrationOptions = {}
+) => {
+  const migrationType = migration.type
+  let tenantId: string
+  if (migrationType !== MigrationType.INSTALLATION) {
+    tenantId = getTenantId()
+  }
+  const migrationName = migration.name
+  const silent = migration.silent
+
+  const log = (message: string) => {
+    if (!silent) {
+      console.log(message)
+    }
+  }
+
+  // get the db to store the migration in
+  let dbNames
+  if (migrationType === MigrationType.GLOBAL) {
+    dbNames = [getGlobalDBName()]
+  } else if (migrationType === MigrationType.APP) {
+    if (options.noOp) {
+      dbNames = [options.noOp.appId]
+    } else {
+      const apps = await getAllApps(migration.appOpts)
+      dbNames = apps.map(app => app.appId)
+    }
+  } else if (migrationType === MigrationType.INSTALLATION) {
+    dbNames = [StaticDatabases.PLATFORM_INFO.name]
+  } else {
+    throw new Error(`Unrecognised migration type [${migrationType}]`)
+  }
+
+  const length = dbNames.length
+  let count = 0
+
+  // run the migration against each db
+  for (const dbName of dbNames) {
+    count++
+    const lengthStatement = length > 1 ? `[${count}/${length}]` : ""
+
+    await doWithDB(dbName, async (db: any) => {
+      try {
+        const doc = await exports.getMigrationsDoc(db)
+
+        // the migration has already been run
+        if (doc[migrationName]) {
+          // check for force
+          if (
+            options.force &&
+            options.force[migrationType] &&
+            options.force[migrationType].includes(migrationName)
+          ) {
+            log(
+              `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Forcing`
+            )
+          } else {
+            // no force, exit
+            return
+          }
+        }
+
+        // check if the migration is not a no-op
+        if (!options.noOp) {
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Running ${lengthStatement}`
+          )
+
+          if (migration.preventRetry) {
+            // eagerly set the completion date
+            // so that we never run this migration twice even upon failure
+            doc[migrationName] = Date.now()
+            const response = await db.put(doc)
+            doc._rev = response.rev
+          }
+
+          // run the migration
+          if (migrationType === MigrationType.APP) {
+            await context.doInAppContext(db.name, async () => {
+              await migration.fn(db)
+            })
+          } else {
+            await migration.fn(db)
+          }
+
+          log(
+            `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Complete`
+          )
+        }
+
+        // mark as complete
+        doc[migrationName] = Date.now()
+        await db.put(doc)
+      } catch (err) {
+        console.error(
+          `[Tenant: ${tenantId}] [Migration: ${migrationName}] [DB: ${dbName}] Error: `,
+          err
+        )
+        throw err
+      }
+    })
+  }
+}
+
+export const runMigrations = async (
+  migrations: Migration[],
+  options: MigrationOptions = {}
+) => {
+  let tenantIds
+
+  if (environment.MULTI_TENANCY) {
+    if (options.noOp) {
+      tenantIds = [options.noOp.tenantId]
+    } else if (!options.tenantIds || !options.tenantIds.length) {
+      // run for all tenants
+      tenantIds = await getTenantIds()
+    } else {
+      tenantIds = options.tenantIds
+    }
+  } else {
+    // single tenancy
+    tenantIds = [DEFAULT_TENANT_ID]
+  }
+
+  if (tenantIds.length > 1) {
+    console.log(`Checking migrations for ${tenantIds.length} tenants`)
+  } else {
+    console.log("Checking migrations")
+  }
+
+  let count = 0
+  // for all tenants
+  for (const tenantId of tenantIds) {
+    count++
+    if (tenantIds.length > 1) {
+      console.log(`Progress [${count}/${tenantIds.length}]`)
+    }
+    // for all migrations
+    for (const migration of migrations) {
+      // run the migration
+      await doInTenant(tenantId, () => runMigration(migration, options))
+    }
+  }
+  console.log("Migrations complete")
+}

packages/backend-core/src/migrations/tests/__snapshots__/index.spec.js.snap

@@ -3,7 +3,9 @@
 exports[`migrations should match snapshot 1`] = `
 Object {
   "_id": "migrations",
-  "_rev": "1-6277abc4e3db950221768e5a2618a059",
+  "_rev": "1-a32b0b708e59eeb006ed5e063cfeb36a",
-  "test": 1487076708000,
+  "createdAt": "2020-01-01T00:00:00.000Z",
+  "test": 1577836800000,
+  "updatedAt": "2020-01-01T00:00:00.000Z",
 }
 `;

packages/backend-core/src/migrations/tests/index.spec.js

@@ -1,12 +1,10 @@
-require("../../tests/utilities/dbConfig")
+require("../../../tests/utilities/TestConfiguration")
-
 const { runMigrations, getMigrationsDoc } = require("../index")
 const { dangerousGetDB } = require("../../db")
 const {
   StaticDatabases,
 } = require("../../db/utils")
 
-Date.now = jest.fn(() => 1487076708000)
 let db
 
 describe("migrations", () => {

packages/backend-core/src/objectStore/index.ts

@@ -1,16 +1,16 @@
 const sanitize = require("sanitize-s3-objectkey")
-const AWS = require("aws-sdk")
+import AWS from "aws-sdk"
-const stream = require("stream")
+import stream from "stream"
-const fetch = require("node-fetch")
+import fetch from "node-fetch"
-const tar = require("tar-fs")
+import tar from "tar-fs"
 const zlib = require("zlib")
-const { promisify } = require("util")
+import { promisify } from "util"
-const { join } = require("path")
+import { join } from "path"
-const fs = require("fs")
+import fs from "fs"
-const env = require("../environment")
+import env from "../environment"
-const { budibaseTempDir, ObjectStoreBuckets } = require("./utils")
+import { budibaseTempDir, ObjectStoreBuckets } from "./utils"
-const { v4 } = require("uuid")
+import { v4 } from "uuid"
-const { APP_PREFIX, APP_DEV_PREFIX } = require("../db/utils")
+import { APP_PREFIX, APP_DEV_PREFIX } from "../db/utils"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -18,7 +18,7 @@ const STATE = {
   bucketCreationPromises: {},
 }
 
-const CONTENT_TYPE_MAP = {
+const CONTENT_TYPE_MAP: any = {
   html: "text/html",
   css: "text/css",
   js: "application/javascript",
@@ -32,20 +32,16 @@ const STRING_CONTENT_TYPES = [
 ]
 
 // does normal sanitization and then swaps dev apps to apps
-function sanitizeKey(input) {
+export function sanitizeKey(input: any) {
   return sanitize(sanitizeBucket(input)).replace(/\\/g, "/")
 }
 
-exports.sanitizeKey = sanitizeKey
-
 // simply handles the dev app to app conversion
-function sanitizeBucket(input) {
+export function sanitizeBucket(input: any) {
   return input.replace(new RegExp(APP_DEV_PREFIX, "g"), APP_PREFIX)
 }
 
-exports.sanitizeBucket = sanitizeBucket
+function publicPolicy(bucketName: any) {
-
-function publicPolicy(bucketName) {
   return {
     Version: "2012-10-17",
     Statement: [
@@ -69,13 +65,13 @@ const PUBLIC_BUCKETS = [ObjectStoreBuckets.APPS, ObjectStoreBuckets.GLOBAL]
  * @return {Object} an S3 object store object, check S3 Nodejs SDK for usage.
  * @constructor
  */
-exports.ObjectStore = bucket => {
+export const ObjectStore = (bucket: any) => {
   AWS.config.update({
     accessKeyId: env.MINIO_ACCESS_KEY,
     secretAccessKey: env.MINIO_SECRET_KEY,
     region: env.AWS_REGION,
   })
-  const config = {
+  const config: any = {
     s3ForcePathStyle: true,
     signatureVersion: "v4",
     apiVersion: "2006-03-01",
@@ -93,7 +89,7 @@ exports.ObjectStore = bucket => {
  * Given an object store and a bucket name this will make sure the bucket exists,
  * if it does not exist then it will create it.
  */
-exports.makeSureBucketExists = async (client, bucketName) => {
+export const makeSureBucketExists = async (client: any, bucketName: any) => {
   bucketName = sanitizeBucket(bucketName)
   try {
     await client
@@ -101,8 +97,8 @@ exports.makeSureBucketExists = async (client, bucketName) => {
         Bucket: bucketName,
       })
       .promise()
-  } catch (err) {
+  } catch (err: any) {
-    const promises = STATE.bucketCreationPromises
+    const promises: any = STATE.bucketCreationPromises
     const doesntExist = err.statusCode === 404,
       noAccess = err.statusCode === 403
     if (promises[bucketName]) {
@@ -138,20 +134,20 @@ exports.makeSureBucketExists = async (client, bucketName) => {
  * Uploads the contents of a file given the required parameters, useful when
  * temp files in use (for example file uploaded as an attachment).
  */
-exports.upload = async ({
+export const upload = async ({
   bucket: bucketName,
   filename,
   path,
   type,
   metadata,
-}) => {
+}: any) => {
   const extension = [...filename.split(".")].pop()
   const fileBytes = fs.readFileSync(path)
 
-  const objectStore = exports.ObjectStore(bucketName)
+  const objectStore = ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
 
-  const config = {
+  const config: any = {
     // windows file paths need to be converted to forward slashes for s3
     Key: sanitizeKey(filename),
     Body: fileBytes,
@@ -167,9 +163,14 @@ exports.upload = async ({
  * Similar to the upload function but can be used to send a file stream
  * through to the object store.
  */
-exports.streamUpload = async (bucketName, filename, stream, extra = {}) => {
+export const streamUpload = async (
-  const objectStore = exports.ObjectStore(bucketName)
+  bucketName: any,
-  await exports.makeSureBucketExists(objectStore, bucketName)
+  filename: any,
+  stream: any,
+  extra = {}
+) => {
+  const objectStore = ObjectStore(bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
 
   const params = {
     Bucket: sanitizeBucket(bucketName),
@@ -184,13 +185,13 @@ exports.streamUpload = async (bucketName, filename, stream, extra = {}) => {
  * retrieves the contents of a file from the object store, if it is a known content type it
  * will be converted, otherwise it will be returned as a buffer stream.
  */
-exports.retrieve = async (bucketName, filepath) => {
+export const retrieve = async (bucketName: any, filepath: any) => {
-  const objectStore = exports.ObjectStore(bucketName)
+  const objectStore = ObjectStore(bucketName)
   const params = {
     Bucket: sanitizeBucket(bucketName),
     Key: sanitizeKey(filepath),
   }
-  const response = await objectStore.getObject(params).promise()
+  const response: any = await objectStore.getObject(params).promise()
   // currently these are all strings
   if (STRING_CONTENT_TYPES.includes(response.ContentType)) {
     return response.Body.toString("utf8")
@@ -202,10 +203,10 @@ exports.retrieve = async (bucketName, filepath) => {
 /**
  * Same as retrieval function but puts to a temporary file.
  */
-exports.retrieveToTmp = async (bucketName, filepath) => {
+export const retrieveToTmp = async (bucketName: any, filepath: any) => {
   bucketName = sanitizeBucket(bucketName)
   filepath = sanitizeKey(filepath)
-  const data = await exports.retrieve(bucketName, filepath)
+  const data = await retrieve(bucketName, filepath)
   const outputPath = join(budibaseTempDir(), v4())
   fs.writeFileSync(outputPath, data)
   return outputPath
@@ -214,9 +215,9 @@ exports.retrieveToTmp = async (bucketName, filepath) => {
 /**
  * Delete a single file.
  */
-exports.deleteFile = async (bucketName, filepath) => {
+export const deleteFile = async (bucketName: any, filepath: any) => {
-  const objectStore = exports.ObjectStore(bucketName)
+  const objectStore = ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Key: filepath,
@@ -224,13 +225,13 @@ exports.deleteFile = async (bucketName, filepath) => {
   return objectStore.deleteObject(params)
 }
 
-exports.deleteFiles = async (bucketName, filepaths) => {
+export const deleteFiles = async (bucketName: any, filepaths: any) => {
-  const objectStore = exports.ObjectStore(bucketName)
+  const objectStore = ObjectStore(bucketName)
-  await exports.makeSureBucketExists(objectStore, bucketName)
+  await makeSureBucketExists(objectStore, bucketName)
   const params = {
     Bucket: bucketName,
     Delete: {
-      Objects: filepaths.map(path => ({ Key: path })),
+      Objects: filepaths.map((path: any) => ({ Key: path })),
     },
   }
   return objectStore.deleteObjects(params).promise()
@@ -239,38 +240,45 @@ exports.deleteFiles = async (bucketName, filepaths) => {
 /**
  * Delete a path, including everything within.
  */
-exports.deleteFolder = async (bucketName, folder) => {
+export const deleteFolder = async (
+  bucketName: any,
+  folder: any
+): Promise<any> => {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)
-  const client = exports.ObjectStore(bucketName)
+  const client = ObjectStore(bucketName)
   const listParams = {
     Bucket: bucketName,
     Prefix: folder,
   }
 
-  let response = await client.listObjects(listParams).promise()
+  let response: any = await client.listObjects(listParams).promise()
   if (response.Contents.length === 0) {
     return
   }
-  const deleteParams = {
+  const deleteParams: any = {
     Bucket: bucketName,
     Delete: {
       Objects: [],
     },
   }
 
-  response.Contents.forEach(content => {
+  response.Contents.forEach((content: any) => {
     deleteParams.Delete.Objects.push({ Key: content.Key })
   })
 
   response = await client.deleteObjects(deleteParams).promise()
   // can only empty 1000 items at once
   if (response.Deleted.length === 1000) {
-    return exports.deleteFolder(bucketName, folder)
+    return deleteFolder(bucketName, folder)
   }
 }
 
-exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
+export const uploadDirectory = async (
+  bucketName: any,
+  localPath: any,
+  bucketPath: any
+) => {
   bucketName = sanitizeBucket(bucketName)
   let uploads = []
   const files = fs.readdirSync(localPath, { withFileTypes: true })
@@ -278,17 +286,15 @@ exports.uploadDirectory = async (bucketName, localPath, bucketPath) => {
     const path = sanitizeKey(join(bucketPath, file.name))
     const local = join(localPath, file.name)
     if (file.isDirectory()) {
-      uploads.push(exports.uploadDirectory(bucketName, local, path))
+      uploads.push(uploadDirectory(bucketName, local, path))
     } else {
-      uploads.push(
+      uploads.push(streamUpload(bucketName, path, fs.createReadStream(local)))
-        exports.streamUpload(bucketName, path, fs.createReadStream(local))
-      )
     }
   }
   await Promise.all(uploads)
 }
 
-exports.downloadTarball = async (url, bucketName, path) => {
+export const downloadTarball = async (url: any, bucketName: any, path: any) => {
   bucketName = sanitizeBucket(bucketName)
   path = sanitizeKey(path)
   const response = await fetch(url)
@@ -299,7 +305,7 @@ exports.downloadTarball = async (url, bucketName, path) => {
   const tmpPath = join(budibaseTempDir(), path)
   await streamPipeline(response.body, zlib.Unzip(), tar.extract(tmpPath))
   if (!env.isTest() && env.SELF_HOSTED) {
-    await exports.uploadDirectory(bucketName, tmpPath, path)
+    await uploadDirectory(bucketName, tmpPath, path)
   }
   // return the temporary path incase there is a use for it
   return tmpPath

packages/backend-core/src/pkg/cache.ts

@@ -0,0 +1,11 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import * as generic from "../cache/generic"
+import * as user from "../cache/user"
+import * as app from "../cache/appMetadata"
+
+export = {
+  app,
+  user,
+  ...generic,
+}

packages/backend-core/src/pkg/context.ts

@@ -0,0 +1,24 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+  doInTenant,
+} from "../context"
+
+import * as identity from "../context/identity"
+
+export = {
+  getAppDB,
+  getDevAppDB,
+  getProdAppDB,
+  getAppId,
+  updateAppId,
+  doInAppContext,
+  doInTenant,
+  identity,
+}

packages/backend-core/src/pkg/db.ts

@@ -0,0 +1,7 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../db"
+export * from "../db/utils"
+export * from "../db/views"
+export * from "../db/pouch"
+export * from "../db/constants"

packages/backend-core/src/pkg/objectStore.ts

@@ -0,0 +1,4 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../objectStore"
+export * from "../objectStore/utils"

packages/backend-core/src/pkg/redis.ts

@@ -0,0 +1,11 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+import Client from "../redis"
+import utils from "../redis/utils"
+import clients from "../redis/authRedis"
+
+export = {
+  Client,
+  utils,
+  clients,
+}

packages/backend-core/src/pkg/utils.ts

@@ -0,0 +1,4 @@
+// Mimic the outer package export for usage in index.ts
+// The outer exports can't be used as they now reference dist directly
+export * from "../utils"
+export * from "../hashing"

packages/backend-core/src/redis/authRedis.js

@@ -1,13 +1,23 @@
 const Client = require("./index")
 const utils = require("./utils")
+const { getRedlock } = require("./redlock")
 
 let userClient, sessionClient, appClient, cacheClient
+let migrationsRedlock
+
+// turn retry off so that only one instance can ever hold the lock
+const migrationsRedlockConfig = { retryCount: 0 }
 
 async function init() {
   userClient = await new Client(utils.Databases.USER_CACHE).init()
   sessionClient = await new Client(utils.Databases.SESSIONS).init()
   appClient = await new Client(utils.Databases.APP_METADATA).init()
   cacheClient = await new Client(utils.Databases.GENERIC_CACHE).init()
+  // pass the underlying ioredis client to redlock
+  migrationsRedlock = getRedlock(
+    cacheClient.getClient(),
+    migrationsRedlockConfig
+  )
 }
 
 process.on("exit", async () => {
@@ -42,4 +52,10 @@ module.exports = {
     }
     return cacheClient
   },
+  getMigrationsRedlock: async () => {
+    if (!migrationsRedlock) {
+      await init()
+    }
+    return migrationsRedlock
+  },
 }

packages/backend-core/src/redis/index.js

@@ -139,6 +139,10 @@ class RedisWrapper {
     this._db = db
   }
 
+  getClient() {
+    return CLIENT
+  }
+
   async init() {
     CLOSED = false
     init()
@@ -164,6 +168,11 @@ class RedisWrapper {
     return promisifyStream(stream)
   }
 
+  async keys(pattern) {
+    const db = this._db
+    return CLIENT.keys(addDbPrefix(db, pattern))
+  }
+
   async get(key) {
     const db = this._db
     let response = await CLIENT.get(addDbPrefix(db, key))

packages/backend-core/src/redis/redlock.ts

@@ -0,0 +1,21 @@
+import Redlock from "redlock"
+
+export const getRedlock = (redisClient: any, opts = { retryCount: 10 }) => {
+  return new Redlock([redisClient], {
+    // the expected clock drift; for more details
+    // see http://redis.io/topics/distlock
+    driftFactor: 0.01, // multiplied by lock ttl to determine drift time
+
+    // the max number of times Redlock will attempt
+    // to lock a resource before erroring
+    retryCount: opts.retryCount,
+
+    // the time in ms between attempts
+    retryDelay: 200, // time in ms
+
+    // the max time in ms randomly added to retries
+    // to improve performance under high contention
+    // see https://www.awsarchitectureblog.com/2015/03/backoff.html
+    retryJitter: 200, // time in ms
+  })
+}

packages/backend-core/src/tenancy/index.js

@@ -1,4 +0,0 @@
-module.exports = {
-  ...require("../context"),
-  ...require("./tenancy"),
-}

packages/backend-core/src/tenancy/index.ts

@@ -0,0 +1,9 @@
+import * as context from "../context"
+import * as tenancy from "./tenancy"
+
+const pkg = {
+  ...context,
+  ...tenancy,
+}
+
+export = pkg

packages/backend-core/src/tenancy/tenancy.ts

@@ -1,18 +1,18 @@
-const { doWithDB } = require("../db")
+import { doWithDB } from "../db"
-const { StaticDatabases } = require("../db/constants")
+import { StaticDatabases } from "../db/constants"
-const { baseGlobalDBName } = require("./utils")
+import { baseGlobalDBName } from "./utils"
-const {
+import {
   getTenantId,
   DEFAULT_TENANT_ID,
   isMultiTenant,
   getTenantIDFromAppID,
-} = require("../context")
+} from "../context"
-const env = require("../environment")
+import env from "../environment"
 
 const TENANT_DOC = StaticDatabases.PLATFORM_INFO.docs.tenants
 const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
 
-exports.addTenantToUrl = url => {
+export const addTenantToUrl = (url: string) => {
   const tenantId = getTenantId()
 
   if (isMultiTenant()) {
@@ -23,8 +23,8 @@ exports.addTenantToUrl = url => {
   return url
 }
 
-exports.doesTenantExist = async tenantId => {
+export const doesTenantExist = async (tenantId: string) => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)
@@ -40,9 +40,14 @@ exports.doesTenantExist = async tenantId => {
   })
 }
 
-exports.tryAddTenant = async (tenantId, userId, email) => {
+export const tryAddTenant = async (
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+  tenantId: string,
-    const getDoc = async id => {
+  userId: string,
+  email: string,
+  afterCreateTenant: () => Promise<void>
+) => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
+    const getDoc = async (id: string) => {
       if (!id) {
         return null
       }
@@ -76,12 +81,13 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
     if (tenants.tenantIds.indexOf(tenantId) === -1) {
       tenants.tenantIds.push(tenantId)
       promises.push(db.put(tenants))
+      await afterCreateTenant()
     }
     await Promise.all(promises)
   })
 }
 
-exports.getGlobalDBName = (tenantId = null) => {
+export const getGlobalDBName = (tenantId?: string) => {
   // tenant ID can be set externally, for example user API where
   // new tenants are being created, this may be the case
   if (!tenantId) {
@@ -90,12 +96,12 @@ exports.getGlobalDBName = (tenantId = null) => {
   return baseGlobalDBName(tenantId)
 }
 
-exports.doWithGlobalDB = (tenantId, cb) => {
+export const doWithGlobalDB = (tenantId: string, cb: any) => {
-  return doWithDB(exports.getGlobalDBName(tenantId), cb)
+  return doWithDB(getGlobalDBName(tenantId), cb)
 }
 
-exports.lookupTenantId = async userId => {
+export const lookupTenantId = async (userId: string) => {
-  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async db => {
+  return doWithDB(StaticDatabases.PLATFORM_INFO.name, async (db: any) => {
     let tenantId = env.MULTI_TENANCY ? DEFAULT_TENANT_ID : null
     try {
       const doc = await db.get(userId)
@@ -110,8 +116,8 @@ exports.lookupTenantId = async userId => {
 }
 
 // lookup, could be email or userId, either will return a doc
-exports.getTenantUser = async identifier => {
+export const getTenantUser = async (identifier: string) => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     try {
       return await db.get(identifier)
     } catch (err) {
@@ -120,7 +126,7 @@ exports.getTenantUser = async identifier => {
   })
 }
 
-exports.isUserInAppTenant = (appId, user = null) => {
+export const isUserInAppTenant = (appId: string, user: any) => {
   let userTenantId
   if (user) {
     userTenantId = user.tenantId || DEFAULT_TENANT_ID
@@ -131,8 +137,8 @@ exports.isUserInAppTenant = (appId, user = null) => {
   return tenantId === userTenantId
 }
 
-exports.getTenantIds = async () => {
+export const getTenantIds = async () => {
-  return doWithDB(PLATFORM_INFO_DB, async db => {
+  return doWithDB(PLATFORM_INFO_DB, async (db: any) => {
     let tenants
     try {
       tenants = await db.get(TENANT_DOC)

packages/backend-core/src/tests/utils.spec.js

@@ -0,0 +1,17 @@
+require("../../tests/utilities/TestConfiguration")
+const { structures } = require("../../tests/utilities")
+const utils = require("../utils")
+const events = require("../events")
+const { doInTenant, DEFAULT_TENANT_ID }= require("../context")
+
+describe("utils", () => {
+  describe("platformLogout", () => {
+    it("should call platform logout", async () => {
+      await doInTenant(DEFAULT_TENANT_ID, async () => {
+        const ctx = structures.koa.newContext()
+        await utils.platformLogout({ ctx, userId: "test" })
+        expect(events.auth.logout).toBeCalledTimes(1)
+      })
+    })
+  })
+})

packages/backend-core/src/users.js

@@ -0,0 +1,21 @@
+const { ViewNames } = require("./db/utils")
+const { queryGlobalView } = require("./db/views")
+
+/**
+ * Given an email address this will use a view to search through
+ * all the users to find one with this email address.
+ * @param {string} email the email to lookup the user by.
+ * @return {Promise<object|null>}
+ */
+exports.getGlobalUserByEmail = async email => {
+  if (email == null) {
+    throw "Must supply an email address to view"
+  }
+
+  const response = await queryGlobalView(ViewNames.USER_BY_EMAIL, {
+    key: email.toLowerCase(),
+    include_docs: true,
+  })
+
+  return response
+}

packages/backend-core/src/utils.js

@@ -2,25 +2,16 @@ const {
   DocumentTypes,
   SEPARATOR,
   ViewNames,
-  generateGlobalUserID,
   getAllApps,
 } = require("./db/utils")
 const jwt = require("jsonwebtoken")
 const { options } = require("./middleware/passport/jwt")
 const { queryGlobalView } = require("./db/views")
-const { Headers, UserStatus, Cookies, MAX_VALID_DATE } = require("./constants")
+const { Headers, Cookies, MAX_VALID_DATE } = require("./constants")
-const {
-  doWithGlobalDB,
-  updateTenantId,
-  getTenantUser,
-  tryAddTenant,
-} = require("./tenancy")
-const environment = require("./environment")
-const accounts = require("./cloud/accounts")
-const { hash } = require("./hashing")
-const userCache = require("./cache/user")
 const env = require("./environment")
+const userCache = require("./cache/user")
 const { getUserSessions, invalidateSessions } = require("./security/sessions")
+const events = require("./events")
 const tenancy = require("./tenancy")
 
 const APP_PREFIX = DocumentTypes.APP + SEPARATOR
@@ -135,8 +126,8 @@ exports.setCookie = (ctx, value, name = "builder", opts = { sign: true }) => {
     overwrite: true,
   }
 
-  if (environment.COOKIE_DOMAIN) {
+  if (env.COOKIE_DOMAIN) {
-    config.domain = environment.COOKIE_DOMAIN
+    config.domain = env.COOKIE_DOMAIN
   }
 
   ctx.cookies.set(name, value, config)
@@ -159,23 +150,6 @@ exports.isClient = ctx => {
   return ctx.headers[Headers.TYPE] === "client"
 }
 
-/**
- * Given an email address this will use a view to search through
- * all the users to find one with this email address.
- * @param {string} email the email to lookup the user by.
- * @return {Promise<object|null>}
- */
-exports.getGlobalUserByEmail = async email => {
-  if (email == null) {
-    throw "Must supply an email address to view"
-  }
-
-  return queryGlobalView(ViewNames.USER_BY_EMAIL, {
-    key: email.toLowerCase(),
-    include_docs: true,
-  })
-}
-
 const getBuilders = async () => {
   const builders = await queryGlobalView(ViewNames.USER_BY_BUILDERS, {
     include_docs: false,
@@ -197,124 +171,6 @@ exports.getBuildersCount = async () => {
   return builders.length
 }
 
-const DEFAULT_SAVE_USER = {
-  hashPassword: true,
-  requirePassword: true,
-  bulkCreate: false,
-}
-
-exports.internalSaveUser = async (
-  user,
-  tenantId,
-  { hashPassword, requirePassword, bulkCreate } = DEFAULT_SAVE_USER
-) => {
-  if (!tenantId) {
-    throw "No tenancy specified."
-  }
-  // need to set the context for this request, as specified
-  updateTenantId(tenantId)
-  // specify the tenancy incase we're making a new admin user (public)
-  return doWithGlobalDB(tenantId, async db => {
-    let { email, password, _id } = user
-    // make sure another user isn't using the same email
-    let dbUser
-    // user can't exist in bulk creation
-    if (bulkCreate) {
-      dbUser = null
-    } else if (email) {
-      // check budibase users inside the tenant
-      dbUser = await exports.getGlobalUserByEmail(email)
-      if (dbUser != null && (dbUser._id !== _id || Array.isArray(dbUser))) {
-        throw `Email address ${email} already in use.`
-      }
-
-      // check budibase users in other tenants
-      if (env.MULTI_TENANCY) {
-        const tenantUser = await getTenantUser(email)
-        if (tenantUser != null && tenantUser.tenantId !== tenantId) {
-          throw `Email address ${email} already in use.`
-        }
-      }
-
-      // check root account users in account portal
-      if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
-        const account = await accounts.getAccount(email)
-        if (account && account.verified && account.tenantId !== tenantId) {
-          throw `Email address ${email} already in use.`
-        }
-      }
-    } else {
-      dbUser = await db.get(_id)
-    }
-
-    // get the password, make sure one is defined
-    let hashedPassword
-    if (password) {
-      hashedPassword = hashPassword ? await hash(password) : password
-    } else if (dbUser) {
-      hashedPassword = dbUser.password
-    } else if (requirePassword) {
-      throw "Password must be specified."
-    }
-
-    _id = _id || generateGlobalUserID()
-    user = {
-      createdAt: Date.now(),
-      ...dbUser,
-      ...user,
-      _id,
-      password: hashedPassword,
-      tenantId,
-    }
-    // make sure the roles object is always present
-    if (!user.roles) {
-      user.roles = {}
-    }
-    // add the active status to a user if its not provided
-    if (user.status == null) {
-      user.status = UserStatus.ACTIVE
-    }
-    try {
-      const putOpts = {
-        password: hashedPassword,
-        ...user,
-      }
-      if (bulkCreate) {
-        return putOpts
-      }
-      const response = await db.put(putOpts)
-      if (env.MULTI_TENANCY) {
-        await tryAddTenant(tenantId, _id, email)
-      }
-      await userCache.invalidateUser(response.id)
-      return {
-        _id: response.id,
-        _rev: response.rev,
-        email,
-      }
-    } catch (err) {
-      if (err.status === 409) {
-        throw "User exists already"
-      } else {
-        throw err
-      }
-    }
-  })
-}
-
-// maintained for api compat, don't want to change function signature
-exports.saveUser = async (
-  user,
-  tenantId,
-  hashPassword = true,
-  requirePassword = true
-) => {
-  return exports.internalSaveUser(user, tenantId, {
-    hashPassword,
-    requirePassword,
-  })
-}
-
 /**
  * Logs a user out from budibase. Re-used across account portal and builder.
  */
@@ -338,5 +194,6 @@ exports.platformLogout = async ({ ctx, userId, keepActiveSession }) => {
     userId,
     sessions.map(({ sessionId }) => sessionId)
   )
+  await events.auth.logout()
   await userCache.invalidateUser(userId)
 }

packages/backend-core/tests/index.ts

@@ -0,0 +1 @@
+export * from "./utilities"