Identity tenant and installation groups, property updates

packages/backend-core/context.js

@@ -5,10 +5,11 @@ const {
   getAppId,
   updateAppId,
   doInAppContext,
-  doInUserContext,
   doInTenant,
 } = require("./src/context")
 
+const identity = require("./src/context/identity")
+
 module.exports = {
   getAppDB,
   getDevAppDB,
@@ -16,6 +17,6 @@ module.exports = {
   getAppId,
   updateAppId,
   doInAppContext,
-  doInUserContext,
   doInTenant,
+  identity,
 }

packages/backend-core/package.json

@@ -32,6 +32,7 @@
     "pouchdb-find": "^7.2.2",
     "pouchdb-replication-stream": "^1.2.9",
     "sanitize-s3-objectkey": "^0.0.1",
+    "semver": "^7.0.0",
     "tar-fs": "^2.1.1",
     "uuid": "^8.3.2",
     "zlib": "^1.0.5"
@@ -52,6 +53,7 @@
     "@types/node-fetch": "^2.6.1",
     "@types/tar-fs": "^2.0.1",
     "@types/uuid": "^8.3.4",
+    "@types/semver": "^7.0.0",
     "ioredis-mock": "^5.5.5",
     "jest": "^27.0.3",
     "koa": "2.7.0",

packages/backend-core/src/context/identity.ts

@@ -0,0 +1,50 @@
+import {
+  IdentityContext,
+  IdentityType,
+  User,
+  UserContext,
+  isCloudAccount,
+  Account,
+  AccountUserContext,
+} from "@budibase/types"
+import * as context from "."
+
+export const getIdentity = (): IdentityContext | undefined => {
+  return context.getIdentity()
+}
+
+export const doInIdentityContext = (identity: IdentityContext, task: any) => {
+  return context.doInIdentityContext(identity, task)
+}
+
+export const doInUserContext = (user: User, task: any) => {
+  const userContext: UserContext = {
+    ...user,
+    _id: user._id as string,
+    type: IdentityType.USER,
+  }
+  return doInIdentityContext(userContext, task)
+}
+
+export const doInAccountContext = (account: Account, task: any) => {
+  const _id = getAccountUserId(account)
+  const tenantId = account.tenantId
+  const accountContext: AccountUserContext = {
+    _id,
+    type: IdentityType.USER,
+    tenantId,
+    account,
+  }
+  return doInIdentityContext(accountContext, task)
+}
+
+export const getAccountUserId = (account: Account) => {
+  let userId: string
+  if (isCloudAccount(account)) {
+    userId = account.budibaseUserId
+  } else {
+    // use account id as user id for self hosting
+    userId = account.accountId
+  }
+  return userId
+}

packages/backend-core/src/context/index.js

@@ -15,7 +15,7 @@ const ContextKeys = {
   TENANT_ID: "tenantId",
   GLOBAL_DB: "globalDb",
   APP_ID: "appId",
-  USER: "user",
+  IDENTITY: "identity",
   // whatever the request app DB was
   CURRENT_DB: "currentDb",
   // get the prod app DB from the request
@@ -138,19 +138,19 @@ exports.doInAppContext = (appId, task) => {
     throw new Error("appId is required")
   }
 
-  const user = exports.getUser()
+  const identity = exports.getIdentity()
 
   // the internal function is so that we can re-use an existing
   // context - don't want to close DB on a parent context
-  async function internal(opts = { existing: false, user: undefined }) {
+  async function internal(opts = { existing: false }) {
     // set the app tenant id
     if (!opts.existing) {
       setAppTenantId(appId)
     }
     // set the app ID
     cls.setOnContext(ContextKeys.APP_ID, appId)
-    // preserve the user
-    exports.setUser(user)
+    // preserve the identity
+    exports.setIdentity(identity)
     try {
       // invoke the task
       return await task()
@@ -175,17 +175,17 @@ exports.doInAppContext = (appId, task) => {
   }
 }
 
-exports.doInUserContext = (user, task) => {
-  if (!user) {
-    throw new Error("user is required")
+exports.doInIdentityContext = (identity, task) => {
+  if (!identity) {
+    throw new Error("identity is required")
   }
 
   async function internal(opts = { existing: false }) {
     if (!opts.existing) {
-      cls.setOnContext(ContextKeys.USER, user)
-      // set the tenant so that doInTenant will preserve user
-      if (user.tenantId) {
-        exports.updateTenantId(user.tenantId)
+      cls.setOnContext(ContextKeys.IDENTITY, identity)
+      // set the tenant so that doInTenant will preserve identity
+      if (identity.tenantId) {
+        exports.updateTenantId(identity.tenantId)
       }
     }
 
@@ -195,16 +195,16 @@ exports.doInUserContext = (user, task) => {
     } finally {
       const using = cls.getFromContext(ContextKeys.IN_USE)
       if (!using || using <= 1) {
-        exports.setUser(null)
+        exports.setIdentity(null)
       } else {
         cls.setOnContext(using - 1)
       }
     }
   }
 
-  const existing = cls.getFromContext(ContextKeys.USER)
+  const existing = cls.getFromContext(ContextKeys.IDENTITY)
   const using = cls.getFromContext(ContextKeys.IN_USE)
-  if (using && existing && existing._id === user._id) {
+  if (using && existing && existing._id === identity._id) {
     cls.setOnContext(ContextKeys.IN_USE, using + 1)
     return internal({ existing: true })
   } else {
@@ -215,15 +215,15 @@ exports.doInUserContext = (user, task) => {
   }
 }
 
-exports.setUser = user => {
-  cls.setOnContext(ContextKeys.USER, user)
+exports.setIdentity = identity => {
+  cls.setOnContext(ContextKeys.IDENTITY, identity)
 }
 
-exports.getUser = () => {
+exports.getIdentity = () => {
   try {
-    return cls.getFromContext(ContextKeys.USER)
+    return cls.getFromContext(ContextKeys.IDENTITY)
   } catch (e) {
-    // do nothing - user is not in context
+    // do nothing - identity is not in context
   }
 }
 

packages/backend-core/src/environment.ts

@@ -52,6 +52,7 @@ const env: any = {
     process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads",
   USE_COUCH: process.env.USE_COUCH || true,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
+  SERVICE: process.env.SERVICE || "budibase",
   _set(key: any, value: any) {
     process.env[key] = value
     module.exports[key] = value

packages/backend-core/src/events/analytics.ts

@@ -3,6 +3,7 @@ import * as tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
 import { Configs } from "../constants"
 
+// TODO: cache in redis
 export const enabled = async () => {
   // cloud - always use the environment variable
   if (!env.SELF_HOSTED) {

packages/backend-core/src/events/identification.ts

@@ -1,56 +1,239 @@
 import * as context from "../context"
+import * as identityCtx from "../context/identity"
 import env from "../environment"
 import {
   Hosting,
   User,
-  SessionUser,
   Identity,
   IdentityType,
   Account,
-  BudibaseIdentity,
   isCloudAccount,
   isSSOAccount,
-  TenantIdentity,
+  TenantGroup,
   SettingsConfig,
   CloudAccount,
   UserIdentity,
-  InstallationIdentity,
-  Installation,
-  isInstallation,
+  InstallationGroup,
+  isSelfHostAccount,
+  UserContext,
+  Group,
 } from "@budibase/types"
 import { processors } from "./processors"
 import * as dbUtils from "../db/utils"
 import { Configs } from "../constants"
 import * as hashing from "../hashing"
+import * as installation from "../installation"
 
 const pkg = require("../../package.json")
 
+/**
+ * An identity can be:
+ * - account user (Self host)
+ * - budibase user
+ * - tenant
+ * - installation
+ */
 export const getCurrentIdentity = async (): Promise<Identity> => {
-  const user: SessionUser | undefined = context.getUser()
+  let identityContext = identityCtx.getIdentity()
 
-  const tenantId = await getGlobalTenantId(context.getTenantId())
-  let id: string
-  let type: IdentityType
+  let identityType
 
-  if (user) {
-    id = user._id
-    type = IdentityType.USER
+  if (!identityContext) {
+    identityType = IdentityType.TENANT
   } else {
-    id = tenantId
-    type = IdentityType.TENANT
+    identityType = identityContext.type
   }
 
-  if (user && isInstallation(user)) {
-    type = IdentityType.INSTALLATION
-  }
+  if (identityType === IdentityType.INSTALLATION) {
+    const installationId = await getInstallationId()
+    return {
+      id: formatDistinctId(installationId, identityType),
+      type: identityType,
+      installationId,
+    }
+  } else if (identityType === IdentityType.TENANT) {
+    const installationId = await getInstallationId()
+    const tenantId = await getCurrentTenantId()
 
-  return {
-    id,
-    tenantId,
-    type,
+    return {
+      id: formatDistinctId(tenantId, identityType),
+      type: identityType,
+      installationId,
+      tenantId,
+    }
+  } else if (identityType === IdentityType.USER) {
+    const userContext = identityContext as UserContext
+    const tenantId = await getCurrentTenantId()
+    let installationId: string | undefined
+
+    // self host account users won't have installation
+    if (!userContext.account || !isSelfHostAccount(userContext.account)) {
+      installationId = await getInstallationId()
+    }
+
+    return {
+      id: userContext._id,
+      type: identityType,
+      installationId,
+      tenantId,
+    }
+  } else {
+    throw new Error("Unknown identity type")
   }
 }
 
+export const identifyInstallationGroup = async (
+  installId: string,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = installId
+  const type = IdentityType.INSTALLATION
+  const hosting = getHostingFromEnv()
+  const version = pkg.version
+
+  const group: InstallationGroup = {
+    id,
+    type,
+    hosting,
+    version,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the empty auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyTenantGroup = async (
+  tenantId: string,
+  account: Account | undefined,
+  timestamp?: string | number
+): Promise<void> => {
+  const id = await getGlobalTenantId(tenantId)
+  const type = IdentityType.TENANT
+
+  let hosting: Hosting
+  let profession: string | undefined
+  let companySize: string | undefined
+
+  if (account) {
+    profession = account.profession
+    companySize = account.size
+    hosting = account.hosting
+  } else {
+    hosting = getHostingFromEnv()
+  }
+
+  const group: TenantGroup = {
+    id,
+    type,
+    hosting,
+    profession,
+    companySize,
+  }
+
+  await identifyGroup(group, timestamp)
+  // need to create a normal identity for the group to be able to query it globally
+  // match the posthog syntax to link this identity to the auto generated one
+  await identify({ ...group, id: `$${type}_${id}` }, timestamp)
+}
+
+export const identifyUser = async (
+  user: User,
+  account: CloudAccount | undefined,
+  timestamp?: string | number
+) => {
+  const id = user._id as string
+  const tenantId = await getGlobalTenantId(user.tenantId)
+  const type = IdentityType.USER
+  let builder = user.builder?.global || false
+  let admin = user.admin?.global || false
+  let providerType = user.providerType
+  const accountHolder = account?.budibaseUserId === user._id || false
+  const verified =
+    account && account?.budibaseUserId === user._id ? account.verified : false
+  const installationId = await getInstallationId()
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    installationId,
+    tenantId,
+    verified,
+    accountHolder,
+    providerType,
+    builder,
+    admin,
+  }
+
+  await identify(identity, timestamp)
+}
+
+export const identifyAccount = async (account: Account) => {
+  let id = account.accountId
+  const tenantId = account.tenantId
+  let type = IdentityType.USER
+  let providerType = isSSOAccount(account) ? account.providerType : undefined
+  const verified = account.verified
+  const accountHolder = true
+
+  if (isCloudAccount(account)) {
+    if (account.budibaseUserId) {
+      // use the budibase user as the id if set
+      id = account.budibaseUserId
+    }
+  }
+
+  const identity: UserIdentity = {
+    id,
+    type,
+    tenantId,
+    providerType,
+    verified,
+    accountHolder,
+  }
+
+  await identify(identity)
+}
+
+export const identify = async (
+  identity: Identity,
+  timestamp?: string | number
+) => {
+  await processors.identify(identity, timestamp)
+}
+
+export const identifyGroup = async (
+  group: Group,
+  timestamp?: string | number
+) => {
+  await processors.identifyGroup(group, timestamp)
+}
+
+const getHostingFromEnv = () => {
+  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
+}
+
+export const getCurrentTenantId = () => getGlobalTenantId(context.getTenantId())
+
+export const getInstallationId = async () => {
+  if (isAccountPortal()) {
+    return "account-portal"
+  }
+  const install = await installation.getInstall()
+  return install.installId
+}
+
+const getGlobalTenantId = async (tenantId: string): Promise<string> => {
+  if (env.SELF_HOSTED) {
+    return getGlobalId(tenantId)
+  } else {
+    // tenant id's in the cloud are already unique
+    return tenantId
+  }
+}
+
+// TODO: cache in redis
 const getGlobalId = async (tenantId: string): Promise<string> => {
   const db = context.getGlobalDB()
   const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
@@ -68,134 +251,14 @@ const getGlobalId = async (tenantId: string): Promise<string> => {
   }
 }
 
-const getGlobalTenantId = async (tenantId: string): Promise<string> => {
-  if (env.SELF_HOSTED) {
-    return getGlobalId(tenantId)
+const isAccountPortal = () => {
+  return env.SERVICE === "account-portal"
+}
+
+const formatDistinctId = (id: string, type: IdentityType) => {
+  if (type === IdentityType.INSTALLATION || type === IdentityType.TENANT) {
+    return `$${type}_${id}`
   } else {
-    // tenant id's in the cloud are already unique
-    return tenantId
+    return id
   }
 }
-
-const getHostingFromEnv = () => {
-  return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
-}
-
-export const identifyInstallation = async (
-  install: Installation,
-  timestamp: string | number
-) => {
-  const id = install.installId
-  // the default tenant id, so we can match installations to other events
-  const tenantId = await getGlobalTenantId(context.getTenantId())
-  const version: string = pkg.version as string
-  const type = IdentityType.INSTALLATION
-  const hosting = getHostingFromEnv()
-
-  const identity: InstallationIdentity = {
-    id,
-    tenantId,
-    type,
-    version,
-    hosting,
-  }
-  await identify(identity, timestamp)
-}
-
-export const identifyTenant = async (
-  tenantId: string,
-  account: CloudAccount | undefined,
-  timestamp?: string | number
-) => {
-  const globalTenantId = await getGlobalTenantId(tenantId)
-  const id = globalTenantId
-  const hosting = getHostingFromEnv()
-  const type = IdentityType.TENANT
-  const profession = account?.profession
-  const companySize = account?.size
-
-  const identity: TenantIdentity = {
-    id,
-    tenantId: globalTenantId,
-    hosting,
-    type,
-    profession,
-    companySize,
-  }
-  await identify(identity, timestamp)
-}
-
-export const identifyUser = async (
-  user: User,
-  account: CloudAccount | undefined,
-  timestamp?: string | number
-) => {
-  const id = user._id as string
-  const tenantId = user.tenantId
-  const hosting = env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
-  const type = IdentityType.USER
-  let builder = user.builder?.global
-  let admin = user.admin?.global
-  let providerType = user.providerType
-  const accountHolder = account?.budibaseUserId === user._id
-  const verified =
-    account && account?.budibaseUserId === user._id ? account.verified : false
-  const profession = account?.profession
-  const companySize = account?.size
-
-  const identity: BudibaseIdentity = {
-    id,
-    tenantId,
-    hosting,
-    type,
-    builder,
-    admin,
-    providerType,
-    accountHolder,
-    verified,
-    profession,
-    companySize,
-  }
-
-  await identify(identity, timestamp)
-}
-
-export const identifyAccount = async (account: Account) => {
-  let id = account.accountId
-  const tenantId = account.tenantId
-  const hosting = account.hosting
-  let type = IdentityType.USER
-  let providerType = isSSOAccount(account) ? account.providerType : undefined
-  const verified = account.verified
-  const profession = account.profession
-  const companySize = account.size
-  const accountHolder = true
-
-  if (isCloudAccount(account)) {
-    if (account.budibaseUserId) {
-      // use the budibase user as the id if set
-      id = account.budibaseUserId
-    }
-  }
-
-  const identity: UserIdentity = {
-    id,
-    tenantId,
-    hosting,
-    type,
-    providerType,
-    verified,
-    profession,
-    companySize,
-    accountHolder,
-  }
-
-  await identify(identity)
-}
-
-export const identify = async (
-  identity: Identity,
-  timestamp?: string | number
-) => {
-  await processors.identify(identity, timestamp)
-}

packages/backend-core/src/events/processors/AnalyticsProcessor.ts

@@ -1,9 +1,15 @@
-import { Event, Identity } from "@budibase/types"
+import { Event, Identity, Group, IdentityType } from "@budibase/types"
 import { EventProcessor } from "./types"
 import env from "../../environment"
 import * as analytics from "../analytics"
 import PosthogProcessor from "./PosthogProcessor"
 
+/**
+ * Events that are always captured.
+ */
+const EVENT_WHITELIST = [Event.VERSION_UPGRADED, Event.VERSION_DOWNGRADED]
+const IDENTITY_WHITELIST = [IdentityType.INSTALLATION, IdentityType.TENANT]
+
 export default class AnalyticsProcessor implements EventProcessor {
   posthog: PosthogProcessor | undefined
 
@@ -19,7 +25,7 @@ export default class AnalyticsProcessor implements EventProcessor {
     properties: any,
     timestamp?: string | number
   ): Promise<void> {
-    if (!(await analytics.enabled())) {
+    if (!EVENT_WHITELIST.includes(event) && !(await analytics.enabled())) {
       return
     }
     if (this.posthog) {
@@ -28,7 +34,11 @@ export default class AnalyticsProcessor implements EventProcessor {
   }
 
   async identify(identity: Identity, timestamp?: string | number) {
-    if (!(await analytics.enabled())) {
+    // Group indentifications (tenant and installation) always on
+    if (
+      !IDENTITY_WHITELIST.includes(identity.type) &&
+      !(await analytics.enabled())
+    ) {
       return
     }
     if (this.posthog) {
@@ -36,6 +46,13 @@ export default class AnalyticsProcessor implements EventProcessor {
     }
   }
 
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    // Group indentifications (tenant and installation) always on
+    if (this.posthog) {
+      this.posthog.identifyGroup(group, timestamp)
+    }
+  }
+
   shutdown() {
     if (this.posthog) {
       this.posthog.shutdown()

packages/backend-core/src/events/processors/LoggingProcessor.ts

@@ -1,7 +1,17 @@
-import { Event, Identity } from "@budibase/types"
+import { Event, Identity, Group } from "@budibase/types"
 import { EventProcessor } from "./types"
 import env from "../../environment"
 
+const getTimestampString = (timestamp?: string | number) => {
+  let timestampString = ""
+  if (timestamp) {
+    timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
+  }
+  return timestampString
+}
+
+const skipLogging = env.SELF_HOSTED && !env.isDev()
+
 export default class LoggingProcessor implements EventProcessor {
   async processEvent(
     event: Event,
@@ -9,34 +19,35 @@ export default class LoggingProcessor implements EventProcessor {
     properties: any,
     timestamp?: string
   ): Promise<void> {
-    if (env.SELF_HOSTED && !env.isDev()) {
+    if (skipLogging) {
       return
     }
-    let timestampString = ""
-    if (timestamp) {
-      timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
-    }
-
+    let timestampString = getTimestampString(timestamp)
     console.log(
       `[audit] [tenant=${identity.tenantId}] [identityType=${identity.type}] [identity=${identity.id}] ${timestampString} ${event} `
     )
   }
 
   async identify(identity: Identity, timestamp?: string | number) {
-    if (env.SELF_HOSTED && !env.isDev()) {
+    if (skipLogging) {
       return
     }
-
-    let timestampString = ""
-    if (timestamp) {
-      timestampString = `[timestamp=${new Date(timestamp).toISOString()}]`
-    }
-
+    let timestampString = getTimestampString(timestamp)
     console.log(
       `[audit] [${JSON.stringify(identity)}] ${timestampString} identified`
     )
   }
 
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    if (skipLogging) {
+      return
+    }
+    let timestampString = getTimestampString(timestamp)
+    console.log(
+      `[audit] [${JSON.stringify(group)}] ${timestampString} group identified`
+    )
+  }
+
   shutdown(): void {
     // no-op
   }

packages/backend-core/src/events/processors/PosthogProcessor.ts

@@ -1,6 +1,8 @@
 import PostHog from "posthog-node"
-import { Event, Identity } from "@budibase/types"
+import { Event, Identity, Group } from "@budibase/types"
 import { EventProcessor } from "./types"
+import env from "../../environment"
+const pkg = require("../../../package.json")
 
 export default class PosthogProcessor implements EventProcessor {
   posthog: PostHog
@@ -18,10 +20,24 @@ export default class PosthogProcessor implements EventProcessor {
     properties: any,
     timestamp?: string | number
   ): Promise<void> {
+    properties.version = pkg.version
+    properties.service = env.SERVICE
     const payload: any = { distinctId: identity.id, event, properties }
     if (timestamp) {
       payload.timestamp = new Date(timestamp)
     }
+
+    // add groups to the event
+    if (identity.installationId || identity.tenantId) {
+      payload.groups = {}
+      if (identity.installationId) {
+        payload.groups.installation = identity.installationId
+      }
+      if (identity.tenantId) {
+        payload.groups.tenant = identity.tenantId
+      }
+    }
+
     this.posthog.capture(payload)
   }
 
@@ -33,6 +49,20 @@ export default class PosthogProcessor implements EventProcessor {
     this.posthog.identify(payload)
   }
 
+  async identifyGroup(group: Group, timestamp?: string | number) {
+    const payload: any = {
+      distinctId: group.id,
+      groupType: group.type,
+      groupKey: group.id,
+      properties: group,
+    }
+
+    if (timestamp) {
+      payload.timestamp = new Date(timestamp)
+    }
+    this.posthog.groupIdentify(payload)
+  }
+
   shutdown() {
     this.posthog.shutdown()
   }

packages/backend-core/src/events/processors/Processors.ts

@@ -1,4 +1,4 @@
-import { Event, Identity } from "@budibase/types"
+import { Event, Identity, Group } from "@budibase/types"
 import { EventProcessor } from "./types"
 
 export default class Processor implements EventProcessor {
@@ -29,6 +29,15 @@ export default class Processor implements EventProcessor {
     }
   }
 
+  async identifyGroup(
+    identity: Group,
+    timestamp?: string | number
+  ): Promise<void> {
+    for (const eventProcessor of this.processors) {
+      await eventProcessor.identifyGroup(identity, timestamp)
+    }
+  }
+
   shutdown() {
     for (const eventProcessor of this.processors) {
       eventProcessor.shutdown()

packages/backend-core/src/events/processors/types.ts

@@ -1,4 +1,4 @@
-import { Event, Identity } from "@budibase/types"
+import { Event, Identity, Group } from "@budibase/types"
 
 export enum EventProcessorType {
   POSTHOG = "posthog",
@@ -13,5 +13,6 @@ export interface EventProcessor {
     timestamp?: string | number
   ): Promise<void>
   identify(identity: Identity, timestamp?: string | number): Promise<void>
+  identifyGroup(group: Group, timestamp?: string | number): Promise<void>
   shutdown(): void
 }

packages/backend-core/src/events/publishers/layout.ts

@@ -13,9 +13,9 @@ export async function created(layout: Layout, timestamp?: string) {
   await publishEvent(Event.LAYOUT_CREATED, properties, timestamp)
 }
 
-export async function deleted(layout: Layout) {
+export async function deleted(layoutId: string) {
   const properties: LayoutDeletedEvent = {
-    layoutId: layout._id as string,
+    layoutId,
   }
   await publishEvent(Event.LAYOUT_DELETED, properties)
 }

packages/backend-core/src/events/publishers/serve.ts

@@ -9,17 +9,23 @@ import {
 
 /* eslint-disable */
 
-export async function servedBuilder(version: number) {
+export async function servedBuilder() {
   const properties: BuilderServedEvent = {}
   await publishEvent(Event.SERVED_BUILDER, properties)
 }
 
 export async function servedApp(app: App) {
-  const properties: AppServedEvent = {}
+  const properties: AppServedEvent = {
+    appId: app.appId,
+    appVersion: app.version,
+  }
   await publishEvent(Event.SERVED_APP, properties)
 }
 
 export async function servedAppPreview(app: App) {
-  const properties: AppPreviewServedEvent = {}
+  const properties: AppPreviewServedEvent = {
+    appId: app.appId,
+    appVersion: app.version,
+  }
   await publishEvent(Event.SERVED_APP_PREVIEW, properties)
 }

packages/backend-core/src/index.ts

@@ -4,6 +4,7 @@ import * as events from "./events"
 import * as migrations from "./migrations"
 import * as users from "./users"
 import * as accounts from "./cloud/accounts"
+import * as installation from "./installation"
 import env from "./environment"
 import tenancy from "./tenancy"
 import featureFlags from "./featureFlags"
@@ -46,4 +47,5 @@ export = {
   events,
   sessions,
   deprovisioning,
+  installation,
 }

packages/backend-core/src/installation.ts

@@ -1,29 +1,27 @@
-import {
-  db as dbUtils,
-  events,
-  utils,
-  context,
-  tenancy,
-} from "@budibase/backend-core"
-import { Installation } from "@budibase/types"
+import * as hashing from "./hashing"
+import * as events from "./events"
+import { StaticDatabases } from "./db/constants"
+import { doWithDB } from "./db"
+import { Installation, IdentityType } from "@budibase/types"
+import * as context from "./context"
 import semver from "semver"
 
 const pkg = require("../package.json")
 
 export const getInstall = async (): Promise<Installation> => {
-  return dbUtils.doWithDB(
-    dbUtils.StaticDatabases.PLATFORM_INFO.name,
+  return doWithDB(
+    StaticDatabases.PLATFORM_INFO.name,
     async (platformDb: any) => {
       let install: Installation
       try {
         install = await platformDb.get(
-          dbUtils.StaticDatabases.PLATFORM_INFO.docs.install
+          StaticDatabases.PLATFORM_INFO.docs.install
         )
       } catch (e: any) {
         if (e.status === 404) {
           install = {
-            _id: dbUtils.StaticDatabases.PLATFORM_INFO.docs.install,
-            installId: utils.newid(),
+            _id: StaticDatabases.PLATFORM_INFO.docs.install,
+            installId: hashing.newid(),
             version: pkg.version,
           }
           const resp = await platformDb.put(install)
@@ -40,8 +38,8 @@ export const getInstall = async (): Promise<Installation> => {
 
 const updateVersion = async (version: string): Promise<boolean> => {
   try {
-    await dbUtils.doWithDB(
-      dbUtils.StaticDatabases.PLATFORM_INFO.name,
+    await doWithDB(
+      StaticDatabases.PLATFORM_INFO.name,
       async (platformDb: any) => {
         const install = await getInstall()
         install.version = version
@@ -73,11 +71,10 @@ export const checkInstallVersion = async (): Promise<void> => {
     const success = await updateVersion(newVersion)
 
     if (success) {
-      await context.doInUserContext(
+      await context.doInIdentityContext(
         {
           _id: install.installId,
-          isInstall: true,
-          tenantId: tenancy.DEFAULT_TENANT_ID,
+          type: IdentityType.INSTALLATION,
         },
         async () => {
           if (isUpgrade) {
@@ -87,6 +84,7 @@ export const checkInstallVersion = async (): Promise<void> => {
           }
         }
       )
+      await events.identification.identifyInstallationGroup(install.installId)
     }
   }
 }

packages/backend-core/src/middleware/authenticated.js

@@ -7,7 +7,7 @@ const env = require("../environment")
 const { SEPARATOR, ViewNames, queryGlobalView } = require("../../db")
 const { getGlobalDB, doInTenant } = require("../tenancy")
 const { decrypt } = require("../security/encryption")
-const context = require("../context")
+const identity = require("../context/identity")
 
 function finalise(
   ctx,
@@ -135,7 +135,7 @@ module.exports = (
       finalise(ctx, { authenticated, user, internal, version, publicEndpoint })
 
       if (user && user.email) {
-        return context.doInUserContext(user, next)
+        return identity.doInUserContext(user, next)
       } else {
         return next()
       }
@@ -147,7 +147,7 @@ module.exports = (
       // allow configuring for public access
       if ((opts && opts.publicAllowed) || publicEndpoint) {
         finalise(ctx, { authenticated: false, version, publicEndpoint })
-        return context.doInUserContext({ _id: "public_user" }, next)
+        return next()
       } else {
         ctx.throw(err.status || 403, err)
       }

packages/backend-core/src/tests/utilities/mocks/events.js

@@ -56,9 +56,11 @@ jest.mock("../../../events", () => {
       nameUpdated: jest.fn(),
       logoUpdated: jest.fn(),
       platformURLUpdated: jest.fn(),
-      versionChecked: jest.fn(),
       analyticsOptOut: jest.fn(),
     },
+    version: {
+      checked: jest.fn(),
+    },
     query: {
       created: jest.fn(),
       updated: jest.fn(),

packages/backend-core/yarn.lock

@@ -971,6 +971,11 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/semver@^7.0.0":
+  version "7.3.9"
+  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
+  integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ==
+
 "@types/serve-static@*":
   version "1.13.10"
   resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9"
@@ -5012,7 +5017,7 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-semver@7.x, semver@^7.3.4:
+semver@7.x, semver@^7.0.0, semver@^7.3.4:
   version "7.3.7"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
   integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==

packages/server/package.json

@@ -129,7 +129,6 @@
     "pouchdb-find": "^7.2.2",
     "pouchdb-replication-stream": "1.2.9",
     "redis": "4",
-    "semver": "^7.0.0",
     "server-destroy": "1.0.1",
     "svelte": "^3.38.2",
     "swagger-parser": "^10.0.3",
@@ -160,7 +159,6 @@
     "@types/node": "^15.12.4",
     "@types/oracledb": "^5.2.1",
     "@types/redis": "^4.0.11",
-    "@types/semver": "^7.0.0",
     "@typescript-eslint/parser": "5.12.0",
     "apidoc": "^0.50.2",
     "babel-jest": "^27.0.2",

packages/server/src/api/controllers/dev.js

@@ -131,5 +131,5 @@ exports.getBudibaseVersion = async ctx => {
   ctx.body = {
     version,
   }
-  await events.version.versionChecked(version)
+  await events.version.checked(version)
 }

packages/server/src/api/controllers/layout.js

@@ -20,7 +20,7 @@ exports.save = async function (ctx) {
 
   layout._id = layout._id || generateLayoutID()
   const response = await db.put(layout)
-  await events.layout.created()
+  await events.layout.created(layout)
   layout._rev = response.rev
 
   ctx.body = layout
@@ -48,7 +48,7 @@ exports.destroy = async function (ctx) {
   }
 
   await db.remove(layoutId, layoutRev)
-  await events.layout.deleted()
+  await events.layout.deleted(layoutId)
   ctx.body = { message: "Layout deleted successfully" }
   ctx.status = 200
 }

packages/server/src/api/controllers/static/index.js

@@ -19,7 +19,6 @@ const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 const AWS = require("aws-sdk")
 const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 const { events } = require("@budibase/backend-core")
-const version = require("../../../../package.json").version
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -43,7 +42,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
 exports.serveBuilder = async function (ctx) {
   let builderPath = resolve(TOP_LEVEL_PATH, "builder")
   await send(ctx, ctx.file, { root: builderPath })
-  await events.serve.servedBuilder(version)
+  await events.serve.servedBuilder()
 }
 
 exports.uploadFile = async function (ctx) {

packages/server/src/api/routes/tests/dev.spec.js

@@ -33,8 +33,8 @@ describe("/dev", () => {
         .expect(200)
 
       expect(res.body.version).toBe(version)
-      expect(events.org.versionChecked).toBeCalledTimes(1)
-      expect(events.org.versionChecked).toBeCalledWith(version)
+      expect(events.version.checked).toBeCalledTimes(1)
+      expect(events.version.checked).toBeCalledWith(version)
     })
   })
 })

packages/server/src/app.ts

@@ -16,8 +16,7 @@ const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
-import { events } from "@budibase/backend-core"
-import * as installation from "./installation"
+import { events, installation } from "@budibase/backend-core"
 
 const app = new Koa()
 

packages/server/src/migrations/functions/backfill/global.ts

@@ -2,9 +2,8 @@ import * as users from "./global/users"
 import * as rows from "./global/rows"
 import * as configs from "./global/configs"
 import { tenancy, events, migrations, accounts } from "@budibase/backend-core"
-import { CloudAccount, Installation } from "@budibase/types"
+import { CloudAccount } from "@budibase/types"
 import env from "../../../environment"
-import * as installation from "../../../installation"
 
 /**
  * Date:
@@ -22,7 +21,7 @@ export const run = async (db: any) => {
     account = await accounts.getAccountByTenantId(tenantId)
   }
 
-  await events.identification.identifyTenant(
+  await events.identification.identifyTenantGroup(
     tenantId,
     account,
     installTimestamp

packages/server/src/migrations/functions/backfill/installation.ts

@@ -1,6 +1,5 @@
-import { events, tenancy } from "@budibase/backend-core"
+import { events, tenancy, installation } from "@budibase/backend-core"
 import { Installation } from "@budibase/types"
-import * as installation from "../../../installation"
 import * as global from "./global"
 
 /**
@@ -17,6 +16,9 @@ export const run = async () => {
     const db = tenancy.getGlobalDB()
     const installTimestamp = (await global.getInstallTimestamp(db)) as number
     const install: Installation = await installation.getInstall()
-    await events.identification.identifyInstallation(install, installTimestamp)
+    await events.identification.identifyInstallationGroup(
+      install.installId,
+      installTimestamp
+    )
   })
 }

packages/server/yarn.lock

@@ -2754,11 +2754,6 @@
     "@types/tough-cookie" "*"
     form-data "^2.5.0"
 
-"@types/semver@^7.0.0":
-  version "7.3.9"
-  resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
-  integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ==
-
 "@types/serve-static@*":
   version "1.13.10"
   resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.10.tgz#f5e0ce8797d2d7cc5ebeda48a52c96c4fa47a8d9"
@@ -11650,13 +11645,6 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d"
   integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==
 
-semver@^7.0.0:
-  version "7.3.7"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
-  integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==
-  dependencies:
-    lru-cache "^6.0.0"
-
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"

packages/types/src/core/context.ts

@@ -0,0 +1,19 @@
+import { User, Account } from "../documents"
+import { IdentityType } from "./identification"
+
+export interface BaseContext {
+  _id: string
+  type: IdentityType
+}
+
+export interface AccountUserContext extends BaseContext {
+  tenantId: string
+  account: Account
+}
+
+export interface UserContext extends BaseContext, User {
+  _id: string
+  account?: Account
+}
+
+export type IdentityContext = BaseContext | AccountUserContext | UserContext

packages/types/src/core/identification.ts

@@ -0,0 +1,49 @@
+import { Hosting } from "."
+
+// GROUPS
+
+export enum GroupType {
+  TENANT = "tenant",
+  INSTALLATION = "installation",
+}
+
+export interface Group {
+  id: string
+  type: IdentityType
+}
+
+export interface TenantGroup extends Group {
+  // account level information is associated with the tenant group
+  // as we don't have this at the user level
+  profession?: string // only available in cloud
+  companySize?: string // only available in cloud
+  hosting: Hosting // need hosting at the tenant level for cloud self host accounts
+}
+
+export interface InstallationGroup extends Group {
+  version: string
+  hosting: Hosting
+}
+
+// IDENTITIES
+
+export enum IdentityType {
+  USER = "user",
+  TENANT = "tenant",
+  INSTALLATION = "installation",
+}
+
+export interface Identity {
+  id: string
+  type: IdentityType
+  installationId?: string
+  tenantId?: string
+}
+
+export interface UserIdentity extends Identity {
+  verified: boolean
+  accountHolder: boolean
+  providerType?: string
+  builder?: boolean
+  admin?: boolean
+}

packages/types/src/core/index.ts

@@ -1,2 +1,3 @@
 export * from "./hosting"
-export * from "./sessions"
+export * from "./context"
+export * from "./identification"

packages/types/src/core/sessions.ts

@@ -1,49 +0,0 @@
-import { User, Account } from "../documents"
-import { Hosting } from "./hosting"
-
-/**
- * Account portal user session. Used for self hosted accounts only.
- */
-export interface AccountUserSession {
-  _id: string
-  email: string
-  tenantId: string
-  accountPortalAccess: boolean
-  account: Account
-}
-
-/**
- * Budibase user session.
- */
-export interface BudibaseUserSession extends User {
-  _id: string // overwrite potentially undefined
-  account?: Account
-  accountPortalAccess?: boolean
-}
-
-export const isAccountSession = (
-  user: AccountUserSession | BudibaseUserSession
-): user is AccountUserSession => {
-  return user.account?.hosting === Hosting.SELF
-}
-
-export const isUserSession = (
-  user: AccountUserSession | BudibaseUserSession
-): user is BudibaseUserSession => {
-  return !user.account || user.account?.hosting === Hosting.CLOUD
-}
-
-// not technically a session, but used to identify the installation
-export interface InstallationSession {
-  _id: string
-  isInstallation: boolean
-}
-
-export const isInstallation = (user: any): user is InstallationSession => {
-  return !!user.isInstallation
-}
-
-export type SessionUser =
-  | AccountUserSession
-  | BudibaseUserSession
-  | InstallationSession

packages/types/src/events/auth.ts

@@ -1,4 +1,4 @@
-export type LoginSource = "local" | "google" | "oidc"
+export type LoginSource = "local" | "google" | "oidc" | "google-internal"
 export type SSOType = "oidc" | "google"
 
 export interface LoginEvent {

packages/types/src/events/event.ts

@@ -8,7 +8,7 @@ export enum Event {
   USER_PERMISSION_ADMIN_ASSIGNED = "user:admin:assigned",
   USER_PERMISSION_ADMIN_REMOVED = "user:admin:removed",
   USER_PERMISSION_BUILDER_ASSIGNED = "user:builder:assigned",
-  USER_PERMISSION_BUILDER_REMOVED = "userbuilder:removed",
+  USER_PERMISSION_BUILDER_REMOVED = "user:builder:removed",
 
   // USER / INVITE
   USER_INVITED = "user:invited",

packages/types/src/events/identification.ts

@@ -1,37 +0,0 @@
-import { Hosting } from "../core"
-
-export enum IdentityType {
-  USER = "user",
-  TENANT = "tenant",
-  INSTALLATION = "installation",
-}
-
-export interface Identity {
-  id: string
-  tenantId: string
-  type: IdentityType
-}
-
-export interface InstallationIdentity extends Identity {
-  version: string
-  hosting: Hosting
-}
-
-export interface TenantIdentity extends Identity {
-  hosting: Hosting
-  profession?: string
-  companySize?: string
-}
-
-export interface UserIdentity extends TenantIdentity {
-  hosting: Hosting
-  type: IdentityType
-  verified: boolean
-  accountHolder: boolean
-  providerType?: string
-}
-
-export interface BudibaseIdentity extends UserIdentity {
-  builder?: boolean
-  admin?: boolean
-}

packages/types/src/events/index.ts

@@ -15,5 +15,4 @@ export * from "./serve"
 export * from "./table"
 export * from "./user"
 export * from "./view"
-export * from "./identification"
 export * from "./account"

packages/types/src/events/serve.ts

@@ -1,5 +1,11 @@
 export interface BuilderServedEvent {}
 
-export interface AppServedEvent {}
+export interface AppServedEvent {
+  appId: string
+  appVersion: string
+}
 
-export interface AppPreviewServedEvent {}
+export interface AppPreviewServedEvent {
+  appId: string
+  appVersion: string
+}

packages/worker/src/api/controllers/global/auth.ts

@@ -70,9 +70,9 @@ async function authInternal(ctx: any, user: any, err = null, info = null) {
 export const authenticate = async (ctx: any, next: any) => {
   return passport.authenticate(
     "local",
-    async (err: any, user: any, info: any) => {
+    async (err: any, user: User, info: any) => {
       await authInternal(ctx, user, err, info)
-      await context.doInUserContext(user, async () => {
+      await context.identity.doInUserContext(user, async () => {
         await events.auth.login("local")
       })
       ctx.status = 200
@@ -213,10 +213,10 @@ export const googleAuth = async (ctx: any, next: any) => {
   return passport.authenticate(
     strategy,
     { successRedirect: "/", failureRedirect: "/error" },
-    async (err: any, user: any, info: any) => {
+    async (err: any, user: User, info: any) => {
       await authInternal(ctx, user, err, info)
-      await context.doInUserContext(user, async () => {
-        await events.auth.login("google")
+      await context.identity.doInUserContext(user, async () => {
+        await events.auth.login("google-internal")
       })
       ctx.redirect("/")
     }
@@ -261,7 +261,7 @@ export const oidcAuth = async (ctx: any, next: any) => {
     { successRedirect: "/", failureRedirect: "/error" },
     async (err: any, user: any, info: any) => {
       await authInternal(ctx, user, err, info)
-      await context.doInUserContext(user, async () => {
+      await context.identity.doInUserContext(user, async () => {
         await events.auth.login("oidc")
       })
       ctx.redirect("/")

packages/worker/src/api/controllers/global/users.ts

@@ -69,7 +69,7 @@ export const adminUser = async (ctx: any) => {
     if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
       account = await accounts.getAccountByTenantId(tenantId)
     }
-    await events.identification.identifyTenant(tenantId, account)
+    await events.identification.identifyTenantGroup(tenantId, account)
   } catch (err: any) {
     ctx.throw(err.status || 400, err)
   }

scripts/link-dependencies.sh

@@ -8,6 +8,12 @@ cd packages/string-templates
 yarn link
 cd -
 
+echo "Linking types"
+cd packages/types 
+yarn link
+cd -
+
+
 if [ -d "../budibase-pro" ]; then
   cd ../budibase-pro
   yarn bootstrap
@@ -38,6 +44,9 @@ if [ -d "../account-portal" ]; then
   echo "Linking string-templates to account-portal"
   yarn link "@budibase/string-templates"
 
+  echo "Linking types to account-portal"
+  yarn link "@budibase/types"
+
   if [ -d "../../../budibase-pro" ]; then
     echo "Linking pro to account-portal"
     yarn link "@budibase/pro"