Lots of failure handling and caching

package.json

@@ -24,7 +24,7 @@
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
     "bootstrap": "lerna link && lerna bootstrap && ./scripts/link-dependencies.sh",
     "build": "lerna run build",
-    "build:watch": "lerna run build:watch --stream --parallel",
+    "build:watch": "lerna run build:watch --ignore @budibase/backend-core --stream --parallel",
     "release": "lerna publish patch --yes --force-publish && yarn release:pro",
     "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop",
     "release:pro": "bash scripts/pro/release.sh",

packages/backend-core/package.json

@@ -8,6 +8,7 @@
   "license": "GPL-3.0",
   "scripts": {
     "build": "rimraf dist/ && tsc -p tsconfig.build.json",
+    "build:watch": "rimraf dist/ && tsc --build tsconfig.build.json --watch --preserveWatchOutput",
     "test": "jest",
     "test:watch": "jest --watchAll"
   },

packages/backend-core/src/cache/generic.js

@@ -4,6 +4,9 @@ const { getTenantId } = require("../context")
 
 exports.CacheKeys = {
   CHECKLIST: "checklist",
+  INSTALLATION: "installation",
+  ANALYTICS_ENABLED: "analyticsEnabled",
+  UNIQUE_TENANT_ID: "uniqueTenantId",
 }
 
 exports.TTL = {
@@ -17,8 +20,8 @@ function generateTenantKey(key) {
   return `${key}:${tenantId}`
 }
 
-exports.withCache = async (key, ttl, fetchFn) => {
-  key = generateTenantKey(key)
+exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
+  key = opts.useTenancy ? generateTenantKey(key) : key
   const client = await redis.getCacheClient()
   const cachedValue = await client.get(key)
   if (cachedValue) {

packages/backend-core/src/db/utils.ts

@@ -391,6 +391,7 @@ export const getScopedFullConfig = async function (
       // defaults
       scopedConfig = {
         doc: {
+          _id: generateConfigID({ type, user, workspace }),
           config: {
             platformUrl: await getPlatformUrl(),
             analyticsEnabled: await events.analytics.enabled(),

packages/backend-core/src/events/analytics.ts

@@ -2,8 +2,8 @@ import env from "../environment"
 import * as tenancy from "../tenancy"
 import * as dbUtils from "../db/utils"
 import { Configs } from "../constants"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
 
-// TODO: cache in redis
 export const enabled = async () => {
   // cloud - always use the environment variable
   if (!env.SELF_HOSTED) {
@@ -11,14 +11,25 @@ export const enabled = async () => {
   }
 
   // self host - prefer the settings doc
-  // check for explicit true/false values to support
-  // backwards compatibility where setting may not exist
+  // use cache as events have high throughput
+  const enabledInDB = await withCache(
+    CacheKeys.ANALYTICS_ENABLED,
+    TTL.ONE_DAY,
+    async () => {
       const settings = await getSettingsDoc()
+
+      // need to do explicit checks in case the field is not set
       if (settings?.config?.analyticsEnabled === false) {
         return false
       } else if (settings?.config?.analyticsEnabled === true) {
         return true
       }
+    }
+  )
+
+  if (enabledInDB !== undefined) {
+    return enabledInDB
+  }
 
   // fallback to the environment variable
   // explicitly check for 0 or false here, undefined or otherwise is treated as true

packages/backend-core/src/events/identification.ts

@@ -23,6 +23,7 @@ import * as dbUtils from "../db/utils"
 import { Configs } from "../constants"
 import * as hashing from "../hashing"
 import * as installation from "../installation"
+import { withCache, TTL, CacheKeys } from "../cache/generic"
 
 const pkg = require("../../package.json")
 
@@ -53,7 +54,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
     }
   } else if (identityType === IdentityType.TENANT) {
     const installationId = await getInstallationId()
-    const tenantId = await getCurrentTenantId()
+    const tenantId = await getEventTenantId(context.getTenantId())
 
     return {
       id: formatDistinctId(tenantId, identityType),
@@ -63,7 +64,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
     }
   } else if (identityType === IdentityType.USER) {
     const userContext = identityContext as UserContext
-    const tenantId = await getCurrentTenantId()
+    const tenantId = await getEventTenantId(context.getTenantId())
     let installationId: string | undefined
 
     // self host account users won't have installation
@@ -109,7 +110,7 @@ export const identifyTenantGroup = async (
   account: Account | undefined,
   timestamp?: string | number
 ): Promise<void> => {
-  const id = await getGlobalTenantId(tenantId)
+  const id = await getEventTenantId(tenantId)
   const type = IdentityType.TENANT
 
   let hosting: Hosting
@@ -144,7 +145,7 @@ export const identifyUser = async (
   timestamp?: string | number
 ) => {
   const id = user._id as string
-  const tenantId = await getGlobalTenantId(user.tenantId)
+  const tenantId = await getEventTenantId(user.tenantId)
   const type = IdentityType.USER
   let builder = user.builder?.global || false
   let admin = user.admin?.global || false
@@ -214,8 +215,6 @@ const getHostingFromEnv = () => {
   return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
 }
 
-export const getCurrentTenantId = () => getGlobalTenantId(context.getTenantId())
-
 export const getInstallationId = async () => {
   if (isAccountPortal()) {
     return "account-portal"
@@ -224,31 +223,35 @@ export const getInstallationId = async () => {
   return install.installId
 }
 
-const getGlobalTenantId = async (tenantId: string): Promise<string> => {
+const getEventTenantId = async (tenantId: string): Promise<string> => {
   if (env.SELF_HOSTED) {
-    return getGlobalId(tenantId)
+    return getUniqueTenantId(tenantId)
   } else {
     // tenant id's in the cloud are already unique
     return tenantId
   }
 }
 
-// TODO: cache in redis
-const getGlobalId = async (tenantId: string): Promise<string> => {
+const getUniqueTenantId = async (tenantId: string): Promise<string> => {
+  // make sure this tenantId always matches the tenantId in context
+  return context.doInTenant(tenantId, () => {
+    return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
       const db = context.getGlobalDB()
       const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
         type: Configs.SETTINGS,
       })
 
-  let globalId: string
-  if (config.config.globalId) {
-    return config.config.globalId
+      let uniqueTenantId: string
+      if (config.config.uniqueTenantId) {
+        return config.config.uniqueTenantId
       } else {
-    globalId = `${hashing.newid()}_${tenantId}`
-    config.config.globalId = globalId
+        uniqueTenantId = `${hashing.newid()}_${tenantId}`
+        config.config.uniqueTenantId = uniqueTenantId
         await db.put(config)
-    return globalId
+        return uniqueTenantId
       }
+    })
+  })
 }
 
 const isAccountPortal = () => {

packages/backend-core/src/events/publishers/automation.ts

@@ -9,6 +9,7 @@ import {
   AutomationStepCreatedEvent,
   AutomationStepDeletedEvent,
   AutomationTriggerUpdatedEvent,
+  AutomationsRunEvent,
 } from "@budibase/types"
 
 export async function created(automation: Automation, timestamp?: string) {
@@ -51,6 +52,13 @@ export async function tested(automation: Automation) {
   await publishEvent(Event.AUTOMATION_TESTED, properties)
 }
 
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: AutomationsRunEvent = {
+    count,
+  }
+  await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
+}
+
 export async function stepCreated(
   automation: Automation,
   step: AutomationStep,

packages/backend-core/src/events/publishers/backfill.ts

@@ -0,0 +1,67 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  AppBackfillSucceededEvent,
+  AppBackfillFailedEvent,
+  TenantBackfillSucceededEvent,
+  TenantBackfillFailedEvent,
+  InstallationBackfillSucceededEvent,
+  InstallationBackfillFailedEvent,
+} from "@budibase/types"
+const env = require("../../environment")
+
+const shouldSkip = !env.SELF_HOSTED && !env.isDev()
+
+export async function appSucceeded(properties: AppBackfillSucceededEvent) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function appFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: AppBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.APP_BACKFILL_FAILED, properties)
+}
+
+export async function tenantSucceeded(
+  properties: TenantBackfillSucceededEvent
+) {
+  if (shouldSkip) {
+    return
+  }
+  await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function tenantFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: TenantBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
+}
+
+export async function installationSucceeded() {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillSucceededEvent = {}
+  await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
+}
+
+export async function installationFailed(error: any) {
+  if (shouldSkip) {
+    return
+  }
+  const properties: InstallationBackfillFailedEvent = {
+    error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
+  }
+  await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
+}

packages/backend-core/src/events/publishers/index.ts

@@ -16,3 +16,4 @@ export * as serve from "./serve"
 export * as user from "./user"
 export * as view from "./view"
 export * as version from "./version"
+export * as backfill from "./backfill"

packages/backend-core/src/events/publishers/query.ts

@@ -8,6 +8,7 @@ import {
   QueryDeletedEvent,
   QueryImportedEvent,
   QueryPreviewedEvent,
+  QueriesRunEvent,
 } from "@budibase/types"
 
 /* eslint-disable */
@@ -40,6 +41,13 @@ export const imported = async (
   await publishEvent(Event.QUERY_IMPORT, properties)
 }
 
+export const run = async (count: number, timestamp?: string | number) => {
+  const properties: QueriesRunEvent = {
+    count,
+  }
+  await publishEvent(Event.QUERIES_RUN, properties, timestamp)
+}
+
 export const previewed = async (datasource: Datasource) => {
   const properties: QueryPreviewedEvent = {}
   await publishEvent(Event.QUERY_PREVIEWED, properties)

packages/backend-core/src/installation.ts

@@ -5,10 +5,17 @@ import { doWithDB } from "./db"
 import { Installation, IdentityType } from "@budibase/types"
 import * as context from "./context"
 import semver from "semver"
+import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic"
 
 const pkg = require("../package.json")
 
 export const getInstall = async (): Promise<Installation> => {
+  return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
+    useTenancy: false,
+  })
+}
+
+const getInstallFromDB = async (): Promise<Installation> => {
   return doWithDB(
     StaticDatabases.PLATFORM_INFO.name,
     async (platformDb: any) => {
@@ -44,6 +51,7 @@ const updateVersion = async (version: string): Promise<boolean> => {
         const install = await getInstall()
         install.version = version
         await platformDb.put(install)
+        await bustCache(CacheKeys.INSTALLATION)
       },
       {}
     )

packages/backend-core/src/tests/utilities/mocks/events.js

@@ -1,135 +1,106 @@
-jest.mock("../../../events", () => {
-  return {
-    identification: {
-      identifyTenantGroup: jest.fn(),
-      identifyUser: jest.fn(),
-    },
-    analytics: {
-      enabled: () => false,
-    },
-    shutdown: () => {},
-    account: {
-      created: jest.fn(),
-      deleted: jest.fn(),
-      verified: jest.fn(),
-    },
-    app: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      published: jest.fn(),
-      unpublished: jest.fn(),
-      templateImported: jest.fn(),
-      fileImported: jest.fn(),
-      versionUpdated: jest.fn(),
-      versionReverted: jest.fn(),
-      reverted: jest.fn(),
-      exported: jest.fn(),
-    },
-    auth: {
-      login: jest.fn(),
-      logout: jest.fn(),
-      SSOCreated: jest.fn(),
-      SSOUpdated: jest.fn(),
-      SSOActivated: jest.fn(),
-      SSODeactivated: jest.fn(),
-    },
-    automation: {
-      created: jest.fn(),
-      deleted: jest.fn(),
-      tested: jest.fn(),
-      // run: jest.fn(),
-      stepCreated: jest.fn(),
-      stepDeleted: jest.fn(),
-      triggerUpdated: jest.fn(),
-    },
-    datasource: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-    },
-    email: {
-      SMTPCreated: jest.fn(),
-      SMTPUpdated: jest.fn(),
-    },
-    layout: {
-      created: jest.fn(),
-      deleted: jest.fn(),
-    },
-    org: {
-      nameUpdated: jest.fn(),
-      logoUpdated: jest.fn(),
-      platformURLUpdated: jest.fn(),
-      analyticsOptOut: jest.fn(),
-    },
-    version: {
-      checked: jest.fn(),
-    },
-    query: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      imported: jest.fn(),
-      previewed: jest.fn(),
-    },
-    role: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      assigned: jest.fn(),
-      unassigned: jest.fn(),
-    },
-    rows: {
-      imported: jest.fn(),
-      created: jest.fn(),
-    },
-    screen: {
-      created: jest.fn(),
-      deleted: jest.fn(),
-    },
-    user: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      permissionAdminAssigned: jest.fn(),
-      permissionAdminRemoved: jest.fn(),
-      permissionBuilderAssigned: jest.fn(),
-      permissionBuilderRemoved: jest.fn(),
-      invited: jest.fn(),
-      inviteAccepted: jest.fn(),
-      passwordForceReset: jest.fn(),
-      passwordUpdated: jest.fn(),
-      passwordResetRequested: jest.fn(),
-      passwordReset: jest.fn(),
-    },
-    serve: {
-      servedBuilder: jest.fn(),
-      servedApp: jest.fn(),
-      servedAppPreview: jest.fn(),
-    },
-    table: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      exported: jest.fn(),
-      imported: jest.fn(),
-      permissionUpdated: jest.fn(),
-    },
-    view: {
-      created: jest.fn(),
-      updated: jest.fn(),
-      deleted: jest.fn(),
-      exported: jest.fn(),
-      filterCreated: jest.fn(),
-      filterUpdated: jest.fn(),
-      filterDeleted: jest.fn(),
-      calculationCreated: jest.fn(),
-      calculationUpdated: jest.fn(),
-      calculationDeleted: jest.fn(),
-    },
-  }
-})
-
 const events = require("../../../events")
 
+jest.spyOn(events.identification, "identifyTenantGroup")
+jest.spyOn(events.identification, "identifyUser")
+
+jest.spyOn(events.account, "created")
+jest.spyOn(events.account, "deleted")
+jest.spyOn(events.account, "verified")
+
+jest.spyOn(events.app, "created")
+jest.spyOn(events.app, "updated")
+jest.spyOn(events.app, "deleted")
+jest.spyOn(events.app, "published")
+jest.spyOn(events.app, "unpublished")
+jest.spyOn(events.app, "templateImported")
+jest.spyOn(events.app, "fileImported")
+jest.spyOn(events.app, "versionUpdated")
+jest.spyOn(events.app, "versionReverted")
+jest.spyOn(events.app, "reverted")
+jest.spyOn(events.app, "exported")
+
+jest.spyOn(events.auth, "login")
+jest.spyOn(events.auth, "logout")
+jest.spyOn(events.auth, "SSOCreated")
+jest.spyOn(events.auth, "SSOUpdated")
+jest.spyOn(events.auth, "SSOActivated")
+jest.spyOn(events.auth, "SSODeactivated")
+
+jest.spyOn(events.automation, "created")
+jest.spyOn(events.automation, "deleted")
+jest.spyOn(events.automation, "tested")
+jest.spyOn(events.automation, "stepCreated")
+jest.spyOn(events.automation, "stepDeleted")
+jest.spyOn(events.automation, "triggerUpdated")
+
+jest.spyOn(events.datasource, "created")
+jest.spyOn(events.datasource, "updated")
+jest.spyOn(events.datasource, "deleted")
+
+jest.spyOn(events.email, "SMTPCreated")
+jest.spyOn(events.email, "SMTPUpdated")
+
+jest.spyOn(events.layout, "created")
+jest.spyOn(events.layout, "deleted")
+
+jest.spyOn(events.org, "nameUpdated")
+jest.spyOn(events.org, "logoUpdated")
+jest.spyOn(events.org, "platformURLUpdated")
+jest.spyOn(events.org, "analyticsOptOut")
+
+jest.spyOn(events.version, "checked")
+
+jest.spyOn(events.query, "created")
+jest.spyOn(events.query, "updated")
+jest.spyOn(events.query, "deleted")
+jest.spyOn(events.query, "imported")
+jest.spyOn(events.query, "previewed")
+
+jest.spyOn(events.role, "created")
+jest.spyOn(events.role, "updated")
+jest.spyOn(events.role, "deleted")
+jest.spyOn(events.role, "assigned")
+jest.spyOn(events.role, "unassigned")
+
+jest.spyOn(events.rows, "imported")
+jest.spyOn(events.rows, "created")
+
+jest.spyOn(events.screen, "created")
+jest.spyOn(events.screen, "deleted")
+
+jest.spyOn(events.user, "created")
+jest.spyOn(events.user, "updated")
+jest.spyOn(events.user, "deleted")
+jest.spyOn(events.user, "permissionAdminAssigned")
+jest.spyOn(events.user, "permissionAdminRemoved")
+jest.spyOn(events.user, "permissionBuilderAssigned")
+jest.spyOn(events.user, "permissionBuilderRemoved")
+jest.spyOn(events.user, "invited")
+jest.spyOn(events.user, "inviteAccepted")
+jest.spyOn(events.user, "passwordForceReset")
+jest.spyOn(events.user, "passwordUpdated")
+jest.spyOn(events.user, "passwordResetRequested")
+jest.spyOn(events.user, "passwordReset")
+
+jest.spyOn(events.serve, "servedBuilder")
+jest.spyOn(events.serve, "servedApp")
+jest.spyOn(events.serve, "servedAppPreview")
+
+jest.spyOn(events.table, "created")
+jest.spyOn(events.table, "updated")
+jest.spyOn(events.table, "deleted")
+jest.spyOn(events.table, "exported")
+jest.spyOn(events.table, "imported")
+
+jest.spyOn(events.view, "created")
+jest.spyOn(events.view, "updated")
+jest.spyOn(events.view, "deleted")
+jest.spyOn(events.view, "exported")
+jest.spyOn(events.view, "filterCreated")
+jest.spyOn(events.view, "filterUpdated")
+jest.spyOn(events.view, "filterDeleted")
+jest.spyOn(events.view, "calculationCreated")
+jest.spyOn(events.view, "calculationUpdated")
+jest.spyOn(events.view, "calculationDeleted")
+
 module.exports = events

packages/backend-core/tsconfig.build.json

@@ -1,6 +1,9 @@
 {
   // Used for building with tsc
   "extends": "./tsconfig.json",
+  "references": [
+    { "path": "../types" }
+  ],
   "exclude": [
     "node_modules",
     "dist/**/*",

packages/backend-core/yarn.lock

@@ -895,6 +895,13 @@
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
   integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
 
+"@types/ioredis@^4.27.1":
+  version "4.28.10"
+  resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
+  integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
@@ -1092,11 +1099,6 @@ acorn-walk@^7.1.1:
   resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
   integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
 
-acorn@^5.2.1:
-  version "5.7.4"
-  resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e"
-  integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==
-
 acorn@^7.1.1:
   version "7.4.1"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
@@ -1132,11 +1134,6 @@ ajv@^6.12.3:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-amdefine@>=0.0.4:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
-  integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=
-
 ansi-align@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59"
@@ -1228,11 +1225,6 @@ assert-plus@^0.2.0:
   resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234"
   integrity sha1-104bh+ev/A24qttwIfP+SBAasjQ=
 
-ast-types@0.9.6:
-  version "0.9.6"
-  resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
-  integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk=
-
 async@~2.1.4:
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc"
@@ -1361,11 +1353,6 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
-base62@^1.1.0:
-  version "1.2.8"
-  resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428"
-  integrity sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA==
-
 base64-js@^1.0.2, base64-js@^1.3.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@@ -1721,26 +1708,6 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
   dependencies:
     delayed-stream "~1.0.0"
 
-commander@^2.5.0:
-  version "2.20.3"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
-  integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
-
-commoner@^0.10.1:
-  version "0.10.8"
-  resolved "https://registry.yarnpkg.com/commoner/-/commoner-0.10.8.tgz#34fc3672cd24393e8bb47e70caa0293811f4f2c5"
-  integrity sha1-NPw2cs0kOT6LtH5wyqApOBH08sU=
-  dependencies:
-    commander "^2.5.0"
-    detective "^4.3.1"
-    glob "^5.0.15"
-    graceful-fs "^4.1.2"
-    iconv-lite "^0.4.5"
-    mkdirp "^0.5.0"
-    private "^0.1.6"
-    q "^1.1.2"
-    recast "^0.11.17"
-
 component-type@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9"
@@ -1945,11 +1912,6 @@ deferred-leveldown@~5.3.0:
     abstract-leveldown "~6.2.1"
     inherits "^2.0.3"
 
-defined@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693"
-  integrity sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=
-
 delayed-stream@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@@ -1990,14 +1952,6 @@ detect-newline@^3.0.0:
   resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
   integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==
 
-detective@^4.3.1:
-  version "4.7.1"
-  resolved "https://registry.yarnpkg.com/detective/-/detective-4.7.1.tgz#0eca7314338442febb6d65da54c10bb1c82b246e"
-  integrity sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig==
-  dependencies:
-    acorn "^5.2.1"
-    defined "^1.0.0"
-
 diff-sequences@^27.5.1:
   version "27.5.1"
   resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
@@ -2134,15 +2088,6 @@ error-inject@^1.0.0:
   resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37"
   integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc=
 
-es3ify@^0.2.2:
-  version "0.2.2"
-  resolved "https://registry.yarnpkg.com/es3ify/-/es3ify-0.2.2.tgz#5dae3e650e5be3684b88066513d528d092629862"
-  integrity sha1-Xa4+ZQ5b42hLiAZlE9Uo0JJimGI=
-  dependencies:
-    esprima "^2.7.1"
-    jstransform "~11.0.0"
-    through "~2.3.4"
-
 escalade@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@@ -2180,26 +2125,11 @@ escodegen@^2.0.0:
   optionalDependencies:
     source-map "~0.6.1"
 
-esprima-fb@^15001.1.0-dev-harmony-fb:
-  version "15001.1.0-dev-harmony-fb"
-  resolved "https://registry.yarnpkg.com/esprima-fb/-/esprima-fb-15001.1.0-dev-harmony-fb.tgz#30a947303c6b8d5e955bee2b99b1d233206a6901"
-  integrity sha512-59dDGQo2b3M/JfKIws0/z8dcXH2mnVHkfSPRhCYS91JNGfGNwr7GsSF6qzWZuOGvw5Ii0w9TtylrX07MGmlOoQ==
-
-esprima@^2.7.1:
-  version "2.7.3"
-  resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581"
-  integrity sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A==
-
 esprima@^4.0.0, esprima@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
   integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
 
-esprima@~3.1.0:
-  version "3.1.3"
-  resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633"
-  integrity sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg==
-
 estraverse@^5.2.0:
   version "5.3.0"
   resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123"
@@ -2467,17 +2397,6 @@ glob-parent@~5.1.2:
   dependencies:
     is-glob "^4.0.1"
 
-glob@^5.0.15:
-  version "5.0.15"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1"
-  integrity sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=
-  dependencies:
-    inflight "^1.0.4"
-    inherits "2"
-    minimatch "2 || 3"
-    once "^1.3.0"
-    path-is-absolute "^1.0.0"
-
 glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
@@ -2716,7 +2635,7 @@ human-signals@^2.1.0:
   resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
   integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
 
-iconv-lite@0.4.24, iconv-lite@^0.4.5:
+iconv-lite@0.4.24:
   version "0.4.24"
   resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
   integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
@@ -3550,17 +3469,6 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.3.6"
 
-jstransform@~11.0.0:
-  version "11.0.3"
-  resolved "https://registry.yarnpkg.com/jstransform/-/jstransform-11.0.3.tgz#09a78993e0ae4d4ef4487f6155a91f6190cb4223"
-  integrity sha1-CaeJk+CuTU70SH9hVakfYZDLQiM=
-  dependencies:
-    base62 "^1.1.0"
-    commoner "^0.10.1"
-    esprima-fb "^15001.1.0-dev-harmony-fb"
-    object-assign "^2.0.0"
-    source-map "^0.4.2"
-
 jwa@^1.1.4:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.1.5.tgz#a0552ce0220742cd52e153774a32905c30e756e5"
@@ -4069,13 +3977,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
   integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
 
-"minimatch@2 || 3":
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
-  integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
-  dependencies:
-    brace-expansion "^1.1.7"
-
 minimatch@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@@ -4083,7 +3984,7 @@ minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
+minimist@^1.2.0, minimist@^1.2.5:
   version "1.2.6"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
   integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
@@ -4108,13 +4009,6 @@ mkdirp-classic@^0.5.2:
   resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
   integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
 
-mkdirp@^0.5.0:
-  version "0.5.6"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6"
-  integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==
-  dependencies:
-    minimist "^1.2.6"
-
 mkdirp@^1.0.3:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
@@ -4295,11 +4189,6 @@ oauth@0.9.x, oauth@^0.9.15:
   resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
   integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
 
-object-assign@^2.0.0:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa"
-  integrity sha1-Q8NuXVaf+OSBbE76i+AtJpZ8GKo=
-
 object-assign@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
@@ -4624,17 +4513,6 @@ pouchdb-adapter-utils@7.2.2:
     pouchdb-merge "7.2.2"
     pouchdb-utils "7.2.2"
 
-pouchdb-all-dbs@^1.0.2:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/pouchdb-all-dbs/-/pouchdb-all-dbs-1.1.1.tgz#85f04a39cafda52497ec49abf1c93bb5e72813f6"
-  integrity sha512-UUnsdmcnRSQ8MAOYSJjfTwKkQNb/6fvOfd/f7dNNivWZ2YDYVuMfgw1WQdL634yEtcXTxAENZ/EyLRdzPCB41A==
-  dependencies:
-    argsarray "0.0.1"
-    es3ify "^0.2.2"
-    inherits "~2.0.1"
-    pouchdb-promise "6.4.3"
-    tiny-queue "^0.2.0"
-
 pouchdb-binary-utils@7.2.2:
   version "7.2.2"
   resolved "https://registry.yarnpkg.com/pouchdb-binary-utils/-/pouchdb-binary-utils-7.2.2.tgz#0690b348052c543b1e67f032f47092ca82bcb10e"
@@ -4711,7 +4589,7 @@ pouchdb-merge@7.2.2:
   resolved "https://registry.yarnpkg.com/pouchdb-merge/-/pouchdb-merge-7.2.2.tgz#940d85a2b532d6a93a6cab4b250f5648511bcc16"
   integrity sha512-6yzKJfjIchBaS7Tusuk8280WJdESzFfQ0sb4jeMUNnrqs4Cx3b0DIEOYTRRD9EJDM+je7D3AZZ4AT0tFw8gb4A==
 
-pouchdb-promise@6.4.3, pouchdb-promise@^6.0.4:
+pouchdb-promise@^6.0.4:
   version "6.4.3"
   resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3"
   integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw==
@@ -4798,11 +4676,6 @@ pretty-format@^27.0.0, pretty-format@^27.5.1:
     ansi-styles "^5.0.0"
     react-is "^17.0.1"
 
-private@^0.1.6, private@~0.1.5:
-  version "0.1.8"
-  resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
-  integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==
-
 process-nextick-args@~2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
@@ -4861,11 +4734,6 @@ pupa@^2.1.1:
   dependencies:
     escape-goat "^2.0.0"
 
-q@^1.1.2:
-  version "1.5.1"
-  resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
-  integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=
-
 qs@~6.4.0:
   version "6.4.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
@@ -4950,16 +4818,6 @@ readline-sync@^1.4.9:
   resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
   integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==
 
-recast@^0.11.17:
-  version "0.11.23"
-  resolved "https://registry.yarnpkg.com/recast/-/recast-0.11.23.tgz#451fd3004ab1e4df9b4e4b66376b2a21912462d3"
-  integrity sha1-RR/TAEqx5N+bTktmN2sqIZEkYtM=
-  dependencies:
-    ast-types "0.9.6"
-    esprima "~3.1.0"
-    private "~0.1.5"
-    source-map "~0.5.0"
-
 redis-commands@1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
@@ -5230,14 +5088,7 @@ source-map-support@^0.5.6:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
-source-map@^0.4.2:
-  version "0.4.4"
-  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
-  integrity sha1-66T12pwNyZneaAMti092FzZSA2s=
-  dependencies:
-    amdefine ">=0.0.4"
-
-source-map@^0.5.0, source-map@~0.5.0:
+source-map@^0.5.0:
   version "0.5.7"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
@@ -5510,21 +5361,11 @@ through2@^2.0.0, through2@^2.0.2, through2@^2.0.3:
     readable-stream "~2.3.6"
     xtend "~4.0.1"
 
-through@~2.3.4:
-  version "2.3.8"
-  resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
-  integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
-
 timekeeper@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368"
   integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==
 
-tiny-queue@^0.2.0:
-  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/tiny-queue/-/tiny-queue-0.2.1.tgz#25a67f2c6e253b2ca941977b5ef7442ef97a6046"
-  integrity sha1-JaZ/LG4lOyypQZd7XvdELvl6YEY=
-
 tmp@^0.0.33:
   version "0.0.33"
   resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"

packages/server/scripts/jestSetup.js

@@ -1,11 +1,5 @@
 const { tmpdir } = require("os")
 const env = require("../src/environment")
-const { mocks } = require("@budibase/backend-core/testUtils")
-
-// mock all dates to 2020-01-01T00:00:00.000Z
-// use tk.reset() to use real dates in individual tests
-const tk = require("timekeeper")
-tk.freeze(mocks.date.MOCK_DATE)
 
 env._set("SELF_HOSTED", "1")
 env._set("NODE_ENV", "jest")
@@ -15,4 +9,11 @@ env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
 env._set("PORT", 0)
 
+const { mocks } = require("@budibase/backend-core/testUtils")
+
+// mock all dates to 2020-01-01T00:00:00.000Z
+// use tk.reset() to use real dates in individual tests
+const tk = require("timekeeper")
+tk.freeze(mocks.date.MOCK_DATE)
+
 global.console.log = jest.fn() // console.log are ignored in tests

packages/server/src/api/controllers/automation.js

@@ -71,9 +71,9 @@ exports.create = async function (ctx) {
     newAuto: automation,
   })
   const response = await db.put(automation)
-  await events.automation.created()
+  await events.automation.created(automation)
   for (let step of automation.definition.steps) {
-    await events.automation.stepCreated(step)
+    await events.automation.stepCreated(automation, step)
   }
   automation._rev = response.rev
 

packages/server/src/api/controllers/datasource.js

@@ -144,7 +144,7 @@ exports.save = async function (ctx) {
   }
 
   const dbResp = await db.put(datasource)
-  await events.datasource.created()
+  await events.datasource.created(datasource)
   datasource._rev = dbResp.rev
 
   // Drain connection pools when configuration is changed

packages/server/src/api/controllers/query/import/tests/index.spec.js

@@ -53,9 +53,11 @@ describe("Rest Importer", () => {
   }
 
   const runTest = async (test, assertions) => {
+    config.doInContext(config.appId, async () => {
       for (let [key, data] of Object.entries(datasets)) {
         await test(key, data, assertions)
       }
+    })
   }
 
   const testGetInfo = async (key, data, assertions) => {

packages/server/src/app.ts

@@ -83,6 +83,22 @@ module.exports = server.listen(env.PORT || 0, async () => {
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+
+  // run migrations on startup if not done via http
+  // not recommended in a clustered environment
+  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
+    try {
+      await migrations.migrate()
+    } catch (e) {
+      console.error("Error performing migrations. Exiting.\n", e)
+      shutdown()
+    }
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // done last - this will never complete
   await automations.init()
 })
 
@@ -99,15 +115,3 @@ process.on("uncaughtException", err => {
 process.on("SIGTERM", () => {
   shutdown()
 })
-
-// run migrations on startup if not done via http
-// not recommended in a clustered environment
-if (!env.HTTP_MIGRATIONS && !env.isTest()) {
-  migrations.migrate().catch(err => {
-    console.error("Error performing migrations. Exiting.\n", err)
-    shutdown()
-  })
-}
-
-// check for version updates
-installation.checkInstallVersion()

packages/server/src/migrations/functions/backfill/app.ts

@@ -6,8 +6,21 @@ import * as roles from "./app/roles"
 import * as tables from "./app/tables"
 import * as screens from "./app/screens"
 import * as global from "./global"
-import { App } from "@budibase/types"
+import { App, AppBackfillSucceededEvent } from "@budibase/types"
 import { db as dbUtils, events } from "@budibase/backend-core"
+import env from "../../../environment"
+
+const failGraceful = env.SELF_HOSTED && !env.isDev()
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
 
 /**
  * Date:
@@ -18,6 +31,7 @@ import { db as dbUtils, events } from "@budibase/backend-core"
  */
 
 export const run = async (appDb: any) => {
+  try {
     if (await global.isComplete()) {
       // make sure new apps aren't backfilled
       // return if the global migration for this tenant is complete
@@ -28,18 +42,81 @@ export const run = async (appDb: any) => {
     const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
     const timestamp = app.createdAt as string
 
-  if (dbUtils.isDevAppID(app.appId)) {
-    await events.app.created(app, timestamp)
-    await automations.backfill(appDb, timestamp)
-    await datasources.backfill(appDb, timestamp)
-    await layouts.backfill(appDb, timestamp)
-    await queries.backfill(appDb, timestamp)
-    await roles.backfill(appDb, timestamp)
-    await tables.backfill(appDb, timestamp)
-    await screens.backfill(appDb, timestamp)
-  }
-
     if (dbUtils.isProdAppID(app.appId)) {
       await events.app.published(app, timestamp)
     }
+
+    const totals: any = {}
+    const errors: any = []
+
+    if (dbUtils.isDevAppID(app.appId)) {
+      await events.app.created(app, timestamp)
+      try {
+        totals.automations = await automations.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.datasources = await datasources.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.layouts = await layouts.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.queries = await queries.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.roles = await roles.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.screens = await screens.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+
+      try {
+        totals.tables = await tables.backfill(appDb, timestamp)
+      } catch (e) {
+        handleError(e, errors)
+      }
+    }
+
+    const properties: AppBackfillSucceededEvent = {
+      appId: app.appId,
+      automations: totals.automations,
+      datasources: totals.datasources,
+      layouts: totals.layouts,
+      queries: totals.queries,
+      roles: totals.roles,
+      tables: totals.tables,
+      screens: totals.screens,
+    }
+
+    if (errors.length) {
+      properties.errors = errors.map((e: any) =>
+        JSON.stringify(e, Object.getOwnPropertyNames(e))
+      )
+      properties.errorCount = errors.length
+    } else {
+      properties.errorCount = 0
+    }
+
+    await events.backfill.appSucceeded(properties)
+  } catch (e) {
+    handleError(e)
+    await events.backfill.appFailed(e)
+  }
 }

packages/server/src/migrations/functions/backfill/app/automations.ts

@@ -21,4 +21,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
       await events.automation.stepCreated(automation, step, timestamp)
     }
   }
+
+  return automations.length
 }

packages/server/src/migrations/functions/backfill/app/datasources.ts

@@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
   for (const datasource of datasources) {
     await events.datasource.created(datasource, timestamp)
   }
+
+  return datasources.length
 }

packages/server/src/migrations/functions/backfill/app/layouts.ts

@@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
   for (const layout of layouts) {
     await events.layout.created(layout, timestamp)
   }
+
+  return layouts.length
 }

packages/server/src/migrations/functions/backfill/app/queries.ts

@@ -28,4 +28,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
     )
     await events.query.created(datasource, query, timestamp)
   }
+
+  return queries.length
 }

packages/server/src/migrations/functions/backfill/app/roles.ts

@@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
   for (const role of roles) {
     await events.role.created(role, timestamp)
   }
+
+  return roles.length
 }

packages/server/src/migrations/functions/backfill/app/screens.ts

@@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
   for (const screen of screens) {
     await events.screen.created(screen, timestamp)
   }
+
+  return screens.length
 }

packages/server/src/migrations/functions/backfill/app/tables.ts

@@ -31,4 +31,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
       }
     }
   }
+
+  return tables.length
 }

packages/server/src/migrations/functions/backfill/global.ts

@@ -1,10 +1,62 @@
+import { TenantBackfillSucceededEvent } from "./../../../../../types/src/events/backfill"
 import * as users from "./global/users"
-import * as rows from "./global/rows"
 import * as configs from "./global/configs"
-import { tenancy, events, migrations, accounts } from "@budibase/backend-core"
-import { CloudAccount } from "@budibase/types"
+import * as quotas from "./global/quotas"
+import {
+  tenancy,
+  events,
+  migrations,
+  accounts,
+  db as dbUtils,
+} from "@budibase/backend-core"
+import { QuotaUsage } from "@budibase/pro"
+import { CloudAccount, App } from "@budibase/types"
 import env from "../../../environment"
 
+const failGraceful = env.SELF_HOSTED && !env.isDev()
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
+
+const formatUsage = (usage: QuotaUsage) => {
+  let maxAutomations = 0
+  let maxQueries = 0
+  let rows = 0
+  let developers = 0
+
+  if (usage) {
+    if (usage.usageQuota) {
+      rows = usage.usageQuota.rows
+      developers = usage.usageQuota.developers
+    }
+
+    if (usage.monthly) {
+      for (const value of Object.values(usage.monthly)) {
+        if (value.automations > maxAutomations) {
+          maxAutomations = value.automations
+        }
+        if (value.queries > maxQueries) {
+          maxQueries = value.queries
+        }
+      }
+    }
+  }
+
+  return {
+    maxAutomations,
+    maxQueries,
+    rows,
+    developers,
+  }
+}
+
 /**
  * Date:
  * May 2022
@@ -14,23 +66,75 @@ import env from "../../../environment"
  */
 
 export const run = async (db: any) => {
+  try {
     const tenantId = tenancy.getTenantId()
-  const installTimestamp = (await getInstallTimestamp(db)) as number
+    let installTimestamp
+
+    const totals: any = {}
+    const errors: any = []
+
+    try {
+      installTimestamp = await getInstallTimestamp(db)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
     let account: CloudAccount | undefined
     if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
       account = await accounts.getAccountByTenantId(tenantId)
     }
 
+    try {
       await events.identification.identifyTenantGroup(
         tenantId,
         account,
         installTimestamp
       )
-  await configs.backfill(db, installTimestamp)
+    } catch (e) {
+      handleError(e, errors)
+    }
 
-  // users and rows provide their own timestamp
-  await users.backfill(db, account)
-  await rows.backfill()
+    try {
+      await configs.backfill(db, installTimestamp)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    try {
+      totals.users = await users.backfill(db, account)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    try {
+      const allApps: App[] = await dbUtils.getAllApps({ dev: true })
+      totals.apps = allApps.length
+
+      totals.usage = await quotas.backfill(allApps)
+    } catch (e) {
+      handleError(e, errors)
+    }
+
+    const properties: TenantBackfillSucceededEvent = {
+      apps: totals.apps,
+      users: totals.users,
+      ...formatUsage(totals.usage),
+      usage: totals.usage,
+    }
+
+    if (errors.length) {
+      properties.errors = errors.map((e: any) =>
+        JSON.stringify(e, Object.getOwnPropertyNames(e))
+      )
+      properties.errorCount = errors.length
+    } else {
+      properties.errorCount = 0
+    }
+    await events.backfill.tenantSucceeded(properties)
+  } catch (e) {
+    handleError(e)
+    await events.backfill.tenantFailed(e)
+  }
 }
 
 export const isComplete = async (): Promise<boolean> => {

packages/server/src/migrations/functions/backfill/global/quotas.ts

@@ -0,0 +1,53 @@
+import { events } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
+import { App } from "@budibase/types"
+
+const getOldestCreatedAt = (allApps: App[]): string | undefined => {
+  const timestamps = allApps
+    .filter(app => !!app.createdAt)
+    .map(app => app.createdAt as string)
+    .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
+
+  if (timestamps.length) {
+    return timestamps[0]
+  }
+}
+
+const getMonthTimestamp = (monthString: string): number => {
+  const parts = monthString.split("-")
+  const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
+  const year = parseInt(parts[1])
+
+  // using 0 as the day in next month gives us last day in previous month
+  const date = new Date(year, month + 1, 0).getTime()
+  const now = new Date().getTime()
+
+  if (date > now) {
+    return now
+  } else {
+    return date
+  }
+}
+
+export const backfill = async (allApps: App[]) => {
+  const usage = await quotas.getQuotaUsage()
+
+  const rows = usage.usageQuota.rows
+  const rowsTimestamp = getOldestCreatedAt(allApps)
+  await events.rows.created(rows, rowsTimestamp)
+
+  for (const [monthString, quotas] of Object.entries(usage.monthly)) {
+    if (monthString === "current") {
+      continue
+    }
+    const monthTimestamp = getMonthTimestamp(monthString)
+
+    const queries = quotas.queries
+    await events.query.run(queries, monthTimestamp)
+
+    const automations = quotas.automations
+    await events.automation.run(automations, monthTimestamp)
+  }
+
+  return usage
+}

packages/server/src/migrations/functions/backfill/global/rows.ts

@@ -1,28 +0,0 @@
-import { events, db as dbUtils } from "@budibase/backend-core"
-import { Row, App } from "@budibase/types"
-import { getUniqueRows } from "../../../../utilities/usageQuota/rows"
-
-// Rows is a special circumstance where we get rows across all apps
-// therefore migration is performed at the global level
-
-const getOldestCreatedAt = (allApps: App[]): string | undefined => {
-  const timestamps = allApps
-    .filter(app => !!app.createdAt)
-    .map(app => app.createdAt as string)
-    .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
-
-  if (timestamps.length) {
-    return timestamps[0]
-  }
-}
-
-export const backfill = async () => {
-  const allApps: App[] = await dbUtils.getAllApps({ dev: true })
-  const timestamp = getOldestCreatedAt(allApps)
-  const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
-  const rows: Row[] = await getUniqueRows(appIds)
-  const rowCount = rows ? rows.length : 0
-  if (rowCount) {
-    await events.rows.created(rowCount, timestamp)
-  }
-}

packages/server/src/migrations/functions/backfill/global/users.ts

@@ -40,4 +40,6 @@ export const backfill = async (
       }
     }
   }
+
+  return users.length
 }

packages/server/src/migrations/functions/backfill/installation.ts

@@ -1,6 +1,19 @@
 import { events, tenancy, installation } from "@budibase/backend-core"
 import { Installation } from "@budibase/types"
 import * as global from "./global"
+import env from "../../../environment"
+
+const failGraceful = env.SELF_HOSTED
+
+const handleError = (e: any, errors?: any) => {
+  if (failGraceful) {
+    if (errors) {
+      errors.push(e)
+    }
+    return
+  }
+  throw e
+}
 
 /**
  * Date:
@@ -11,6 +24,7 @@ import * as global from "./global"
  */
 
 export const run = async () => {
+  try {
     // need to use the default tenant to try to get the installation time
     await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
       const db = tenancy.getGlobalDB()
@@ -21,4 +35,10 @@ export const run = async () => {
         installTimestamp
       )
     })
+    await events.backfill.installationSucceeded()
+    throw new Error("fail")
+  } catch (e) {
+    handleError(e)
+    await events.backfill.installationFailed(e)
+  }
 }

packages/types/src/documents/global/config.ts

@@ -19,7 +19,7 @@ export interface SettingsConfig extends Config {
     company: string
     logoUrl: string
     platformUrl: string
-    globalId?: string
+    uniqueTenantId?: string
   }
 }
 

packages/types/src/events/automation.ts

@@ -43,3 +43,7 @@ export interface AutomationStepDeletedEvent {
   stepId: string
   stepType: string
 }
+
+export interface AutomationsRunEvent {
+  count: number
+}

packages/types/src/events/backfill.ts

@@ -0,0 +1,35 @@
+export interface AppBackfillSucceededEvent {
+  appId: string
+  automations: number
+  datasources: number
+  layouts: number
+  queries: number
+  roles: number
+  tables: number
+  screens: number
+  errors?: string[]
+  errorCount?: number
+}
+
+export interface AppBackfillFailedEvent {
+  error: string
+}
+
+export interface TenantBackfillSucceededEvent {
+  apps: number
+  users: number
+
+  usage: any
+  errors?: [string]
+  errorCount?: number
+}
+
+export interface TenantBackfillFailedEvent {
+  error: string
+}
+
+export interface InstallationBackfillSucceededEvent {}
+
+export interface InstallationBackfillFailedEvent {
+  error: string
+}

packages/types/src/events/event.ts

@@ -81,7 +81,7 @@ export enum Event {
   QUERY_UPDATED = "query:updated",
   QUERY_DELETED = "query:deleted",
   QUERY_IMPORT = "query:import",
-  // QUERY_RUN = "query:run",
+  QUERIES_RUN = "queries:run",
   QUERY_PREVIEWED = "query:previewed",
 
   // TABLE
@@ -124,7 +124,7 @@ export enum Event {
   AUTOMATION_CREATED = "automation:created",
   AUTOMATION_DELETED = "automation:deleted",
   AUTOMATION_TESTED = "automation:tested",
-  // AUTOMATION_RUN = "automation:run",
+  AUTOMATIONS_RUN = "automations:run",
   AUTOMATION_STEP_CREATED = "automation:step:created",
   AUTOMATION_STEP_DELETED = "automation:step:deleted",
   AUTOMATION_TRIGGER_UPDATED = "automation:trigger:updated",
@@ -140,6 +140,14 @@ export enum Event {
   ACCOUNT_CREATED = "account:created",
   ACCOUNT_DELETED = "account:deleted",
   ACCOUNT_VERIFIED = "account:verified",
+
+  // BACKFILL
+  APP_BACKFILL_SUCCEEDED = "app:backfill:succeeded",
+  APP_BACKFILL_FAILED = "app:backfill:failed",
+  TENANT_BACKFILL_SUCCEEDED = "tenant:backfill:succeeded",
+  TENANT_BACKFILL_FAILED = "tenant:backfill:failed",
+  INSTALLATION_BACKFILL_SUCCEEDED = "installation:backfill:succeeded",
+  INSTALLATION_BACKFILL_FAILED = "installation:backfill:failed",
 }
 
 export type RowImportFormat = "csv"

packages/types/src/events/index.ts

@@ -16,3 +16,4 @@ export * from "./table"
 export * from "./user"
 export * from "./view"
 export * from "./account"
+export * from "./backfill"

packages/types/src/events/query.ts

@@ -7,3 +7,7 @@ export interface QueryDeletedEvent {}
 export interface QueryImportedEvent {}
 
 export interface QueryPreviewedEvent {}
+
+export interface QueriesRunEvent {
+  count: number
+}

packages/worker/src/api/controllers/global/configs.js

@@ -167,6 +167,7 @@ exports.save = async function (ctx) {
   try {
     const response = await db.put(ctx.request.body)
     await bustCache(CacheKeys.CHECKLIST)
+    await bustCache(CacheKeys.ANALYTICS_ENABLED)
 
     for (const fn of eventFns) {
       await fn()