From fd845284d39660ab73a032c807046b4f6068f08d Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 30 May 2022 21:46:08 +0100 Subject: [PATCH] Lots of failure handling and caching --- package.json | 2 +- packages/backend-core/package.json | 1 + packages/backend-core/src/cache/generic.js | 7 +- packages/backend-core/src/db/utils.ts | 1 + packages/backend-core/src/events/analytics.ts | 27 +- .../backend-core/src/events/identification.ts | 49 ++-- .../src/events/publishers/automation.ts | 8 + .../src/events/publishers/backfill.ts | 67 +++++ .../src/events/publishers/index.ts | 1 + .../src/events/publishers/query.ts | 8 + packages/backend-core/src/installation.ts | 8 + .../src/tests/utilities/mocks/events.js | 235 ++++++++---------- packages/backend-core/tsconfig.build.json | 3 + packages/backend-core/yarn.lock | 181 +------------- packages/server/scripts/jestSetup.js | 13 +- .../server/src/api/controllers/automation.js | 4 +- .../server/src/api/controllers/datasource.js | 2 +- .../query/import/tests/index.spec.js | 8 +- packages/server/src/app.ts | 28 ++- .../src/migrations/functions/backfill/app.ts | 119 +++++++-- .../functions/backfill/app/automations.ts | 2 + .../functions/backfill/app/datasources.ts | 2 + .../functions/backfill/app/layouts.ts | 2 + .../functions/backfill/app/queries.ts | 2 + .../functions/backfill/app/roles.ts | 2 + .../functions/backfill/app/screens.ts | 2 + .../functions/backfill/app/tables.ts | 2 + .../migrations/functions/backfill/global.ts | 142 +++++++++-- .../functions/backfill/global/quotas.ts | 53 ++++ .../functions/backfill/global/rows.ts | 28 --- .../functions/backfill/global/users.ts | 2 + .../functions/backfill/installation.ts | 40 ++- packages/types/src/documents/global/config.ts | 2 +- packages/types/src/events/automation.ts | 4 + packages/types/src/events/backfill.ts | 35 +++ packages/types/src/events/event.ts | 12 +- packages/types/src/events/index.ts | 1 + packages/types/src/events/query.ts | 4 + .../src/api/controllers/global/configs.js | 1 + 39 files changed, 669 insertions(+), 441 deletions(-) create mode 100644 packages/backend-core/src/events/publishers/backfill.ts create mode 100644 packages/server/src/migrations/functions/backfill/global/quotas.ts delete mode 100644 packages/server/src/migrations/functions/backfill/global/rows.ts create mode 100644 packages/types/src/events/backfill.ts diff --git a/package.json b/package.json index 7a410482db..bf9fd72e3c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev", "bootstrap": "lerna link && lerna bootstrap && ./scripts/link-dependencies.sh", "build": "lerna run build", - "build:watch": "lerna run build:watch --stream --parallel", + "build:watch": "lerna run build:watch --ignore @budibase/backend-core --stream --parallel", "release": "lerna publish patch --yes --force-publish && yarn release:pro", "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop", "release:pro": "bash scripts/pro/release.sh", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 8f3a8a9f52..6c5063eb01 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -8,6 +8,7 @@ "license": "GPL-3.0", "scripts": { "build": "rimraf dist/ && tsc -p tsconfig.build.json", + "build:watch": "rimraf dist/ && tsc --build tsconfig.build.json --watch --preserveWatchOutput", "test": "jest", "test:watch": "jest --watchAll" }, diff --git a/packages/backend-core/src/cache/generic.js b/packages/backend-core/src/cache/generic.js index b23568f8b9..9ddde7f99a 100644 --- a/packages/backend-core/src/cache/generic.js +++ b/packages/backend-core/src/cache/generic.js @@ -4,6 +4,9 @@ const { getTenantId } = require("../context") exports.CacheKeys = { CHECKLIST: "checklist", + INSTALLATION: "installation", + ANALYTICS_ENABLED: "analyticsEnabled", + UNIQUE_TENANT_ID: "uniqueTenantId", } exports.TTL = { @@ -17,8 +20,8 @@ function generateTenantKey(key) { return `${key}:${tenantId}` } -exports.withCache = async (key, ttl, fetchFn) => { - key = generateTenantKey(key) +exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => { + key = opts.useTenancy ? generateTenantKey(key) : key const client = await redis.getCacheClient() const cachedValue = await client.get(key) if (cachedValue) { diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 9bf1abd18c..7e95b7df97 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -391,6 +391,7 @@ export const getScopedFullConfig = async function ( // defaults scopedConfig = { doc: { + _id: generateConfigID({ type, user, workspace }), config: { platformUrl: await getPlatformUrl(), analyticsEnabled: await events.analytics.enabled(), diff --git a/packages/backend-core/src/events/analytics.ts b/packages/backend-core/src/events/analytics.ts index 58c49714f7..a7e2067fd6 100644 --- a/packages/backend-core/src/events/analytics.ts +++ b/packages/backend-core/src/events/analytics.ts @@ -2,8 +2,8 @@ import env from "../environment" import * as tenancy from "../tenancy" import * as dbUtils from "../db/utils" import { Configs } from "../constants" +import { withCache, TTL, CacheKeys } from "../cache/generic" -// TODO: cache in redis export const enabled = async () => { // cloud - always use the environment variable if (!env.SELF_HOSTED) { @@ -11,13 +11,24 @@ export const enabled = async () => { } // self host - prefer the settings doc - // check for explicit true/false values to support - // backwards compatibility where setting may not exist - const settings = await getSettingsDoc() - if (settings?.config?.analyticsEnabled === false) { - return false - } else if (settings?.config?.analyticsEnabled === true) { - return true + // use cache as events have high throughput + const enabledInDB = await withCache( + CacheKeys.ANALYTICS_ENABLED, + TTL.ONE_DAY, + async () => { + const settings = await getSettingsDoc() + + // need to do explicit checks in case the field is not set + if (settings?.config?.analyticsEnabled === false) { + return false + } else if (settings?.config?.analyticsEnabled === true) { + return true + } + } + ) + + if (enabledInDB !== undefined) { + return enabledInDB } // fallback to the environment variable diff --git a/packages/backend-core/src/events/identification.ts b/packages/backend-core/src/events/identification.ts index 6157b28916..b8800bfd12 100644 --- a/packages/backend-core/src/events/identification.ts +++ b/packages/backend-core/src/events/identification.ts @@ -23,6 +23,7 @@ import * as dbUtils from "../db/utils" import { Configs } from "../constants" import * as hashing from "../hashing" import * as installation from "../installation" +import { withCache, TTL, CacheKeys } from "../cache/generic" const pkg = require("../../package.json") @@ -53,7 +54,7 @@ export const getCurrentIdentity = async (): Promise => { } } else if (identityType === IdentityType.TENANT) { const installationId = await getInstallationId() - const tenantId = await getCurrentTenantId() + const tenantId = await getEventTenantId(context.getTenantId()) return { id: formatDistinctId(tenantId, identityType), @@ -63,7 +64,7 @@ export const getCurrentIdentity = async (): Promise => { } } else if (identityType === IdentityType.USER) { const userContext = identityContext as UserContext - const tenantId = await getCurrentTenantId() + const tenantId = await getEventTenantId(context.getTenantId()) let installationId: string | undefined // self host account users won't have installation @@ -109,7 +110,7 @@ export const identifyTenantGroup = async ( account: Account | undefined, timestamp?: string | number ): Promise => { - const id = await getGlobalTenantId(tenantId) + const id = await getEventTenantId(tenantId) const type = IdentityType.TENANT let hosting: Hosting @@ -144,7 +145,7 @@ export const identifyUser = async ( timestamp?: string | number ) => { const id = user._id as string - const tenantId = await getGlobalTenantId(user.tenantId) + const tenantId = await getEventTenantId(user.tenantId) const type = IdentityType.USER let builder = user.builder?.global || false let admin = user.admin?.global || false @@ -214,8 +215,6 @@ const getHostingFromEnv = () => { return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD } -export const getCurrentTenantId = () => getGlobalTenantId(context.getTenantId()) - export const getInstallationId = async () => { if (isAccountPortal()) { return "account-portal" @@ -224,31 +223,35 @@ export const getInstallationId = async () => { return install.installId } -const getGlobalTenantId = async (tenantId: string): Promise => { +const getEventTenantId = async (tenantId: string): Promise => { if (env.SELF_HOSTED) { - return getGlobalId(tenantId) + return getUniqueTenantId(tenantId) } else { // tenant id's in the cloud are already unique return tenantId } } -// TODO: cache in redis -const getGlobalId = async (tenantId: string): Promise => { - const db = context.getGlobalDB() - const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { - type: Configs.SETTINGS, - }) +const getUniqueTenantId = async (tenantId: string): Promise => { + // make sure this tenantId always matches the tenantId in context + return context.doInTenant(tenantId, () => { + return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => { + const db = context.getGlobalDB() + const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, { + type: Configs.SETTINGS, + }) - let globalId: string - if (config.config.globalId) { - return config.config.globalId - } else { - globalId = `${hashing.newid()}_${tenantId}` - config.config.globalId = globalId - await db.put(config) - return globalId - } + let uniqueTenantId: string + if (config.config.uniqueTenantId) { + return config.config.uniqueTenantId + } else { + uniqueTenantId = `${hashing.newid()}_${tenantId}` + config.config.uniqueTenantId = uniqueTenantId + await db.put(config) + return uniqueTenantId + } + }) + }) } const isAccountPortal = () => { diff --git a/packages/backend-core/src/events/publishers/automation.ts b/packages/backend-core/src/events/publishers/automation.ts index daaa06f571..e1df28ead8 100644 --- a/packages/backend-core/src/events/publishers/automation.ts +++ b/packages/backend-core/src/events/publishers/automation.ts @@ -9,6 +9,7 @@ import { AutomationStepCreatedEvent, AutomationStepDeletedEvent, AutomationTriggerUpdatedEvent, + AutomationsRunEvent, } from "@budibase/types" export async function created(automation: Automation, timestamp?: string) { @@ -51,6 +52,13 @@ export async function tested(automation: Automation) { await publishEvent(Event.AUTOMATION_TESTED, properties) } +export const run = async (count: number, timestamp?: string | number) => { + const properties: AutomationsRunEvent = { + count, + } + await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp) +} + export async function stepCreated( automation: Automation, step: AutomationStep, diff --git a/packages/backend-core/src/events/publishers/backfill.ts b/packages/backend-core/src/events/publishers/backfill.ts new file mode 100644 index 0000000000..c16b3cc9dc --- /dev/null +++ b/packages/backend-core/src/events/publishers/backfill.ts @@ -0,0 +1,67 @@ +import { publishEvent } from "../events" +import { + Event, + AppBackfillSucceededEvent, + AppBackfillFailedEvent, + TenantBackfillSucceededEvent, + TenantBackfillFailedEvent, + InstallationBackfillSucceededEvent, + InstallationBackfillFailedEvent, +} from "@budibase/types" +const env = require("../../environment") + +const shouldSkip = !env.SELF_HOSTED && !env.isDev() + +export async function appSucceeded(properties: AppBackfillSucceededEvent) { + if (shouldSkip) { + return + } + await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties) +} + +export async function appFailed(error: any) { + if (shouldSkip) { + return + } + const properties: AppBackfillFailedEvent = { + error: JSON.stringify(error, Object.getOwnPropertyNames(error)), + } + await publishEvent(Event.APP_BACKFILL_FAILED, properties) +} + +export async function tenantSucceeded( + properties: TenantBackfillSucceededEvent +) { + if (shouldSkip) { + return + } + await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties) +} + +export async function tenantFailed(error: any) { + if (shouldSkip) { + return + } + const properties: TenantBackfillFailedEvent = { + error: JSON.stringify(error, Object.getOwnPropertyNames(error)), + } + await publishEvent(Event.TENANT_BACKFILL_FAILED, properties) +} + +export async function installationSucceeded() { + if (shouldSkip) { + return + } + const properties: InstallationBackfillSucceededEvent = {} + await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties) +} + +export async function installationFailed(error: any) { + if (shouldSkip) { + return + } + const properties: InstallationBackfillFailedEvent = { + error: JSON.stringify(error, Object.getOwnPropertyNames(error)), + } + await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties) +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 99069aede8..48b2387c5a 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -16,3 +16,4 @@ export * as serve from "./serve" export * as user from "./user" export * as view from "./view" export * as version from "./version" +export * as backfill from "./backfill" diff --git a/packages/backend-core/src/events/publishers/query.ts b/packages/backend-core/src/events/publishers/query.ts index e659f5c2be..1a567bb051 100644 --- a/packages/backend-core/src/events/publishers/query.ts +++ b/packages/backend-core/src/events/publishers/query.ts @@ -8,6 +8,7 @@ import { QueryDeletedEvent, QueryImportedEvent, QueryPreviewedEvent, + QueriesRunEvent, } from "@budibase/types" /* eslint-disable */ @@ -40,6 +41,13 @@ export const imported = async ( await publishEvent(Event.QUERY_IMPORT, properties) } +export const run = async (count: number, timestamp?: string | number) => { + const properties: QueriesRunEvent = { + count, + } + await publishEvent(Event.QUERIES_RUN, properties, timestamp) +} + export const previewed = async (datasource: Datasource) => { const properties: QueryPreviewedEvent = {} await publishEvent(Event.QUERY_PREVIEWED, properties) diff --git a/packages/backend-core/src/installation.ts b/packages/backend-core/src/installation.ts index 25e4a311ed..372877ec36 100644 --- a/packages/backend-core/src/installation.ts +++ b/packages/backend-core/src/installation.ts @@ -5,10 +5,17 @@ import { doWithDB } from "./db" import { Installation, IdentityType } from "@budibase/types" import * as context from "./context" import semver from "semver" +import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic" const pkg = require("../package.json") export const getInstall = async (): Promise => { + return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, { + useTenancy: false, + }) +} + +const getInstallFromDB = async (): Promise => { return doWithDB( StaticDatabases.PLATFORM_INFO.name, async (platformDb: any) => { @@ -44,6 +51,7 @@ const updateVersion = async (version: string): Promise => { const install = await getInstall() install.version = version await platformDb.put(install) + await bustCache(CacheKeys.INSTALLATION) }, {} ) diff --git a/packages/backend-core/src/tests/utilities/mocks/events.js b/packages/backend-core/src/tests/utilities/mocks/events.js index 2a40f89b8a..c1964d7ca7 100644 --- a/packages/backend-core/src/tests/utilities/mocks/events.js +++ b/packages/backend-core/src/tests/utilities/mocks/events.js @@ -1,135 +1,106 @@ -jest.mock("../../../events", () => { - return { - identification: { - identifyTenantGroup: jest.fn(), - identifyUser: jest.fn(), - }, - analytics: { - enabled: () => false, - }, - shutdown: () => {}, - account: { - created: jest.fn(), - deleted: jest.fn(), - verified: jest.fn(), - }, - app: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - published: jest.fn(), - unpublished: jest.fn(), - templateImported: jest.fn(), - fileImported: jest.fn(), - versionUpdated: jest.fn(), - versionReverted: jest.fn(), - reverted: jest.fn(), - exported: jest.fn(), - }, - auth: { - login: jest.fn(), - logout: jest.fn(), - SSOCreated: jest.fn(), - SSOUpdated: jest.fn(), - SSOActivated: jest.fn(), - SSODeactivated: jest.fn(), - }, - automation: { - created: jest.fn(), - deleted: jest.fn(), - tested: jest.fn(), - // run: jest.fn(), - stepCreated: jest.fn(), - stepDeleted: jest.fn(), - triggerUpdated: jest.fn(), - }, - datasource: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - }, - email: { - SMTPCreated: jest.fn(), - SMTPUpdated: jest.fn(), - }, - layout: { - created: jest.fn(), - deleted: jest.fn(), - }, - org: { - nameUpdated: jest.fn(), - logoUpdated: jest.fn(), - platformURLUpdated: jest.fn(), - analyticsOptOut: jest.fn(), - }, - version: { - checked: jest.fn(), - }, - query: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - imported: jest.fn(), - previewed: jest.fn(), - }, - role: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - assigned: jest.fn(), - unassigned: jest.fn(), - }, - rows: { - imported: jest.fn(), - created: jest.fn(), - }, - screen: { - created: jest.fn(), - deleted: jest.fn(), - }, - user: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - permissionAdminAssigned: jest.fn(), - permissionAdminRemoved: jest.fn(), - permissionBuilderAssigned: jest.fn(), - permissionBuilderRemoved: jest.fn(), - invited: jest.fn(), - inviteAccepted: jest.fn(), - passwordForceReset: jest.fn(), - passwordUpdated: jest.fn(), - passwordResetRequested: jest.fn(), - passwordReset: jest.fn(), - }, - serve: { - servedBuilder: jest.fn(), - servedApp: jest.fn(), - servedAppPreview: jest.fn(), - }, - table: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - exported: jest.fn(), - imported: jest.fn(), - permissionUpdated: jest.fn(), - }, - view: { - created: jest.fn(), - updated: jest.fn(), - deleted: jest.fn(), - exported: jest.fn(), - filterCreated: jest.fn(), - filterUpdated: jest.fn(), - filterDeleted: jest.fn(), - calculationCreated: jest.fn(), - calculationUpdated: jest.fn(), - calculationDeleted: jest.fn(), - }, - } -}) - const events = require("../../../events") +jest.spyOn(events.identification, "identifyTenantGroup") +jest.spyOn(events.identification, "identifyUser") + +jest.spyOn(events.account, "created") +jest.spyOn(events.account, "deleted") +jest.spyOn(events.account, "verified") + +jest.spyOn(events.app, "created") +jest.spyOn(events.app, "updated") +jest.spyOn(events.app, "deleted") +jest.spyOn(events.app, "published") +jest.spyOn(events.app, "unpublished") +jest.spyOn(events.app, "templateImported") +jest.spyOn(events.app, "fileImported") +jest.spyOn(events.app, "versionUpdated") +jest.spyOn(events.app, "versionReverted") +jest.spyOn(events.app, "reverted") +jest.spyOn(events.app, "exported") + +jest.spyOn(events.auth, "login") +jest.spyOn(events.auth, "logout") +jest.spyOn(events.auth, "SSOCreated") +jest.spyOn(events.auth, "SSOUpdated") +jest.spyOn(events.auth, "SSOActivated") +jest.spyOn(events.auth, "SSODeactivated") + +jest.spyOn(events.automation, "created") +jest.spyOn(events.automation, "deleted") +jest.spyOn(events.automation, "tested") +jest.spyOn(events.automation, "stepCreated") +jest.spyOn(events.automation, "stepDeleted") +jest.spyOn(events.automation, "triggerUpdated") + +jest.spyOn(events.datasource, "created") +jest.spyOn(events.datasource, "updated") +jest.spyOn(events.datasource, "deleted") + +jest.spyOn(events.email, "SMTPCreated") +jest.spyOn(events.email, "SMTPUpdated") + +jest.spyOn(events.layout, "created") +jest.spyOn(events.layout, "deleted") + +jest.spyOn(events.org, "nameUpdated") +jest.spyOn(events.org, "logoUpdated") +jest.spyOn(events.org, "platformURLUpdated") +jest.spyOn(events.org, "analyticsOptOut") + +jest.spyOn(events.version, "checked") + +jest.spyOn(events.query, "created") +jest.spyOn(events.query, "updated") +jest.spyOn(events.query, "deleted") +jest.spyOn(events.query, "imported") +jest.spyOn(events.query, "previewed") + +jest.spyOn(events.role, "created") +jest.spyOn(events.role, "updated") +jest.spyOn(events.role, "deleted") +jest.spyOn(events.role, "assigned") +jest.spyOn(events.role, "unassigned") + +jest.spyOn(events.rows, "imported") +jest.spyOn(events.rows, "created") + +jest.spyOn(events.screen, "created") +jest.spyOn(events.screen, "deleted") + +jest.spyOn(events.user, "created") +jest.spyOn(events.user, "updated") +jest.spyOn(events.user, "deleted") +jest.spyOn(events.user, "permissionAdminAssigned") +jest.spyOn(events.user, "permissionAdminRemoved") +jest.spyOn(events.user, "permissionBuilderAssigned") +jest.spyOn(events.user, "permissionBuilderRemoved") +jest.spyOn(events.user, "invited") +jest.spyOn(events.user, "inviteAccepted") +jest.spyOn(events.user, "passwordForceReset") +jest.spyOn(events.user, "passwordUpdated") +jest.spyOn(events.user, "passwordResetRequested") +jest.spyOn(events.user, "passwordReset") + +jest.spyOn(events.serve, "servedBuilder") +jest.spyOn(events.serve, "servedApp") +jest.spyOn(events.serve, "servedAppPreview") + +jest.spyOn(events.table, "created") +jest.spyOn(events.table, "updated") +jest.spyOn(events.table, "deleted") +jest.spyOn(events.table, "exported") +jest.spyOn(events.table, "imported") + +jest.spyOn(events.view, "created") +jest.spyOn(events.view, "updated") +jest.spyOn(events.view, "deleted") +jest.spyOn(events.view, "exported") +jest.spyOn(events.view, "filterCreated") +jest.spyOn(events.view, "filterUpdated") +jest.spyOn(events.view, "filterDeleted") +jest.spyOn(events.view, "calculationCreated") +jest.spyOn(events.view, "calculationUpdated") +jest.spyOn(events.view, "calculationDeleted") + module.exports = events diff --git a/packages/backend-core/tsconfig.build.json b/packages/backend-core/tsconfig.build.json index 602d4b3a7d..c18128c16e 100644 --- a/packages/backend-core/tsconfig.build.json +++ b/packages/backend-core/tsconfig.build.json @@ -1,6 +1,9 @@ { // Used for building with tsc "extends": "./tsconfig.json", + "references": [ + { "path": "../types" } + ], "exclude": [ "node_modules", "dist/**/*", diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index 606e978ab1..98c82da865 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -895,6 +895,13 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1" integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w== +"@types/ioredis@^4.27.1": + version "4.28.10" + resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff" + integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ== + dependencies: + "@types/node" "*" + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762" @@ -1092,11 +1099,6 @@ acorn-walk@^7.1.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== -acorn@^5.2.1: - version "5.7.4" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e" - integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg== - acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" @@ -1132,11 +1134,6 @@ ajv@^6.12.3: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -amdefine@>=0.0.4: - version "1.0.1" - resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5" - integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU= - ansi-align@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59" @@ -1228,11 +1225,6 @@ assert-plus@^0.2.0: resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234" integrity sha1-104bh+ev/A24qttwIfP+SBAasjQ= -ast-types@0.9.6: - version "0.9.6" - resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9" - integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk= - async@~2.1.4: version "2.1.5" resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc" @@ -1361,11 +1353,6 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== -base62@^1.1.0: - version "1.2.8" - resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428" - integrity sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA== - base64-js@^1.0.2, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -1721,26 +1708,6 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -commander@^2.5.0: - version "2.20.3" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" - integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== - -commoner@^0.10.1: - version "0.10.8" - resolved "https://registry.yarnpkg.com/commoner/-/commoner-0.10.8.tgz#34fc3672cd24393e8bb47e70caa0293811f4f2c5" - integrity sha1-NPw2cs0kOT6LtH5wyqApOBH08sU= - dependencies: - commander "^2.5.0" - detective "^4.3.1" - glob "^5.0.15" - graceful-fs "^4.1.2" - iconv-lite "^0.4.5" - mkdirp "^0.5.0" - private "^0.1.6" - q "^1.1.2" - recast "^0.11.17" - component-type@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9" @@ -1945,11 +1912,6 @@ deferred-leveldown@~5.3.0: abstract-leveldown "~6.2.1" inherits "^2.0.3" -defined@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693" - integrity sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM= - delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -1990,14 +1952,6 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -detective@^4.3.1: - version "4.7.1" - resolved "https://registry.yarnpkg.com/detective/-/detective-4.7.1.tgz#0eca7314338442febb6d65da54c10bb1c82b246e" - integrity sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig== - dependencies: - acorn "^5.2.1" - defined "^1.0.0" - diff-sequences@^27.5.1: version "27.5.1" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" @@ -2134,15 +2088,6 @@ error-inject@^1.0.0: resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37" integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc= -es3ify@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/es3ify/-/es3ify-0.2.2.tgz#5dae3e650e5be3684b88066513d528d092629862" - integrity sha1-Xa4+ZQ5b42hLiAZlE9Uo0JJimGI= - dependencies: - esprima "^2.7.1" - jstransform "~11.0.0" - through "~2.3.4" - escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -2180,26 +2125,11 @@ escodegen@^2.0.0: optionalDependencies: source-map "~0.6.1" -esprima-fb@^15001.1.0-dev-harmony-fb: - version "15001.1.0-dev-harmony-fb" - resolved "https://registry.yarnpkg.com/esprima-fb/-/esprima-fb-15001.1.0-dev-harmony-fb.tgz#30a947303c6b8d5e955bee2b99b1d233206a6901" - integrity sha512-59dDGQo2b3M/JfKIws0/z8dcXH2mnVHkfSPRhCYS91JNGfGNwr7GsSF6qzWZuOGvw5Ii0w9TtylrX07MGmlOoQ== - -esprima@^2.7.1: - version "2.7.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" - integrity sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A== - esprima@^4.0.0, esprima@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== -esprima@~3.1.0: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" - integrity sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg== - estraverse@^5.2.0: version "5.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" @@ -2467,17 +2397,6 @@ glob-parent@~5.1.2: dependencies: is-glob "^4.0.1" -glob@^5.0.15: - version "5.0.15" - resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1" - integrity sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E= - dependencies: - inflight "^1.0.4" - inherits "2" - minimatch "2 || 3" - once "^1.3.0" - path-is-absolute "^1.0.0" - glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" @@ -2716,7 +2635,7 @@ human-signals@^2.1.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== -iconv-lite@0.4.24, iconv-lite@^0.4.5: +iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== @@ -3550,17 +3469,6 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.3.6" -jstransform@~11.0.0: - version "11.0.3" - resolved "https://registry.yarnpkg.com/jstransform/-/jstransform-11.0.3.tgz#09a78993e0ae4d4ef4487f6155a91f6190cb4223" - integrity sha1-CaeJk+CuTU70SH9hVakfYZDLQiM= - dependencies: - base62 "^1.1.0" - commoner "^0.10.1" - esprima-fb "^15001.1.0-dev-harmony-fb" - object-assign "^2.0.0" - source-map "^0.4.2" - jwa@^1.1.4: version "1.1.5" resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.1.5.tgz#a0552ce0220742cd52e153774a32905c30e756e5" @@ -4069,13 +3977,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1: resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== -"minimatch@2 || 3": - version "3.1.2" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" - integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== - dependencies: - brace-expansion "^1.1.7" - minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -4083,7 +3984,7 @@ minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6: +minimist@^1.2.0, minimist@^1.2.5: version "1.2.6" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== @@ -4108,13 +4009,6 @@ mkdirp-classic@^0.5.2: resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== -mkdirp@^0.5.0: - version "0.5.6" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6" - integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw== - dependencies: - minimist "^1.2.6" - mkdirp@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" @@ -4295,11 +4189,6 @@ oauth@0.9.x, oauth@^0.9.15: resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1" integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE= -object-assign@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa" - integrity sha1-Q8NuXVaf+OSBbE76i+AtJpZ8GKo= - object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -4624,17 +4513,6 @@ pouchdb-adapter-utils@7.2.2: pouchdb-merge "7.2.2" pouchdb-utils "7.2.2" -pouchdb-all-dbs@^1.0.2: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pouchdb-all-dbs/-/pouchdb-all-dbs-1.1.1.tgz#85f04a39cafda52497ec49abf1c93bb5e72813f6" - integrity sha512-UUnsdmcnRSQ8MAOYSJjfTwKkQNb/6fvOfd/f7dNNivWZ2YDYVuMfgw1WQdL634yEtcXTxAENZ/EyLRdzPCB41A== - dependencies: - argsarray "0.0.1" - es3ify "^0.2.2" - inherits "~2.0.1" - pouchdb-promise "6.4.3" - tiny-queue "^0.2.0" - pouchdb-binary-utils@7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/pouchdb-binary-utils/-/pouchdb-binary-utils-7.2.2.tgz#0690b348052c543b1e67f032f47092ca82bcb10e" @@ -4711,7 +4589,7 @@ pouchdb-merge@7.2.2: resolved "https://registry.yarnpkg.com/pouchdb-merge/-/pouchdb-merge-7.2.2.tgz#940d85a2b532d6a93a6cab4b250f5648511bcc16" integrity sha512-6yzKJfjIchBaS7Tusuk8280WJdESzFfQ0sb4jeMUNnrqs4Cx3b0DIEOYTRRD9EJDM+je7D3AZZ4AT0tFw8gb4A== -pouchdb-promise@6.4.3, pouchdb-promise@^6.0.4: +pouchdb-promise@^6.0.4: version "6.4.3" resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3" integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw== @@ -4798,11 +4676,6 @@ pretty-format@^27.0.0, pretty-format@^27.5.1: ansi-styles "^5.0.0" react-is "^17.0.1" -private@^0.1.6, private@~0.1.5: - version "0.1.8" - resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff" - integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg== - process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" @@ -4861,11 +4734,6 @@ pupa@^2.1.1: dependencies: escape-goat "^2.0.0" -q@^1.1.2: - version "1.5.1" - resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" - integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= - qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" @@ -4950,16 +4818,6 @@ readline-sync@^1.4.9: resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b" integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw== -recast@^0.11.17: - version "0.11.23" - resolved "https://registry.yarnpkg.com/recast/-/recast-0.11.23.tgz#451fd3004ab1e4df9b4e4b66376b2a21912462d3" - integrity sha1-RR/TAEqx5N+bTktmN2sqIZEkYtM= - dependencies: - ast-types "0.9.6" - esprima "~3.1.0" - private "~0.1.5" - source-map "~0.5.0" - redis-commands@1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" @@ -5230,14 +5088,7 @@ source-map-support@^0.5.6: buffer-from "^1.0.0" source-map "^0.6.0" -source-map@^0.4.2: - version "0.4.4" - resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" - integrity sha1-66T12pwNyZneaAMti092FzZSA2s= - dependencies: - amdefine ">=0.0.4" - -source-map@^0.5.0, source-map@~0.5.0: +source-map@^0.5.0: version "0.5.7" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= @@ -5510,21 +5361,11 @@ through2@^2.0.0, through2@^2.0.2, through2@^2.0.3: readable-stream "~2.3.6" xtend "~4.0.1" -through@~2.3.4: - version "2.3.8" - resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= - timekeeper@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368" integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A== -tiny-queue@^0.2.0: - version "0.2.1" - resolved "https://registry.yarnpkg.com/tiny-queue/-/tiny-queue-0.2.1.tgz#25a67f2c6e253b2ca941977b5ef7442ef97a6046" - integrity sha1-JaZ/LG4lOyypQZd7XvdELvl6YEY= - tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" diff --git a/packages/server/scripts/jestSetup.js b/packages/server/scripts/jestSetup.js index ef5ab8e9b5..f0241a03ff 100644 --- a/packages/server/scripts/jestSetup.js +++ b/packages/server/scripts/jestSetup.js @@ -1,11 +1,5 @@ const { tmpdir } = require("os") const env = require("../src/environment") -const { mocks } = require("@budibase/backend-core/testUtils") - -// mock all dates to 2020-01-01T00:00:00.000Z -// use tk.reset() to use real dates in individual tests -const tk = require("timekeeper") -tk.freeze(mocks.date.MOCK_DATE) env._set("SELF_HOSTED", "1") env._set("NODE_ENV", "jest") @@ -15,4 +9,11 @@ env._set("BUDIBASE_DIR", tmpdir("budibase-unittests")) env._set("LOG_LEVEL", "silent") env._set("PORT", 0) +const { mocks } = require("@budibase/backend-core/testUtils") + +// mock all dates to 2020-01-01T00:00:00.000Z +// use tk.reset() to use real dates in individual tests +const tk = require("timekeeper") +tk.freeze(mocks.date.MOCK_DATE) + global.console.log = jest.fn() // console.log are ignored in tests diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js index ddea76beb4..2469e132d3 100644 --- a/packages/server/src/api/controllers/automation.js +++ b/packages/server/src/api/controllers/automation.js @@ -71,9 +71,9 @@ exports.create = async function (ctx) { newAuto: automation, }) const response = await db.put(automation) - await events.automation.created() + await events.automation.created(automation) for (let step of automation.definition.steps) { - await events.automation.stepCreated(step) + await events.automation.stepCreated(automation, step) } automation._rev = response.rev diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js index 63c51fb92c..e217dd2c47 100644 --- a/packages/server/src/api/controllers/datasource.js +++ b/packages/server/src/api/controllers/datasource.js @@ -144,7 +144,7 @@ exports.save = async function (ctx) { } const dbResp = await db.put(datasource) - await events.datasource.created() + await events.datasource.created(datasource) datasource._rev = dbResp.rev // Drain connection pools when configuration is changed diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js index 0e9c5f7543..00d7daa43f 100644 --- a/packages/server/src/api/controllers/query/import/tests/index.spec.js +++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js @@ -53,9 +53,11 @@ describe("Rest Importer", () => { } const runTest = async (test, assertions) => { - for (let [key, data] of Object.entries(datasets)) { - await test(key, data, assertions) - } + config.doInContext(config.appId, async () => { + for (let [key, data] of Object.entries(datasets)) { + await test(key, data, assertions) + } + }) } const testGetInfo = async (key, data, assertions) => { diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 63dc3f1a6f..149a2389bb 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -83,6 +83,22 @@ module.exports = server.listen(env.PORT || 0, async () => { eventEmitter.emitPort(env.PORT) fileSystem.init() await redis.init() + + // run migrations on startup if not done via http + // not recommended in a clustered environment + if (!env.HTTP_MIGRATIONS && !env.isTest()) { + try { + await migrations.migrate() + } catch (e) { + console.error("Error performing migrations. Exiting.\n", e) + shutdown() + } + } + + // check for version updates + await installation.checkInstallVersion() + + // done last - this will never complete await automations.init() }) @@ -99,15 +115,3 @@ process.on("uncaughtException", err => { process.on("SIGTERM", () => { shutdown() }) - -// run migrations on startup if not done via http -// not recommended in a clustered environment -if (!env.HTTP_MIGRATIONS && !env.isTest()) { - migrations.migrate().catch(err => { - console.error("Error performing migrations. Exiting.\n", err) - shutdown() - }) -} - -// check for version updates -installation.checkInstallVersion() diff --git a/packages/server/src/migrations/functions/backfill/app.ts b/packages/server/src/migrations/functions/backfill/app.ts index 60ff953aa0..83f99a1c16 100644 --- a/packages/server/src/migrations/functions/backfill/app.ts +++ b/packages/server/src/migrations/functions/backfill/app.ts @@ -6,8 +6,21 @@ import * as roles from "./app/roles" import * as tables from "./app/tables" import * as screens from "./app/screens" import * as global from "./global" -import { App } from "@budibase/types" +import { App, AppBackfillSucceededEvent } from "@budibase/types" import { db as dbUtils, events } from "@budibase/backend-core" +import env from "../../../environment" + +const failGraceful = env.SELF_HOSTED && !env.isDev() + +const handleError = (e: any, errors?: any) => { + if (failGraceful) { + if (errors) { + errors.push(e) + } + return + } + throw e +} /** * Date: @@ -18,28 +31,92 @@ import { db as dbUtils, events } from "@budibase/backend-core" */ export const run = async (appDb: any) => { - if (await global.isComplete()) { - // make sure new apps aren't backfilled - // return if the global migration for this tenant is complete - // which runs after the app migrations - return - } + try { + if (await global.isComplete()) { + // make sure new apps aren't backfilled + // return if the global migration for this tenant is complete + // which runs after the app migrations + return + } - const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA) - const timestamp = app.createdAt as string + const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA) + const timestamp = app.createdAt as string - if (dbUtils.isDevAppID(app.appId)) { - await events.app.created(app, timestamp) - await automations.backfill(appDb, timestamp) - await datasources.backfill(appDb, timestamp) - await layouts.backfill(appDb, timestamp) - await queries.backfill(appDb, timestamp) - await roles.backfill(appDb, timestamp) - await tables.backfill(appDb, timestamp) - await screens.backfill(appDb, timestamp) - } + if (dbUtils.isProdAppID(app.appId)) { + await events.app.published(app, timestamp) + } - if (dbUtils.isProdAppID(app.appId)) { - await events.app.published(app, timestamp) + const totals: any = {} + const errors: any = [] + + if (dbUtils.isDevAppID(app.appId)) { + await events.app.created(app, timestamp) + try { + totals.automations = await automations.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.datasources = await datasources.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.layouts = await layouts.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.queries = await queries.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.roles = await roles.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.screens = await screens.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.tables = await tables.backfill(appDb, timestamp) + } catch (e) { + handleError(e, errors) + } + } + + const properties: AppBackfillSucceededEvent = { + appId: app.appId, + automations: totals.automations, + datasources: totals.datasources, + layouts: totals.layouts, + queries: totals.queries, + roles: totals.roles, + tables: totals.tables, + screens: totals.screens, + } + + if (errors.length) { + properties.errors = errors.map((e: any) => + JSON.stringify(e, Object.getOwnPropertyNames(e)) + ) + properties.errorCount = errors.length + } else { + properties.errorCount = 0 + } + + await events.backfill.appSucceeded(properties) + } catch (e) { + handleError(e) + await events.backfill.appFailed(e) } } diff --git a/packages/server/src/migrations/functions/backfill/app/automations.ts b/packages/server/src/migrations/functions/backfill/app/automations.ts index f455e10fc8..e4c3ad19f3 100644 --- a/packages/server/src/migrations/functions/backfill/app/automations.ts +++ b/packages/server/src/migrations/functions/backfill/app/automations.ts @@ -21,4 +21,6 @@ export const backfill = async (appDb: any, timestamp: string) => { await events.automation.stepCreated(automation, step, timestamp) } } + + return automations.length } diff --git a/packages/server/src/migrations/functions/backfill/app/datasources.ts b/packages/server/src/migrations/functions/backfill/app/datasources.ts index 1919231a88..1245e5d745 100644 --- a/packages/server/src/migrations/functions/backfill/app/datasources.ts +++ b/packages/server/src/migrations/functions/backfill/app/datasources.ts @@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => { for (const datasource of datasources) { await events.datasource.created(datasource, timestamp) } + + return datasources.length } diff --git a/packages/server/src/migrations/functions/backfill/app/layouts.ts b/packages/server/src/migrations/functions/backfill/app/layouts.ts index a02eeb7123..ec36d59b30 100644 --- a/packages/server/src/migrations/functions/backfill/app/layouts.ts +++ b/packages/server/src/migrations/functions/backfill/app/layouts.ts @@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => { for (const layout of layouts) { await events.layout.created(layout, timestamp) } + + return layouts.length } diff --git a/packages/server/src/migrations/functions/backfill/app/queries.ts b/packages/server/src/migrations/functions/backfill/app/queries.ts index 9e83e6a371..fb0e561363 100644 --- a/packages/server/src/migrations/functions/backfill/app/queries.ts +++ b/packages/server/src/migrations/functions/backfill/app/queries.ts @@ -28,4 +28,6 @@ export const backfill = async (appDb: any, timestamp: string) => { ) await events.query.created(datasource, query, timestamp) } + + return queries.length } diff --git a/packages/server/src/migrations/functions/backfill/app/roles.ts b/packages/server/src/migrations/functions/backfill/app/roles.ts index 4fa57b4165..1c90b0f658 100644 --- a/packages/server/src/migrations/functions/backfill/app/roles.ts +++ b/packages/server/src/migrations/functions/backfill/app/roles.ts @@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => { for (const role of roles) { await events.role.created(role, timestamp) } + + return roles.length } diff --git a/packages/server/src/migrations/functions/backfill/app/screens.ts b/packages/server/src/migrations/functions/backfill/app/screens.ts index b1d9791c2d..a17be76ea3 100644 --- a/packages/server/src/migrations/functions/backfill/app/screens.ts +++ b/packages/server/src/migrations/functions/backfill/app/screens.ts @@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => { for (const screen of screens) { await events.screen.created(screen, timestamp) } + + return screens.length } diff --git a/packages/server/src/migrations/functions/backfill/app/tables.ts b/packages/server/src/migrations/functions/backfill/app/tables.ts index 23d0dff702..ea41cd896a 100644 --- a/packages/server/src/migrations/functions/backfill/app/tables.ts +++ b/packages/server/src/migrations/functions/backfill/app/tables.ts @@ -31,4 +31,6 @@ export const backfill = async (appDb: any, timestamp: string) => { } } } + + return tables.length } diff --git a/packages/server/src/migrations/functions/backfill/global.ts b/packages/server/src/migrations/functions/backfill/global.ts index a03ae67f41..7031b0f61f 100644 --- a/packages/server/src/migrations/functions/backfill/global.ts +++ b/packages/server/src/migrations/functions/backfill/global.ts @@ -1,10 +1,62 @@ +import { TenantBackfillSucceededEvent } from "./../../../../../types/src/events/backfill" import * as users from "./global/users" -import * as rows from "./global/rows" import * as configs from "./global/configs" -import { tenancy, events, migrations, accounts } from "@budibase/backend-core" -import { CloudAccount } from "@budibase/types" +import * as quotas from "./global/quotas" +import { + tenancy, + events, + migrations, + accounts, + db as dbUtils, +} from "@budibase/backend-core" +import { QuotaUsage } from "@budibase/pro" +import { CloudAccount, App } from "@budibase/types" import env from "../../../environment" +const failGraceful = env.SELF_HOSTED && !env.isDev() + +const handleError = (e: any, errors?: any) => { + if (failGraceful) { + if (errors) { + errors.push(e) + } + return + } + throw e +} + +const formatUsage = (usage: QuotaUsage) => { + let maxAutomations = 0 + let maxQueries = 0 + let rows = 0 + let developers = 0 + + if (usage) { + if (usage.usageQuota) { + rows = usage.usageQuota.rows + developers = usage.usageQuota.developers + } + + if (usage.monthly) { + for (const value of Object.values(usage.monthly)) { + if (value.automations > maxAutomations) { + maxAutomations = value.automations + } + if (value.queries > maxQueries) { + maxQueries = value.queries + } + } + } + } + + return { + maxAutomations, + maxQueries, + rows, + developers, + } +} + /** * Date: * May 2022 @@ -14,23 +66,75 @@ import env from "../../../environment" */ export const run = async (db: any) => { - const tenantId = tenancy.getTenantId() - const installTimestamp = (await getInstallTimestamp(db)) as number - let account: CloudAccount | undefined - if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { - account = await accounts.getAccountByTenantId(tenantId) + try { + const tenantId = tenancy.getTenantId() + let installTimestamp + + const totals: any = {} + const errors: any = [] + + try { + installTimestamp = await getInstallTimestamp(db) + } catch (e) { + handleError(e, errors) + } + + let account: CloudAccount | undefined + if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) { + account = await accounts.getAccountByTenantId(tenantId) + } + + try { + await events.identification.identifyTenantGroup( + tenantId, + account, + installTimestamp + ) + } catch (e) { + handleError(e, errors) + } + + try { + await configs.backfill(db, installTimestamp) + } catch (e) { + handleError(e, errors) + } + + try { + totals.users = await users.backfill(db, account) + } catch (e) { + handleError(e, errors) + } + + try { + const allApps: App[] = await dbUtils.getAllApps({ dev: true }) + totals.apps = allApps.length + + totals.usage = await quotas.backfill(allApps) + } catch (e) { + handleError(e, errors) + } + + const properties: TenantBackfillSucceededEvent = { + apps: totals.apps, + users: totals.users, + ...formatUsage(totals.usage), + usage: totals.usage, + } + + if (errors.length) { + properties.errors = errors.map((e: any) => + JSON.stringify(e, Object.getOwnPropertyNames(e)) + ) + properties.errorCount = errors.length + } else { + properties.errorCount = 0 + } + await events.backfill.tenantSucceeded(properties) + } catch (e) { + handleError(e) + await events.backfill.tenantFailed(e) } - - await events.identification.identifyTenantGroup( - tenantId, - account, - installTimestamp - ) - await configs.backfill(db, installTimestamp) - - // users and rows provide their own timestamp - await users.backfill(db, account) - await rows.backfill() } export const isComplete = async (): Promise => { diff --git a/packages/server/src/migrations/functions/backfill/global/quotas.ts b/packages/server/src/migrations/functions/backfill/global/quotas.ts new file mode 100644 index 0000000000..274804adad --- /dev/null +++ b/packages/server/src/migrations/functions/backfill/global/quotas.ts @@ -0,0 +1,53 @@ +import { events } from "@budibase/backend-core" +import { quotas } from "@budibase/pro" +import { App } from "@budibase/types" + +const getOldestCreatedAt = (allApps: App[]): string | undefined => { + const timestamps = allApps + .filter(app => !!app.createdAt) + .map(app => app.createdAt as string) + .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) + + if (timestamps.length) { + return timestamps[0] + } +} + +const getMonthTimestamp = (monthString: string): number => { + const parts = monthString.split("-") + const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation + const year = parseInt(parts[1]) + + // using 0 as the day in next month gives us last day in previous month + const date = new Date(year, month + 1, 0).getTime() + const now = new Date().getTime() + + if (date > now) { + return now + } else { + return date + } +} + +export const backfill = async (allApps: App[]) => { + const usage = await quotas.getQuotaUsage() + + const rows = usage.usageQuota.rows + const rowsTimestamp = getOldestCreatedAt(allApps) + await events.rows.created(rows, rowsTimestamp) + + for (const [monthString, quotas] of Object.entries(usage.monthly)) { + if (monthString === "current") { + continue + } + const monthTimestamp = getMonthTimestamp(monthString) + + const queries = quotas.queries + await events.query.run(queries, monthTimestamp) + + const automations = quotas.automations + await events.automation.run(automations, monthTimestamp) + } + + return usage +} diff --git a/packages/server/src/migrations/functions/backfill/global/rows.ts b/packages/server/src/migrations/functions/backfill/global/rows.ts deleted file mode 100644 index b336668d64..0000000000 --- a/packages/server/src/migrations/functions/backfill/global/rows.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { events, db as dbUtils } from "@budibase/backend-core" -import { Row, App } from "@budibase/types" -import { getUniqueRows } from "../../../../utilities/usageQuota/rows" - -// Rows is a special circumstance where we get rows across all apps -// therefore migration is performed at the global level - -const getOldestCreatedAt = (allApps: App[]): string | undefined => { - const timestamps = allApps - .filter(app => !!app.createdAt) - .map(app => app.createdAt as string) - .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) - - if (timestamps.length) { - return timestamps[0] - } -} - -export const backfill = async () => { - const allApps: App[] = await dbUtils.getAllApps({ dev: true }) - const timestamp = getOldestCreatedAt(allApps) - const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : [] - const rows: Row[] = await getUniqueRows(appIds) - const rowCount = rows ? rows.length : 0 - if (rowCount) { - await events.rows.created(rowCount, timestamp) - } -} diff --git a/packages/server/src/migrations/functions/backfill/global/users.ts b/packages/server/src/migrations/functions/backfill/global/users.ts index 5ffbfd056c..0a59bec630 100644 --- a/packages/server/src/migrations/functions/backfill/global/users.ts +++ b/packages/server/src/migrations/functions/backfill/global/users.ts @@ -40,4 +40,6 @@ export const backfill = async ( } } } + + return users.length } diff --git a/packages/server/src/migrations/functions/backfill/installation.ts b/packages/server/src/migrations/functions/backfill/installation.ts index b807d650e8..425ec80f0d 100644 --- a/packages/server/src/migrations/functions/backfill/installation.ts +++ b/packages/server/src/migrations/functions/backfill/installation.ts @@ -1,6 +1,19 @@ import { events, tenancy, installation } from "@budibase/backend-core" import { Installation } from "@budibase/types" import * as global from "./global" +import env from "../../../environment" + +const failGraceful = env.SELF_HOSTED + +const handleError = (e: any, errors?: any) => { + if (failGraceful) { + if (errors) { + errors.push(e) + } + return + } + throw e +} /** * Date: @@ -11,14 +24,21 @@ import * as global from "./global" */ export const run = async () => { - // need to use the default tenant to try to get the installation time - await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { - const db = tenancy.getGlobalDB() - const installTimestamp = (await global.getInstallTimestamp(db)) as number - const install: Installation = await installation.getInstall() - await events.identification.identifyInstallationGroup( - install.installId, - installTimestamp - ) - }) + try { + // need to use the default tenant to try to get the installation time + await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => { + const db = tenancy.getGlobalDB() + const installTimestamp = (await global.getInstallTimestamp(db)) as number + const install: Installation = await installation.getInstall() + await events.identification.identifyInstallationGroup( + install.installId, + installTimestamp + ) + }) + await events.backfill.installationSucceeded() + throw new Error("fail") + } catch (e) { + handleError(e) + await events.backfill.installationFailed(e) + } } diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 9ad80e4eef..f62f0a12a8 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -19,7 +19,7 @@ export interface SettingsConfig extends Config { company: string logoUrl: string platformUrl: string - globalId?: string + uniqueTenantId?: string } } diff --git a/packages/types/src/events/automation.ts b/packages/types/src/events/automation.ts index 4bd40bee92..108894cec6 100644 --- a/packages/types/src/events/automation.ts +++ b/packages/types/src/events/automation.ts @@ -43,3 +43,7 @@ export interface AutomationStepDeletedEvent { stepId: string stepType: string } + +export interface AutomationsRunEvent { + count: number +} diff --git a/packages/types/src/events/backfill.ts b/packages/types/src/events/backfill.ts new file mode 100644 index 0000000000..35e921dece --- /dev/null +++ b/packages/types/src/events/backfill.ts @@ -0,0 +1,35 @@ +export interface AppBackfillSucceededEvent { + appId: string + automations: number + datasources: number + layouts: number + queries: number + roles: number + tables: number + screens: number + errors?: string[] + errorCount?: number +} + +export interface AppBackfillFailedEvent { + error: string +} + +export interface TenantBackfillSucceededEvent { + apps: number + users: number + + usage: any + errors?: [string] + errorCount?: number +} + +export interface TenantBackfillFailedEvent { + error: string +} + +export interface InstallationBackfillSucceededEvent {} + +export interface InstallationBackfillFailedEvent { + error: string +} diff --git a/packages/types/src/events/event.ts b/packages/types/src/events/event.ts index d5528ea114..9c8e10b9ad 100644 --- a/packages/types/src/events/event.ts +++ b/packages/types/src/events/event.ts @@ -81,7 +81,7 @@ export enum Event { QUERY_UPDATED = "query:updated", QUERY_DELETED = "query:deleted", QUERY_IMPORT = "query:import", - // QUERY_RUN = "query:run", + QUERIES_RUN = "queries:run", QUERY_PREVIEWED = "query:previewed", // TABLE @@ -124,7 +124,7 @@ export enum Event { AUTOMATION_CREATED = "automation:created", AUTOMATION_DELETED = "automation:deleted", AUTOMATION_TESTED = "automation:tested", - // AUTOMATION_RUN = "automation:run", + AUTOMATIONS_RUN = "automations:run", AUTOMATION_STEP_CREATED = "automation:step:created", AUTOMATION_STEP_DELETED = "automation:step:deleted", AUTOMATION_TRIGGER_UPDATED = "automation:trigger:updated", @@ -140,6 +140,14 @@ export enum Event { ACCOUNT_CREATED = "account:created", ACCOUNT_DELETED = "account:deleted", ACCOUNT_VERIFIED = "account:verified", + + // BACKFILL + APP_BACKFILL_SUCCEEDED = "app:backfill:succeeded", + APP_BACKFILL_FAILED = "app:backfill:failed", + TENANT_BACKFILL_SUCCEEDED = "tenant:backfill:succeeded", + TENANT_BACKFILL_FAILED = "tenant:backfill:failed", + INSTALLATION_BACKFILL_SUCCEEDED = "installation:backfill:succeeded", + INSTALLATION_BACKFILL_FAILED = "installation:backfill:failed", } export type RowImportFormat = "csv" diff --git a/packages/types/src/events/index.ts b/packages/types/src/events/index.ts index 44f6f297a6..d87558332a 100644 --- a/packages/types/src/events/index.ts +++ b/packages/types/src/events/index.ts @@ -16,3 +16,4 @@ export * from "./table" export * from "./user" export * from "./view" export * from "./account" +export * from "./backfill" diff --git a/packages/types/src/events/query.ts b/packages/types/src/events/query.ts index 4a89c38f79..864298e7e2 100644 --- a/packages/types/src/events/query.ts +++ b/packages/types/src/events/query.ts @@ -7,3 +7,7 @@ export interface QueryDeletedEvent {} export interface QueryImportedEvent {} export interface QueryPreviewedEvent {} + +export interface QueriesRunEvent { + count: number +} diff --git a/packages/worker/src/api/controllers/global/configs.js b/packages/worker/src/api/controllers/global/configs.js index 5ce3fd0bac..227b86d181 100644 --- a/packages/worker/src/api/controllers/global/configs.js +++ b/packages/worker/src/api/controllers/global/configs.js @@ -167,6 +167,7 @@ exports.save = async function (ctx) { try { const response = await db.put(ctx.request.body) await bustCache(CacheKeys.CHECKLIST) + await bustCache(CacheKeys.ANALYTICS_ENABLED) for (const fn of eventFns) { await fn()