From ceeeb0ad8dd3bd5ca85ab5994bf0c99e93c0888a Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Tue, 16 May 2023 15:05:12 +0000 Subject: [PATCH 01/14] v2.6.16 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 6 +++--- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 12 ++++++------ packages/frontend-core/package.json | 6 +++--- packages/sdk/package.json | 2 +- packages/server/package.json | 12 ++++++------ packages/shared-core/package.json | 4 ++-- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lerna.json b/lerna.json index 377742661b..643023f2b1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.15", + "version": "2.6.16", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index a99417d355..94b9e9e745 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.6.15", + "@budibase/types": "^2.6.16", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 422fbe2a7f..a27fe96ebf 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.6.15", + "version": "2.6.16", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.6.15", - "@budibase/string-templates": "^2.6.15", + "@budibase/shared-core": "^2.6.16", + "@budibase/string-templates": "^2.6.16", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 8c04bebcfa..a0a14498f6 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.6.15", + "version": "2.6.16", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.6.15", - "@budibase/frontend-core": "^2.6.15", - "@budibase/shared-core": "^2.6.15", - "@budibase/string-templates": "^2.6.15", + "@budibase/bbui": "^2.6.16", + "@budibase/frontend-core": "^2.6.16", + "@budibase/shared-core": "^2.6.16", + "@budibase/string-templates": "^2.6.16", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 7e9e6db9db..ed843dc18e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,9 +29,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.6.15", - "@budibase/string-templates": "^2.6.15", - "@budibase/types": "^2.6.15", + "@budibase/backend-core": "^2.6.16", + "@budibase/string-templates": "^2.6.16", + "@budibase/types": "^2.6.16", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index f833da2725..0871d05302 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.6.15", + "version": "2.6.16", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.6.15", - "@budibase/frontend-core": "^2.6.15", - "@budibase/shared-core": "^2.6.15", - "@budibase/string-templates": "^2.6.15", - "@budibase/types": "^2.6.15", + "@budibase/bbui": "^2.6.16", + "@budibase/frontend-core": "^2.6.16", + "@budibase/shared-core": "^2.6.16", + "@budibase/string-templates": "^2.6.16", + "@budibase/types": "^2.6.16", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 2e62535f0d..b2053a8fa0 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.6.15", - "@budibase/shared-core": "^2.6.15", + "@budibase/bbui": "^2.6.16", + "@budibase/shared-core": "^2.6.16", "dayjs": "^1.11.7", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 2828e38aa3..41a02e71a9 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index d1d31a5e82..25df882a64 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.6.15", - "@budibase/client": "^2.6.15", + "@budibase/backend-core": "^2.6.16", + "@budibase/client": "^2.6.16", "@budibase/pro": "2.6.15", - "@budibase/shared-core": "^2.6.15", - "@budibase/string-templates": "^2.6.15", - "@budibase/types": "^2.6.15", + "@budibase/shared-core": "^2.6.16", + "@budibase/string-templates": "^2.6.16", + "@budibase/types": "^2.6.16", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index ab625a89c3..b69e8105e6 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.6.15", + "version": "2.6.16", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "^2.6.15" + "@budibase/types": "^2.6.16" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index d5aad72ccb..79ffb2b0e4 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.6.15", + "version": "2.6.16", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index ac63e4577f..db3bc658d4 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index 23fe510793..46e5d990fd 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.6.15", + "version": "2.6.16", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.6.15", + "@budibase/backend-core": "^2.6.16", "@budibase/pro": "2.6.15", - "@budibase/string-templates": "^2.6.15", - "@budibase/types": "^2.6.15", + "@budibase/string-templates": "^2.6.16", + "@budibase/types": "^2.6.16", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 58cb15623b1efaf9cdb453e3f1c76ed6059e7806 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Tue, 16 May 2023 15:06:49 +0000 Subject: [PATCH 02/14] Update pro version to 2.6.16 --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- yarn.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 25df882a64..9f6c33f0e2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.6.16", "@budibase/client": "^2.6.16", - "@budibase/pro": "2.6.15", + "@budibase/pro": "2.6.16", "@budibase/shared-core": "^2.6.16", "@budibase/string-templates": "^2.6.16", "@budibase/types": "^2.6.16", diff --git a/packages/worker/package.json b/packages/worker/package.json index 46e5d990fd..a2a4ebafaf 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.6.16", - "@budibase/pro": "2.6.15", + "@budibase/pro": "2.6.16", "@budibase/string-templates": "^2.6.16", "@budibase/types": "^2.6.16", "@koa/router": "8.0.8", diff --git a/yarn.lock b/yarn.lock index aa9bd8b159..6d606cb461 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.6.14": - version "2.6.14" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.14.tgz#88345d2aa6630e4aef4943874c56b66f017c83e1" - integrity sha512-86mo0QeBgCGqsSAesBZjOBEwJEcYdcnsJFXzw0UY3jdpMwu+nOdZGwBpLNI10qy88S/cpe44MrtPslQLIJRIcQ== +"@budibase/pro@2.6.15": + version "2.6.15" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.15.tgz#527a0e6fe85d0413382433a23b0342d77670db1d" + integrity sha512-oXssIt4VvFEde8ftRnk7XPYMOXI/E7YN8GseSyhAxzozPYdGatd1AyqnBAJFgrYtAwQCLQOZBJFWRWhCpCCPhA== dependencies: - "@budibase/backend-core" "2.6.14" + "@budibase/backend-core" "2.6.15" "@budibase/shared-core" "2.5.9" "@budibase/string-templates" "2.5.9" - "@budibase/types" "2.6.14" + "@budibase/types" "2.6.15" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" From 42f472b038adacb1b62b3d9752a17c19e2057c0b Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 12:26:07 +0100 Subject: [PATCH 03/14] Type updates for automation jobs --- packages/server/src/automations/triggers.ts | 8 +++++--- packages/server/src/definitions/automations.ts | 5 +---- packages/server/src/threads/automation.ts | 11 +++++------ packages/server/src/threads/definitions.ts | 2 -- packages/server/src/threads/index.ts | 6 ++++-- packages/types/src/documents/app/automation.ts | 5 +++++ packages/types/src/sdk/automations/index.ts | 15 +++++++++++++++ packages/types/src/sdk/index.ts | 1 + 8 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 packages/types/src/sdk/automations/index.ts diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 78f8a87b0c..3436c1be7a 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -9,7 +9,7 @@ import { checkTestFlag } from "../utilities/redis" import * as utils from "./utils" import env from "../environment" import { context, db as dbCore } from "@budibase/backend-core" -import { Automation, Row } from "@budibase/types" +import { Automation, Row, AutomationData, AutomationJob } from "@budibase/types" export const TRIGGER_DEFINITIONS = definitions const JOB_OPTS = { @@ -109,14 +109,16 @@ export async function externalTrigger( } params.fields = coercedFields } - const data: Record = { automation, event: params } + + const data: AutomationData = { automation, event: params as any } if (getResponses) { data.event = { ...data.event, appId: context.getAppId(), automation, } - return utils.processEvent({ data }) + const job = { data } as AutomationJob + return utils.processEvent(job) } else { return automationQueue.add(data, JOB_OPTS) } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index 79cbe6e4ac..d7a7b42095 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -1,4 +1,4 @@ -import { AutomationResults, AutomationStep, Document } from "@budibase/types" +import { AutomationResults, AutomationStep } from "@budibase/types" export enum LoopStepType { ARRAY = "Array", @@ -28,6 +28,3 @@ export interface AutomationContext extends AutomationResults { trigger: any } -export interface AutomationMetadata extends Document { - errorCount?: number -} diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index adc1a83af0..6bfb6afc25 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -13,13 +13,12 @@ import { generateAutomationMetadataID, isProdAppID } from "../db/utils" import { definitions as triggerDefs } from "../automations/triggerInfo" import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants" import { storeLog } from "../automations/logging" -import { Automation, AutomationStep, AutomationStatus } from "@budibase/types" +import { Automation, AutomationStep, AutomationStatus, AutomationMetadata, AutomationJob } from "@budibase/types" import { LoopStep, LoopInput, TriggerOutput, AutomationContext, - AutomationMetadata, } from "../definitions/automations" import { WorkerCallback } from "./definitions" import { context, logging } from "@budibase/backend-core" @@ -60,11 +59,11 @@ class Orchestrator { _job: Job executionOutput: AutomationContext - constructor(job: Job) { - let automation = job.data.automation, - triggerOutput = job.data.event + constructor(job: AutomationJob) { + let automation = job.data.automation + let triggerOutput = job.data.event const metadata = triggerOutput.metadata - this._chainCount = metadata ? metadata.automationChainCount : 0 + this._chainCount = metadata ? metadata.automationChainCount! : 0 this._appId = triggerOutput.appId as string this._job = job const triggerStepId = automation.definition.trigger.stepId diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 2cf5d8066c..21e7ce0b69 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -1,5 +1,3 @@ -import { EnvironmentVariablesDecrypted } from "@budibase/types" - export type WorkerCallback = (error: any, response?: any) => void export interface QueryEvent { diff --git a/packages/server/src/threads/index.ts b/packages/server/src/threads/index.ts index 6876f1a07c..9b6bffa867 100644 --- a/packages/server/src/threads/index.ts +++ b/packages/server/src/threads/index.ts @@ -1,5 +1,7 @@ import workerFarm from "worker-farm" import env from "../environment" +import { AutomationJob } from "@budibase/types" +import { QueryEvent } from "./definitions" export const ThreadType = { QUERY: "query", @@ -64,11 +66,11 @@ export class Thread { ) } - run(data: any) { + run(job: AutomationJob | QueryEvent) { const timeout = this.timeoutMs return new Promise((resolve, reject) => { function fire(worker: any) { - worker.execute(data, (err: any, response: any) => { + worker.execute(job, (err: any, response: any) => { if (err && err.type === "TimeoutError") { reject( new Error(`Query response time exceeded ${timeout}ms timeout.`) diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index 110a2c0642..f726e70b21 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -177,3 +177,8 @@ export type AutomationStepInput = { appId: string apiKey?: string } + +export interface AutomationMetadata extends Document { + errorCount?: number + automationChainCount?: number +} \ No newline at end of file diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts new file mode 100644 index 0000000000..1025220791 --- /dev/null +++ b/packages/types/src/sdk/automations/index.ts @@ -0,0 +1,15 @@ +import { Automation, AutomationMetadata } from "../../documents" +import { Job } from "bull" + +export interface AutomationDataEvent { + appId?: string + metadata?: AutomationMetadata + automation?: Automation +} + +export interface AutomationData { + event: AutomationDataEvent + automation: Automation +} + +export type AutomationJob = Job \ No newline at end of file diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index 8fc5fb287e..ed44c13667 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -1,3 +1,4 @@ +export * from "./automations" export * from "./hosting" export * from "./context" export * from "./events" From ad37186665a0609764891936b6d7f2d59e8ec11f Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 13:54:20 +0100 Subject: [PATCH 04/14] Logging updates for automations --- .../backend-core/src/context/mainContext.ts | 17 ++ packages/backend-core/src/context/types.ts | 1 + .../backend-core/src/logging/pino/logger.ts | 12 ++ packages/backend-core/src/queue/listeners.ts | 157 ++++++++++++------ packages/server/src/automations/utils.ts | 39 +++-- 5 files changed, 160 insertions(+), 66 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 861777b679..9aa887166e 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -104,6 +104,18 @@ async function newContext(updates: ContextMap, task: any) { return Context.run(context, task) } +export async function doInAutomationContext(appId: string, automationId: string, task: any): Promise { + const tenantId = getTenantIDFromAppID(appId) + return newContext( + { + tenantId, + appId, + automationId + }, + task + ) +} + export async function doInContext(appId: string, task: any): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( @@ -187,6 +199,11 @@ export function getTenantId(): string { return tenantId } +export function getAutomationId(): string | undefined { + const context = Context.get() + return context?.automationId +} + export function getAppId(): string | undefined { const context = Context.get() const foundId = context?.appId diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index 727dad80bc..d687a93594 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -7,4 +7,5 @@ export type ContextMap = { identity?: IdentityContext environmentVariables?: Record isScim?: boolean + automationId?: string } diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index 276377eb00..14c8e7e5a9 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -39,6 +39,7 @@ if (!env.DISABLE_PINO_LOGGER) { objects?: any[] tenantId?: string appId?: string + automationId?: string identityId?: string identityType?: IdentityType correlationId?: string @@ -86,6 +87,7 @@ if (!env.DISABLE_PINO_LOGGER) { contextObject = { tenantId: getTenantId(), appId: getAppId(), + automationId: getAutomationId(), identityId: identity?._id, identityType: identity?.type, correlationId: correlation.getId(), @@ -159,6 +161,16 @@ if (!env.DISABLE_PINO_LOGGER) { return appId } + const getAutomationId = () => { + let appId + try { + appId = context.getAutomationId() + } catch (e) { + // do nothing + } + return appId + } + const getIdentity = () => { let identity try { diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 331b690fe9..fd59bec911 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -1,5 +1,6 @@ import { Job, JobId, Queue } from "bull" import { JobQueue } from "./constants" +import * as context from "../context" export type StalledFn = (job: Job) => Promise @@ -31,77 +32,135 @@ function handleStalled(queue: Queue, removeStalledCb?: StalledFn) { }) } -function logging(queue: Queue, jobQueue: JobQueue) { - let eventType: string - switch (jobQueue) { - case JobQueue.AUTOMATION: - eventType = "automation-event" - break - case JobQueue.APP_BACKUP: - eventType = "app-backup-event" - break - case JobQueue.AUDIT_LOG: - eventType = "audit-log-event" - break - case JobQueue.SYSTEM_EVENT_QUEUE: - eventType = "system-event" - break +function getLogParams( + eventType: QueueEventType, + event: BullEvent, + opts: { + job?: Job, + jobId?: JobId, + error?: Error + } = {}, + extra: any = {} +) { + const message = `[BULL] ${eventType}=${event}` + const err = opts.error + + const data = { + eventType, + event, + job: opts.job, + jobId: opts.jobId || opts.job?.id, + ...extra } + + return [message, err, data] +} + +enum BullEvent { + ERROR="error", + WAITING="waiting", + ACTIVE="active", + STALLED="stalled", + PROGRESS="progress", + COMPLETED="completed", + FAILED="failed", + PAUSED="paused", + RESUMED="resumed", + CLEANED="cleaned", + DRAINED="drained", + REMOVED="removed", +} + +enum QueueEventType { + AUTOMATION_EVENT="automation-event", + APP_BACKUP_EVENT="app-backup-event", + AUDIT_LOG_EVENT="audit-log-event", + SYSTEM_EVENT="system-event" +} + +const EventTypeMap: { [key in JobQueue]: QueueEventType } = { + [JobQueue.AUTOMATION]: QueueEventType.AUTOMATION_EVENT, + [JobQueue.APP_BACKUP]: QueueEventType.APP_BACKUP_EVENT, + [JobQueue.AUDIT_LOG]: QueueEventType.AUDIT_LOG_EVENT, + [JobQueue.SYSTEM_EVENT_QUEUE]: QueueEventType.SYSTEM_EVENT, +} + +function logging(queue: Queue, jobQueue: JobQueue) { + const eventType = EventTypeMap[jobQueue] + + function doInJobContext(job: Job, task: any) { + // if this is an automation job try to get the app id + const appId = job.data.event?.appId + if (appId) { + return context.doInContext(appId, task) + } else { + task() + } + } + + queue + .on(BullEvent.STALLED, async (job: Job) => { + // A job has been marked as stalled. This is useful for debugging job + // workers that crash or pause the event loop. + await doInJobContext(job, () => { + console.error(...getLogParams(eventType, BullEvent.STALLED, { job })) + }) + }) + .on(BullEvent.ERROR, (error: any) => { + // An error occurred. + console.error(...getLogParams(eventType, BullEvent.ERROR, { error })) + }) + if (process.env.NODE_DEBUG?.includes("bull")) { queue - .on("error", (error: any) => { - // An error occurred. - console.error(`${eventType}=error error=${JSON.stringify(error)}`) - }) - .on("waiting", (jobId: JobId) => { + .on(BullEvent.WAITING, (jobId: JobId) => { // A Job is waiting to be processed as soon as a worker is idling. - console.log(`${eventType}=waiting jobId=${jobId}`) + console.info(...getLogParams(eventType, BullEvent.WAITING, { jobId })) }) - .on("active", (job: Job, jobPromise: any) => { + .on(BullEvent.ACTIVE, async (job: Job, jobPromise: any) => { // A job has started. You can use `jobPromise.cancel()`` to abort it. - console.log(`${eventType}=active jobId=${job.id}`) + await doInJobContext(job, () => { + console.info(...getLogParams(eventType, BullEvent.ACTIVE, { job })) + }) }) - .on("stalled", (job: Job) => { - // A job has been marked as stalled. This is useful for debugging job - // workers that crash or pause the event loop. - console.error( - `${eventType}=stalled jobId=${job.id} job=${JSON.stringify(job)}` - ) + .on(BullEvent.PROGRESS, async (job: Job, progress: any) => { + // A job's progress was updated + await doInJobContext(job, () => { + console.info(...getLogParams(eventType, BullEvent.PROGRESS, { job }, { progress })) + }) }) - .on("progress", (job: Job, progress: any) => { - // A job's progress was updated! - console.log( - `${eventType}=progress jobId=${job.id} progress=${progress}` - ) - }) - .on("completed", (job: Job, result) => { + .on(BullEvent.COMPLETED, async (job: Job, result) => { // A job successfully completed with a `result`. - console.log(`${eventType}=completed jobId=${job.id} result=${result}`) + await doInJobContext(job, () => { + console.info(...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result })) + }) }) - .on("failed", (job, err: any) => { + .on(BullEvent.FAILED, async (job: Job, error: any) => { // A job failed with reason `err`! - console.log(`${eventType}=failed jobId=${job.id} error=${err}`) + await doInJobContext(job, () => { + console.error(...getLogParams(eventType, BullEvent.FAILED, { job, error })) + }) }) - .on("paused", () => { + .on(BullEvent.PAUSED, () => { // The queue has been paused. - console.log(`${eventType}=paused`) + console.info(...getLogParams(eventType, BullEvent.PAUSED)) }) - .on("resumed", (job: Job) => { + .on(BullEvent.RESUMED, () => { // The queue has been resumed. - console.log(`${eventType}=paused jobId=${job.id}`) + console.info(...getLogParams(eventType, BullEvent.RESUMED)) }) - .on("cleaned", (jobs: Job[], type: string) => { + .on(BullEvent.CLEANED, (jobs: Job[], type: string) => { // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned // jobs, and `type` is the type of jobs cleaned. - console.log(`${eventType}=cleaned length=${jobs.length} type=${type}`) + console.info(...getLogParams(eventType, BullEvent.CLEANED, {}, { length: jobs.length, type } )) }) - .on("drained", () => { + .on(BullEvent.DRAINED, () => { // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) - console.log(`${eventType}=drained`) + console.info(...getLogParams(eventType, BullEvent.DRAINED )) }) - .on("removed", (job: Job) => { + .on(BullEvent.REMOVED, (job: Job) => { // A job successfully removed. - console.log(`${eventType}=removed jobId=${job.id}`) + console.info(...getLogParams(eventType, BullEvent.REMOVED, { job } )) }) } } diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 5296a0fa50..50c9b77818 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -8,7 +8,7 @@ import { db as dbCore, context } from "@budibase/backend-core" import { getAutomationMetadataParams } from "../db/utils" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" -import { Automation, WebhookActionType } from "@budibase/types" +import { Automation, AutomationJob, WebhookActionType } from "@budibase/types" import sdk from "../sdk" const REBOOT_CRON = "@reboot" @@ -16,27 +16,32 @@ const WH_STEP_ID = definitions.WEBHOOK.stepId const CRON_STEP_ID = definitions.CRON.stepId const Runner = new Thread(ThreadType.AUTOMATION) -const jobMessage = (job: any, message: string) => { - return `app=${job.data.event.appId} automation=${job.data.automation._id} jobId=${job.id} trigger=${job.data.automation.definition.trigger.event} : ${message}` +function loggingArgs(job: AutomationJob) { + return { + jobId: job.id, + trigger: job.data.automation.definition.trigger.event + } } -export async function processEvent(job: any) { - try { - const automationId = job.data.automation._id - console.log(jobMessage(job, "running")) - // need to actually await these so that an error can be captured properly - return await context.doInContext(job.data.event.appId, async () => { +export async function processEvent(job: AutomationJob) { + const appId = job.data.event.appId! + const automationId = job.data.automation._id! + return await context.doInAutomationContext(appId, automationId, async () => { + try { + // need to actually await these so that an error can be captured properly + console.log("automation running", loggingArgs(job)) + const runFn = () => Runner.run(job) - return quotas.addAutomation(runFn, { + const result = await quotas.addAutomation(runFn, { automationId, }) - }) - } catch (err) { - const errJson = JSON.stringify(err) - console.error(jobMessage(job, `was unable to run - ${errJson}`)) - console.trace(err) - return { err } - } + console.log("automation completed", loggingArgs(job)) + return result + } catch (err) { + console.error(`automation was unable to run`, err, loggingArgs(job)) + return { err } + } + }) } export async function updateTestHistory( From 578b5994288661ec57aa98d1b903cf5b70078382 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 14:10:22 +0100 Subject: [PATCH 05/14] Lint --- .../backend-core/src/context/mainContext.ts | 8 +- packages/backend-core/src/queue/listeners.ts | 68 +++++---- .../console-control-strings/README.md~ | 140 ++++++++++++++++++ .../node_modules/copy-concurrently/README.md~ | 127 ++++++++++++++++ .../node_modules/move-concurrently/README.md~ | 52 +++++++ packages/pro/node_modules/umask/index.js~ | 68 +++++++++ .../pro/node_modules/umask/test/ChangeLog~ | 5 + .../pro/node_modules/umask/test/simple.js~ | 14 ++ packages/server/src/automations/utils.ts | 2 +- .../server/src/definitions/automations.ts | 1 - packages/server/src/threads/automation.ts | 8 +- .../types/src/documents/app/automation.ts | 2 +- packages/types/src/sdk/automations/index.ts | 2 +- 13 files changed, 465 insertions(+), 32 deletions(-) create mode 100644 packages/pro/node_modules/console-control-strings/README.md~ create mode 100644 packages/pro/node_modules/copy-concurrently/README.md~ create mode 100644 packages/pro/node_modules/move-concurrently/README.md~ create mode 100644 packages/pro/node_modules/umask/index.js~ create mode 100644 packages/pro/node_modules/umask/test/ChangeLog~ create mode 100644 packages/pro/node_modules/umask/test/simple.js~ diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 9aa887166e..492e0abb10 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -104,13 +104,17 @@ async function newContext(updates: ContextMap, task: any) { return Context.run(context, task) } -export async function doInAutomationContext(appId: string, automationId: string, task: any): Promise { +export async function doInAutomationContext( + appId: string, + automationId: string, + task: any +): Promise { const tenantId = getTenantIDFromAppID(appId) return newContext( { tenantId, appId, - automationId + automationId, }, task ) diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index fd59bec911..6732bfaaa9 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -36,8 +36,8 @@ function getLogParams( eventType: QueueEventType, event: BullEvent, opts: { - job?: Job, - jobId?: JobId, + job?: Job + jobId?: JobId error?: Error } = {}, extra: any = {} @@ -50,32 +50,32 @@ function getLogParams( event, job: opts.job, jobId: opts.jobId || opts.job?.id, - ...extra + ...extra, } return [message, err, data] } enum BullEvent { - ERROR="error", - WAITING="waiting", - ACTIVE="active", - STALLED="stalled", - PROGRESS="progress", - COMPLETED="completed", - FAILED="failed", - PAUSED="paused", - RESUMED="resumed", - CLEANED="cleaned", - DRAINED="drained", - REMOVED="removed", + ERROR = "error", + WAITING = "waiting", + ACTIVE = "active", + STALLED = "stalled", + PROGRESS = "progress", + COMPLETED = "completed", + FAILED = "failed", + PAUSED = "paused", + RESUMED = "resumed", + CLEANED = "cleaned", + DRAINED = "drained", + REMOVED = "removed", } enum QueueEventType { - AUTOMATION_EVENT="automation-event", - APP_BACKUP_EVENT="app-backup-event", - AUDIT_LOG_EVENT="audit-log-event", - SYSTEM_EVENT="system-event" + AUTOMATION_EVENT = "automation-event", + APP_BACKUP_EVENT = "app-backup-event", + AUDIT_LOG_EVENT = "audit-log-event", + SYSTEM_EVENT = "system-event", } const EventTypeMap: { [key in JobQueue]: QueueEventType } = { @@ -126,19 +126,30 @@ function logging(queue: Queue, jobQueue: JobQueue) { .on(BullEvent.PROGRESS, async (job: Job, progress: any) => { // A job's progress was updated await doInJobContext(job, () => { - console.info(...getLogParams(eventType, BullEvent.PROGRESS, { job }, { progress })) + console.info( + ...getLogParams( + eventType, + BullEvent.PROGRESS, + { job }, + { progress } + ) + ) }) }) .on(BullEvent.COMPLETED, async (job: Job, result) => { // A job successfully completed with a `result`. await doInJobContext(job, () => { - console.info(...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result })) + console.info( + ...getLogParams(eventType, BullEvent.COMPLETED, { job }, { result }) + ) }) }) .on(BullEvent.FAILED, async (job: Job, error: any) => { // A job failed with reason `err`! await doInJobContext(job, () => { - console.error(...getLogParams(eventType, BullEvent.FAILED, { job, error })) + console.error( + ...getLogParams(eventType, BullEvent.FAILED, { job, error }) + ) }) }) .on(BullEvent.PAUSED, () => { @@ -152,15 +163,22 @@ function logging(queue: Queue, jobQueue: JobQueue) { .on(BullEvent.CLEANED, (jobs: Job[], type: string) => { // Old jobs have been cleaned from the queue. `jobs` is an array of cleaned // jobs, and `type` is the type of jobs cleaned. - console.info(...getLogParams(eventType, BullEvent.CLEANED, {}, { length: jobs.length, type } )) + console.info( + ...getLogParams( + eventType, + BullEvent.CLEANED, + {}, + { length: jobs.length, type } + ) + ) }) .on(BullEvent.DRAINED, () => { // Emitted every time the queue has processed all the waiting jobs (even if there can be some delayed jobs not yet processed) - console.info(...getLogParams(eventType, BullEvent.DRAINED )) + console.info(...getLogParams(eventType, BullEvent.DRAINED)) }) .on(BullEvent.REMOVED, (job: Job) => { // A job successfully removed. - console.info(...getLogParams(eventType, BullEvent.REMOVED, { job } )) + console.info(...getLogParams(eventType, BullEvent.REMOVED, { job })) }) } } diff --git a/packages/pro/node_modules/console-control-strings/README.md~ b/packages/pro/node_modules/console-control-strings/README.md~ new file mode 100644 index 0000000000..6eb34e89d1 --- /dev/null +++ b/packages/pro/node_modules/console-control-strings/README.md~ @@ -0,0 +1,140 @@ +# Console Control Strings + +A library of cross-platform tested terminal/console command strings for +doing things like color and cursor positioning. This is a subset of both +ansi and vt100. All control codes included work on both Windows & Unix-like +OSes, except where noted. + +## Usage + +```js +var consoleControl = require('console-control-strings') + +console.log(consoleControl.color('blue','bgRed', 'bold') + 'hi there' + consoleControl.color('reset')) +process.stdout.write(consoleControl.goto(75, 10)) +``` + +## Why Another? + +There are tons of libraries similar to this one. I wanted one that was: + +1. Very clear about compatibility goals. +2. Could emit, for instance, a start color code without an end one. +3. Returned strings w/o writing to streams. +4. Was not weighed down with other unrelated baggage. + +## Functions + +### var code = consoleControl.up(_num = 1_) + +Returns the escape sequence to move _num_ lines up. + +### var code = consoleControl.down(_num = 1_) + +Returns the escape sequence to move _num_ lines down. + +### var code = consoleControl.forward(_num = 1_) + +Returns the escape sequence to move _num_ lines righ. + +### var code = consoleControl.back(_num = 1_) + +Returns the escape sequence to move _num_ lines left. + +### var code = consoleControl.nextLine(_num = 1_) + +Returns the escape sequence to move _num_ lines down and to the beginning of +the line. + +### var code = consoleControl.previousLine(_num = 1_) + +Returns the escape sequence to move _num_ lines up and to the beginning of +the line. + +### var code = consoleControl.eraseData() + +Returns the escape sequence to erase everything from the current cursor +position to the bottom right of the screen. This is line based, so it +erases the remainder of the current line and all following lines. + +### var code = consoleControl.eraseLine() + +Returns the escape sequence to erase to the end of the current line. + +### var code = consoleControl.goto(_x_, _y_) + +Returns the escape sequence to move the cursor to the designated position. +Note that the origin is _1, 1_ not _0, 0_. + +### var code = consoleControl.gotoSOL() + +Returns the escape sequence to move the cursor to the beginning of the +current line. (That is, it returns a carriage return, `\r`.) + +### var code = consoleControl.hideCursor() + +Returns the escape sequence to hide the cursor. + +### var code = consoleControl.showCursor() + +Returns the escape sequence to show the cursor. + +### var code = consoleControl.color(_colors = []_) + +### var code = consoleControl.color(_color1_, _color2_, _…_, _colorn_) + +Returns the escape sequence to set the current terminal display attributes +(mostly colors). Arguments can either be a list of attributes or an array +of attributes. The difference between passing in an array or list of colors +and calling `.color` separately for each one, is that in the former case a +single escape sequence will be produced where as in the latter each change +will have its own distinct escape sequence. Each attribute can be one of: + +* Reset: + * **reset** – Reset all attributes to the terminal default. +* Styles: + * **bold** – Display text as bold. In some terminals this means using a + bold font, in others this means changing the color. In some it means + both. + * **italic** – Display text as italic. This is not available in most Windows terminals. + * **underline** – Underline text. This is not available in most Windows Terminals. + * **inverse** – Invert the foreground and background colors. + * **stopBold** – Do not display text as bold. + * **stopItalic** – Do not display text as italic. + * **stopUnderline** – Do not underline text. + * **stopInverse** – Do not invert foreground and background. +* Colors: + * **white** + * **black** + * **blue** + * **cyan** + * **green** + * **magenta** + * **red** + * **yellow** + * **grey** / **brightBlack** + * **brightRed** + * **brightGreen** + * **brightYellow** + * **brightBlue** + * **brightMagenta** + * **brightCyan** + * **brightWhite** +* Background Colors: + * **bgWhite** + * **bgBlack** + * **bgBlue** + * **bgCyan** + * **bgGreen** + * **bgMagenta** + * **bgRed** + * **bgYellow** + * **bgGrey** / **bgBrightBlack** + * **bgBrightRed** + * **bgBrightGreen** + * **bgBrightYellow** + * **bgBrightBlue** + * **bgBrightMagenta** + * **bgBrightCyan** + * **bgBrightWhite** + diff --git a/packages/pro/node_modules/copy-concurrently/README.md~ b/packages/pro/node_modules/copy-concurrently/README.md~ new file mode 100644 index 0000000000..7f93b2ad26 --- /dev/null +++ b/packages/pro/node_modules/copy-concurrently/README.md~ @@ -0,0 +1,127 @@ +# copy-concurrently + +Copy files, directories and symlinks + +``` +const copy = require('copy-concurrently') +copy('/path/to/thing', '/new/path/thing').then(() => { + // this is now copied +}).catch(err => { + // oh noooo +}) +``` + +Copies files, directories and symlinks. Ownership is maintained when +running as root, permissions are always maintained. On Windows, if symlinks +are unavailable then junctions will be used. + +## PUBLIC INTERFACE + +### copy(from, to, [options]) → Promise + +Recursively copies `from` to `to` and resolves its promise when finished. +If `to` already exists then the promise will be rejected with an `EEXIST` +error. + +Options are: + +* maxConcurrency – (Default: `1`) The maximum number of concurrent copies to do at once. +* recurseWith - (Default: `copy.item`) The function to call on each file after recursing into a directory. +* isWindows - (Default: `process.platform === 'win32'`) If true enables Windows symlink semantics. This requires + an extra `stat` to determine if the destination of a symlink is a file or directory. If symlinking a directory + fails then we'll try making a junction instead. + +Options can also include dependency injection: + +* Promise - (Default: `global.Promise`) The promise implementation to use, defaults to Node's. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* writeStreamAtomic - (Default: `require('fs-write-stream-atomic')`) The + implementation of `writeStreamAtomic` to use. Used to inject a mock. +* getuid - (Default: `process.getuid`) A function that returns the current UID. Used to inject a mock. + +## EXTENSION INTERFACE + +Ordinarily you'd only call `copy` above. But it's possible to use it's +component functions directly. This is useful if, say, you're writing +[move-concurently](https://npmjs.com/package/move-concurrently). + +### copy.file(from, to, options) → Promise + +Copies a ordinary file `from` to destination `to`. Uses +`fs-write-stream-atomic` to ensure that the file is entirely copied or not +at all. + +Options are: + +* uid, gid - (Optional) If `getuid()` is `0` then this and gid will be used to + set the user and group of `to`. If uid is present then gid must be too. +* mode - (Optional) If set then `to` will have its perms set to `mode`. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* Promise - (Default: `global.Promise`) The promise implementation to use, defaults to Node's. +* writeStreamAtomic - (Default `require('fs-write-stream-atomic')`) The + implementation of `writeStreamAtomic` to use. Used to inject a mock. + +### copy.symlink(from, to, options) → Promise + +Copies a symlink `from` to destination `to`. If on Windows then if +symlinking fails, a junction will be used instead. + +Options are: + +* top - The top level the copy is being run from. This is used to determine + if the symlink destination is within the set of files we're copying or + outside it. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* Promise - (Default: `global.Promise`) The promise implementation to use, defaults to Node's. +* isWindows - (Default: `process.platform === 'win32'`) If true enables Windows symlink semantics. This requires + an extra `stat` to determine if the destination of a symlink is a file or directory. If symlinking a directory + fails then we'll try making a junction instead. + +### copy.recurse(from, to, options) → Promise + +Reads all of the files in directory `from` and adds them to the `queue` +using `recurseWith` (by default `copy.item`). + +Options are: + +* queue - A [`run-queue`](https://npmjs.com/package/run-queue) object to add files found inside `from` to. +* recurseWith - (Default: `copy.item`) The function to call on each file after recursing into a directory. +* uid, gid - (Optional) If `getuid()` is `0` then this and gid will be used to + set the user and group of `to`. If uid is present then gid must be too. +* mode - (Optional) If set then `to` will have its perms set to `mode`. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* getuid - (Default: `process.getuid`) A function that returns the current UID. Used to inject a mock. + +### copy.item(from, to, options) → Promise + +Copies some kind of `from` to destination `to`. This looks at the filetype +and calls `copy.file`, `copy.symlink` or `copy.recurse` as appropriate. + +Symlink copies are queued with a priority such that they happen after all +file and directory copies as you can't create a junction on windows to a +file that doesn't exist yet. + +Options are: + +* top - The top level the copy is being run from. This is used to determine + if the symlink destination is within the set of files we're copying or + outside it. +* queue - The [`run-queue`](https://npmjs.com/package/run-queue) object to + pass to `copy.recurse` if `from` is a directory. +* recurseWith - (Default: `copy.item`) The function to call on each file after recursing into a directory. +* uid, gid - (Optional) If `getuid()` is `0` then this and gid will be used to + set the user and group of `to`. If uid is present then gid must be too. +* mode - (Optional) If set then `to` will have its perms set to `mode`. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* getuid - (Default: `process.getuid`) A function that returns the current UID. Used to inject a mock. +* isWindows - (Default: `process.platform === 'win32'`) If true enables Windows symlink semantics. This requires + an extra `stat` to determine if the destination of a symlink is a file or directory. If symlinking a directory + fails then we'll try making a junction instead. +* Promise - (Default: `global.Promise`) The promise implementation to use, defaults to Node's. +* writeStreamAtomic - (Default `require('fs-write-stream-atomic')`) The + implementation of `writeStreamAtomic` to use. Used to inject a mock. diff --git a/packages/pro/node_modules/move-concurrently/README.md~ b/packages/pro/node_modules/move-concurrently/README.md~ new file mode 100644 index 0000000000..7d04d45955 --- /dev/null +++ b/packages/pro/node_modules/move-concurrently/README.md~ @@ -0,0 +1,52 @@ +# move-concurrently + +Move files and directories. + +``` +const move = require('move-concurrently') +move('/path/to/thing', '/new/path/thing'), err => { + if (err) throw err + // thing is now moved! +}) +``` + +Uses `rename` to move things as fast as possible. + +If you `move` across devices or on filesystems that don't support renaming +large directories. That is, situations that result in `rename` returning +the `EXDEV` error, then `move` will fallback to copy + delete. + +When recursively copying directories it will first try to rename the +contents before falling back to copying. While this will be slightly slower +in true cross-device scenarios, it is MUCH faster in cases where the +filesystem can't handle directory renames. + +When copying ownership is maintained when running as root. Permissions are +always maintained. On Windows, if symlinks are unavailable then junctions +will be used. + +## INTERFACE + +### move(from, to, options) → Promise + +Recursively moves `from` to `to` and resolves its promise when finished. +If `to` already exists then the promise will be rejected with an `EEXIST` +error. + +Starts by trying to rename `from` to `to`. + +Options are: + +* maxConcurrency – (Default: `1`) The maximum number of concurrent copies to do at once. +* isWindows - (Default: `process.platform === 'win32'`) If true enables Windows symlink semantics. This requires + an extra `stat` to determine if the destination of a symlink is a file or directory. If symlinking a directory + fails then we'll try making a junction instead. + +Options can also include dependency injection: + +* Promise - (Default: `global.Promise`) The promise implementation to use, defaults to Node's. +* fs - (Default: `require('fs')`) The filesystem module to use. Can be used + to use `graceful-fs` or to inject a mock. +* writeStreamAtomic - (Default: `require('fs-write-stream-atomic')`) The + implementation of `writeStreamAtomic` to use. Used to inject a mock. +* getuid - (Default: `process.getuid`) A function that returns the current UID. Used to inject a mock. diff --git a/packages/pro/node_modules/umask/index.js~ b/packages/pro/node_modules/umask/index.js~ new file mode 100644 index 0000000000..87738e4a10 --- /dev/null +++ b/packages/pro/node_modules/umask/index.js~ @@ -0,0 +1,68 @@ +// commands for handling umask + +var util = require("util") +var log = require("npmlog") + +var defaultUmask = 0022 +var defaultUmaskString = toString(defaultUmask) + +exports.toString = toString +exports.fromString = fromString +exports.validate = validate + +function toString(val) { + val = val.toString(8) + while (val.length < 4) val = "0" + val + return val +} + +function validate (data, k, val) { + // must be either an integer or an octal string. + if (typeof val === "number") { + data[k] = val + return true + } + + if (typeof val === "string") { + if (val.charAt(0) !== "0" || isNaN(val)) return false + data[k] = parseInt(val, 8) + return true + } + + return false +} + +function fromString(val, cb) { + + // synchronous callback, no zalgo + _fromString(val, cb || function (err, result) { + + if (err) log.warn("invalid umask", err.message) + val = result + }) + + return val +} + +function _fromString(val, cb) { + if(typeof val === "string") { + if (!/^[0-7]+$/.test(val)) { + return cb(new Error(util.format("Expected octal string, got %j, defaulting to %j", + val, defaultUmaskString)), + defaultUmask) + } + + val = parseInt(val, 8) + } else if(typeof val !== "number") { + return cb(new Error(util.format("Expected number or octal string, got %j, defaulting to %j", + val, defaultUmaskString)), + defaultUmask) + } + + if ((val < 0) || (val > 511)) { + return cb(new Error(util.format("Must be in range 0..511 (0000..0777), got %j", val)), + defaultUmask) + } + + cb(null, val) +} \ No newline at end of file diff --git a/packages/pro/node_modules/umask/test/ChangeLog~ b/packages/pro/node_modules/umask/test/ChangeLog~ new file mode 100644 index 0000000000..72ade111ec --- /dev/null +++ b/packages/pro/node_modules/umask/test/ChangeLog~ @@ -0,0 +1,5 @@ +2015-01-15 Sam Mikes + + * index.js: (convert_fromString) accept decimal strings provided they + don't begin with '0' + diff --git a/packages/pro/node_modules/umask/test/simple.js~ b/packages/pro/node_modules/umask/test/simple.js~ new file mode 100644 index 0000000000..a2184c3cd4 --- /dev/null +++ b/packages/pro/node_modules/umask/test/simple.js~ @@ -0,0 +1,14 @@ +'use strict'; + +var Purefts = require('..'); + +var Code = require('code'); +var Lab = require('lab'); +var lab = Lab.script(); +exports.lab = lab; + +var describe = lab.describe; +var it = lab.it; +var expect = Code.expect; + +describe('create objects', function () { diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 50c9b77818..50a3af2cbb 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -19,7 +19,7 @@ const Runner = new Thread(ThreadType.AUTOMATION) function loggingArgs(job: AutomationJob) { return { jobId: job.id, - trigger: job.data.automation.definition.trigger.event + trigger: job.data.automation.definition.trigger.event, } } diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index d7a7b42095..7e86608bf3 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -27,4 +27,3 @@ export interface AutomationContext extends AutomationResults { env?: Record trigger: any } - diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 6bfb6afc25..e271ebba0b 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -13,7 +13,13 @@ import { generateAutomationMetadataID, isProdAppID } from "../db/utils" import { definitions as triggerDefs } from "../automations/triggerInfo" import { AutomationErrors, MAX_AUTOMATION_RECURRING_ERRORS } from "../constants" import { storeLog } from "../automations/logging" -import { Automation, AutomationStep, AutomationStatus, AutomationMetadata, AutomationJob } from "@budibase/types" +import { + Automation, + AutomationStep, + AutomationStatus, + AutomationMetadata, + AutomationJob, +} from "@budibase/types" import { LoopStep, LoopInput, diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index f726e70b21..07d4654433 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -181,4 +181,4 @@ export type AutomationStepInput = { export interface AutomationMetadata extends Document { errorCount?: number automationChainCount?: number -} \ No newline at end of file +} diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts index 1025220791..f1e3d38101 100644 --- a/packages/types/src/sdk/automations/index.ts +++ b/packages/types/src/sdk/automations/index.ts @@ -12,4 +12,4 @@ export interface AutomationData { automation: Automation } -export type AutomationJob = Job \ No newline at end of file +export type AutomationJob = Job From 6c412c7cd914f1a77bc32616dfe3478cd1b278d1 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 14:22:23 +0100 Subject: [PATCH 06/14] Test fix --- packages/backend-core/src/queue/inMemoryQueue.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index b80aece418..ec1d9d4a90 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -128,6 +128,7 @@ class InMemoryQueue { on() { // do nothing + return this } async waitForCompletion() { From f41b134d315b992a1118d1445ed0304fe4dfee8e Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 14:57:29 +0100 Subject: [PATCH 07/14] Refactor to use param for doInAutomationContext --- packages/backend-core/src/context/mainContext.ts | 12 ++++++------ packages/server/src/automations/utils.ts | 6 ++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 492e0abb10..41e6579384 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -104,19 +104,19 @@ async function newContext(updates: ContextMap, task: any) { return Context.run(context, task) } -export async function doInAutomationContext( +export async function doInAutomationContext(params: { appId: string, automationId: string, task: any -): Promise { - const tenantId = getTenantIDFromAppID(appId) +}): Promise { + const tenantId = getTenantIDFromAppID(params.appId) return newContext( { tenantId, - appId, - automationId, + appId: params.appId, + automationId: params.automationId, }, - task + params.task ) } diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index 50a3af2cbb..d9f19013ee 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -26,7 +26,7 @@ function loggingArgs(job: AutomationJob) { export async function processEvent(job: AutomationJob) { const appId = job.data.event.appId! const automationId = job.data.automation._id! - return await context.doInAutomationContext(appId, automationId, async () => { + const task = async () => { try { // need to actually await these so that an error can be captured properly console.log("automation running", loggingArgs(job)) @@ -41,7 +41,9 @@ export async function processEvent(job: AutomationJob) { console.error(`automation was unable to run`, err, loggingArgs(job)) return { err } } - }) + } + + return await context.doInAutomationContext({ appId, automationId, task }) } export async function updateTestHistory( From 78daed6ec1e956cdb042b423fe884be52615c61b Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 15:31:14 +0100 Subject: [PATCH 08/14] Lint --- packages/backend-core/src/context/mainContext.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 41e6579384..61d96bb4b0 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -105,8 +105,8 @@ async function newContext(updates: ContextMap, task: any) { } export async function doInAutomationContext(params: { - appId: string, - automationId: string, + appId: string + automationId: string task: any }): Promise { const tenantId = getTenantIDFromAppID(params.appId) From e4835d5f917a543cf559e5a6bc0312085c0ebefb Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 17 May 2023 14:39:18 +0000 Subject: [PATCH 09/14] v2.6.17 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 6 +++--- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 12 ++++++------ packages/frontend-core/package.json | 6 +++--- packages/sdk/package.json | 2 +- packages/server/package.json | 12 ++++++------ packages/shared-core/package.json | 4 ++-- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lerna.json b/lerna.json index 643023f2b1..3efa044add 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.16", + "version": "2.6.17", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 94b9e9e745..9cbb2f0cc9 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.6.16", + "@budibase/types": "^2.6.17", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index a27fe96ebf..6a01ebe616 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.6.16", + "version": "2.6.17", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.6.16", - "@budibase/string-templates": "^2.6.16", + "@budibase/shared-core": "^2.6.17", + "@budibase/string-templates": "^2.6.17", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index a0a14498f6..315ece71c8 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.6.16", + "version": "2.6.17", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.6.16", - "@budibase/frontend-core": "^2.6.16", - "@budibase/shared-core": "^2.6.16", - "@budibase/string-templates": "^2.6.16", + "@budibase/bbui": "^2.6.17", + "@budibase/frontend-core": "^2.6.17", + "@budibase/shared-core": "^2.6.17", + "@budibase/string-templates": "^2.6.17", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index ed843dc18e..720aa6af7b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,9 +29,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.6.16", - "@budibase/string-templates": "^2.6.16", - "@budibase/types": "^2.6.16", + "@budibase/backend-core": "^2.6.17", + "@budibase/string-templates": "^2.6.17", + "@budibase/types": "^2.6.17", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index 0871d05302..a85cc1d613 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.6.16", + "version": "2.6.17", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.6.16", - "@budibase/frontend-core": "^2.6.16", - "@budibase/shared-core": "^2.6.16", - "@budibase/string-templates": "^2.6.16", - "@budibase/types": "^2.6.16", + "@budibase/bbui": "^2.6.17", + "@budibase/frontend-core": "^2.6.17", + "@budibase/shared-core": "^2.6.17", + "@budibase/string-templates": "^2.6.17", + "@budibase/types": "^2.6.17", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index b2053a8fa0..569d4fcf8c 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.6.16", - "@budibase/shared-core": "^2.6.16", + "@budibase/bbui": "^2.6.17", + "@budibase/shared-core": "^2.6.17", "dayjs": "^1.11.7", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 41a02e71a9..a51e825987 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 9f6c33f0e2..80a7159650 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.6.16", - "@budibase/client": "^2.6.16", + "@budibase/backend-core": "^2.6.17", + "@budibase/client": "^2.6.17", "@budibase/pro": "2.6.16", - "@budibase/shared-core": "^2.6.16", - "@budibase/string-templates": "^2.6.16", - "@budibase/types": "^2.6.16", + "@budibase/shared-core": "^2.6.17", + "@budibase/string-templates": "^2.6.17", + "@budibase/types": "^2.6.17", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index b69e8105e6..e82cc467bf 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.6.16", + "version": "2.6.17", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "^2.6.16" + "@budibase/types": "^2.6.17" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 79ffb2b0e4..419417a8c8 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.6.16", + "version": "2.6.17", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index db3bc658d4..570c3e764e 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index a2a4ebafaf..2709951668 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.6.16", + "version": "2.6.17", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.6.16", + "@budibase/backend-core": "^2.6.17", "@budibase/pro": "2.6.16", - "@budibase/string-templates": "^2.6.16", - "@budibase/types": "^2.6.16", + "@budibase/string-templates": "^2.6.17", + "@budibase/types": "^2.6.17", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From d84284db2884f4636871f6425786dcd04258ed86 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 17 May 2023 14:40:47 +0000 Subject: [PATCH 10/14] Update pro version to 2.6.17 --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- yarn.lock | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 80a7159650..3a755ab84c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.6.17", "@budibase/client": "^2.6.17", - "@budibase/pro": "2.6.16", + "@budibase/pro": "2.6.17", "@budibase/shared-core": "^2.6.17", "@budibase/string-templates": "^2.6.17", "@budibase/types": "^2.6.17", diff --git a/packages/worker/package.json b/packages/worker/package.json index 2709951668..b3bbdbd2a3 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.6.17", - "@budibase/pro": "2.6.16", + "@budibase/pro": "2.6.17", "@budibase/string-templates": "^2.6.17", "@budibase/types": "^2.6.17", "@koa/router": "8.0.8", diff --git a/yarn.lock b/yarn.lock index 6d606cb461..52fbdb4712 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.6.15": - version "2.6.15" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.15.tgz#527a0e6fe85d0413382433a23b0342d77670db1d" - integrity sha512-oXssIt4VvFEde8ftRnk7XPYMOXI/E7YN8GseSyhAxzozPYdGatd1AyqnBAJFgrYtAwQCLQOZBJFWRWhCpCCPhA== +"@budibase/pro@2.6.16": + version "2.6.16" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.16.tgz#c861c68787e8b85a28dcfe9bfe7f96225e493069" + integrity sha512-mmxfZe5eiqQMYlmVT9+t2NiWd9Cpy2Z0VBE6nPgv98lHNZTcA1S6Msb+PV/24FEzSFnE0IHQlEsgYFJi5lLXlg== dependencies: - "@budibase/backend-core" "2.6.15" + "@budibase/backend-core" "2.6.16" "@budibase/shared-core" "2.5.9" "@budibase/string-templates" "2.5.9" - "@budibase/types" "2.6.15" + "@budibase/types" "2.6.16" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" From 26201a9d76e7b98d74d5a5beabd60d603030e1f8 Mon Sep 17 00:00:00 2001 From: andz-bb Date: Wed, 17 May 2023 16:14:50 +0100 Subject: [PATCH 11/14] hotfix for time-only date picker problems (reverting flatpickr to previous version) --- packages/bbui/package.json | 2 +- yarn.lock | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 6a01ebe616..eddcd2381d 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -84,7 +84,7 @@ "@spectrum-css/vars": "3.0.1", "dayjs": "^1.10.4", "easymde": "^2.16.1", - "svelte-flatpickr": "^3.3.2", + "svelte-flatpickr": "3.2.3", "svelte-portal": "^1.0.0" }, "resolutions": { diff --git a/yarn.lock b/yarn.lock index 52fbdb4712..4eaf610f15 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1486,15 +1486,15 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@2.6.16": - version "2.6.16" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.16.tgz#c861c68787e8b85a28dcfe9bfe7f96225e493069" - integrity sha512-mmxfZe5eiqQMYlmVT9+t2NiWd9Cpy2Z0VBE6nPgv98lHNZTcA1S6Msb+PV/24FEzSFnE0IHQlEsgYFJi5lLXlg== +"@budibase/pro@2.6.17": + version "2.6.17" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.6.17.tgz#1f238370d2aa92a650e35798a3b00c5c0aadf1ad" + integrity sha512-WWvgicSlqkYpx0cNbAhYi/W/CjIQyjzS7We7aXncwHtWWJIeYfItmxxZsX7aPOEYygG7650wCA/+j57oKfzYmA== dependencies: - "@budibase/backend-core" "2.6.16" + "@budibase/backend-core" "2.6.17" "@budibase/shared-core" "2.5.9" "@budibase/string-templates" "2.5.9" - "@budibase/types" "2.6.16" + "@budibase/types" "2.6.17" "@koa/router" "8.0.8" bull "4.10.1" joi "17.6.0" @@ -22952,7 +22952,14 @@ svelte-dnd-action@^0.9.8: resolved "https://registry.yarnpkg.com/svelte-dnd-action/-/svelte-dnd-action-0.9.22.tgz#003eee9dddb31d8c782f6832aec8b1507fff194d" integrity sha512-lOQJsNLM1QWv5mdxIkCVtk6k4lHCtLgfE59y8rs7iOM6erchbLC9hMEFYSveZz7biJV0mpg7yDSs4bj/RT/YkA== -svelte-flatpickr@^3.1.0, svelte-flatpickr@^3.2.3, svelte-flatpickr@^3.3.2: +svelte-flatpickr@3.2.3: + version "3.2.3" + resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.2.3.tgz#db5dd7ad832ef83262b45e09737955ad3d591fc8" + integrity sha512-PNkqK4Napx8nTvCwkaUXdnKo8dISThaxEOK+szTUXcY6H0dQM0TSyuoMaVWY2yX7pM+PN5cpCQCcVe8YvTRFSw== + dependencies: + flatpickr "^4.5.2" + +svelte-flatpickr@^3.1.0, svelte-flatpickr@^3.2.3: version "3.3.2" resolved "https://registry.yarnpkg.com/svelte-flatpickr/-/svelte-flatpickr-3.3.2.tgz#f08bcde83d439cb30df6fd07b974d87371f130c1" integrity sha512-VNJLYyLRDplI63oWX5hJylzAJc2VhTh3z9SNecfjtuPZmP6FZPpg9Fw7rXpkEV2DPovIWj2PtaVxB6Kp9r423w== From 30dd224a34da17bb4219b57418e3a696419dc7cd Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 17 May 2023 16:13:35 +0000 Subject: [PATCH 12/14] v2.6.18 --- lerna.json | 2 +- packages/backend-core/package.json | 4 ++-- packages/bbui/package.json | 6 +++--- packages/builder/package.json | 10 +++++----- packages/cli/package.json | 8 ++++---- packages/client/package.json | 12 ++++++------ packages/frontend-core/package.json | 6 +++--- packages/sdk/package.json | 2 +- packages/server/package.json | 12 ++++++------ packages/shared-core/package.json | 4 ++-- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 8 ++++---- 13 files changed, 39 insertions(+), 39 deletions(-) diff --git a/lerna.json b/lerna.json index 3efa044add..81677cb994 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.6.17", + "version": "2.6.18", "npmClient": "yarn", "useWorkspaces": true, "packages": ["packages/*"], diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 9cbb2f0cc9..c48e11efc1 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -24,7 +24,7 @@ "dependencies": { "@budibase/nano": "10.1.2", "@budibase/pouchdb-replication-stream": "1.2.10", - "@budibase/types": "^2.6.17", + "@budibase/types": "^2.6.18", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index eddcd2381d..3ab8642bc7 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.6.17", + "version": "2.6.18", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,8 +38,8 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/shared-core": "^2.6.17", - "@budibase/string-templates": "^2.6.17", + "@budibase/shared-core": "^2.6.18", + "@budibase/string-templates": "^2.6.18", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/builder/package.json b/packages/builder/package.json index 315ece71c8..07712b6f3b 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.6.17", + "version": "2.6.18", "license": "GPL-3.0", "private": true, "scripts": { @@ -58,10 +58,10 @@ } }, "dependencies": { - "@budibase/bbui": "^2.6.17", - "@budibase/frontend-core": "^2.6.17", - "@budibase/shared-core": "^2.6.17", - "@budibase/string-templates": "^2.6.17", + "@budibase/bbui": "^2.6.18", + "@budibase/frontend-core": "^2.6.18", + "@budibase/shared-core": "^2.6.18", + "@budibase/string-templates": "^2.6.18", "@fortawesome/fontawesome-svg-core": "^6.2.1", "@fortawesome/free-brands-svg-icons": "^6.2.1", "@fortawesome/free-solid-svg-icons": "^6.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 720aa6af7b..38c51d80b9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "dist/index.js", "bin": { @@ -29,9 +29,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "^2.6.17", - "@budibase/string-templates": "^2.6.17", - "@budibase/types": "^2.6.17", + "@budibase/backend-core": "^2.6.18", + "@budibase/string-templates": "^2.6.18", + "@budibase/types": "^2.6.18", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index a85cc1d613..affca2cf28 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.6.17", + "version": "2.6.18", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,11 +19,11 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^2.6.17", - "@budibase/frontend-core": "^2.6.17", - "@budibase/shared-core": "^2.6.17", - "@budibase/string-templates": "^2.6.17", - "@budibase/types": "^2.6.17", + "@budibase/bbui": "^2.6.18", + "@budibase/frontend-core": "^2.6.18", + "@budibase/shared-core": "^2.6.18", + "@budibase/string-templates": "^2.6.18", + "@budibase/types": "^2.6.18", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 569d4fcf8c..fb45a88805 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,13 +1,13 @@ { "name": "@budibase/frontend-core", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^2.6.17", - "@budibase/shared-core": "^2.6.17", + "@budibase/bbui": "^2.6.18", + "@budibase/shared-core": "^2.6.18", "dayjs": "^1.11.7", "lodash": "^4.17.21", "socket.io-client": "^4.6.1", diff --git a/packages/sdk/package.json b/packages/sdk/package.json index a51e825987..7d6461c3e5 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index 3a755ab84c..56f75a7b1f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -45,12 +45,12 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^2.6.17", - "@budibase/client": "^2.6.17", + "@budibase/backend-core": "^2.6.18", + "@budibase/client": "^2.6.18", "@budibase/pro": "2.6.17", - "@budibase/shared-core": "^2.6.17", - "@budibase/string-templates": "^2.6.17", - "@budibase/types": "^2.6.17", + "@budibase/shared-core": "^2.6.18", + "@budibase/string-templates": "^2.6.18", + "@budibase/types": "^2.6.18", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/shared-core/package.json b/packages/shared-core/package.json index e82cc467bf..d9d7d4ab31 100644 --- a/packages/shared-core/package.json +++ b/packages/shared-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/shared-core", - "version": "2.6.17", + "version": "2.6.18", "description": "Shared data utils", "main": "dist/cjs/src/index.js", "types": "dist/mjs/src/index.d.ts", @@ -20,7 +20,7 @@ "dev:builder": "yarn prebuild && concurrently \"tsc -p tsconfig.build.json --watch\" \"tsc -p tsconfig-cjs.build.json --watch\"" }, "dependencies": { - "@budibase/types": "^2.6.17" + "@budibase/types": "^2.6.18" }, "devDependencies": { "concurrently": "^7.6.0", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 419417a8c8..30c67b7d68 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.6.17", + "version": "2.6.18", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 570c3e764e..487f5b9767 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase types", "main": "dist/cjs/index.js", "types": "dist/mjs/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index b3bbdbd2a3..e25230266c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.6.17", + "version": "2.6.18", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -37,10 +37,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^2.6.17", + "@budibase/backend-core": "^2.6.18", "@budibase/pro": "2.6.17", - "@budibase/string-templates": "^2.6.17", - "@budibase/types": "^2.6.17", + "@budibase/string-templates": "^2.6.18", + "@budibase/types": "^2.6.18", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", From 0595b4485e1ca166226f7f17bbd6167b31bb0769 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Wed, 17 May 2023 16:15:07 +0000 Subject: [PATCH 13/14] Update pro version to 2.6.18 --- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 56f75a7b1f..e1afcafac0 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -47,7 +47,7 @@ "@apidevtools/swagger-parser": "10.0.3", "@budibase/backend-core": "^2.6.18", "@budibase/client": "^2.6.18", - "@budibase/pro": "2.6.17", + "@budibase/pro": "2.6.18", "@budibase/shared-core": "^2.6.18", "@budibase/string-templates": "^2.6.18", "@budibase/types": "^2.6.18", diff --git a/packages/worker/package.json b/packages/worker/package.json index e25230266c..9a82edfb4e 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -38,7 +38,7 @@ "license": "GPL-3.0", "dependencies": { "@budibase/backend-core": "^2.6.18", - "@budibase/pro": "2.6.17", + "@budibase/pro": "2.6.18", "@budibase/string-templates": "^2.6.18", "@budibase/types": "^2.6.18", "@koa/router": "8.0.8", From 55ee94892a0d9cf109115678059eb0890f5fd82b Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 17 May 2023 22:18:50 +0100 Subject: [PATCH 14/14] Support setting object name for logged json objects --- .../backend-core/src/logging/pino/logger.ts | 29 +++++++++++++++++-- packages/backend-core/src/queue/listeners.ts | 15 ++++++++-- packages/server/src/automations/utils.ts | 20 ++++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/logging/pino/logger.ts b/packages/backend-core/src/logging/pino/logger.ts index 14c8e7e5a9..cebc78ffc7 100644 --- a/packages/backend-core/src/logging/pino/logger.ts +++ b/packages/backend-core/src/logging/pino/logger.ts @@ -94,12 +94,37 @@ if (!env.DISABLE_PINO_LOGGER) { } } - const mergingObject = { - objects: objects.length ? objects : undefined, + const mergingObject: any = { err: error, ...contextObject, } + if (objects.length) { + // init generic data object for params supplied that don't have a + // '_logKey' field. This prints an object using argument index as the key + // e.g. { 0: {}, 1: {} } + const data: any = {} + let dataIndex = 0 + + for (let i = 0; i < objects.length; i++) { + const object = objects[i] + // the object has specified a log key + // use this instead of generic key + const logKey = object._logKey + if (logKey) { + delete object._logKey + mergingObject[logKey] = object + } else { + data[dataIndex] = object + dataIndex++ + } + } + + if (Object.keys(data).length) { + mergingObject.data = data + } + } + return [mergingObject, message] } diff --git a/packages/backend-core/src/queue/listeners.ts b/packages/backend-core/src/queue/listeners.ts index 6732bfaaa9..42e3172364 100644 --- a/packages/backend-core/src/queue/listeners.ts +++ b/packages/backend-core/src/queue/listeners.ts @@ -45,7 +45,8 @@ function getLogParams( const message = `[BULL] ${eventType}=${event}` const err = opts.error - const data = { + const bullLog = { + _logKey: "bull", eventType, event, job: opts.job, @@ -53,7 +54,17 @@ function getLogParams( ...extra, } - return [message, err, data] + let automationLog + if (opts.job?.data?.automation) { + automationLog = { + _logKey: "automation", + trigger: opts.job + ? opts.job.data.automation.definition.trigger.event + : undefined, + } + } + + return [message, err, bullLog, automationLog] } enum BullEvent { diff --git a/packages/server/src/automations/utils.ts b/packages/server/src/automations/utils.ts index d9f19013ee..dc2f02c821 100644 --- a/packages/server/src/automations/utils.ts +++ b/packages/server/src/automations/utils.ts @@ -17,10 +17,16 @@ const CRON_STEP_ID = definitions.CRON.stepId const Runner = new Thread(ThreadType.AUTOMATION) function loggingArgs(job: AutomationJob) { - return { - jobId: job.id, - trigger: job.data.automation.definition.trigger.event, - } + return [ + { + _logKey: "automation", + trigger: job.data.automation.definition.trigger.event, + }, + { + _logKey: "bull", + jobId: job.id, + }, + ] } export async function processEvent(job: AutomationJob) { @@ -29,16 +35,16 @@ export async function processEvent(job: AutomationJob) { const task = async () => { try { // need to actually await these so that an error can be captured properly - console.log("automation running", loggingArgs(job)) + console.log("automation running", ...loggingArgs(job)) const runFn = () => Runner.run(job) const result = await quotas.addAutomation(runFn, { automationId, }) - console.log("automation completed", loggingArgs(job)) + console.log("automation completed", ...loggingArgs(job)) return result } catch (err) { - console.error(`automation was unable to run`, err, loggingArgs(job)) + console.error(`automation was unable to run`, err, ...loggingArgs(job)) return { err } } }