From 169fec29c6e23ecc5b5c26711406d58f5c6de282 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 13 Feb 2024 18:21:41 +0000 Subject: [PATCH 01/25] Some quick fixes, making sure that automation queries respect timeout, they will timeout within the usual range. --- packages/backend-core/src/utils/utils.ts | 15 +++++++++++++-- .../components/integration/RestQueryViewer.svelte | 2 +- packages/server/src/automations/bullboard.ts | 2 +- .../server/src/automations/steps/executeQuery.ts | 7 ++++++- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 30cf55b149..046f8aaf94 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -226,8 +226,19 @@ export function isClient(ctx: Ctx) { return ctx.headers[Header.TYPE] === "client" } -export function timeout(timeMs: number) { - return new Promise(resolve => setTimeout(resolve, timeMs)) +export function timeout( + timeMs: number, + opts?: { reject?: boolean } +): Promise { + return new Promise((resolve, reject) => { + setTimeout(() => { + if (opts?.reject) { + reject(new Error(`timed out - ${timeMs}ms`)) + } else { + resolve() + } + }, timeMs) + }) } export function isAudited(event: Event) { diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index d6a8fe6fc3..74aaa5386f 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -159,7 +159,7 @@ newQuery.fields.queryString = queryString newQuery.fields.authConfigId = authConfigId newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders) - newQuery.schema = schema + newQuery.schema = schema || {} return newQuery } diff --git a/packages/server/src/automations/bullboard.ts b/packages/server/src/automations/bullboard.ts index df784eacff..34f18754a2 100644 --- a/packages/server/src/automations/bullboard.ts +++ b/packages/server/src/automations/bullboard.ts @@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin" export async function init() { // Set up queues for bull board admin - const backupQueue = await backups.getBackupQueue() + const backupQueue = backups.getBackupQueue() const queues = [automationQueue] if (backupQueue) { queues.push(backupQueue) diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index a9517b01a0..d3dc6a219c 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -10,6 +10,8 @@ import { AutomationStepSchema, AutomationStepType, } from "@budibase/types" +import { utils } from "@budibase/backend-core" +import env from "../../environment" export const definition: AutomationStepSchema = { name: "External Data Connector", @@ -84,7 +86,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { }) try { - await queryController.executeV2(ctx, { isAutomation: true }) + await Promise.race([ + queryController.executeV2(ctx, { isAutomation: true }), + utils.timeout(env.QUERY_THREAD_TIMEOUT, { reject: true }), + ]) const { data, ...rest } = ctx.body return { From cfd1c98c8571cdb1301a236dd0210ad63dd10ebb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 14 Feb 2024 11:44:07 +0000 Subject: [PATCH 02/25] Updating timeouts for automations, making it easier to manage/find the various timeout limits. Also adding a new environment variable AUTOMATION_THREAD_TIMEOUT which can be used to control how long automations can run for. --- .../server/src/api/controllers/automation.ts | 31 +++++++++---------- .../server/src/api/controllers/query/index.ts | 2 +- packages/server/src/automations/steps/bash.ts | 2 +- .../src/automations/steps/executeQuery.ts | 5 +-- .../automations/steps/triggerAutomationRun.ts | 5 +-- .../tests/triggerAutomationRun.spec.ts | 13 ++++++-- packages/server/src/constants/automations.ts | 4 +++ packages/server/src/constants/index.ts | 2 ++ packages/server/src/environment.ts | 4 ++- packages/server/src/threads/automation.ts | 4 +-- 10 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 packages/server/src/constants/automations.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 6feba9fb2d..212ef5668f 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -1,4 +1,3 @@ -import * as actions from "../../automations/actions" import * as triggers from "../../automations/triggers" import { getAutomationParams, @@ -11,7 +10,7 @@ import { removeDeprecated, } from "../../automations/utils" import { deleteEntityMetadata } from "../../utilities" -import { MetadataTypes } from "../../constants" +import { MetadataTypes, AUTOMATION_SYNC_TIMEOUT } from "../../constants" import { setTestFlag, clearTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" @@ -20,7 +19,7 @@ import { Automation, AutomationActionStepId, AutomationResults, - BBContext, + UserCtx, } from "@budibase/types" import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" @@ -72,7 +71,7 @@ function cleanAutomationInputs(automation: Automation) { return automation } -export async function create(ctx: BBContext) { +export async function create(ctx: UserCtx) { const db = context.getAppDB() let automation = ctx.request.body automation.appId = ctx.appId @@ -141,7 +140,7 @@ export async function handleStepEvents( } } -export async function update(ctx: BBContext) { +export async function update(ctx: UserCtx) { const db = context.getAppDB() let automation = ctx.request.body automation.appId = ctx.appId @@ -192,7 +191,7 @@ export async function update(ctx: BBContext) { builderSocket?.emitAutomationUpdate(ctx, automation) } -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { const db = context.getAppDB() const response = await db.allDocs( getAutomationParams(null, { @@ -202,12 +201,12 @@ export async function fetch(ctx: BBContext) { ctx.body = response.rows.map(row => row.doc) } -export async function find(ctx: BBContext) { +export async function find(ctx: UserCtx) { const db = context.getAppDB() ctx.body = await db.get(ctx.params.id) } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const automationId = ctx.params.id const oldAutomation = await db.get(automationId) @@ -221,11 +220,11 @@ export async function destroy(ctx: BBContext) { builderSocket?.emitAutomationDeletion(ctx, automationId) } -export async function logSearch(ctx: BBContext) { +export async function logSearch(ctx: UserCtx) { ctx.body = await automations.logs.logSearch(ctx.request.body) } -export async function clearLogError(ctx: BBContext) { +export async function clearLogError(ctx: UserCtx) { const { automationId, appId } = ctx.request.body await context.doInAppContext(appId, async () => { const db = context.getProdAppDB() @@ -244,15 +243,15 @@ export async function clearLogError(ctx: BBContext) { }) } -export async function getActionList(ctx: BBContext) { +export async function getActionList(ctx: UserCtx) { ctx.body = await getActionDefinitions() } -export async function getTriggerList(ctx: BBContext) { +export async function getTriggerList(ctx: UserCtx) { ctx.body = getTriggerDefinitions() } -export async function getDefinitionList(ctx: BBContext) { +export async function getDefinitionList(ctx: UserCtx) { ctx.body = { trigger: getTriggerDefinitions(), action: await getActionDefinitions(), @@ -265,7 +264,7 @@ export async function getDefinitionList(ctx: BBContext) { * * *********************/ -export async function trigger(ctx: BBContext) { +export async function trigger(ctx: UserCtx) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) @@ -275,7 +274,7 @@ export async function trigger(ctx: BBContext) { automation, { fields: ctx.request.body.fields, - timeout: ctx.request.body.timeout * 1000 || 120000, + timeout: ctx.request.body.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) @@ -310,7 +309,7 @@ function prepareTestInput(input: any) { return input } -export async function test(ctx: BBContext) { +export async function test(ctx: UserCtx) { const db = context.getAppDB() let automation = await db.get(ctx.params.id) await setTestFlag(automation._id!) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 1be836b169..8dabe5b3cc 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -22,7 +22,7 @@ import { import { ValidQueryNameRegex } from "@budibase/shared-core" const Runner = new Thread(ThreadType.QUERY, { - timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, + timeoutMs: env.QUERY_THREAD_TIMEOUT, }) // simple function to append "readable" to all read queries diff --git a/packages/server/src/automations/steps/bash.ts b/packages/server/src/automations/steps/bash.ts index 61d446f12c..1a13f651ec 100644 --- a/packages/server/src/automations/steps/bash.ts +++ b/packages/server/src/automations/steps/bash.ts @@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) { success = true try { stdout = execSync(command, { - timeout: environment.QUERY_THREAD_TIMEOUT || 500, + timeout: environment.QUERY_THREAD_TIMEOUT, }).toString() } catch (err: any) { stdout = err.message diff --git a/packages/server/src/automations/steps/executeQuery.ts b/packages/server/src/automations/steps/executeQuery.ts index d3dc6a219c..ea0737c86a 100644 --- a/packages/server/src/automations/steps/executeQuery.ts +++ b/packages/server/src/automations/steps/executeQuery.ts @@ -86,10 +86,7 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { }) try { - await Promise.race([ - queryController.executeV2(ctx, { isAutomation: true }), - utils.timeout(env.QUERY_THREAD_TIMEOUT, { reject: true }), - ]) + await queryController.executeV2(ctx, { isAutomation: true }) const { data, ...rest } = ctx.body return { diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index cb6126ca01..73c9b3adf1 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -9,8 +9,9 @@ import { AutomationCustomIOType, } from "@budibase/types" import * as triggers from "../triggers" -import { db as dbCore, context } from "@budibase/backend-core" +import { context } from "@budibase/backend-core" import { features } from "@budibase/pro" +import { AUTOMATION_SYNC_TIMEOUT } from "../../constants" export const definition: AutomationStepSchema = { name: "Trigger an automation", @@ -76,7 +77,7 @@ export async function run({ inputs }: AutomationStepInput) { automation, { fields: { ...fieldParams }, - timeout: inputs.timeout * 1000 || 120000, + timeout: inputs.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) diff --git a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts index f8cf647e79..83e3c20bbb 100644 --- a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts +++ b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts @@ -3,6 +3,7 @@ jest.spyOn(global.console, "error") import * as setup from "./utilities" import * as automation from "../index" import { serverLogAutomation } from "../../tests/utilities/structures" +import { AUTOMATION_ASYNC_TIMEOUT } from "../../constants" describe("Test triggering an automation from another automation", () => { let config = setup.getConfig() @@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => { let newAutomation = await config.createAutomation(automation) const inputs: any = { - automation: { automationId: newAutomation._id, timeout: 12000 }, + automation: { + automationId: newAutomation._id, + timeout: AUTOMATION_ASYNC_TIMEOUT, + }, } const res = await setup.runStep( setup.actions.TRIGGER_AUTOMATION_RUN.stepId, @@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => { }) it("should fail gracefully if the automation id is incorrect", async () => { - const inputs: any = { automation: { automationId: null, timeout: 12000 } } + const inputs: any = { + automation: { + automationId: null, + timeout: AUTOMATION_ASYNC_TIMEOUT, + }, + } const res = await setup.runStep( setup.actions.TRIGGER_AUTOMATION_RUN.stepId, inputs diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts new file mode 100644 index 0000000000..5d83369f87 --- /dev/null +++ b/packages/server/src/constants/automations.ts @@ -0,0 +1,4 @@ +import { Duration } from "@budibase/backend-core" + +export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs() +export const AUTOMATION_ASYNC_TIMEOUT = Duration.fromSeconds(12).toMs() diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 49f1d01afb..530cc536ca 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -6,6 +6,8 @@ import { TableSourceType, } from "@budibase/types" +export * from "./automations" + export enum FilterTypes { STRING = "string", FUZZY = "fuzzy", diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 8e6866d5e4..b3d75070a6 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -53,7 +53,9 @@ const environment = { parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, - QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT), + QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || 10000, + AUTOMATION_THREAD_TIMEOUT: + parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || 12000, SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index a828af5d19..a4938bb138 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -303,7 +303,7 @@ class Orchestrator { if (timeout) { setTimeout(() => { timeoutFlag = true - }, timeout || 12000) + }, timeout || env.AUTOMATION_THREAD_TIMEOUT) } stepCount++ @@ -621,7 +621,7 @@ export async function executeInThread(job: Job) { const timeoutPromise = new Promise((resolve, reject) => { setTimeout(() => { reject(new Error("Timeout exceeded")) - }, job.data.event.timeout || 12000) + }, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT) }) return await context.doInAppContext(appId, async () => { From 77225e6eb9c76982c2d07b79462dc0985c759db9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 14 Feb 2024 15:04:08 +0000 Subject: [PATCH 03/25] Updating how default environment variables are handled, so that the defaults are easier to access. --- packages/server/src/constants/automations.ts | 4 +- packages/server/src/environment.ts | 51 +++++++++++++++----- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts index 5d83369f87..6688cbedb7 100644 --- a/packages/server/src/constants/automations.ts +++ b/packages/server/src/constants/automations.ts @@ -1,4 +1,6 @@ import { Duration } from "@budibase/backend-core" +import env from "../environment" +const defaults = env.getDefaults() export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs() -export const AUTOMATION_ASYNC_TIMEOUT = Duration.fromSeconds(12).toMs() +export const AUTOMATION_ASYNC_TIMEOUT = defaults.QUERY_THREAD_TIMEOUT diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index b3d75070a6..ec70ced66c 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -18,6 +18,20 @@ function parseIntSafe(number?: string) { } } +const DEFAULTS = { + QUERY_THREAD_TIMEOUT: 10000, + AUTOMATION_THREAD_TIMEOUT: 12000, + AUTOMATION_MAX_ITERATIONS: 200, + JS_PER_EXECUTION_TIME_LIMIT_MS: 1000, + TEMPLATE_REPOSITORY: "app", + PLUGINS_DIR: "/plugins", + FORKED_PROCESS_NAME: "main", + JS_RUNNER_MEMORY_LIMIT: 64, +} + +const QUERY_THREAD_TIMEOUT = + parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || + DEFAULTS.QUERY_THREAD_TIMEOUT const environment = { // features APP_FEATURES: process.env.APP_FEATURES, @@ -42,7 +56,8 @@ const environment = { JEST_WORKER_ID: process.env.JEST_WORKER_ID, BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, - TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app", + TEMPLATE_REPOSITORY: + process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY, DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC, SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD, // minor @@ -50,16 +65,20 @@ const environment = { LOGGER: process.env.LOGGER, ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL, AUTOMATION_MAX_ITERATIONS: - parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200, + parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || + DEFAULTS.AUTOMATION_MAX_ITERATIONS, SENDGRID_API_KEY: process.env.SENDGRID_API_KEY, DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT, - QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) || 10000, + QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT, AUTOMATION_THREAD_TIMEOUT: - parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || 12000, + parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) || + DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT + ? DEFAULTS.AUTOMATION_THREAD_TIMEOUT + : QUERY_THREAD_TIMEOUT, SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL, BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD, - PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins", + PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR, OPENAI_API_KEY: process.env.OPENAI_API_KEY, MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB, SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS, @@ -72,12 +91,21 @@ const environment = { ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, SELF_HOSTED: process.env.SELF_HOSTED, HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT, - FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main", + FORKED_PROCESS_NAME: + process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME, JS_PER_INVOCATION_TIMEOUT_MS: - parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000, + parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || + DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS, JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe( process.env.JS_PER_REQUEST_TIME_LIMIT_MS ), + TOP_LEVEL_PATH: + process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH, + APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT), + JS_RUNNER_MEMORY_LIMIT: + parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || + DEFAULTS.JS_RUNNER_MEMORY_LIMIT, + LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { @@ -94,12 +122,9 @@ const environment = { isInThread: () => { return process.env.FORKED_PROCESS }, - TOP_LEVEL_PATH: - process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH, - APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT), - JS_RUNNER_MEMORY_LIMIT: - parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64, - LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, + getDefaults: () => { + return DEFAULTS + }, } // clean up any environment variable edge cases From fa585fe69df421c0f0b90c697f0376267e7d0795 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 14 Feb 2024 15:05:08 +0000 Subject: [PATCH 04/25] Removing timeout reject option. --- packages/backend-core/src/utils/utils.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index 046f8aaf94..30cf55b149 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -226,19 +226,8 @@ export function isClient(ctx: Ctx) { return ctx.headers[Header.TYPE] === "client" } -export function timeout( - timeMs: number, - opts?: { reject?: boolean } -): Promise { - return new Promise((resolve, reject) => { - setTimeout(() => { - if (opts?.reject) { - reject(new Error(`timed out - ${timeMs}ms`)) - } else { - resolve() - } - }, timeMs) - }) +export function timeout(timeMs: number) { + return new Promise(resolve => setTimeout(resolve, timeMs)) } export function isAudited(event: Event) { From ea5d04e1d3dbaf22337402c7b736684ff44ad5e7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 14 Feb 2024 17:15:42 +0000 Subject: [PATCH 05/25] Addressing PR comments. --- packages/server/src/api/controllers/automation.ts | 7 +++++-- .../server/src/automations/steps/triggerAutomationRun.ts | 5 +++-- .../src/automations/tests/triggerAutomationRun.spec.ts | 6 +++--- packages/server/src/constants/automations.ts | 6 ------ packages/server/src/constants/index.ts | 2 -- packages/server/src/environment.ts | 1 + 6 files changed, 12 insertions(+), 15 deletions(-) delete mode 100644 packages/server/src/constants/automations.ts diff --git a/packages/server/src/api/controllers/automation.ts b/packages/server/src/api/controllers/automation.ts index 212ef5668f..186b68f3b7 100644 --- a/packages/server/src/api/controllers/automation.ts +++ b/packages/server/src/api/controllers/automation.ts @@ -10,7 +10,7 @@ import { removeDeprecated, } from "../../automations/utils" import { deleteEntityMetadata } from "../../utilities" -import { MetadataTypes, AUTOMATION_SYNC_TIMEOUT } from "../../constants" +import { MetadataTypes } from "../../constants" import { setTestFlag, clearTestFlag } from "../../utilities/redis" import { context, cache, events, db as dbCore } from "@budibase/backend-core" import { automations, features } from "@budibase/pro" @@ -24,6 +24,7 @@ import { import { getActionDefinitions as actionDefs } from "../../automations/actions" import sdk from "../../sdk" import { builderSocket } from "../../websockets" +import env from "../../environment" async function getActionDefinitions() { return removeDeprecated(await actionDefs()) @@ -274,7 +275,9 @@ export async function trigger(ctx: UserCtx) { automation, { fields: ctx.request.body.fields, - timeout: ctx.request.body.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT, + timeout: + ctx.request.body.timeout * 1000 || + env.getDefaults().AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) diff --git a/packages/server/src/automations/steps/triggerAutomationRun.ts b/packages/server/src/automations/steps/triggerAutomationRun.ts index 73c9b3adf1..83e1722877 100644 --- a/packages/server/src/automations/steps/triggerAutomationRun.ts +++ b/packages/server/src/automations/steps/triggerAutomationRun.ts @@ -11,7 +11,7 @@ import { import * as triggers from "../triggers" import { context } from "@budibase/backend-core" import { features } from "@budibase/pro" -import { AUTOMATION_SYNC_TIMEOUT } from "../../constants" +import env from "../../environment" export const definition: AutomationStepSchema = { name: "Trigger an automation", @@ -77,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) { automation, { fields: { ...fieldParams }, - timeout: inputs.timeout * 1000 || AUTOMATION_SYNC_TIMEOUT, + timeout: + inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT, }, { getResponses: true } ) diff --git a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts index 83e3c20bbb..9d699e15fa 100644 --- a/packages/server/src/automations/tests/triggerAutomationRun.spec.ts +++ b/packages/server/src/automations/tests/triggerAutomationRun.spec.ts @@ -3,7 +3,7 @@ jest.spyOn(global.console, "error") import * as setup from "./utilities" import * as automation from "../index" import { serverLogAutomation } from "../../tests/utilities/structures" -import { AUTOMATION_ASYNC_TIMEOUT } from "../../constants" +import env from "../../environment" describe("Test triggering an automation from another automation", () => { let config = setup.getConfig() @@ -25,7 +25,7 @@ describe("Test triggering an automation from another automation", () => { const inputs: any = { automation: { automationId: newAutomation._id, - timeout: AUTOMATION_ASYNC_TIMEOUT, + timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT, }, } const res = await setup.runStep( @@ -40,7 +40,7 @@ describe("Test triggering an automation from another automation", () => { const inputs: any = { automation: { automationId: null, - timeout: AUTOMATION_ASYNC_TIMEOUT, + timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT, }, } const res = await setup.runStep( diff --git a/packages/server/src/constants/automations.ts b/packages/server/src/constants/automations.ts deleted file mode 100644 index 6688cbedb7..0000000000 --- a/packages/server/src/constants/automations.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Duration } from "@budibase/backend-core" -import env from "../environment" - -const defaults = env.getDefaults() -export const AUTOMATION_SYNC_TIMEOUT = Duration.fromMinutes(2).toMs() -export const AUTOMATION_ASYNC_TIMEOUT = defaults.QUERY_THREAD_TIMEOUT diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 530cc536ca..49f1d01afb 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -6,8 +6,6 @@ import { TableSourceType, } from "@budibase/types" -export * from "./automations" - export enum FilterTypes { STRING = "string", FUZZY = "fuzzy", diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index ec70ced66c..20142776b8 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -21,6 +21,7 @@ function parseIntSafe(number?: string) { const DEFAULTS = { QUERY_THREAD_TIMEOUT: 10000, AUTOMATION_THREAD_TIMEOUT: 12000, + AUTOMATION_SYNC_TIMEOUT: 120000, AUTOMATION_MAX_ITERATIONS: 200, JS_PER_EXECUTION_TIME_LIMIT_MS: 1000, TEMPLATE_REPOSITORY: "app", From b12aa639d3098c2a79c04871d50e23d3e6503e6d Mon Sep 17 00:00:00 2001 From: Gerard Burns Date: Thu, 15 Feb 2024 10:53:58 +0000 Subject: [PATCH 06/25] Allow Collapsing Selected Components, Add Hotkeys for Collapsing Components (#12764) * wip * fix spelling * wip * linting * change order of fix version of linting * lint fix * linting --- package.json | 2 +- .../src/builderStore/store/frontend.js | 19 ++++++-- .../ComponentDropdownMenu.svelte | 34 ++++++++++++++ .../ComponentList/ComponentKeyHandler.svelte | 35 ++++++++++++++ .../ComponentList/ComponentTree.svelte | 32 ++++--------- .../stores/portal/componentTreeNodesStore.js | 36 +++++++++++++++ packages/frontend-core/src/stores/index.js | 1 + .../src/stores/sessionStorage.js | 46 +++++++++++++++++++ 8 files changed, 178 insertions(+), 27 deletions(-) create mode 100644 packages/builder/src/stores/portal/componentTreeNodesStore.js create mode 100644 packages/frontend-core/src/stores/sessionStorage.js diff --git a/package.json b/package.json index 499952a441..4407fd33f3 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "lint": "yarn run lint:eslint && yarn run lint:prettier", "lint:fix:eslint": "eslint --fix --max-warnings=0 packages qa-core", "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --write \"qa-core/**/*.{js,ts,svelte}\"", - "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", + "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier", "build:specs": "lerna run --stream specs", "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild", "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild", diff --git a/packages/builder/src/builderStore/store/frontend.js b/packages/builder/src/builderStore/store/frontend.js index 55208bb97e..fd492cca0b 100644 --- a/packages/builder/src/builderStore/store/frontend.js +++ b/packages/builder/src/builderStore/store/frontend.js @@ -39,6 +39,7 @@ import { makePropSafe as safe } from "@budibase/string-templates" import { getComponentFieldOptions } from "helpers/formFields" import { createBuilderWebsocket } from "builderStore/websocket" import { BuilderSocketEvent } from "@budibase/shared-core" +import componentTreeNodesStore from "stores/portal/componentTreeNodesStore" const INITIAL_FRONTEND_STATE = { initialised: false, @@ -1053,6 +1054,7 @@ export const getFrontendStore = () => { const screen = get(selectedScreen) const parent = findComponentParent(screen.props, componentId) const index = parent?._children.findIndex(x => x._id === componentId) + const componentTreeNodes = get(componentTreeNodesStore) // Check for screen and navigation component edge cases const screenComponentId = `${screen._id}-screen` @@ -1071,9 +1073,15 @@ export const getFrontendStore = () => { if (index > 0) { // If sibling before us accepts children, select a descendant const previousSibling = parent._children[index - 1] - if (previousSibling._children?.length) { + if ( + previousSibling._children?.length && + componentTreeNodes[`nodeOpen-${previousSibling._id}`] + ) { let target = previousSibling - while (target._children?.length) { + while ( + target._children?.length && + componentTreeNodes[`nodeOpen-${target._id}`] + ) { target = target._children[target._children.length - 1] } return target._id @@ -1093,6 +1101,7 @@ export const getFrontendStore = () => { const screen = get(selectedScreen) const parent = findComponentParent(screen.props, componentId) const index = parent?._children.findIndex(x => x._id === componentId) + const componentTreeNodes = get(componentTreeNodesStore) // Check for screen and navigation component edge cases const screenComponentId = `${screen._id}-screen` @@ -1102,7 +1111,11 @@ export const getFrontendStore = () => { } // If we have children, select first child - if (component._children?.length) { + if ( + component._children?.length && + (state.selectedComponentId === navComponentId || + componentTreeNodes[`nodeOpen-${component._id}`]) + ) { return component._children[0]._id } else if (!parent) { return null diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte index 4645ee0d41..baaa561679 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentDropdownMenu.svelte @@ -3,6 +3,7 @@ import { ActionMenu, MenuItem, Icon } from "@budibase/bbui" export let component + export let opened $: definition = componentStore.getDefinition(component?._component) $: noPaste = !$componentStore.componentToPaste @@ -85,6 +86,39 @@ > Paste + + {#if component?._children?.length} + keyboardEvent("ArrowRight", false)} + disabled={opened} + > + Expand + + keyboardEvent("ArrowLeft", false)} + disabled={!opened} + > + Collapse + + keyboardEvent("ArrowRight", true)} + > + Expand All + + keyboardEvent("ArrowLeft", true)} + > + Collapse All + + {/if}