MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/cluster-mode

This commit is contained in:
Michael Drury 2024-02-15 13:43:52 +00:00 committed by GitHub
parent d1da8ad49f c84c7f9299
commit 4bd361921b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/builder/src/components/integration/RestQueryViewer.svelte
View File

@ -159,7 +159,7 @@
newQuery.fields.queryString = queryString
newQuery.fields.authConfigId = authConfigId
newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders)
newQuery.schema = schema
newQuery.schema = schema || {}
return newQuery
}

32
packages/server/src/api/controllers/automation.ts
View File

@ -1,4 +1,3 @@
import * as actions from "../../automations/actions"
import * as triggers from "../../automations/triggers"
import {
getAutomationParams,
@ -20,11 +19,12 @@ import {
Automation,
AutomationActionStepId,
AutomationResults,
BBContext,
UserCtx,
} from "@budibase/types"
import { getActionDefinitions as actionDefs } from "../../automations/actions"
import sdk from "../../sdk"
import { builderSocket } from "../../websockets"
import env from "../../environment"
async function getActionDefinitions() {
return removeDeprecated(await actionDefs())
@ -72,7 +72,7 @@ function cleanAutomationInputs(automation: Automation) {
return automation
}
export async function create(ctx: BBContext) {
export async function create(ctx: UserCtx) {
const db = context.getAppDB()
let automation = ctx.request.body
automation.appId = ctx.appId
@ -141,7 +141,7 @@ export async function handleStepEvents(
}
}
export async function update(ctx: BBContext) {
export async function update(ctx: UserCtx) {
const db = context.getAppDB()
let automation = ctx.request.body
automation.appId = ctx.appId
@ -192,7 +192,7 @@ export async function update(ctx: BBContext) {
builderSocket?.emitAutomationUpdate(ctx, automation)
}
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const response = await db.allDocs(
getAutomationParams(null, {
@ -202,12 +202,12 @@ export async function fetch(ctx: BBContext) {
ctx.body = response.rows.map(row => row.doc)
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const db = context.getAppDB()
ctx.body = await db.get(ctx.params.id)
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const db = context.getAppDB()
const automationId = ctx.params.id
const oldAutomation = await db.get<Automation>(automationId)
@ -221,11 +221,11 @@ export async function destroy(ctx: BBContext) {
builderSocket?.emitAutomationDeletion(ctx, automationId)
}
export async function logSearch(ctx: BBContext) {
export async function logSearch(ctx: UserCtx) {
ctx.body = await automations.logs.logSearch(ctx.request.body)
}
export async function clearLogError(ctx: BBContext) {
export async function clearLogError(ctx: UserCtx) {
const { automationId, appId } = ctx.request.body
await context.doInAppContext(appId, async () => {
const db = context.getProdAppDB()
@ -244,15 +244,15 @@ export async function clearLogError(ctx: BBContext) {
})
}
export async function getActionList(ctx: BBContext) {
export async function getActionList(ctx: UserCtx) {
ctx.body = await getActionDefinitions()
}
export async function getTriggerList(ctx: BBContext) {
export async function getTriggerList(ctx: UserCtx) {
ctx.body = getTriggerDefinitions()
}
export async function getDefinitionList(ctx: BBContext) {
export async function getDefinitionList(ctx: UserCtx) {
ctx.body = {
trigger: getTriggerDefinitions(),
action: await getActionDefinitions(),
@ -265,7 +265,7 @@ export async function getDefinitionList(ctx: BBContext) {
* *
*********************/
export async function trigger(ctx: BBContext) {
export async function trigger(ctx: UserCtx) {
const db = context.getAppDB()
let automation = await db.get<Automation>(ctx.params.id)
@ -275,7 +275,9 @@ export async function trigger(ctx: BBContext) {
automation,
{
fields: ctx.request.body.fields,
timeout: ctx.request.body.timeout * 1000 || 120000,
timeout:
ctx.request.body.timeout * 1000 ||
env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
},
{ getResponses: true }
)
@ -310,7 +312,7 @@ function prepareTestInput(input: any) {
return input
}
export async function test(ctx: BBContext) {
export async function test(ctx: UserCtx) {
const db = context.getAppDB()
let automation = await db.get<Automation>(ctx.params.id)
await setTestFlag(automation._id!)

2
packages/server/src/api/controllers/query/index.ts
View File

@ -22,7 +22,7 @@ import {
import { ValidQueryNameRegex } from "@budibase/shared-core"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
timeoutMs: env.QUERY_THREAD_TIMEOUT,
})
// simple function to append "readable" to all read queries

2
packages/server/src/automations/bullboard.ts
View File

@ -15,7 +15,7 @@ const PATH_PREFIX = "/bulladmin"
export async function init() {
// Set up queues for bull board admin
const backupQueue = await backups.getBackupQueue()
const backupQueue = backups.getBackupQueue()
const queues = [automationQueue]
if (backupQueue) {
queues.push(backupQueue)

2
packages/server/src/automations/steps/bash.ts
View File

@ -65,7 +65,7 @@ export async function run({ inputs, context }: AutomationStepInput) {
success = true
try {
stdout = execSync(command, {
timeout: environment.QUERY_THREAD_TIMEOUT || 500,
timeout: environment.QUERY_THREAD_TIMEOUT,
}).toString()
} catch (err: any) {
stdout = err.message

2
packages/server/src/automations/steps/executeQuery.ts
View File

@ -10,6 +10,8 @@ import {
AutomationStepSchema,
AutomationStepType,
} from "@budibase/types"
import { utils } from "@budibase/backend-core"
import env from "../../environment"
export const definition: AutomationStepSchema = {
name: "External Data Connector",

6
packages/server/src/automations/steps/triggerAutomationRun.ts
View File

@ -9,8 +9,9 @@ import {
AutomationCustomIOType,
} from "@budibase/types"
import * as triggers from "../triggers"
import { db as dbCore, context } from "@budibase/backend-core"
import { context } from "@budibase/backend-core"
import { features } from "@budibase/pro"
import env from "../../environment"
export const definition: AutomationStepSchema = {
name: "Trigger an automation",
@ -76,7 +77,8 @@ export async function run({ inputs }: AutomationStepInput) {
automation,
{
fields: { ...fieldParams },
timeout: inputs.timeout * 1000 || 120000,
timeout:
inputs.timeout * 1000 || env.getDefaults().AUTOMATION_SYNC_TIMEOUT,
},
{ getResponses: true }
)

13
packages/server/src/automations/tests/triggerAutomationRun.spec.ts
View File

@ -3,6 +3,7 @@ jest.spyOn(global.console, "error")
import * as setup from "./utilities"
import * as automation from "../index"
import { serverLogAutomation } from "../../tests/utilities/structures"
import env from "../../environment"
describe("Test triggering an automation from another automation", () => {
let config = setup.getConfig()
@ -22,7 +23,10 @@ describe("Test triggering an automation from another automation", () => {
let newAutomation = await config.createAutomation(automation)
const inputs: any = {
automation: { automationId: newAutomation._id, timeout: 12000 },
automation: {
automationId: newAutomation._id,
timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
},
}
const res = await setup.runStep(
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
@ -33,7 +37,12 @@ describe("Test triggering an automation from another automation", () => {
})
it("should fail gracefully if the automation id is incorrect", async () => {
const inputs: any = { automation: { automationId: null, timeout: 12000 } }
const inputs: any = {
automation: {
automationId: null,
timeout: env.getDefaults().AUTOMATION_THREAD_TIMEOUT,
},
}
const res = await setup.runStep(
setup.actions.TRIGGER_AUTOMATION_RUN.stepId,
inputs

52
packages/server/src/environment.ts
View File

@ -18,6 +18,21 @@ function parseIntSafe(number?: string) {
}
}
const DEFAULTS = {
QUERY_THREAD_TIMEOUT: 10000,
AUTOMATION_THREAD_TIMEOUT: 12000,
AUTOMATION_SYNC_TIMEOUT: 120000,
AUTOMATION_MAX_ITERATIONS: 200,
JS_PER_EXECUTION_TIME_LIMIT_MS: 1000,
TEMPLATE_REPOSITORY: "app",
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
}
const QUERY_THREAD_TIMEOUT =
parseIntSafe(process.env.QUERY_THREAD_TIMEOUT) ||
DEFAULTS.QUERY_THREAD_TIMEOUT
const environment = {
// features
APP_FEATURES: process.env.APP_FEATURES,
@ -42,7 +57,8 @@ const environment = {
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
TEMPLATE_REPOSITORY: process.env.TEMPLATE_REPOSITORY || "app",
TEMPLATE_REPOSITORY:
process.env.TEMPLATE_REPOSITORY || DEFAULTS.TEMPLATE_REPOSITORY,
DISABLE_AUTO_PROD_APP_SYNC: process.env.DISABLE_AUTO_PROD_APP_SYNC,
SESSION_UPDATE_PERIOD: process.env.SESSION_UPDATE_PERIOD,
// minor
@ -50,14 +66,20 @@ const environment = {
LOGGER: process.env.LOGGER,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
AUTOMATION_MAX_ITERATIONS:
parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) ||
DEFAULTS.AUTOMATION_MAX_ITERATIONS,
SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
QUERY_THREAD_TIMEOUT: QUERY_THREAD_TIMEOUT,
AUTOMATION_THREAD_TIMEOUT:
parseIntSafe(process.env.AUTOMATION_THREAD_TIMEOUT) ||
DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT
? DEFAULTS.AUTOMATION_THREAD_TIMEOUT
: QUERY_THREAD_TIMEOUT,
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
@ -70,12 +92,21 @@ const environment = {
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
SELF_HOSTED: process.env.SELF_HOSTED,
HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
FORKED_PROCESS_NAME:
process.env.FORKED_PROCESS_NAME || DEFAULTS.FORKED_PROCESS_NAME,
JS_PER_INVOCATION_TIMEOUT_MS:
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) ||
DEFAULTS.JS_PER_EXECUTION_TIME_LIMIT_MS,
JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
process.env.JS_PER_REQUEST_TIME_LIMIT_MS
),
TOP_LEVEL_PATH:
process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
JS_RUNNER_MEMORY_LIMIT:
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
// old
CLIENT_ID: process.env.CLIENT_ID,
_set(key: string, value: any) {
@ -92,12 +123,9 @@ const environment = {
isInThread: () => {
return process.env.FORKED_PROCESS
},
TOP_LEVEL_PATH:
process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
JS_RUNNER_MEMORY_LIMIT:
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
getDefaults: () => {
return DEFAULTS
},
}
// clean up any environment variable edge cases

4
packages/server/src/threads/automation.ts
View File

@ -303,7 +303,7 @@ class Orchestrator {
if (timeout) {
setTimeout(() => {
timeoutFlag = true
}, timeout || 12000)
}, timeout || env.AUTOMATION_THREAD_TIMEOUT)
}
stepCount++
@ -621,7 +621,7 @@ export async function executeInThread(job: Job<AutomationData>) {
const timeoutPromise = new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error("Timeout exceeded"))
}, job.data.event.timeout || 12000)
}, job.data.event.timeout || env.AUTOMATION_THREAD_TIMEOUT)
})
return await context.doInAppContext(appId, async () => {