MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/handle-manual-prices

This commit is contained in:
jvcalderon 2023-12-20 13:28:49 +01:00
parent 1c1e305f03 9c78b86902
commit 3c2ab7d2ec
12 changed files with 475 additions and 321 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/budibase_ci.yml vendored
View File

@ -76,6 +76,18 @@ jobs:
yarn check:types
fi
helm-lint:
runs-on: ubuntu-latest
steps:
- name: Checkout repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Use Node.js 18.x
uses: azure/setup-helm@v3
- run: cd charts/budibase && helm lint .
test-libraries:
runs-on: ubuntu-latest
steps:

16
charts/budibase/templates/app-service-deployment.yaml
View File

@ -227,6 +227,14 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{ if .Values.services.apps.command }}
command:
{{- toYaml .Values.services.apps.command | nindent 10 }}
{{ end }}
{{ if .Values.services.apps.args }}
args:
{{- toYaml .Values.services.apps.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
@ -244,12 +252,4 @@ spec:
{{ end }}
restartPolicy: Always
serviceAccountName: ""
{{ if .Values.services.apps.command }}
command:
{{- toYaml .Values.services.apps.command | nindent 8 }}
{{ end }}
{{ if .Values.services.apps.args }}
args:
{{- toYaml .Values.services.apps.args | nindent 8 }}
{{ end }}
status: {}

14
charts/budibase/templates/automation-worker-service-deployment.yaml
View File

@ -227,6 +227,13 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
command:
{{- toYaml .Values.services.automationWorkers.command | nindent 10 }}
{{ end }}
{{ if .Values.services.automationWorkers.args }}
args:
{{- toYaml .Values.services.automationWorkers.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
@ -245,12 +252,5 @@ spec:
restartPolicy: Always
serviceAccountName: ""
{{ if .Values.services.automationWorkers.command }}}
command:
{{- toYaml .Values.services.automationWorkers.command | nindent 8 }}
{{ end }}
{{ if .Values.services.automationWorkers.args }}
args:
{{- toYaml .Values.services.automationWorkers.args | nindent 8 }}
{{ end }}
status: {}
{{- end }}

16
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -213,6 +213,14 @@ spec:
resources:
{{- toYaml . | nindent 10 }}
{{ end }}
{{ if .Values.services.worker.command }}
command:
{{- toYaml .Values.services.worker.command | nindent 10 }}
{{ end }}
{{ if .Values.services.worker.args }}
args:
{{- toYaml .Values.services.worker.args | nindent 10 }}
{{ end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
@ -230,12 +238,4 @@ spec:
{{ end }}
restartPolicy: Always
serviceAccountName: ""
{{ if .Values.services.worker.command }}
command:
{{- toYaml .Values.services.worker.command | nindent 8 }}
{{ end }}
{{ if .Values.services.worker.args }}
args:
{{- toYaml .Values.services.worker.args | nindent 8 }}
{{ end }}
status: {}

2
hosting/single/runner.sh
View File

@ -79,7 +79,7 @@ redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
/bbcouch-runner.sh &
# only start minio if use s3 isn't passed
if [[ -z "${USE_S3}"]]; then
if [[ -z "${USE_S3}" ]]; then
/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
fi

1
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -15,6 +15,7 @@ function newJob(queue: string, message: any) {
timestamp: Date.now(),
queue: queue,
data: message,
opts: {},
}
}

6
packages/backend-core/src/timers/timers.ts
View File

@ -30,7 +30,7 @@ export class ExecutionTimeTracker {
return new ExecutionTimeTracker(limitMs)
}
constructor(private limitMs: number) {}
constructor(readonly limitMs: number) {}
private totalTimeMs = 0
@ -46,6 +46,10 @@ export class ExecutionTimeTracker {
}
}
get elapsedMS() {
return this.totalTimeMs
}
private checkLimit() {
if (this.totalTimeMs > this.limitMs) {
throw new ExecutionTimeoutError(

39
packages/server/src/automations/utils.ts
View File

@ -16,6 +16,7 @@ import {
} from "@budibase/types"
import sdk from "../sdk"
import { automationsEnabled } from "../features"
import tracer from "dd-trace"
const REBOOT_CRON = "@reboot"
const WH_STEP_ID = definitions.WEBHOOK.stepId
@ -39,8 +40,37 @@ function loggingArgs(job: AutomationJob) {
}
export async function processEvent(job: AutomationJob) {
return tracer.trace(
"processEvent",
{ resource: "automation" },
async span => {
const appId = job.data.event.appId!
const automationId = job.data.automation._id!
span?.addTags({
appId,
automationId,
job: {
id: job.id,
name: job.name,
attemptsMade: job.attemptsMade,
opts: {
attempts: job.opts.attempts,
priority: job.opts.priority,
delay: job.opts.delay,
repeat: job.opts.repeat,
backoff: job.opts.backoff,
lifo: job.opts.lifo,
timeout: job.opts.timeout,
jobId: job.opts.jobId,
removeOnComplete: job.opts.removeOnComplete,
removeOnFail: job.opts.removeOnFail,
stackTraceLimit: job.opts.stackTraceLimit,
preventParsingData: job.opts.preventParsingData,
},
},
})
const task = async () => {
try {
// need to actually await these so that an error can be captured properly
@ -53,12 +83,19 @@ export async function processEvent(job: AutomationJob) {
console.log("automation completed", ...loggingArgs(job))
return result
} catch (err) {
console.error(`automation was unable to run`, err, ...loggingArgs(job))
span?.addTags({ error: true })
console.error(
`automation was unable to run`,
err,
...loggingArgs(job)
)
return { err }
}
}
return await context.doInAutomationContext({ appId, automationId, task })
}
)
}
export async function updateTestHistory(

5
packages/server/src/integrations/rest.ts
View File

@ -131,7 +131,10 @@ class RestIntegration implements IntegrationBase {
let data, raw, headers
const contentType = response.headers.get("content-type") || ""
try {
if (contentType.includes("application/json")) {
if (response.status === 204) {
data = []
raw = []
} else if (contentType.includes("application/json")) {
data = await response.json()
raw = JSON.stringify(data)
} else if (

22
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -186,9 +186,15 @@ describe("REST Integration", () => {
})
describe("response", () => {
function buildInput(json: any, text: any, header: any) {
const contentTypes = ["application/json", "text/plain", "application/xml"]
function buildInput(
json: any,
text: any,
header: any,
status: number = 200
) {
return {
status: 200,
status,
json: json ? async () => json : undefined,
text: text ? async () => text : undefined,
headers: {
@ -225,6 +231,18 @@ describe("REST Integration", () => {
expect(output.extra.raw).toEqual(text)
expect(output.extra.headers["content-type"]).toEqual("application/xml")
})
test.each(contentTypes)(
"should not throw an error on 204 no content",
async contentType => {
const input = buildInput(undefined, null, contentType, 204)
const output = await config.integration.parseResponse(input)
expect(output.data).toEqual([])
expect(output.extra.raw).toEqual([])
expect(output.info.code).toEqual(204)
expect(output.extra.headers["content-type"]).toEqual(contentType)
}
)
})
describe("authentication", () => {

9
packages/server/src/jsRunner.ts
View File

@ -2,11 +2,13 @@ import vm from "vm"
import env from "./environment"
import { setJSRunner } from "@budibase/string-templates"
import { context, timers } from "@budibase/backend-core"
import tracer from "dd-trace"
type TrackerFn = <T>(f: () => T) => T
export function init() {
setJSRunner((js: string, ctx: vm.Context) => {
return tracer.trace("runJS", {}, span => {
const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
@ -17,6 +19,12 @@ export function init() {
timers.ExecutionTimeTracker.withLimit(perRequestLimit)
}
track = bbCtx.jsExecutionTracker.track.bind(bbCtx.jsExecutionTracker)
span?.addTags({
js: {
limitMS: bbCtx.jsExecutionTracker.limitMs,
elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
},
})
}
}
@ -33,4 +41,5 @@ export function init() {
})
)
})
})
}

106
packages/server/src/threads/automation.ts
View File

@ -34,6 +34,7 @@ import { cloneDeep } from "lodash/fp"
import { performance } from "perf_hooks"
import * as sdkUtils from "../sdk/utils"
import env from "../environment"
import tracer from "dd-trace"
threadUtils.threadSetup()
const FILTER_STEP_ID = actions.BUILTIN_ACTION_DEFINITIONS.FILTER.stepId
@ -242,6 +243,15 @@ class Orchestrator {
}
async execute(): Promise<any> {
return tracer.trace(
"Orchestrator.execute",
{ resource: "automation" },
async span => {
span?.addTags({
appId: this._appId,
automationId: this._automation._id,
})
// this will retrieve from context created at start of thread
this._context.env = await sdkUtils.getEnvironmentVariables()
let automation = this._automation
@ -257,15 +267,39 @@ class Orchestrator {
let timeout = this._job.data.event.timeout
// check if this is a recurring automation,
if (isProdAppID(this._appId) && isRecurring(automation)) {
span?.addTags({ recurring: true })
metadata = await this.getMetadata()
const shouldStop = await this.checkIfShouldStop(metadata)
if (shouldStop) {
span?.addTags({ shouldStop: true })
return
}
}
const start = performance.now()
for (let step of automation.definition.steps) {
const stepSpan = tracer.startSpan("Orchestrator.execute.step", {
childOf: span,
})
stepSpan.addTags({
resource: "automation",
step: {
stepId: step.stepId,
id: step.id,
name: step.name,
type: step.type,
title: step.stepTitle,
internal: step.internal,
deprecated: step.deprecated,
},
})
let input: any,
iterations = 1,
iterationCount = 0
try {
if (timeoutFlag) {
span?.addTags({ timedOut: true })
break
}
@ -276,10 +310,6 @@ class Orchestrator {
}
stepCount++
let input: any,
iterations = 1,
iterationCount = 0
if (step.stepId === LOOP_STEP_ID) {
loopStep = step
loopStepNumber = stepCount
@ -289,22 +319,31 @@ class Orchestrator {
if (loopStep) {
input = await processObject(loopStep.inputs, this._context)
iterations = getLoopIterations(loopStep as LoopStep)
stepSpan?.addTags({ step: { iterations } })
}
for (let index = 0; index < iterations; index++) {
let originalStepInput = cloneDeep(step.inputs)
// Handle if the user has set a max iteration count or if it reaches the max limit set by us
if (loopStep && input.binding) {
let tempOutput = { items: loopSteps, iterations: iterationCount }
let tempOutput = {
items: loopSteps,
iterations: iterationCount,
}
try {
loopStep.inputs.binding = automationUtils.typecastForLooping(
loopStep as LoopStep,
loopStep.inputs as LoopInput
)
} catch (err) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.INCORRECT_TYPE,
success: false,
})
}
)
loopSteps = undefined
loopStep = undefined
break
@ -349,7 +388,8 @@ class Orchestrator {
}
} else {
if (typeof value === "string") {
originalStepInput[key] = automationUtils.substituteLoopStep(
originalStepInput[key] =
automationUtils.substituteLoopStep(
value,
`steps.${loopStepNumber}`
)
@ -361,30 +401,42 @@ class Orchestrator {
index === env.AUTOMATION_MAX_ITERATIONS ||
index === parseInt(loopStep.inputs.iterations)
) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.MAX_ITERATIONS,
success: true,
})
}
)
loopSteps = undefined
loopStep = undefined
break
}
let isFailure = false
const currentItem = this._context.steps[loopStepNumber]?.currentItem
const currentItem =
this._context.steps[loopStepNumber]?.currentItem
if (currentItem && typeof currentItem === "object") {
isFailure = Object.keys(currentItem).some(value => {
return currentItem[value] === loopStep?.inputs.failure
})
} else {
isFailure = currentItem && currentItem === loopStep.inputs.failure
isFailure =
currentItem && currentItem === loopStep.inputs.failure
}
if (isFailure) {
this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
this.updateContextAndOutput(
loopStepNumber,
step,
tempOutput,
{
status: AutomationErrors.FAILURE_CONDITION,
success: false,
})
}
)
loopSteps = undefined
loopStep = undefined
break
@ -393,14 +445,22 @@ class Orchestrator {
// execution stopped, record state for that
if (stopped) {
this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
this.updateExecutionOutput(
step.id,
step.stepId,
{},
STOPPED_STATUS
)
continue
}
// If it's a loop step, we need to manually add the bindings to the context
let stepFn = await this.getStepFunctionality(step.stepId)
let inputs = await processObject(originalStepInput, this._context)
inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
inputs = automationUtils.cleanInputValues(
inputs,
step.schema.inputs
)
try {
// appId is always passed
@ -416,10 +476,15 @@ class Orchestrator {
// so that we can finish iterating through the steps and record that it stopped
if (step.stepId === FILTER_STEP_ID && !outputs.result) {
stopped = true
this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
this.updateExecutionOutput(
step.id,
step.stepId,
step.inputs,
{
...outputs,
...STOPPED_STATUS,
})
}
)
continue
}
if (loopStep && loopSteps) {
@ -446,6 +511,9 @@ class Orchestrator {
}
}
}
} finally {
stepSpan?.finish()
}
if (loopStep && iterations === 0) {
loopStep = undefined
@ -515,6 +583,8 @@ class Orchestrator {
}
return this.executionOutput
}
)
}
}
export function execute(job: Job<AutomationData>, callback: WorkerCallback) {