MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15674 from Budibase/test-speedup 

Speed up `loop.spec.ts`
This commit is contained in:
Sam Rose 2025-03-05 14:27:57 +00:00 committed by GitHub
parent ded8272bc5 735ec39cfc
commit eaa6997c47
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 345 additions and 340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
globalSetup.ts
View File

@ -88,6 +88,16 @@ export default async function setup() {
content: `
[log]
level = warn
[httpd]
socket_options = [{nodelay, true}]
[couchdb]
single_node = true
[cluster]
n = 1
q = 1
`,
target: "/opt/couchdb/etc/local.d/test-couchdb.ini",
},

8
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -3,7 +3,6 @@ import { newid } from "../utils"
import { Queue, QueueOptions, JobOptions } from "./queue"
import { helpers } from "@budibase/shared-core"
import { Job, JobId, JobInformation } from "bull"
import { cloneDeep } from "lodash"
function jobToJobInformation(job: Job): JobInformation {
let cron = ""
@ -88,9 +87,7 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
*/
async process(concurrencyOrFunc: number | any, func?: any) {
func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
this._emitter.on("message", async msg => {
const message = cloneDeep(msg)
this._emitter.on("message", async message => {
// For the purpose of testing, don't trigger cron jobs immediately.
// Require the test to trigger them manually with timestamps.
if (!message.manualTrigger && message.opts?.repeat != null) {
@ -165,6 +162,9 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
opts,
}
this._messages.push(message)
if (this._messages.length > 1000) {
this._messages.shift()
}
this._addCount++
this._emitter.emit("message", message)
}

3
packages/server/src/api/routes/tests/automation.spec.ts
View File

@ -290,8 +290,7 @@ describe("/automations", () => {
await setup.delay(500)
let elements = await getAllTableRows(config)
// don't test it unless there are values to test
if (elements.length > 1) {
expect(elements.length).toBeGreaterThanOrEqual(MAX_RETRIES)
if (elements.length >= 1) {
expect(elements[0].name).toEqual("Test")
expect(elements[0].description).toEqual("TEST")
return

538
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -166,18 +166,6 @@ if (descriptions.length) {
)
}
const resetRowUsage = async () => {
await config.doInContext(
undefined,
async () =>
await quotas.setUsage(
0,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
)
}
const getRowUsage = async () => {
const { total } = await config.doInContext(undefined, () =>
quotas.getCurrentUsageValues(
@ -188,19 +176,27 @@ if (descriptions.length) {
return total
}
const assertRowUsage = async (expected: number) => {
const usage = await getRowUsage()
async function expectRowUsage(expected: number, f: () => Promise<void>) {
const before = await getRowUsage()
await f()
const after = await getRowUsage()
const usage = after - before
// Because our quota tracking is not perfect, we allow a 10% margin of
// error. This is to account for the fact that parallel writes can result
// in some quota updates getting lost. We don't have any need to solve this
// right now, so we just allow for some error.
// error. This is to account for the fact that parallel writes can
// result in some quota updates getting lost. We don't have any need
// to solve this right now, so we just allow for some error.
if (expected === 0) {
expect(usage).toEqual(0)
return
}
expect(usage).toBeGreaterThan(expected * 0.9)
expect(usage).toBeLessThan(expected * 1.1)
if (usage < 0) {
expect(usage).toBeGreaterThan(expected * 1.1)
expect(usage).toBeLessThan(expected * 0.9)
} else {
expect(usage).toBeGreaterThan(expected * 0.9)
expect(usage).toBeLessThan(expected * 1.1)
}
}
const defaultRowFields = isInternal
@ -215,91 +211,86 @@ if (descriptions.length) {
table = await config.api.table.save(defaultTable())
})
beforeEach(async () => {
await resetRowUsage()
})
describe("create", () => {
it("creates a new row successfully", async () => {
const rowUsage = await getRowUsage()
const row = await config.api.row.save(table._id!, {
name: "Test Contact",
await expectRowUsage(isInternal ? 1 : 0, async () => {
const row = await config.api.row.save(table._id!, {
name: "Test Contact",
})
expect(row.name).toEqual("Test Contact")
expect(row._rev).toBeDefined()
})
expect(row.name).toEqual("Test Contact")
expect(row._rev).toBeDefined()
await assertRowUsage(isInternal ? rowUsage + 1 : rowUsage)
})
it("fails to create a row for a table that does not exist", async () => {
const rowUsage = await getRowUsage()
await config.api.row.save("1234567", {}, { status: 404 })
await assertRowUsage(rowUsage)
await expectRowUsage(0, async () => {
await config.api.row.save("1234567", {}, { status: 404 })
})
})
it("fails to create a row if required fields are missing", async () => {
const rowUsage = await getRowUsage()
const table = await config.api.table.save(
saveTableRequest({
schema: {
required: {
type: FieldType.STRING,
name: "required",
constraints: {
type: "string",
presence: true,
},
},
},
})
)
await config.api.row.save(
table._id!,
{},
{
status: 500,
body: {
validationErrors: {
required: ["can't be blank"],
},
},
}
)
await assertRowUsage(rowUsage)
})
isInternal &&
it("increment row autoId per create row request", async () => {
const rowUsage = await getRowUsage()
const newTable = await config.api.table.save(
await expectRowUsage(0, async () => {
const table = await config.api.table.save(
saveTableRequest({
schema: {
"Row ID": {
name: "Row ID",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
icon: "ri-magic-line",
autocolumn: true,
required: {
type: FieldType.STRING,
name: "required",
constraints: {
type: "number",
type: "string",
presence: true,
numericality: {
greaterThanOrEqualTo: "",
lessThanOrEqualTo: "",
},
},
},
},
})
)
await config.api.row.save(
table._id!,
{},
{
status: 500,
body: {
validationErrors: {
required: ["can't be blank"],
},
},
}
)
})
})
let previousId = 0
for (let i = 0; i < 10; i++) {
const row = await config.api.row.save(newTable._id!, {})
expect(row["Row ID"]).toBeGreaterThan(previousId)
previousId = row["Row ID"]
}
await assertRowUsage(isInternal ? rowUsage + 10 : rowUsage)
isInternal &&
it("increment row autoId per create row request", async () => {
await expectRowUsage(isInternal ? 10 : 0, async () => {
const newTable = await config.api.table.save(
saveTableRequest({
schema: {
"Row ID": {
name: "Row ID",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
icon: "ri-magic-line",
autocolumn: true,
constraints: {
type: "number",
presence: true,
numericality: {
greaterThanOrEqualTo: "",
lessThanOrEqualTo: "",
},
},
},
},
})
)
let previousId = 0
for (let i = 0; i < 10; i++) {
const row = await config.api.row.save(newTable._id!, {})
expect(row["Row ID"]).toBeGreaterThan(previousId)
previousId = row["Row ID"]
}
})
})
isInternal &&
@ -985,16 +976,16 @@ if (descriptions.length) {
describe("update", () => {
it("updates an existing row successfully", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.save(table._id!, {
_id: existing._id,
_rev: existing._rev,
name: "Updated Name",
await expectRowUsage(0, async () => {
const res = await config.api.row.save(table._id!, {
_id: existing._id,
_rev: existing._rev,
name: "Updated Name",
})
expect(res.name).toEqual("Updated Name")
})
expect(res.name).toEqual("Updated Name")
await assertRowUsage(rowUsage)
})
!isInternal &&
@ -1177,23 +1168,22 @@ if (descriptions.length) {
it("should update only the fields that are supplied", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const row = await config.api.row.patch(table._id!, {
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: "Updated Name",
})
const row = await config.api.row.patch(table._id!, {
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: "Updated Name",
expect(row.name).toEqual("Updated Name")
expect(row.description).toEqual(existing.description)
const savedRow = await config.api.row.get(table._id!, row._id!)
expect(savedRow.description).toEqual(existing.description)
expect(savedRow.name).toEqual("Updated Name")
})
expect(row.name).toEqual("Updated Name")
expect(row.description).toEqual(existing.description)
const savedRow = await config.api.row.get(table._id!, row._id!)
expect(savedRow.description).toEqual(existing.description)
expect(savedRow.name).toEqual("Updated Name")
await assertRowUsage(rowUsage)
})
it("should update only the fields that are supplied and emit the correct oldRow", async () => {
@ -1224,20 +1214,19 @@ if (descriptions.length) {
it("should throw an error when given improper types", async () => {
const existing = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await config.api.row.patch(
table._id!,
{
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: 1,
},
{ status: 400 }
)
await assertRowUsage(rowUsage)
await expectRowUsage(0, async () => {
await config.api.row.patch(
table._id!,
{
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: 1,
},
{ status: 400 }
)
})
})
it("should not overwrite links if those links are not set", async () => {
@ -1452,25 +1441,25 @@ if (descriptions.length) {
it("should be able to delete a row", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow],
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow],
})
expect(res[0]._id).toEqual(createdRow._id)
})
expect(res[0]._id).toEqual(createdRow._id)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to delete a row with ID only", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
@ -1560,31 +1549,29 @@ if (descriptions.length) {
})
it("should return no errors on valid row", async () => {
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const res = await config.api.row.validate(table._id!, {
name: "ivan",
})
const res = await config.api.row.validate(table._id!, {
name: "ivan",
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
})
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
await assertRowUsage(rowUsage)
})
it("should errors on invalid row", async () => {
const rowUsage = await getRowUsage()
await expectRowUsage(0, async () => {
const res = await config.api.row.validate(table._id!, { name: 1 })
const res = await config.api.row.validate(table._id!, { name: 1 })
if (isInternal) {
expect(res.valid).toBe(false)
expect(Object.keys(res.errors)).toEqual(["name"])
} else {
// Validation for external is not implemented, so it will always return valid
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
}
await assertRowUsage(rowUsage)
if (isInternal) {
expect(res.valid).toBe(false)
expect(Object.keys(res.errors)).toEqual(["name"])
} else {
// Validation for external is not implemented, so it will always return valid
expect(res.valid).toBe(true)
expect(Object.keys(res.errors)).toEqual([])
}
})
})
})
@ -1596,15 +1583,15 @@ if (descriptions.length) {
it("should be able to delete a bulk set of rows", async () => {
const row1 = await config.api.row.save(table._id!, {})
const row2 = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2],
await expectRowUsage(isInternal ? -2 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2],
})
expect(res.length).toEqual(2)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
expect(res.length).toEqual(2)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
})
it("should be able to delete a variety of row set types", async () => {
@ -1613,41 +1600,42 @@ if (descriptions.length) {
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
])
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2._id!, { _id: row3._id }],
await expectRowUsage(isInternal ? -3 : 0, async () => {
const res = await config.api.row.bulkDelete(table._id!, {
rows: [row1, row2._id!, { _id: row3._id }],
})
expect(res.length).toEqual(3)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
expect(res.length).toEqual(3)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 3 : rowUsage)
})
it("should accept a valid row object and delete the row", async () => {
const row1 = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.delete(table._id!, row1 as DeleteRow)
await expectRowUsage(isInternal ? -1 : 0, async () => {
const res = await config.api.row.delete(
table._id!,
row1 as DeleteRow
)
expect(res.id).toEqual(row1._id)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
expect(res.id).toEqual(row1._id)
await config.api.row.get(table._id!, row1._id!, { status: 404 })
})
})
it.each([{ not: "valid" }, { rows: 123 }, "invalid"])(
"should ignore malformed/invalid delete request: %s",
async (request: any) => {
const rowUsage = await getRowUsage()
await config.api.row.delete(table._id!, request, {
status: 400,
body: {
message: "Invalid delete rows request",
},
await expectRowUsage(0, async () => {
await config.api.row.delete(table._id!, request, {
status: 400,
body: {
message: "Invalid delete rows request",
},
})
})
await assertRowUsage(rowUsage)
}
)
})
@ -1733,31 +1721,29 @@ if (descriptions.length) {
})
)
const rowUsage = await getRowUsage()
await expectRowUsage(isInternal ? 2 : 0, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{
name: "Row 2",
description: "Row 2 description",
},
],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{
name: "Row 2",
description: "Row 2 description",
},
],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(2)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(2)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage)
})
isInternal &&
@ -1782,35 +1768,33 @@ if (descriptions.length) {
description: "Existing description",
})
const rowUsage = await getRowUsage()
await expectRowUsage(2, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
identifierFields: ["_id"],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
identifierFields: ["_id"],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(3)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
expect(rows[2].name).toEqual("Updated existing row")
expect(rows[2].description).toEqual("Existing description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(3)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Row 1")
expect(rows[0].description).toEqual("Row 1 description")
expect(rows[1].name).toEqual("Row 2")
expect(rows[1].description).toEqual("Row 2 description")
expect(rows[2].name).toEqual("Updated existing row")
expect(rows[2].description).toEqual("Existing description")
await assertRowUsage(rowUsage + 2)
})
isInternal &&
@ -1835,36 +1819,34 @@ if (descriptions.length) {
description: "Existing description",
})
const rowUsage = await getRowUsage()
await expectRowUsage(3, async () => {
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Row 1",
description: "Row 1 description",
},
{ ...existingRow, name: "Updated existing row" },
{
name: "Row 2",
description: "Row 2 description",
},
],
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(4)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Existing row")
expect(rows[0].description).toEqual("Existing description")
expect(rows[1].name).toEqual("Row 1")
expect(rows[1].description).toEqual("Row 1 description")
expect(rows[2].name).toEqual("Row 2")
expect(rows[2].description).toEqual("Row 2 description")
expect(rows[3].name).toEqual("Updated existing row")
expect(rows[3].description).toEqual("Existing description")
})
const rows = await config.api.row.fetch(table._id!)
expect(rows.length).toEqual(4)
rows.sort((a, b) => a.name.localeCompare(b.name))
expect(rows[0].name).toEqual("Existing row")
expect(rows[0].description).toEqual("Existing description")
expect(rows[1].name).toEqual("Row 1")
expect(rows[1].description).toEqual("Row 1 description")
expect(rows[2].name).toEqual("Row 2")
expect(rows[2].description).toEqual("Row 2 description")
expect(rows[3].name).toEqual("Updated existing row")
expect(rows[3].description).toEqual("Existing description")
await assertRowUsage(rowUsage + 3)
})
// Upserting isn't yet supported in MSSQL / Oracle, see:
@ -2187,29 +2169,29 @@ if (descriptions.length) {
return { linkedTable, firstRow, secondRow }
}
)
const rowUsage = await getRowUsage()
// test basic enrichment
const resBasic = await config.api.row.get(
linkedTable._id!,
secondRow._id!
)
expect(resBasic.link.length).toBe(1)
expect(resBasic.link[0]).toEqual({
_id: firstRow._id,
primaryDisplay: firstRow.name,
await expectRowUsage(0, async () => {
// test basic enrichment
const resBasic = await config.api.row.get(
linkedTable._id!,
secondRow._id!
)
expect(resBasic.link.length).toBe(1)
expect(resBasic.link[0]).toEqual({
_id: firstRow._id,
primaryDisplay: firstRow.name,
})
// test full enrichment
const resEnriched = await config.api.row.getEnriched(
linkedTable._id!,
secondRow._id!
)
expect(resEnriched.link.length).toBe(1)
expect(resEnriched.link[0]._id).toBe(firstRow._id)
expect(resEnriched.link[0].name).toBe("Test Contact")
expect(resEnriched.link[0].description).toBe("original description")
})
// test full enrichment
const resEnriched = await config.api.row.getEnriched(
linkedTable._id!,
secondRow._id!
)
expect(resEnriched.link.length).toBe(1)
expect(resEnriched.link[0]._id).toBe(firstRow._id)
expect(resEnriched.link[0].name).toBe("Test Contact")
expect(resEnriched.link[0].description).toBe("original description")
await assertRowUsage(rowUsage)
})
})

42
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -2826,34 +2826,44 @@ if (descriptions.length) {
return total
}
const assertRowUsage = async (expected: number) => {
const usage = await getRowUsage()
async function expectRowUsage<T>(
expected: number,
f: () => Promise<T>
): Promise<T> {
const before = await getRowUsage()
const result = await f()
const after = await getRowUsage()
const usage = after - before
expect(usage).toBe(expected)
return result
}
it("should be able to delete a row", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
await config.api.row.bulkDelete(view.id, { rows: [createdRow] })
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
const createdRow = await expectRowUsage(isInternal ? 1 : 0, () =>
config.api.row.save(table._id!, {})
)
await expectRowUsage(isInternal ? -1 : 0, () =>
config.api.row.bulkDelete(view.id, { rows: [createdRow] })
)
await config.api.row.get(table._id!, createdRow._id!, {
status: 404,
})
})
it("should be able to delete multiple rows", async () => {
const rows = await Promise.all([
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
config.api.row.save(table._id!, {}),
])
const rowUsage = await getRowUsage()
await config.api.row.bulkDelete(view.id, {
rows: [rows[0], rows[2]],
const rows = await expectRowUsage(isInternal ? 3 : 0, async () => {
return [
await config.api.row.save(table._id!, {}),
await config.api.row.save(table._id!, {}),
await config.api.row.save(table._id!, {}),
]
})
await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
await expectRowUsage(isInternal ? -2 : 0, async () => {
await config.api.row.bulkDelete(view.id, {
rows: [rows[0], rows[2]],
})
})
await config.api.row.get(table._id!, rows[0]._id!, {
status: 404,

9
packages/server/src/definitions/automations.ts
View File

@ -20,9 +20,12 @@ export interface TriggerOutput {
export interface AutomationContext {
trigger: AutomationTriggerResultOutputs
steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]]
stepsById: Record<string, AutomationStepResultOutputs>
steps: Record<
string,
AutomationStepResultOutputs | AutomationTriggerResultOutputs
>
stepsByName: Record<string, AutomationStepResultOutputs>
stepsById: Record<string, AutomationStepResultOutputs>
env?: Record<string, string>
user?: UserBindings
settings?: {
@ -31,4 +34,6 @@ export interface AutomationContext {
company?: string
}
loop?: { currentItem: any }
_stepIndex: number
_error: boolean
}

2
packages/server/src/events/AutomationEmitter.ts
View File

@ -32,6 +32,8 @@ class AutomationEmitter implements ContextEmitter {
if (chainAutomations === true) {
return MAX_AUTOMATIONS_ALLOWED
} else if (env.isTest()) {
return 0
} else if (chainAutomations === undefined && env.SELF_HOSTED) {
return MAX_AUTOMATIONS_ALLOWED
} else {

2
packages/server/src/tests/jestSetup.ts
View File

@ -23,6 +23,6 @@ nock.enableNetConnect(host => {
testContainerUtils.setupEnv(env, coreEnv)
afterAll(() => {
afterAll(async () => {
timers.cleanup()
})

4
packages/server/src/tests/utilities/api/base.ts
View File

@ -146,8 +146,9 @@ export abstract class TestAPI {
}
}
let resp: Response | undefined = undefined
try {
return await req
resp = await req
} catch (e: any) {
// We've found that occasionally the connection between supertest and the
// server supertest starts gets reset. Not sure why, but retrying it
@ -161,6 +162,7 @@ export abstract class TestAPI {
}
throw e
}
return resp
}
protected async getHeaders(

67
packages/server/src/threads/automation.ts
View File

@ -143,7 +143,6 @@ async function branchMatches(
branch: Readonly<Branch>
): Promise<boolean> {
const toFilter: Record<string, any> = {}
const preparedCtx = prepareContext(ctx)
// Because we allow bindings on both the left and right of each condition in
// automation branches, we can't pass the BranchSearchFilters directly to
@ -160,9 +159,9 @@ async function branchMatches(
filter.conditions = filter.conditions.map(evaluateBindings)
} else {
for (const [field, value] of Object.entries(filter)) {
toFilter[field] = processStringSync(field, preparedCtx)
toFilter[field] = processStringSync(field, ctx)
if (typeof value === "string" && findHBSBlocks(value).length > 0) {
filter[field] = processStringSync(value, preparedCtx)
filter[field] = processStringSync(value, ctx)
}
}
}
@ -178,17 +177,6 @@ async function branchMatches(
return result.length > 0
}
function prepareContext(context: AutomationContext) {
return {
...context,
steps: {
...context.steps,
...context.stepsById,
...context.stepsByName,
},
}
}
async function enrichBaseContext(context: AutomationContext) {
context.env = await sdkUtils.getEnvironmentVariables()
@ -304,41 +292,37 @@ class Orchestrator {
}
hasErrored(context: AutomationContext): boolean {
const [_trigger, ...steps] = context.steps
for (const step of steps) {
if (step.success === false) {
return true
}
}
return false
return context._error === true
}
async execute(): Promise<AutomationResults> {
return await tracer.trace("execute", async span => {
span.addTags({ appId: this.appId, automationId: this.automation._id })
const job = cloneDeep(this.job)
delete job.data.event.appId
delete job.data.event.metadata
const data = cloneDeep(this.job.data)
delete data.event.appId
delete data.event.metadata
if (this.isCron() && !job.data.event.timestamp) {
job.data.event.timestamp = Date.now()
if (this.isCron() && !data.event.timestamp) {
data.event.timestamp = Date.now()
}
const trigger: AutomationTriggerResult = {
id: job.data.automation.definition.trigger.id,
stepId: job.data.automation.definition.trigger.stepId,
id: data.automation.definition.trigger.id,
stepId: data.automation.definition.trigger.stepId,
inputs: null,
outputs: job.data.event,
outputs: data.event,
}
const result: AutomationResults = { trigger, steps: [trigger] }
const ctx: AutomationContext = {
trigger: trigger.outputs,
steps: [trigger.outputs],
stepsById: {},
steps: { "0": trigger.outputs },
stepsByName: {},
stepsById: {},
user: trigger.outputs.user,
_error: false,
_stepIndex: 1,
}
await enrichBaseContext(ctx)
@ -348,7 +332,7 @@ class Orchestrator {
try {
await helpers.withTimeout(timeout, async () => {
const [stepOutputs, executionTime] = await utils.time(() =>
this.executeSteps(ctx, job.data.automation.definition.steps)
this.executeSteps(ctx, data.automation.definition.steps)
)
result.steps.push(...stepOutputs)
@ -400,9 +384,20 @@ class Orchestrator {
step: AutomationStep,
result: AutomationStepResult
) {
ctx.steps.push(result.outputs)
ctx.steps[step.id] = result.outputs
ctx.steps[step.name || step.id] = result.outputs
ctx.stepsById[step.id] = result.outputs
ctx.stepsByName[step.name || step.id] = result.outputs
ctx._stepIndex ||= 0
ctx.steps[ctx._stepIndex] = result.outputs
ctx._stepIndex++
if (result.outputs.success === false) {
ctx._error = true
}
results.push(result)
}
@ -449,7 +444,7 @@ class Orchestrator {
stepToLoop: AutomationStep
): Promise<AutomationStepResult> {
return await tracer.trace("executeLoopStep", async span => {
await processObject(step.inputs, prepareContext(ctx))
await processObject(step.inputs, ctx)
const maxIterations = getLoopMaxIterations(step)
const items: Record<string, any>[] = []
@ -561,7 +556,7 @@ class Orchestrator {
}
const inputs = automationUtils.cleanInputValues(
await processObject(cloneDeep(step.inputs), prepareContext(ctx)),
await processObject(cloneDeep(step.inputs), ctx),
step.schema.inputs.properties
)
@ -569,7 +564,7 @@ class Orchestrator {
inputs,
appId: this.appId,
emitter: this.emitter,
context: prepareContext(ctx),
context: ctx,
})
if (