Speed up server tests.

globalSetup.ts

@@ -88,6 +88,16 @@ export default async function setup() {
           content: `
           [log]
           level = warn
+
+          [httpd]
+          socket_options = [{nodelay, true}]
+
+          [couchdb]
+          single_node = true
+
+          [cluster]
+          n = 1
+          q = 1
         `,
           target: "/opt/couchdb/etc/local.d/test-couchdb.ini",
         },

packages/backend-core/src/queue/inMemoryQueue.ts

@@ -3,7 +3,6 @@ import { newid } from "../utils"
 import { Queue, QueueOptions, JobOptions } from "./queue"
 import { helpers } from "@budibase/shared-core"
 import { Job, JobId, JobInformation } from "bull"
-import { cloneDeep } from "lodash"
 
 function jobToJobInformation(job: Job): JobInformation {
   let cron = ""
@@ -89,7 +88,7 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
   async process(concurrencyOrFunc: number | any, func?: any) {
     func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc
     this._emitter.on("message", async msg => {
-      const message = cloneDeep(msg)
+      const message = msg
 
       // For the purpose of testing, don't trigger cron jobs immediately.
       // Require the test to trigger them manually with timestamps.
@@ -165,6 +164,9 @@ export class InMemoryQueue<T = any> implements Partial<Queue<T>> {
         opts,
       }
       this._messages.push(message)
+      if (this._messages.length > 1000) {
+        this._messages.shift()
+      }
       this._addCount++
       this._emitter.emit("message", message)
     }

packages/pro

@@ -1 +1 @@
-Subproject commit b28dbd549284cf450be7f25ad85aadf614d08f0b
+Subproject commit f8af563b6a78391d6e19fd0c94fc78724c27ee83

packages/server/src/api/routes/tests/row.spec.ts

@@ -166,17 +166,17 @@ if (descriptions.length) {
         )
       }
 
-      const resetRowUsage = async () => {
+      // const resetRowUsage = async () => {
-        await config.doInContext(
+      //   await config.doInContext(
-          undefined,
+      //     undefined,
-          async () =>
+      //     async () =>
-            await quotas.setUsage(
+      //       await quotas.setUsage(
-              0,
+      //         0,
-              StaticQuotaName.ROWS,
+      //         StaticQuotaName.ROWS,
-              QuotaUsageType.STATIC
+      //         QuotaUsageType.STATIC
-            )
+      //       )
-        )
+      //   )
-      }
+      // }
 
       const getRowUsage = async () => {
         const { total } = await config.doInContext(undefined, () =>
@@ -188,20 +188,30 @@ if (descriptions.length) {
         return total
       }
 
-      const assertRowUsage = async (expected: number) => {
+      async function expectRowUsage(expected: number, f: () => Promise<void>) {
-        const usage = await getRowUsage()
+        return await quotas.withEnabled(async () => {
+          const before = await getRowUsage()
+          await f()
+          const after = await getRowUsage()
+          const usage = after - before
 
           // Because our quota tracking is not perfect, we allow a 10% margin of
-        // error.  This is to account for the fact that parallel writes can result
+          // error.  This is to account for the fact that parallel writes can
-        // in some quota updates getting lost. We don't have any need to solve this
+          // result in some quota updates getting lost. We don't have any need
-        // right now, so we just allow for some error.
+          // to solve this right now, so we just allow for some error.
           if (expected === 0) {
             expect(usage).toEqual(0)
             return
           }
+          if (usage < 0) {
+            expect(usage).toBeGreaterThan(expected * 1.1)
+            expect(usage).toBeLessThan(expected * 0.9)
+          } else {
             expect(usage).toBeGreaterThan(expected * 0.9)
             expect(usage).toBeLessThan(expected * 1.1)
           }
+        })
+      }
 
       const defaultRowFields = isInternal
         ? {
@@ -216,28 +226,28 @@ if (descriptions.length) {
       })
 
       beforeEach(async () => {
-        await resetRowUsage()
+        // await resetRowUsage()
       })
 
       describe("create", () => {
         it("creates a new row successfully", async () => {
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(isInternal ? 1 : 0, async () => {
             const row = await config.api.row.save(table._id!, {
               name: "Test Contact",
             })
             expect(row.name).toEqual("Test Contact")
             expect(row._rev).toBeDefined()
-          await assertRowUsage(isInternal ? rowUsage + 1 : rowUsage)
+          })
         })
 
         it("fails to create a row for a table that does not exist", async () => {
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(0, async () => {
             await config.api.row.save("1234567", {}, { status: 404 })
-          await assertRowUsage(rowUsage)
+          })
         })
 
         it("fails to create a row if required fields are missing", async () => {
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(0, async () => {
             const table = await config.api.table.save(
               saveTableRequest({
                 schema: {
@@ -264,13 +274,12 @@ if (descriptions.length) {
                 },
               }
             )
-          await assertRowUsage(rowUsage)
+          })
         })
 
         isInternal &&
           it("increment row autoId per create row request", async () => {
-            const rowUsage = await getRowUsage()
+            await expectRowUsage(isInternal ? 10 : 0, async () => {
-
               const newTable = await config.api.table.save(
                 saveTableRequest({
                   schema: {
@@ -299,7 +308,7 @@ if (descriptions.length) {
                 expect(row["Row ID"]).toBeGreaterThan(previousId)
                 previousId = row["Row ID"]
               }
-            await assertRowUsage(isInternal ? rowUsage + 10 : rowUsage)
+            })
           })
 
         isInternal &&
@@ -985,8 +994,8 @@ if (descriptions.length) {
       describe("update", () => {
         it("updates an existing row successfully", async () => {
           const existing = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(0, async () => {
             const res = await config.api.row.save(table._id!, {
               _id: existing._id,
               _rev: existing._rev,
@@ -994,7 +1003,7 @@ if (descriptions.length) {
             })
 
             expect(res.name).toEqual("Updated Name")
-          await assertRowUsage(rowUsage)
+          })
         })
 
         !isInternal &&
@@ -1177,8 +1186,7 @@ if (descriptions.length) {
         it("should update only the fields that are supplied", async () => {
           const existing = await config.api.row.save(table._id!, {})
 
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(0, async () => {
-
             const row = await config.api.row.patch(table._id!, {
               _id: existing._id!,
               _rev: existing._rev!,
@@ -1193,7 +1201,7 @@ if (descriptions.length) {
 
             expect(savedRow.description).toEqual(existing.description)
             expect(savedRow.name).toEqual("Updated Name")
-          await assertRowUsage(rowUsage)
+          })
         })
 
         it("should update only the fields that are supplied and emit the correct oldRow", async () => {
@@ -1224,8 +1232,8 @@ if (descriptions.length) {
 
         it("should throw an error when given improper types", async () => {
           const existing = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(0, async () => {
             await config.api.row.patch(
               table._id!,
               {
@@ -1236,8 +1244,7 @@ if (descriptions.length) {
               },
               { status: 400 }
             )
-
+          })
-          await assertRowUsage(rowUsage)
         })
 
         it("should not overwrite links if those links are not set", async () => {
@@ -1452,25 +1459,25 @@ if (descriptions.length) {
 
         it("should be able to delete a row", async () => {
           const createdRow = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(isInternal ? -1 : 0, async () => {
             const res = await config.api.row.bulkDelete(table._id!, {
               rows: [createdRow],
             })
             expect(res[0]._id).toEqual(createdRow._id)
-          await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
+          })
         })
 
         it("should be able to delete a row with ID only", async () => {
           const createdRow = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(isInternal ? -1 : 0, async () => {
             const res = await config.api.row.bulkDelete(table._id!, {
               rows: [createdRow._id!],
             })
             expect(res[0]._id).toEqual(createdRow._id)
             expect(res[0].tableId).toEqual(table._id!)
-          await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
+          })
         })
 
         it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
@@ -1560,20 +1567,18 @@ if (descriptions.length) {
         })
 
         it("should return no errors on valid row", async () => {
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(0, async () => {
-
             const res = await config.api.row.validate(table._id!, {
               name: "ivan",
             })
 
             expect(res.valid).toBe(true)
             expect(Object.keys(res.errors)).toEqual([])
-          await assertRowUsage(rowUsage)
+          })
         })
 
         it("should errors on invalid row", async () => {
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(0, async () => {
-
             const res = await config.api.row.validate(table._id!, { name: 1 })
 
             if (isInternal) {
@@ -1584,7 +1589,7 @@ if (descriptions.length) {
               expect(res.valid).toBe(true)
               expect(Object.keys(res.errors)).toEqual([])
             }
-          await assertRowUsage(rowUsage)
+          })
         })
       })
 
@@ -1596,15 +1601,15 @@ if (descriptions.length) {
         it("should be able to delete a bulk set of rows", async () => {
           const row1 = await config.api.row.save(table._id!, {})
           const row2 = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(isInternal ? -2 : 0, async () => {
             const res = await config.api.row.bulkDelete(table._id!, {
               rows: [row1, row2],
             })
 
             expect(res.length).toEqual(2)
             await config.api.row.get(table._id!, row1._id!, { status: 404 })
-          await assertRowUsage(isInternal ? rowUsage - 2 : rowUsage)
+          })
         })
 
         it("should be able to delete a variety of row set types", async () => {
@@ -1613,41 +1618,42 @@ if (descriptions.length) {
             config.api.row.save(table._id!, {}),
             config.api.row.save(table._id!, {}),
           ])
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(isInternal ? -3 : 0, async () => {
             const res = await config.api.row.bulkDelete(table._id!, {
               rows: [row1, row2._id!, { _id: row3._id }],
             })
 
             expect(res.length).toEqual(3)
             await config.api.row.get(table._id!, row1._id!, { status: 404 })
-          await assertRowUsage(isInternal ? rowUsage - 3 : rowUsage)
+          })
         })
 
         it("should accept a valid row object and delete the row", async () => {
           const row1 = await config.api.row.save(table._id!, {})
-          const rowUsage = await getRowUsage()
 
-          const res = await config.api.row.delete(table._id!, row1 as DeleteRow)
+          await expectRowUsage(isInternal ? -1 : 0, async () => {
+            const res = await config.api.row.delete(
+              table._id!,
+              row1 as DeleteRow
+            )
 
             expect(res.id).toEqual(row1._id)
             await config.api.row.get(table._id!, row1._id!, { status: 404 })
-          await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
+          })
         })
 
         it.each([{ not: "valid" }, { rows: 123 }, "invalid"])(
           "should ignore malformed/invalid delete request: %s",
           async (request: any) => {
-            const rowUsage = await getRowUsage()
+            await expectRowUsage(0, async () => {
-
               await config.api.row.delete(table._id!, request, {
                 status: 400,
                 body: {
                   message: "Invalid delete rows request",
                 },
               })
-
+            })
-            await assertRowUsage(rowUsage)
           }
         )
       })
@@ -1733,8 +1739,7 @@ if (descriptions.length) {
             })
           )
 
-          const rowUsage = await getRowUsage()
+          await expectRowUsage(isInternal ? 2 : 0, async () => {
-
             await config.api.row.bulkImport(table._id!, {
               rows: [
                 {
@@ -1756,8 +1761,7 @@ if (descriptions.length) {
             expect(rows[0].description).toEqual("Row 1 description")
             expect(rows[1].name).toEqual("Row 2")
             expect(rows[1].description).toEqual("Row 2 description")
-
+          })
-          await assertRowUsage(isInternal ? rowUsage + 2 : rowUsage)
         })
 
         isInternal &&
@@ -1782,8 +1786,7 @@ if (descriptions.length) {
               description: "Existing description",
             })
 
-            const rowUsage = await getRowUsage()
+            await expectRowUsage(2, async () => {
-
               await config.api.row.bulkImport(table._id!, {
                 rows: [
                   {
@@ -1809,8 +1812,7 @@ if (descriptions.length) {
               expect(rows[1].description).toEqual("Row 2 description")
               expect(rows[2].name).toEqual("Updated existing row")
               expect(rows[2].description).toEqual("Existing description")
-
+            })
-            await assertRowUsage(rowUsage + 2)
           })
 
         isInternal &&
@@ -1835,8 +1837,7 @@ if (descriptions.length) {
               description: "Existing description",
             })
 
-            const rowUsage = await getRowUsage()
+            await expectRowUsage(3, async () => {
-
               await config.api.row.bulkImport(table._id!, {
                 rows: [
                   {
@@ -1863,8 +1864,7 @@ if (descriptions.length) {
               expect(rows[2].description).toEqual("Row 2 description")
               expect(rows[3].name).toEqual("Updated existing row")
               expect(rows[3].description).toEqual("Existing description")
-
+            })
-            await assertRowUsage(rowUsage + 3)
           })
 
         // Upserting isn't yet supported in MSSQL / Oracle, see:
@@ -2187,8 +2187,8 @@ if (descriptions.length) {
               return { linkedTable, firstRow, secondRow }
             }
           )
-          const rowUsage = await getRowUsage()
 
+          await expectRowUsage(0, async () => {
             // test basic enrichment
             const resBasic = await config.api.row.get(
               linkedTable._id!,
@@ -2209,7 +2209,7 @@ if (descriptions.length) {
             expect(resEnriched.link[0]._id).toBe(firstRow._id)
             expect(resEnriched.link[0].name).toBe("Test Contact")
             expect(resEnriched.link[0].description).toBe("original description")
-          await assertRowUsage(rowUsage)
+          })
         })
       })
 

packages/server/src/automations/tests/steps/loop.spec.ts

@@ -13,24 +13,26 @@ import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
 import TestConfiguration from "../../../tests/utilities/TestConfiguration"
 
 describe("Attempt to run a basic loop automation", () => {
-  const config = new TestConfiguration()
+  let config: TestConfiguration
   let table: Table
 
   beforeAll(async () => {
-    await config.init()
     await automation.init()
   })
 
   beforeEach(async () => {
-    await config.api.automation.deleteAll()
+    config = new TestConfiguration()
+    await config.init()
 
     table = await config.api.table.save(basicTable())
     await config.api.row.save(table._id!, {})
   })
+  afterEach(async () => {
+    config.end()
+  })
 
   afterAll(async () => {
     await automation.shutdown()
-    config.end()
   })
 
   it("attempt to run a basic loop", async () => {

packages/server/src/automations/tests/steps/openai.spec.ts

@@ -44,11 +44,13 @@ describe("test the openai action", () => {
   }
 
   const expectAIUsage = async <T>(expected: number, f: () => Promise<T>) => {
+    return await quotas.withEnabled(async () => {
       const before = await getAIUsage()
       const result = await f()
       const after = await getAIUsage()
       expect(after - before).toEqual(expected)
       return result
+    })
   }
 
   it("should be able to receive a response from ChatGPT given a prompt", async () => {

packages/server/src/definitions/automations.ts

@@ -20,9 +20,12 @@ export interface TriggerOutput {
 
 export interface AutomationContext {
   trigger: AutomationTriggerResultOutputs
-  steps: [AutomationTriggerResultOutputs, ...AutomationStepResultOutputs[]]
+  steps: Record<
-  stepsById: Record<string, AutomationStepResultOutputs>
+    string,
+    AutomationStepResultOutputs | AutomationTriggerResultOutputs
+  >
   stepsByName: Record<string, AutomationStepResultOutputs>
+  stepsById: Record<string, AutomationStepResultOutputs>
   env?: Record<string, string>
   user?: UserBindings
   settings?: {
@@ -31,4 +34,6 @@ export interface AutomationContext {
     company?: string
   }
   loop?: { currentItem: any }
+  _stepIndex: number
+  _error: boolean
 }

packages/server/src/events/AutomationEmitter.ts

@@ -32,6 +32,8 @@ class AutomationEmitter implements ContextEmitter {
 
     if (chainAutomations === true) {
       return MAX_AUTOMATIONS_ALLOWED
+    } else if (env.isTest()) {
+      return 0
     } else if (chainAutomations === undefined && env.SELF_HOSTED) {
       return MAX_AUTOMATIONS_ALLOWED
     } else {

packages/server/src/tests/jestSetup.ts

@@ -3,6 +3,7 @@ import * as matchers from "jest-extended"
 import { env as coreEnv, timers } from "@budibase/backend-core"
 import { testContainerUtils } from "@budibase/backend-core/tests"
 import nock from "nock"
+import { quotas } from "@budibase/pro"
 
 expect.extend(matchers)
 if (!process.env.CI) {
@@ -23,6 +24,10 @@ nock.enableNetConnect(host => {
 
 testContainerUtils.setupEnv(env, coreEnv)
 
-afterAll(() => {
+beforeAll(async () => {
+  quotas.disable()
+})
+
+afterAll(async () => {
   timers.cleanup()
 })

packages/server/src/tests/utilities/api/base.ts

@@ -146,8 +146,9 @@ export abstract class TestAPI {
       }
     }
 
+    let resp: Response | undefined = undefined
     try {
-      return await req
+      resp = await req
     } catch (e: any) {
       // We've found that occasionally the connection between supertest and the
       // server supertest starts gets reset. Not sure why, but retrying it
@@ -161,6 +162,7 @@ export abstract class TestAPI {
       }
       throw e
     }
+    return resp
   }
 
   protected async getHeaders(

packages/server/src/threads/automation.ts

@@ -143,7 +143,6 @@ async function branchMatches(
   branch: Readonly<Branch>
 ): Promise<boolean> {
   const toFilter: Record<string, any> = {}
-  const preparedCtx = prepareContext(ctx)
 
   // Because we allow bindings on both the left and right of each condition in
   // automation branches, we can't pass the BranchSearchFilters directly to
@@ -160,9 +159,9 @@ async function branchMatches(
         filter.conditions = filter.conditions.map(evaluateBindings)
       } else {
         for (const [field, value] of Object.entries(filter)) {
-          toFilter[field] = processStringSync(field, preparedCtx)
+          toFilter[field] = processStringSync(field, ctx)
           if (typeof value === "string" && findHBSBlocks(value).length > 0) {
-            filter[field] = processStringSync(value, preparedCtx)
+            filter[field] = processStringSync(value, ctx)
           }
         }
       }
@@ -178,17 +177,6 @@ async function branchMatches(
   return result.length > 0
 }
 
-function prepareContext(context: AutomationContext) {
-  return {
-    ...context,
-    steps: {
-      ...context.steps,
-      ...context.stepsById,
-      ...context.stepsByName,
-    },
-  }
-}
-
 async function enrichBaseContext(context: AutomationContext) {
   context.env = await sdkUtils.getEnvironmentVariables()
 
@@ -304,41 +292,44 @@ class Orchestrator {
   }
 
   hasErrored(context: AutomationContext): boolean {
-    const [_trigger, ...steps] = context.steps
+    return context._error === true
-    for (const step of steps) {
+    // const [_trigger, ...steps] = context.steps
-      if (step.success === false) {
+    // for (const step of steps) {
-        return true
+    //   if (step.success === false) {
-      }
+    //     return true
-    }
+    //   }
-    return false
+    // }
+    // return false
   }
 
   async execute(): Promise<AutomationResults> {
     return await tracer.trace("execute", async span => {
       span.addTags({ appId: this.appId, automationId: this.automation._id })
 
-      const job = cloneDeep(this.job)
+      const data = cloneDeep(this.job.data)
-      delete job.data.event.appId
+      delete data.event.appId
-      delete job.data.event.metadata
+      delete data.event.metadata
 
-      if (this.isCron() && !job.data.event.timestamp) {
+      if (this.isCron() && !data.event.timestamp) {
-        job.data.event.timestamp = Date.now()
+        data.event.timestamp = Date.now()
       }
 
       const trigger: AutomationTriggerResult = {
-        id: job.data.automation.definition.trigger.id,
+        id: data.automation.definition.trigger.id,
-        stepId: job.data.automation.definition.trigger.stepId,
+        stepId: data.automation.definition.trigger.stepId,
         inputs: null,
-        outputs: job.data.event,
+        outputs: data.event,
       }
       const result: AutomationResults = { trigger, steps: [trigger] }
 
       const ctx: AutomationContext = {
         trigger: trigger.outputs,
-        steps: [trigger.outputs],
+        steps: { "0": trigger.outputs },
-        stepsById: {},
         stepsByName: {},
+        stepsById: {},
         user: trigger.outputs.user,
+        _error: false,
+        _stepIndex: 1,
       }
       await enrichBaseContext(ctx)
 
@@ -348,7 +339,7 @@ class Orchestrator {
       try {
         await helpers.withTimeout(timeout, async () => {
           const [stepOutputs, executionTime] = await utils.time(() =>
-            this.executeSteps(ctx, job.data.automation.definition.steps)
+            this.executeSteps(ctx, data.automation.definition.steps)
           )
 
           result.steps.push(...stepOutputs)
@@ -400,9 +391,20 @@ class Orchestrator {
         step: AutomationStep,
         result: AutomationStepResult
       ) {
-        ctx.steps.push(result.outputs)
+        ctx.steps[step.id] = result.outputs
+        ctx.steps[step.name || step.id] = result.outputs
+
         ctx.stepsById[step.id] = result.outputs
         ctx.stepsByName[step.name || step.id] = result.outputs
+
+        ctx._stepIndex ||= 0
+        ctx.steps[ctx._stepIndex] = result.outputs
+        ctx._stepIndex++
+
+        if (result.outputs.success === false) {
+          ctx._error = true
+        }
+
         results.push(result)
       }
 
@@ -449,7 +451,7 @@ class Orchestrator {
     stepToLoop: AutomationStep
   ): Promise<AutomationStepResult> {
     return await tracer.trace("executeLoopStep", async span => {
-      await processObject(step.inputs, prepareContext(ctx))
+      await processObject(step.inputs, ctx)
 
       const maxIterations = getLoopMaxIterations(step)
       const items: Record<string, any>[] = []
@@ -558,7 +560,7 @@ class Orchestrator {
       }
 
       const inputs = automationUtils.cleanInputValues(
-        await processObject(cloneDeep(step.inputs), prepareContext(ctx)),
+        await processObject(cloneDeep(step.inputs), ctx),
         step.schema.inputs.properties
       )
 
@@ -566,7 +568,7 @@ class Orchestrator {
         inputs,
         appId: this.appId,
         emitter: this.emitter,
-        context: prepareContext(ctx),
+        context: ctx,
       })
 
       if (