From 97b1c41313ce88a3a00eddf6981c11038b71590f Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 11:36:59 +0000 Subject: [PATCH 1/5] update openai automation tests --- packages/server/src/automations/actions.ts | 5 ++ .../src/automations/tests/openai.spec.ts | 71 ++++++++++++++----- .../tests/utilities/AutomationTestBuilder.ts | 13 ++++ .../app/automation/StepInputsOutputs.ts | 2 +- 4 files changed, 74 insertions(+), 17 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a8b69fa7d7..edac2ed690 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -96,6 +96,11 @@ if (env.SELF_HOSTED) { ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition + + if (env.isTest()) { + ACTION_IMPLS["OPENAI"] = openai.run + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + } } export async function getActionDefinitions() { diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 8119750f8b..a4d98fc9d3 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -1,7 +1,9 @@ -import { getConfig, runStep, afterAll as _afterAll } from "./utilities" +import { getConfig, afterAll as _afterAll } from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" import { OpenAI } from "openai" import { setEnv as setCoreEnv } from "@budibase/backend-core" import * as pro from "@budibase/pro" +import { Model } from "@budibase/types" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -47,6 +49,7 @@ describe("test the openai action", () => { let resetEnv: () => void | undefined beforeAll(async () => { + setCoreEnv({ SELF_HOSTED: true }) await config.init() }) @@ -62,17 +65,39 @@ describe("test the openai action", () => { afterAll(_afterAll) it("should be able to receive a response from ChatGPT given a prompt", async () => { - const res = await runStep(config, "OPENAI", { prompt: OPENAI_PROMPT }) - expect(res.response).toEqual("This is a test") - expect(res.success).toBeTruthy() + setCoreEnv({ SELF_HOSTED: true }) + + const result = await createAutomationBuilder({ + name: "Test OpenAI Response", + config, + }) + .appAction({ fields: {} }) + .openai( + { prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI }, + { stepName: "Basic OpenAI Query" } + ) + .run() + + expect(result.steps[0].outputs.response).toEqual("This is a test") + expect(result.steps[0].outputs.success).toBeTruthy() }) it("should present the correct error message when a prompt is not provided", async () => { - const res = await runStep(config, "OPENAI", { prompt: null }) - expect(res.response).toEqual( + const result = await createAutomationBuilder({ + name: "Test OpenAI No Prompt", + config, + }) + .appAction({ fields: {} }) + .openai( + { prompt: "", model: Model.GPT_4O_MINI }, + { stepName: "Empty Prompt Query" } + ) + .run() + + expect(result.steps[0].outputs.response).toEqual( "Budibase OpenAI Automation Failed: No prompt supplied" ) - expect(res.success).toBeFalsy() + expect(result.steps[0].outputs.success).toBeFalsy() }) it("should present the correct error message when an error is thrown from the createChatCompletion call", async () => { @@ -91,14 +116,21 @@ describe("test the openai action", () => { } as any) ) - const res = await runStep(config, "OPENAI", { - prompt: OPENAI_PROMPT, + const result = await createAutomationBuilder({ + name: "Test OpenAI Error", + config, }) + .appAction({ fields: {} }) + .openai( + { prompt: OPENAI_PROMPT, model: Model.GPT_4O_MINI }, + { stepName: "Error Producing Query" } + ) + .run() - expect(res.response).toEqual( + expect(result.steps[0].outputs.response).toEqual( "Error: An error occurred while calling createChatCompletion" ) - expect(res.success).toBeFalsy() + expect(result.steps[0].outputs.success).toBeFalsy() }) it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { @@ -106,10 +138,19 @@ describe("test the openai action", () => { jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) const prompt = "What is the meaning of life?" - await runStep(config, "OPENAI", { - model: "gpt-4o-mini", - prompt, + await createAutomationBuilder({ + name: "Test OpenAI Pro Features", + config, }) + .appAction({ fields: {} }) + .openai( + { + model: Model.GPT_4O_MINI, + prompt, + }, + { stepName: "Pro Features Query" } + ) + .run() expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( "gpt-4o-mini" @@ -117,8 +158,6 @@ describe("test the openai action", () => { const llmInstance = mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value - // init does not appear to be called currently - // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 78830adf2c..0c091ea00b 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -35,6 +35,7 @@ import { Branch, FilterStepInputs, ExecuteScriptStepInputs, + OpenAIStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -221,6 +222,18 @@ class BaseStepBuilder { input ) } + + openai( + input: OpenAIStepInputs, + opts?: { stepName?: string; stepId?: string } + ): this { + return this.step( + AutomationActionStepId.OPENAI, + BUILTIN_ACTION_DEFINITIONS.OPENAI, + input, + opts + ) + } } class StepBuilder extends BaseStepBuilder { build(): AutomationStep[] { diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts index 3aadb77108..b2f679edee 100644 --- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts +++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts @@ -150,7 +150,7 @@ export type OpenAIStepInputs = { prompt: string model: Model } -enum Model { +export enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API GPT_4 = "gpt-4", From 3ef854085702fb012540bf28644f1d754e2efe79 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 11:52:00 +0000 Subject: [PATCH 2/5] migrate queryrows --- .../src/automations/tests/queryRows.spec.ts | 200 ++++++++++-------- 1 file changed, 114 insertions(+), 86 deletions(-) diff --git a/packages/server/src/automations/tests/queryRows.spec.ts b/packages/server/src/automations/tests/queryRows.spec.ts index 12611d3f90..18d2e2d6cd 100644 --- a/packages/server/src/automations/tests/queryRows.spec.ts +++ b/packages/server/src/automations/tests/queryRows.spec.ts @@ -1,5 +1,7 @@ -import { Table } from "@budibase/types" +import { EmptyFilterOption, SortOrder, Table } from "@budibase/types" import * as setup from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" +import * as automation from "../index" const NAME = "Test" @@ -8,6 +10,7 @@ describe("Test a query step automation", () => { let config = setup.getConfig() beforeAll(async () => { + await automation.init() await config.init() table = await config.createTable() const row = { @@ -22,107 +25,132 @@ describe("Test a query step automation", () => { afterAll(setup.afterAll) it("should be able to run the query step", async () => { - const inputs = { - tableId: table._id, - filters: { - equal: { - name: NAME, - }, - }, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Basic Query Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) - expect(res.rows[0].name).toBe(NAME) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: { + equal: { + name: NAME, + }, + }, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query All Rows" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) + expect(result.steps[0].outputs.rows[0].name).toBe(NAME) }) it("Returns all rows when onEmptyFilter has no value and no filters are passed", async () => { - const inputs = { - tableId: table._id, - filters: {}, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Empty Filter Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) - expect(res.rows[0].name).toBe(NAME) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Empty Filter" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) + expect(result.steps[0].outputs.rows[0].name).toBe(NAME) }) it("Returns no rows when onEmptyFilter is RETURN_NONE and theres no filters", async () => { - const inputs = { - tableId: table._id, - filters: {}, - "filters-def": [], - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - onEmptyFilter: "none", - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Return None Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(false) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(0) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + "filters-def": [], + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + }, + { stepName: "Query With Return None" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(false) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(0) }) it("Returns no rows when onEmptyFilters RETURN_NONE and a filter is passed with a null value", async () => { - const inputs = { - tableId: table._id, - onEmptyFilter: "none", - filters: {}, - "filters-def": [ - { - value: null, - }, - ], - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Null Filter Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(false) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(0) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + onEmptyFilter: EmptyFilterOption.RETURN_NONE, + filters: {}, + "filters-def": [ + { + value: null, + }, + ], + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Null Filter" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(false) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(0) }) it("Returns rows when onEmptyFilter is RETURN_ALL and no filter is passed", async () => { - const inputs = { - tableId: table._id, - onEmptyFilter: "all", - filters: {}, - sortColumn: "name", - sortOrder: "ascending", - limit: 10, - } - const res = await setup.runStep( + const result = await createAutomationBuilder({ + name: "Return All Test", config, - setup.actions.QUERY_ROWS.stepId, - inputs - ) - expect(res.success).toBe(true) - expect(res.rows).toBeDefined() - expect(res.rows.length).toBe(2) + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + onEmptyFilter: EmptyFilterOption.RETURN_ALL, + filters: {}, + sortColumn: "name", + sortOrder: SortOrder.ASCENDING, + limit: 10, + }, + { stepName: "Query With Return All" } + ) + .run() + + expect(result.steps[0].outputs.success).toBe(true) + expect(result.steps[0].outputs.rows).toBeDefined() + expect(result.steps[0].outputs.rows.length).toBe(2) }) }) From bb254292d9c2f943eae8ee74f7781f316d52dde3 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 28 Nov 2024 15:25:27 +0000 Subject: [PATCH 3/5] migrate bash tests and actually make some useful automations with them --- .../server/src/automations/tests/bash.spec.ts | 152 ++++++++++++++++-- .../tests/utilities/AutomationTestBuilder.ts | 13 ++ 2 files changed, 150 insertions(+), 15 deletions(-) diff --git a/packages/server/src/automations/tests/bash.spec.ts b/packages/server/src/automations/tests/bash.spec.ts index 472d1092d6..12ed784268 100644 --- a/packages/server/src/automations/tests/bash.spec.ts +++ b/packages/server/src/automations/tests/bash.spec.ts @@ -1,26 +1,148 @@ -import { getConfig, afterAll as _afterAll, runStep } from "./utilities" +import { createAutomationBuilder } from "./utilities/AutomationTestBuilder" +import * as automation from "../index" +import * as setup from "./utilities" +import { Table } from "@budibase/types" -describe("test the bash action", () => { - let config = getConfig() +describe("Execute Bash Automations", () => { + let config = setup.getConfig(), + table: Table beforeAll(async () => { + await automation.init() await config.init() - }) - afterAll(_afterAll) - - it("should be able to execute a script", async () => { - let res = await runStep(config, "EXECUTE_BASH", { - code: "echo 'test'", + table = await config.createTable() + await config.createRow({ + name: "test row", + description: "test description", + tableId: table._id!, }) - expect(res.stdout).toEqual("test\n") - expect(res.success).toEqual(true) }) - it("should handle a null value", async () => { - let res = await runStep(config, "EXECUTE_BASH", { - code: null, + afterAll(setup.afterAll) + + it("should use trigger data in bash command and pass output to subsequent steps", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Trigger Data", + config, }) - expect(res.stdout).toEqual( + .appAction({ fields: { command: "hello world" } }) + .bash( + { code: "echo '{{ trigger.fields.command }}'" }, + { stepName: "Echo Command" } + ) + .serverLog( + { text: "Bash output was: {{ steps.[Echo Command].stdout }}" }, + { stepName: "Log Output" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toEqual("hello world\n") + expect(result.steps[1].outputs.message).toContain( + "Bash output was: hello world" + ) + }) + + it("should chain multiple bash commands using previous outputs", async () => { + const result = await createAutomationBuilder({ + name: "Chained Bash Commands", + config, + }) + .appAction({ fields: { filename: "testfile.txt" } }) + .bash( + { code: "echo 'initial content' > {{ trigger.fields.filename }}" }, + { stepName: "Create File" } + ) + .bash( + { code: "cat {{ trigger.fields.filename }} | tr '[a-z]' '[A-Z]'" }, + { stepName: "Transform Content" } + ) + .bash( + { code: "rm {{ trigger.fields.filename }}" }, + { stepName: "Cleanup" } + ) + .run() + + expect(result.steps[1].outputs.stdout).toEqual("INITIAL CONTENT\n") + expect(result.steps[1].outputs.success).toEqual(true) + }) + + it("should integrate bash output with row operations", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Row Operations", + config, + }) + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + filters: {}, + }, + { stepName: "Get Row" } + ) + .bash( + { + code: "echo Row data: {{ steps.[Get Row].rows.[0].name }} - {{ steps.[Get Row].rows.[0].description }}", + }, + { stepName: "Process Row Data" } + ) + .serverLog( + { text: "{{ steps.[Process Row Data].stdout }}" }, + { stepName: "Log Result" } + ) + .run() + + expect(result.steps[1].outputs.stdout).toContain( + "Row data: test row - test description" + ) + expect(result.steps[2].outputs.message).toContain( + "Row data: test row - test description" + ) + }) + + it("should handle bash output in conditional logic", async () => { + const result = await createAutomationBuilder({ + name: "Bash with Conditional", + config, + }) + .appAction({ fields: { threshold: "5" } }) + .bash( + { code: "echo $(( {{ trigger.fields.threshold }} + 5 ))" }, + { stepName: "Calculate Value" } + ) + .executeScript( + { + code: ` + const value = parseInt(steps["Calculate Value"].stdout); + return value > 8 ? "high" : "low"; + `, + }, + { stepName: "Check Value" } + ) + .serverLog( + { text: "Value was {{ steps.[Check Value].value }}" }, + { stepName: "Log Result" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toEqual("10\n") + expect(result.steps[1].outputs.value).toEqual("high") + expect(result.steps[2].outputs.message).toContain("Value was high") + }) + + it("should handle null values gracefully", async () => { + const result = await createAutomationBuilder({ + name: "Null Bash Input", + config, + }) + .appAction({ fields: {} }) + .bash( + //@ts-ignore + { code: null }, + { stepName: "Null Command" } + ) + .run() + + expect(result.steps[0].outputs.stdout).toBe( "Budibase bash automation failed: Invalid inputs" ) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 0c091ea00b..7a0d489f80 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -36,6 +36,7 @@ import { FilterStepInputs, ExecuteScriptStepInputs, OpenAIStepInputs, + BashStepInputs, } from "@budibase/types" import TestConfiguration from "../../../tests/utilities/TestConfiguration" import * as setup from "../utilities" @@ -223,6 +224,18 @@ class BaseStepBuilder { ) } + bash( + input: BashStepInputs, + opts?: { stepName?: string; stepId?: string } + ): this { + return this.step( + AutomationActionStepId.EXECUTE_BASH, + BUILTIN_ACTION_DEFINITIONS.EXECUTE_BASH, + input, + opts + ) + } + openai( input: OpenAIStepInputs, opts?: { stepName?: string; stepId?: string } From 7d0ae9d567ee58914c8f9e584c6205297d196add Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 3 Dec 2024 10:11:57 +0000 Subject: [PATCH 4/5] no need to add run func to definition --- packages/server/src/automations/actions.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index edac2ed690..0a80714032 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -98,7 +98,6 @@ if (env.SELF_HOSTED) { BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition if (env.isTest()) { - ACTION_IMPLS["OPENAI"] = openai.run BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition } } From 10dbb9fa7aa6bfae1b9edfa9b63567d8aa50898d Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 3 Dec 2024 10:33:18 +0000 Subject: [PATCH 5/5] re-add removed comments --- packages/server/src/automations/tests/openai.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index a4d98fc9d3..9f2bc50599 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -158,6 +158,8 @@ describe("test the openai action", () => { const llmInstance = mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) })