From 807ecf603966e81ecc9311478510f59324341904 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 22 Oct 2024 18:24:10 +0100 Subject: [PATCH 1/5] BB AI QA testing fixes --- .../builder/portal/settings/ai/index.svelte | 5 +- packages/pro | 2 +- packages/server/src/automations/actions.ts | 15 +++-- .../src/api/controllers/global/configs.ts | 57 ++++++++++++------- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 42de9f19ae..9a70fd289a 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -32,9 +32,7 @@ async function fetchAIConfig() { try { const aiDoc = await API.getConfig(ConfigTypes.AI) - if (aiDoc._id) { - fullAIConfig = aiDoc - } + fullAIConfig = aiDoc } catch (error) { notifications.error("Error fetching AI config") } @@ -66,6 +64,7 @@ } // Add new or update existing custom AI Config fullAIConfig.config[id] = editingAIConfig + fullAIConfig.type = ConfigTypes.AI } try { diff --git a/packages/pro b/packages/pro index 297fdc937e..7bd0350ed3 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 297fdc937e9c650b4964fc1a942b60022b195865 +Subproject commit 7bd0350ed3f55da30d07a7c67fd746727ce93a9a diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index e921e569c9..a6c9be41c5 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -18,6 +18,7 @@ import * as loop from "./steps/loop" import * as collect from "./steps/collect" import * as branch from "./steps/branch" import * as triggerAutomationRun from "./steps/triggerAutomationRun" +import * as openai from "./steps/openai" import env from "../environment" import { PluginType, @@ -89,21 +90,27 @@ export const BUILTIN_ACTION_DEFINITIONS: Record< // ran at all if (env.SELF_HOSTED) { const bash = require("./steps/bash") - const openai = require("./steps/openai") // @ts-ignore ACTION_IMPLS["EXECUTE_BASH"] = bash.run // @ts-ignore BUILTIN_ACTION_DEFINITIONS["EXECUTE_BASH"] = bash.definition - // @ts-ignore - ACTION_IMPLS.OPENAI = openai.run - BUILTIN_ACTION_DEFINITIONS.OPENAI = openai.definition } export async function getActionDefinitions() { if (await features.flags.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) { BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition } + if ( + env.SELF_HOSTED || + (await features.flags.isEnabled(FeatureFlag.BUDIBASE_AI)) || + (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) + ) { + BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition + // @ts-ignore + ACTION_IMPLS["OPENAI"] = openai.run + } + const actionDefinitions = BUILTIN_ACTION_DEFINITIONS if (env.SELF_HOSTED) { const plugins = await sdk.plugins.fetch(PluginType.AUTOMATION) diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index e6e80ff3a5..6c09da77de 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -35,6 +35,7 @@ import { AIInnerConfig, } from "@budibase/types" import * as pro from "@budibase/pro" +import { generateConfigID } from "@budibase/backend-core/src/configs" const getEventFns = async (config: Config, existing?: Config) => { const fns = [] @@ -44,9 +45,7 @@ const getEventFns = async (config: Config, existing?: Config) => { fns.push(events.email.SMTPCreated) } else if (isAIConfig(config)) { fns.push(() => events.ai.AIConfigCreated) - fns.push(() => - pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length) - ) + fns.push(() => pro.quotas.addCustomAIConfig) } else if (isGoogleConfig(config)) { fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE)) if (config.config.activated) { @@ -85,9 +84,6 @@ const getEventFns = async (config: Config, existing?: Config) => { fns.push(events.email.SMTPUpdated) } else if (isAIConfig(config)) { fns.push(() => events.ai.AIConfigUpdated) - fns.push(() => - pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length) - ) } else if (isGoogleConfig(config)) { fns.push(() => events.auth.SSOUpdated(ConfigType.GOOGLE)) if (!existing.config.activated && config.config.activated) { @@ -253,7 +249,7 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } - await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length) + await pro.quotas.addCustomAIConfig() break } } catch (err: any) { @@ -342,29 +338,43 @@ export async function find(ctx: UserCtx) { let scopedConfig = await configs.getConfig(type) if (scopedConfig) { - if (type === ConfigType.OIDC_LOGOS) { - enrichOIDCLogos(scopedConfig) - } - - if (type === ConfigType.AI) { - await pro.sdk.ai.enrichAIConfig(scopedConfig) - // Strip out the API Keys from the response so they don't show in the UI - for (const key in scopedConfig.config) { - if (scopedConfig.config[key].apiKey) { - scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT - } - } - } - ctx.body = scopedConfig + await handleConfigType(type, scopedConfig) + } else if (type === ConfigType.AI) { + scopedConfig = { config: {} } as AIConfig + await handleAIConfig(scopedConfig) } else { - // don't throw an error, there simply is nothing to return + // If no config found and not AI type, just return an empty body ctx.body = {} + return } + + ctx.body = scopedConfig } catch (err: any) { ctx.throw(err?.status || 400, err) } } +async function handleConfigType(type: ConfigType, config: Config) { + if (type === ConfigType.OIDC_LOGOS) { + enrichOIDCLogos(config) + } else if (type === ConfigType.AI) { + await handleAIConfig(config) + } +} + +async function handleAIConfig(config: AIConfig) { + await pro.sdk.ai.enrichAIConfig(config) + stripApiKeys(config) +} + +function stripApiKeys(config: AIConfig) { + for (const key in config?.config) { + if (config.config[key].apiKey) { + config.config[key].apiKey = PASSWORD_REPLACEMENT + } + } +} + export async function publicOidc(ctx: Ctx) { try { // Find the config with the most granular scope based on context @@ -508,6 +518,9 @@ export async function destroy(ctx: UserCtx) { try { await db.remove(id, rev) await cache.destroy(cache.CacheKey.CHECKLIST) + if (id === generateConfigID(ConfigType.AI)) { + await pro.quotas.removeCustomAIConfig() + } ctx.body = { message: "Config deleted successfully" } } catch (err: any) { ctx.throw(err.status, err) From 29ab9bef7df3728c20d4db9a68ae1bc043f359d0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 22 Oct 2024 18:31:07 +0100 Subject: [PATCH 2/5] fix lint --- packages/pro | 2 +- packages/worker/src/api/controllers/global/configs.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/pro b/packages/pro index 7bd0350ed3..1ca626aa65 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 7bd0350ed3f55da30d07a7c67fd746727ce93a9a +Subproject commit 1ca626aa65bdbddf940a69947be6654b533cb726 diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 6c09da77de..5d9768443b 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -35,7 +35,6 @@ import { AIInnerConfig, } from "@budibase/types" import * as pro from "@budibase/pro" -import { generateConfigID } from "@budibase/backend-core/src/configs" const getEventFns = async (config: Config, existing?: Config) => { const fns = [] @@ -518,7 +517,7 @@ export async function destroy(ctx: UserCtx) { try { await db.remove(id, rev) await cache.destroy(cache.CacheKey.CHECKLIST) - if (id === generateConfigID(ConfigType.AI)) { + if (id === configs.generateConfigID(ConfigType.AI)) { await pro.quotas.removeCustomAIConfig() } ctx.body = { message: "Config deleted successfully" } From 4f5e831b22473e0a9fd17d11b901d40dbbb58a58 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 22 Oct 2024 21:48:03 +0100 Subject: [PATCH 3/5] PR comments --- .../builder/src/pages/builder/portal/settings/ai/index.svelte | 3 +-- packages/worker/src/api/controllers/global/configs.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 9a70fd289a..bbdf46a24e 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -31,8 +31,7 @@ async function fetchAIConfig() { try { - const aiDoc = await API.getConfig(ConfigTypes.AI) - fullAIConfig = aiDoc + fullAIConfig = await API.getConfig(ConfigTypes.AI) } catch (error) { notifications.error("Error fetching AI config") } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 5d9768443b..750b02088c 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -44,7 +44,7 @@ const getEventFns = async (config: Config, existing?: Config) => { fns.push(events.email.SMTPCreated) } else if (isAIConfig(config)) { fns.push(() => events.ai.AIConfigCreated) - fns.push(() => pro.quotas.addCustomAIConfig) + fns.push(() => pro.quotas.addCustomAIConfig()) } else if (isGoogleConfig(config)) { fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE)) if (config.config.activated) { From 6541f6d31f384f90114537d9be9b01485e5b2125 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 22 Oct 2024 21:49:02 +0100 Subject: [PATCH 4/5] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1ca626aa65..6041196305 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1ca626aa65bdbddf940a69947be6654b533cb726 +Subproject commit 60411963053b98e0415a1e9285b721fc26c628c8 From 8f0a234ae3f36a4d66f331292c05da04c6160b4a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Oct 2024 11:13:52 +0100 Subject: [PATCH 5/5] fix tests --- packages/server/src/automations/actions.ts | 3 +-- packages/types/src/documents/app/automation/schema.ts | 8 ++++---- .../src/api/controllers/global/tests/configs.spec.ts | 4 ---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts index a6c9be41c5..a8b69fa7d7 100644 --- a/packages/server/src/automations/actions.ts +++ b/packages/server/src/automations/actions.ts @@ -51,6 +51,7 @@ const ACTION_IMPLS: ActionImplType = { QUERY_ROWS: queryRow.run, COLLECT: collect.run, TRIGGER_AUTOMATION_RUN: triggerAutomationRun.run, + OPENAI: openai.run, // these used to be lowercase step IDs, maintain for backwards compat discord: discord.run, slack: slack.run, @@ -107,8 +108,6 @@ export async function getActionDefinitions() { (await features.flags.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) ) { BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition - // @ts-ignore - ACTION_IMPLS["OPENAI"] = openai.run } const actionDefinitions = BUILTIN_ACTION_DEFINITIONS diff --git a/packages/types/src/documents/app/automation/schema.ts b/packages/types/src/documents/app/automation/schema.ts index 599256694d..b8a19b7b45 100644 --- a/packages/types/src/documents/app/automation/schema.ts +++ b/packages/types/src/documents/app/automation/schema.ts @@ -126,16 +126,16 @@ export type ActionImplementations = { n8nStepInputs, ExternalAppStepOutputs > + [AutomationActionStepId.OPENAI]: ActionImplementation< + OpenAIStepInputs, + OpenAIStepOutputs + > } & (T extends "self" ? { [AutomationActionStepId.EXECUTE_BASH]: ActionImplementation< BashStepInputs, BashStepOutputs > - [AutomationActionStepId.OPENAI]: ActionImplementation< - OpenAIStepInputs, - OpenAIStepOutputs - > } : {}) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 9091f29247..857f499dcc 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -13,10 +13,6 @@ describe("Global configs controller", () => { await config.afterAll() }) - afterEach(() => { - jest.resetAllMocks() - }) - it("Should strip secrets when pulling AI config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data)