MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into budi-8637-googlesheets-issues-automations-row-actions-and-filtering-2

This commit is contained in:
Sam Rose 2024-09-23 11:46:22 +01:00 committed by GitHub
parent 52fba580cb ab0a3ca918
commit 39736c57ca
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 135 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
packages/backend-core/src/configs/configs.ts
View File

@ -1,4 +1,5 @@
import {
AIConfig,
Config,
ConfigType,
GoogleConfig,
@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
return config?.config
}
// AI
export async function getAIConfig(): Promise<AIConfig | undefined> {
return getConfig<AIConfig>(ConfigType.AI)
}

5
packages/builder/src/pages/builder/portal/settings/ai/index.svelte
View File

@ -56,10 +56,13 @@
} else {
// We don't store the default BB AI config in the DB
delete fullAIConfig.config.budibase_ai
// unset the default value from other configs if default is set
if (editingAIConfig.isDefault) {
for (let key in fullAIConfig.config) {
fullAIConfig.config[key].isDefault = false
if (key !== id) {
fullAIConfig.config[key].isDefault = false
}
}
}
// Add new or update existing custom AI Config

2
packages/pro

@ -1 +1 @@
Subproject commit 922431260e90d558a1ca55398475412e75088057
Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733

2
packages/server/package.json
View File

@ -101,7 +101,7 @@
"mysql2": "3.9.8",
"node-fetch": "2.6.7",
"object-sizeof": "2.6.1",
"openai": "^4.52.1",
"openai": "4.59.0",
"openapi-types": "9.3.1",
"oracledb": "6.5.1",
"pg": "8.10.0",

61
packages/server/src/automations/steps/openai.ts
View File

@ -10,6 +10,7 @@ import {
} from "@budibase/types"
import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro"
enum Model {
GPT_35_TURBO = "gpt-3.5-turbo",
@ -62,19 +63,33 @@ export const definition: AutomationStepDefinition = {
},
}
/**
* Maintains backward compatibility with automation steps created before the introduction
* of custom configurations and Budibase AI
* @param inputs - automation inputs from the OpenAI automation step.
*/
async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
return completion?.choices[0]?.message?.content
}
export async function run({
inputs,
}: {
inputs: OpenAIStepInputs
}): Promise<OpenAIStepOutputs> {
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
if (inputs.prompt == null) {
return {
success: false,
@ -83,20 +98,24 @@ export async function run({
}
try {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
let response
const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
const response = completion?.choices[0]?.message?.content
if (budibaseAIEnabled || customConfigsEnabled) {
const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model)
response = await llm.run(inputs.prompt)
} else {
// fallback to the default that uses the environment variable for backwards compat
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
response = await legacyOpenAIPrompt(inputs)
}
return {
response,

34
packages/server/src/automations/tests/openai.spec.ts
View File

@ -4,6 +4,7 @@ import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import * as pro from "@budibase/pro"
jest.mock("openai", () => ({
OpenAI: jest.fn().mockImplementation(() => ({
@ -23,6 +24,20 @@ jest.mock("openai", () => ({
})),
}))
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: jest.fn().mockImplementation(() => ({
init: jest.fn(),
run: jest.fn(),
})),
},
features: {
isAICustomConfigsEnabled: jest.fn(),
isBudibaseAIEnabled: jest.fn(),
},
}))
const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
const OPENAI_PROMPT = "What is the meaning of life?"
@ -41,6 +56,7 @@ describe("test the openai action", () => {
afterEach(() => {
resetEnv()
jest.clearAllMocks()
})
afterAll(_afterAll)
@ -94,4 +110,22 @@ describe("test the openai action", () => {
)
expect(res.success).toBeFalsy()
})
it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => {
jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true)
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?"
await runStep("OPENAI", {
model: "gpt-4o-mini",
prompt,
})
expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini")
// @ts-ignore
const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value
expect(llmInstance.init).toHaveBeenCalled()
expect(llmInstance.run).toHaveBeenCalledWith(prompt)
})
})

2
packages/types/src/documents/global/config.ts
View File

@ -111,7 +111,7 @@ export interface SCIMInnerConfig {
export interface SCIMConfig extends Config<SCIMInnerConfig> {}
type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom"
export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom"
export interface AIInnerConfig {
[key: string]: {

35
packages/worker/src/api/controllers/global/configs.ts
View File

@ -253,6 +253,7 @@ export async function save(ctx: UserCtx<Config>) {
if (existingConfig) {
await verifyAIConfig(config, existingConfig)
}
await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length)
break
}
} catch (err: any) {
@ -334,32 +335,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
)
}
async function enrichAIConfig(aiConfig: AIConfig) {
// Strip out the API Keys from the response so they don't show in the UI
for (const key in aiConfig.config) {
if (aiConfig.config[key].apiKey) {
aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
}
}
// Return the Budibase AI data source as part of the response if licensing allows
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
const defaultConfigExists = Object.keys(aiConfig.config).some(
key => aiConfig.config[key].isDefault
)
if (budibaseAIEnabled) {
aiConfig.config["budibase_ai"] = {
provider: "OpenAI",
active: true,
isDefault: !defaultConfigExists,
defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "",
name: "Budibase AI",
}
}
return aiConfig
}
export async function find(ctx: UserCtx) {
try {
// Find the config with the most granular scope based on context
@ -372,7 +347,13 @@ export async function find(ctx: UserCtx) {
}
if (type === ConfigType.AI) {
await enrichAIConfig(scopedConfig)
await pro.sdk.ai.enrichAIConfig(scopedConfig)
// Strip out the API Keys from the response so they don't show in the UI
for (const key in scopedConfig.config) {
if (scopedConfig.config[key].apiKey) {
scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT
}
}
}
ctx.body = scopedConfig
} else {

50
packages/worker/src/api/controllers/global/tests/configs.spec.ts
View File

@ -1,4 +1,3 @@
import * as pro from "@budibase/pro"
import { verifyAIConfig } from "../configs"
import { TestConfiguration, structures } from "../../../../tests"
import { AIInnerConfig } from "@budibase/types"
@ -35,55 +34,6 @@ describe("Global configs controller", () => {
})
})
it("Should return the default BB AI config when the feature is turned on", async () => {
jest
.spyOn(pro.features, "isBudibaseAIEnabled")
.mockImplementation(() => Promise.resolve(true))
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)
const response = await config.api.configs.getAIConfig()
expect(response.body.config).toEqual({
budibase_ai: {
provider: "OpenAI",
active: true,
isDefault: true,
name: "Budibase AI",
defaultModel: "",
},
ai: {
active: true,
apiKey: "--secret-value--",
baseUrl: "https://api.example.com",
defaultModel: "gpt4",
isDefault: false,
name: "Test",
provider: "OpenAI",
},
})
})
it("Should not not return the default Budibase AI config when on self host", async () => {
jest
.spyOn(pro.features, "isBudibaseAIEnabled")
.mockImplementation(() => Promise.resolve(false))
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)
const response = await config.api.configs.getAIConfig()
expect(response.body.config).toEqual({
ai: {
active: true,
apiKey: "--secret-value--",
baseUrl: "https://api.example.com",
defaultModel: "gpt4",
isDefault: false,
name: "Test",
provider: "OpenAI",
},
})
})
it("Should not update existing secrets when updating an existing AI Config", async () => {
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)

64
yarn.lock
View File

@ -33,6 +33,19 @@
"@jridgewell/gen-mapping" "^0.3.5"
"@jridgewell/trace-mapping" "^0.3.24"
"@anthropic-ai/sdk@^0.27.3":
version "0.27.3"
resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475"
integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
abort-controller "^3.0.0"
agentkeepalive "^4.2.1"
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"
"@apidevtools/json-schema-ref-parser@^9.0.6":
version "9.1.2"
resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8"
@ -2053,7 +2066,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.6":
"@budibase/backend-core@2.32.5":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
@ -2134,14 +2147,14 @@
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.32.6"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f"
integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ==
version "2.32.5"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea"
integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA==
dependencies:
"@budibase/backend-core" "2.32.6"
"@budibase/shared-core" "2.32.6"
"@budibase/string-templates" "2.32.6"
"@budibase/types" "2.32.6"
"@budibase/backend-core" "2.32.5"
"@budibase/shared-core" "2.32.5"
"@budibase/string-templates" "2.32.5"
"@budibase/types" "2.32.5"
"@koa/router" "8.0.8"
bull "4.10.1"
dd-trace "5.2.0"
@ -2153,13 +2166,13 @@
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.6":
"@budibase/shared-core@2.32.5":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.32.6":
"@budibase/string-templates@2.32.5":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2180,7 @@
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.6":
"@budibase/types@2.32.5":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"
@ -6117,6 +6130,11 @@
resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
"@types/qs@^6.9.15":
version "6.9.16"
resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794"
integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==
"@types/range-parser@*":
version "1.2.4"
resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@ -12354,10 +12372,10 @@ google-p12-pem@^4.0.0:
dependencies:
node-forge "^1.3.1"
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5":
version "4.1.5"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f"
integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ==
"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3":
version "4.1.3"
resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad"
integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w==
dependencies:
axios "^1.4.0"
lodash "^4.17.21"
@ -17097,19 +17115,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
is-docker "^2.1.1"
is-wsl "^2.2.0"
openai@^4.52.1:
version "4.52.1"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
openai@4.59.0:
version "4.59.0"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"
integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
"@types/qs" "^6.9.15"
abort-controller "^3.0.0"
agentkeepalive "^4.2.1"
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"
web-streams-polyfill "^3.2.1"
qs "^6.10.3"
openapi-response-validator@^9.2.0:
version "9.3.1"
@ -22599,11 +22618,6 @@ web-streams-polyfill@4.0.0-beta.3:
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
web-streams-polyfill@^3.2.1:
version "3.3.3"
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b"
integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==
web-vitals@^4.0.1:
version "4.2.3"
resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7"