diff --git a/package.json b/package.json index d083dbad90..0bf3c29b2b 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,6 @@ "@budibase/shared-core": "*", "@budibase/string-templates": "*", "@budibase/types": "*", - "@budibase/pro": "npm:@budibase/pro@latest", "tough-cookie": "4.1.3", "node-fetch": "2.6.7", "semver": "7.5.3", diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 9a46758062..0a5fcc0707 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -163,6 +163,7 @@ const environment = { ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app", ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "", + BUDICLOUD_URL: process.env.BUDICLOUD_URL || "https://budibase.app", DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SELF_HOSTED: selfHosted, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 677feed678..ac3c9e30ab 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -261,9 +261,13 @@ export class UserDB { } } - const change = dbUser ? 0 : 1 // no change if there is existing user - + let change = 1 let creatorsChange = 0 + if (opts.isAccountHolder || dbUser) { + change = 0 + creatorsChange = 1 + } + if (dbUser) { const [isDbUserCreator, isUserCreator] = await creatorsInList([ dbUser, diff --git a/packages/builder/src/pages/builder/app/[application]/settings/oauth2/OAuth2ConfigModalContent.svelte b/packages/builder/src/pages/builder/app/[application]/settings/oauth2/OAuth2ConfigModalContent.svelte index 21df65be12..f215b58869 100644 --- a/packages/builder/src/pages/builder/app/[application]/settings/oauth2/OAuth2ConfigModalContent.svelte +++ b/packages/builder/src/pages/builder/app/[application]/settings/oauth2/OAuth2ConfigModalContent.svelte @@ -15,6 +15,7 @@ import type { InsertOAuth2ConfigRequest } from "@budibase/types" import { OAuth2CredentialsMethod, + OAuth2GrantType, PASSWORD_REPLACEMENT, } from "@budibase/types" import type { ZodType } from "zod" @@ -27,6 +28,8 @@ $: data = (config as Partial) ?? {} + $: data.grantType ??= OAuth2GrantType.CLIENT_CREDENTIALS + $: isCreation = !config $: title = isCreation ? "Create new OAuth2 connection" @@ -64,6 +67,9 @@ method: z.nativeEnum(OAuth2CredentialsMethod, { message: "Authentication method is required.", }), + grantType: z.nativeEnum(OAuth2GrantType, { + message: "Grant type is required.", + }), }) satisfies ZodType const validationResult = validator.safeParse(config) @@ -97,6 +103,7 @@ clientId: configData.clientId, clientSecret: configData.clientSecret, method: configData.method, + grantType: configData.grantType, }) if (!connectionValidation.valid) { let message = "Connection settings could not be validated" @@ -158,6 +165,24 @@ access_token property. + + { clientId: c.clientId, clientSecret: c.clientSecret, method: c.method, + grantType: c.grantType, lastUsage: c.lastUsage, })), loading: false, diff --git a/packages/pro b/packages/pro index f709bb6a07..761ec71e15 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f709bb6a07483785c32ebb6f186709450d735ec3 +Subproject commit 761ec71e1543ef04887d6515f99a2c2911999ebf diff --git a/packages/server/package.json b/packages/server/package.json index 38dd746c4f..ee89218c52 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -182,6 +182,9 @@ "yargs": "^13.2.4", "zod": "^3.23.8" }, + "resolutions": { + "@budibase/pro": "npm:@budibase/pro@latest" + }, "nx": { "targets": { "dev": { diff --git a/packages/server/src/api/controllers/oauth2.ts b/packages/server/src/api/controllers/oauth2.ts index 7d44d7bed6..7c4a145655 100644 --- a/packages/server/src/api/controllers/oauth2.ts +++ b/packages/server/src/api/controllers/oauth2.ts @@ -24,6 +24,7 @@ function toFetchOAuth2ConfigsResponse( clientId: config.clientId, clientSecret: PASSWORD_REPLACEMENT, method: config.method, + grantType: config.grantType, } } @@ -54,6 +55,7 @@ export async function create( clientId: body.clientId, clientSecret: body.clientSecret, method: body.method, + grantType: body.grantType, } const config = await sdk.oauth2.create(newConfig) @@ -80,6 +82,7 @@ export async function edit( clientId: body.clientId, clientSecret: body.clientSecret, method: body.method, + grantType: body.grantType, } const config = await sdk.oauth2.update(toUpdate) @@ -104,6 +107,7 @@ export async function validate( clientId: body.clientId, clientSecret: body.clientSecret, method: body.method, + grantType: body.grantType, } if (config.clientSecret === PASSWORD_REPLACEMENT && body._id) { diff --git a/packages/server/src/api/routes/oauth2.ts b/packages/server/src/api/routes/oauth2.ts index 64b4f66248..fd40c1a5f9 100644 --- a/packages/server/src/api/routes/oauth2.ts +++ b/packages/server/src/api/routes/oauth2.ts @@ -1,5 +1,9 @@ import Router from "@koa/router" -import { OAuth2CredentialsMethod, PermissionType } from "@budibase/types" +import { + OAuth2CredentialsMethod, + OAuth2GrantType, + PermissionType, +} from "@budibase/types" import { middleware } from "@budibase/backend-core" import authorized from "../../middleware/authorized" @@ -13,6 +17,9 @@ const baseSchema = { method: Joi.string() .required() .valid(...Object.values(OAuth2CredentialsMethod)), + grantType: Joi.string() + .required() + .valid(...Object.values(OAuth2GrantType)), } const insertSchema = Joi.object({ diff --git a/packages/server/src/api/routes/tests/oauth2.spec.ts b/packages/server/src/api/routes/tests/oauth2.spec.ts index 7acd0cbdb2..ee01acec1d 100644 --- a/packages/server/src/api/routes/tests/oauth2.spec.ts +++ b/packages/server/src/api/routes/tests/oauth2.spec.ts @@ -3,6 +3,7 @@ import { InsertOAuth2ConfigRequest, OAuth2ConfigResponse, OAuth2CredentialsMethod, + OAuth2GrantType, PASSWORD_REPLACEMENT, } from "@budibase/types" import * as setup from "./utilities" @@ -19,6 +20,7 @@ describe("/oauth2", () => { clientId: generator.guid(), clientSecret: generator.hash(), method: generator.pickone(Object.values(OAuth2CredentialsMethod)), + grantType: generator.pickone(Object.values(OAuth2GrantType)), } } @@ -58,6 +60,7 @@ describe("/oauth2", () => { clientId: c.clientId, clientSecret: PASSWORD_REPLACEMENT, method: c.method, + grantType: c.grantType, })) ), }) @@ -80,6 +83,7 @@ describe("/oauth2", () => { clientId: oauth2Config.clientId, clientSecret: PASSWORD_REPLACEMENT, method: oauth2Config.method, + grantType: oauth2Config.grantType, }, ], }) @@ -102,6 +106,7 @@ describe("/oauth2", () => { clientId: oauth2Config.clientId, clientSecret: PASSWORD_REPLACEMENT, method: oauth2Config.method, + grantType: oauth2Config.grantType, }, { _id: expectOAuth2ConfigId, @@ -111,6 +116,7 @@ describe("/oauth2", () => { clientId: oauth2Config2.clientId, clientSecret: PASSWORD_REPLACEMENT, method: oauth2Config2.method, + grantType: oauth2Config2.grantType, }, ]) ) @@ -139,6 +145,7 @@ describe("/oauth2", () => { clientId: oauth2Config.clientId, clientSecret: PASSWORD_REPLACEMENT, method: oauth2Config.method, + grantType: oauth2Config.grantType, }, ]) }) @@ -177,6 +184,7 @@ describe("/oauth2", () => { clientId: configData.clientId, clientSecret: PASSWORD_REPLACEMENT, method: configData.method, + grantType: configData.grantType, }, ]) ) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 4a545b253e..59a9b98ff9 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -16,7 +16,7 @@ const descriptions = datasourceDescribe({ if (descriptions.length) { describe.each(descriptions)( "queries ($dbName)", - ({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => { + ({ config, dsProvider, isOracle, isMSSQL, isPostgres, isMySQL }) => { let rawDatasource: Datasource let datasource: Datasource let client: Knex @@ -217,6 +217,38 @@ if (descriptions.length) { expect(res).toBeDefined() }) }) + + isMySQL && + it("should handle ANSI_QUOTE=off MySQL queries with bindings", async () => { + const query = await createQuery({ + fields: { + sql: client(tableName) + .select("*") + .where({ + name: client.raw("'{{ name }}'"), + }) + .toString(), + }, + parameters: [ + { + name: "name", + default: "", + }, + ], + queryVerb: "read", + }) + const res = await config.api.query.execute( + query._id!, + { + parameters: { name: "one" }, + }, + { + status: 200, + } + ) + expect(res.data.length).toEqual(1) + expect(res.data[0].name).toEqual("one") + }) }) describe("preview", () => { diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 55f57aff7c..fc61af92df 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -3,7 +3,7 @@ import { OpenAI } from "openai" import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" -import * as pro from "@budibase/pro" +import { ai } from "@budibase/pro" /** * Maintains backward compatibility with automation steps created before the introduction @@ -41,18 +41,9 @@ export async function run({ try { let response - const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - - let llmWrapper - if (budibaseAIEnabled || customConfigsEnabled) { - llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( - inputs.model - ) - } - - response = llmWrapper?.llm - ? await llmWrapper.run(inputs.prompt) + const llm = await ai.getLLM(inputs.model) + response = llm + ? (await llm.prompt(inputs.prompt)).message : await legacyOpenAIPrompt(inputs) return { diff --git a/packages/server/src/integrations/queries/sql.ts b/packages/server/src/integrations/queries/sql.ts index f6b0d68e7f..e7419ea403 100644 --- a/packages/server/src/integrations/queries/sql.ts +++ b/packages/server/src/integrations/queries/sql.ts @@ -1,10 +1,33 @@ import { findHBSBlocks } from "@budibase/string-templates" -import { DatasourcePlus } from "@budibase/types" +import { DatasourcePlus, SourceName } from "@budibase/types" import sdk from "../../sdk" -const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") +const MYSQL_CONST_CHAR_REGEX = new RegExp(`"[^"]*"|'[^']*'`, "g") +const CONST_CHAR_REGEX = new RegExp(`'[^']*'`, "g") + +function getConstCharRegex(sourceName: SourceName) { + // MySQL clients support ANSI_QUOTES mode off, this is by default + // but " and ' count as string literals + if (sourceName === SourceName.MYSQL) { + return MYSQL_CONST_CHAR_REGEX + } else { + return CONST_CHAR_REGEX + } +} + +function getBindingWithinConstCharRegex( + sourceName: SourceName, + binding: string +) { + if (sourceName === SourceName.MYSQL) { + return new RegExp(`[^']*${binding}[^']*'|"[^"]*${binding}[^"]*"`, "g") + } else { + return new RegExp(`'[^']*${binding}[^']*'`) + } +} export async function interpolateSQL( + sourceName: SourceName, fields: { sql: string; bindings: any[] }, parameters: { [key: string]: any }, integration: DatasourcePlus, @@ -24,10 +47,10 @@ export async function interpolateSQL( ) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' // start by finding all the instances of const character strings - const charConstMatch = sql.match(CONST_CHAR_REGEX) || [] + const charConstMatch = sql.match(getConstCharRegex(sourceName)) || [] // now look within them to see if a binding is used const charConstBindingMatch = charConstMatch.find((string: any) => - string.match(new RegExp(`'[^']*${binding}[^']*'`)) + string.match(getBindingWithinConstCharRegex(sourceName, binding)) ) if (charConstBindingMatch) { let [part1, part2] = charConstBindingMatch.split(binding) diff --git a/packages/server/src/integrations/tests/rest.spec.ts b/packages/server/src/integrations/tests/rest.spec.ts index 95f4179bc1..71ff711352 100644 --- a/packages/server/src/integrations/tests/rest.spec.ts +++ b/packages/server/src/integrations/tests/rest.spec.ts @@ -6,6 +6,7 @@ import { BearerRestAuthConfig, BodyType, OAuth2CredentialsMethod, + OAuth2GrantType, RestAuthType, } from "@budibase/types" import { Response } from "node-fetch" @@ -286,6 +287,7 @@ describe("REST Integration", () => { clientId: generator.guid(), clientSecret: secret, method: OAuth2CredentialsMethod.HEADER, + grantType: OAuth2GrantType.CLIENT_CREDENTIALS, }) const token = generator.guid() @@ -323,6 +325,7 @@ describe("REST Integration", () => { clientId: generator.guid(), clientSecret: secret, method: OAuth2CredentialsMethod.BODY, + grantType: OAuth2GrantType.CLIENT_CREDENTIALS, }) const token = generator.guid() diff --git a/packages/server/src/sdk/app/oauth2/tests/utils.spec.ts b/packages/server/src/sdk/app/oauth2/tests/utils.spec.ts index b968f3138e..13d8b7e980 100644 --- a/packages/server/src/sdk/app/oauth2/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/oauth2/tests/utils.spec.ts @@ -6,7 +6,7 @@ import { getToken } from "../utils" import path from "path" import { KEYCLOAK_IMAGE } from "../../../../integrations/tests/utils/images" import { startContainer } from "../../../../integrations/tests/utils" -import { OAuth2CredentialsMethod } from "@budibase/types" +import { OAuth2CredentialsMethod, OAuth2GrantType } from "@budibase/types" import { cache } from "@budibase/backend-core" import tk from "timekeeper" @@ -44,169 +44,175 @@ describe("oauth2 utils", () => { keycloakUrl = `http://127.0.0.1:${port}` }) - describe.each(Object.values(OAuth2CredentialsMethod))( - "getToken (in %s)", - method => { - it("successfully generates tokens", async () => { - const response = await config.doInContext(config.appId, async () => { + describe.each( + Object.values(OAuth2CredentialsMethod).flatMap(method => + Object.values(OAuth2GrantType).map< + [OAuth2CredentialsMethod, OAuth2GrantType] + >(grantType => [method, grantType]) + ) + )("generateToken (in %s, grant type %s)", (method, grantType) => { + it("successfully generates tokens", async () => { + const response = await config.doInContext(config.appId, async () => { + const oauthConfig = await sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, + clientId: "my-client", + clientSecret: "my-secret", + method, + grantType, + }) + + const response = await getToken(oauthConfig._id) + return response + }) + + expect(response).toEqual(expect.stringMatching(/^Bearer .+/)) + }) + + it("handles wrong urls", async () => { + await expect( + config.doInContext(config.appId, async () => { const oauthConfig = await sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/wrong/protocol/openid-connect/token`, + clientId: "my-client", + clientSecret: "my-secret", + method, + grantType, + }) + + await getToken(oauthConfig._id) + }) + ).rejects.toThrow("Error fetching oauth2 token: Not Found") + }) + + it("handles wrong client ids", async () => { + await expect( + config.doInContext(config.appId, async () => { + const oauthConfig = await sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, + clientId: "wrong-client-id", + clientSecret: "my-secret", + method, + grantType, + }) + + await getToken(oauthConfig._id) + }) + ).rejects.toThrow( + "Error fetching oauth2 token: Invalid client or Invalid client credentials" + ) + }) + + it("handles wrong secrets", async () => { + await expect( + config.doInContext(config.appId, async () => { + const oauthConfig = await sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, + clientId: "my-client", + clientSecret: "wrong-secret", + method, + grantType, + }) + + await getToken(oauthConfig._id) + }) + ).rejects.toThrow( + "Error fetching oauth2 token: Invalid client or Invalid client credentials" + ) + }) + + describe("track usages", () => { + beforeAll(() => { + tk.freeze(Date.now()) + }) + + it("tracks usages on generation", async () => { + const oauthConfig = await config.doInContext(config.appId, () => + sdk.oauth2.create({ name: generator.guid(), url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, clientId: "my-client", clientSecret: "my-secret", method, + grantType, }) - - const response = await getToken(oauthConfig._id) - return response - }) - - expect(response).toEqual(expect.stringMatching(/^Bearer .+/)) - }) - - it("handles wrong urls", async () => { - await expect( - config.doInContext(config.appId, async () => { - const oauthConfig = await sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/wrong/protocol/openid-connect/token`, - clientId: "my-client", - clientSecret: "my-secret", - method, - }) - - await getToken(oauthConfig._id) - }) - ).rejects.toThrow("Error fetching oauth2 token: Not Found") - }) - - it("handles wrong client ids", async () => { - await expect( - config.doInContext(config.appId, async () => { - const oauthConfig = await sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, - clientId: "wrong-client-id", - clientSecret: "my-secret", - method, - }) - - await getToken(oauthConfig._id) - }) - ).rejects.toThrow( - "Error fetching oauth2 token: Invalid client or Invalid client credentials" ) - }) - it("handles wrong secrets", async () => { - await expect( - config.doInContext(config.appId, async () => { - const oauthConfig = await sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, - clientId: "my-client", - clientSecret: "wrong-secret", - method, - }) - - await getToken(oauthConfig._id) - }) - ).rejects.toThrow( - "Error fetching oauth2 token: Invalid client or Invalid client credentials" + await config.doInContext(config.appId, () => getToken(oauthConfig._id)) + await testUtils.queue.processMessages( + cache.docWritethrough.DocWritethroughProcessor.queue ) + + const usageLog = await config.doInContext(config.appId, () => + sdk.oauth2.getLastUsages([oauthConfig._id]) + ) + + expect(usageLog[oauthConfig._id]).toEqual(Date.now()) }) - describe("track usages", () => { - beforeAll(() => { - tk.freeze(Date.now()) - }) + it("does not track on failed usages", async () => { + const oauthConfig = await config.doInContext(config.appId, () => + sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, + clientId: "wrong-client", + clientSecret: "my-secret", + method, + grantType, + }) + ) - it("tracks usages on generation", async () => { - const oauthConfig = await config.doInContext(config.appId, () => - sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, - clientId: "my-client", - clientSecret: "my-secret", - method, - }) - ) + await expect( + config.doInContext(config.appId, () => getToken(oauthConfig._id)) + ).rejects.toThrow() + await testUtils.queue.processMessages( + cache.docWritethrough.DocWritethroughProcessor.queue + ) - await config.doInContext(config.appId, () => - getToken(oauthConfig._id) - ) - await testUtils.queue.processMessages( - cache.docWritethrough.DocWritethroughProcessor.queue - ) + const usageLog = await config.doInContext(config.appId, () => + sdk.oauth2.getLastUsages([oauthConfig._id]) + ) - const usageLog = await config.doInContext(config.appId, () => + expect(usageLog[oauthConfig._id]).toBeUndefined() + }) + + it("tracks usages between prod and dev, keeping always the latest", async () => { + const oauthConfig = await config.doInContext(config.appId, () => + sdk.oauth2.create({ + name: generator.guid(), + url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, + clientId: "my-client", + clientSecret: "my-secret", + method, + grantType, + }) + ) + + await config.doInContext(config.appId, () => getToken(oauthConfig._id)) + + await config.publish() + + tk.travel(Date.now() + 100) + await config.doInContext(config.prodAppId, () => + getToken(oauthConfig._id) + ) + await testUtils.queue.processMessages( + cache.docWritethrough.DocWritethroughProcessor.queue + ) + + for (const appId of [config.appId, config.prodAppId]) { + const usageLog = await config.doInContext(appId, () => sdk.oauth2.getLastUsages([oauthConfig._id]) ) - expect(usageLog[oauthConfig._id]).toEqual(Date.now()) - }) - - it("does not track on failed usages", async () => { - const oauthConfig = await config.doInContext(config.appId, () => - sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, - clientId: "wrong-client", - clientSecret: "my-secret", - method, - }) - ) - - await expect( - config.doInContext(config.appId, () => getToken(oauthConfig._id)) - ).rejects.toThrow() - await testUtils.queue.processMessages( - cache.docWritethrough.DocWritethroughProcessor.queue - ) - - const usageLog = await config.doInContext(config.appId, () => - sdk.oauth2.getLastUsages([oauthConfig._id]) - ) - - expect(usageLog[oauthConfig._id]).toBeUndefined() - }) - - it("tracks usages between prod and dev, keeping always the latest", async () => { - const oauthConfig = await config.doInContext(config.appId, () => - sdk.oauth2.create({ - name: generator.guid(), - url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, - clientId: "my-client", - clientSecret: "my-secret", - method, - }) - ) - - await config.doInContext(config.appId, () => - getToken(oauthConfig._id) - ) - - await config.publish() - - tk.travel(Date.now() + 100) - await config.doInContext(config.prodAppId, () => - getToken(oauthConfig._id) - ) - await testUtils.queue.processMessages( - cache.docWritethrough.DocWritethroughProcessor.queue - ) - - for (const appId of [config.appId, config.prodAppId]) { - const usageLog = await config.doInContext(appId, () => - sdk.oauth2.getLastUsages([oauthConfig._id]) - ) - - expect(usageLog).toEqual({ - [oauthConfig._id]: Date.now(), - }) - } - }) + expect(usageLog).toEqual({ + [oauthConfig._id]: Date.now(), + }) + } }) - } - ) + }) + }) }) diff --git a/packages/server/src/sdk/app/oauth2/utils.ts b/packages/server/src/sdk/app/oauth2/utils.ts index 349c62f9e7..993fbe83b1 100644 --- a/packages/server/src/sdk/app/oauth2/utils.ts +++ b/packages/server/src/sdk/app/oauth2/utils.ts @@ -1,7 +1,11 @@ import fetch, { RequestInit } from "node-fetch" import { HttpError } from "koa" import { get } from "../oauth2" -import { Document, OAuth2CredentialsMethod } from "@budibase/types" +import { + Document, + OAuth2CredentialsMethod, + OAuth2GrantType, +} from "@budibase/types" import { cache, context, docIds } from "@budibase/backend-core" interface OAuth2LogDocument extends Document { @@ -15,6 +19,7 @@ async function fetchToken(config: { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType }) { const fetchConfig: RequestInit = { method: "POST", @@ -37,7 +42,7 @@ async function fetchToken(config: { } } else { fetchConfig.body = new URLSearchParams({ - grant_type: "client_credentials", + grant_type: config.grantType, client_id: config.clientId, client_secret: config.clientSecret, }) @@ -82,6 +87,7 @@ export async function validateConfig(config: { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType }): Promise<{ valid: boolean; message?: string }> { try { const resp = await fetchToken(config) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 3ba4995b2c..59c4b6dc89 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -112,9 +112,15 @@ class QueryRunner { let query: Record // handle SQL injections by interpolating the variables if (isSQL(datasourceClone)) { - query = await interpolateSQL(fieldsClone, enrichedContext, integration, { - nullDefaultSupport, - }) + query = await interpolateSQL( + datasource.source, + fieldsClone, + enrichedContext, + integration, + { + nullDefaultSupport, + } + ) } else { query = await sdk.queries.enrichContext(fieldsClone, enrichedContext) } diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts index fa674fcc52..49dd05910d 100644 --- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts @@ -1,30 +1,11 @@ -import { fixAutoColumnSubType, processAIColumns } from "../utils" +import { fixAutoColumnSubType } from "../utils" import { AutoFieldDefaultNames } from "../../../constants" import { - AIOperationEnum, AutoFieldSubType, FieldSchema, FieldType, - INTERNAL_TABLE_SOURCE_ID, RelationshipType, - Table, - TableSourceType, } from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" - -const buildPromptMock = jest.fn() - -jest.mock("@budibase/pro", () => ({ - ai: { - LargeLanguageModel: { - forCurrentTenant: async () => ({ - llm: {}, - run: jest.fn(() => "response from LLM"), - buildPromptFromAIOperation: buildPromptMock, - }), - }, - }, -})) describe("rowProcessor utility", () => { describe("fixAutoColumnSubType", () => { @@ -79,59 +60,4 @@ describe("rowProcessor utility", () => { expect(fixAutoColumnSubType(schema)).toEqual(schema) }) }) - - describe("processAIColumns", () => { - it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => { - const table: Table = { - _id: generator.guid(), - name: "AITestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - product: { - type: FieldType.STRING, - name: "product", - constraints: { - presence: true, - type: "string", - }, - }, - aicol: { - type: FieldType.AI, - name: "aicol", - operation: AIOperationEnum.PROMPT, - prompt: "Translate '{{ product }}' into German", - }, - }, - } - - const inputRows = [ - { - product: "Car Battery", - }, - ] - - const result = await processAIColumns(table, inputRows, { - contextRows: inputRows, - }) - expect(buildPromptMock).toHaveBeenCalledWith({ - row: { - product: "Car Battery", - }, - schema: { - name: "aicol", - operation: "PROMPT", - prompt: "Translate 'Car Battery' into German", - type: "ai", - }, - }) - expect(result).toEqual([ - { - aicol: "response from LLM", - product: "Car Battery", - }, - ]) - }) - }) }) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 7d2f8b49f4..8344ca5ce1 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -15,7 +15,7 @@ import { import { OperationFields } from "@budibase/shared-core" import tracer from "dd-trace" import { context } from "@budibase/backend-core" -import * as pro from "@budibase/pro" +import { ai } from "@budibase/pro" import { coerce } from "./index" interface FormulaOpts { @@ -126,10 +126,8 @@ export async function processAIColumns( const numRows = Array.isArray(inputRows) ? inputRows.length : 1 span?.addTags({ table_id: table._id, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( - "gpt-4o-mini" - ) - if (rows && llmWrapper.llm) { + const llm = await ai.getLLM() + if (rows && llm) { // Ensure we have snippet context await context.ensureSnippetContext() @@ -153,17 +151,12 @@ export async function processAIColumns( } } - const prompt = llmWrapper.buildPromptFromAIOperation({ - schema: aiSchema, - row, - }) - return tracer.trace("processAIColumn", {}, async span => { span?.addTags({ table_id: table._id, column }) - const llmResponse = await llmWrapper.run(prompt) + const llmResponse = await llm.operation(aiSchema, row) return { ...row, - [column]: llmResponse, + [column]: llmResponse.message, } }) }) diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 342e4882e6..9f3a96c6bd 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -6,6 +6,7 @@ export interface GetLicenseRequest { // All fields should be optional to cater for // historical versions of budibase quotaUsage?: QuotaUsage + tenantId?: string install: { id: string tenantId: string diff --git a/packages/types/src/api/web/app/oauth2.ts b/packages/types/src/api/web/app/oauth2.ts index 13e8cf3adc..966f29035a 100644 --- a/packages/types/src/api/web/app/oauth2.ts +++ b/packages/types/src/api/web/app/oauth2.ts @@ -1,4 +1,4 @@ -import { OAuth2CredentialsMethod } from "@budibase/types" +import { OAuth2CredentialsMethod, OAuth2GrantType } from "@budibase/types" export interface OAuth2ConfigResponse { _id: string @@ -8,6 +8,7 @@ export interface OAuth2ConfigResponse { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType } export interface FetchOAuth2ConfigsResponse { @@ -20,6 +21,7 @@ export interface InsertOAuth2ConfigRequest { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType } export interface InsertOAuth2ConfigResponse { @@ -34,6 +36,7 @@ export interface UpdateOAuth2ConfigRequest { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType } export interface UpdateOAuth2ConfigResponse { @@ -46,6 +49,7 @@ export interface ValidateConfigRequest { clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType } export interface ValidateConfigResponse { diff --git a/packages/types/src/documents/app/oauth2.ts b/packages/types/src/documents/app/oauth2.ts index d2ad895529..791fcf2ce2 100644 --- a/packages/types/src/documents/app/oauth2.ts +++ b/packages/types/src/documents/app/oauth2.ts @@ -5,10 +5,15 @@ export enum OAuth2CredentialsMethod { BODY = "BODY", } +export enum OAuth2GrantType { + CLIENT_CREDENTIALS = "client_credentials", +} + export interface OAuth2Config extends Document { name: string url: string clientId: string clientSecret: string method: OAuth2CredentialsMethod + grantType: OAuth2GrantType } diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index c5190354d8..bd0340595c 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,18 +111,26 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" +export type AIProvider = + | "OpenAI" + | "Anthropic" + | "AzureOpenAI" + | "TogetherAI" + | "Custom" + +export interface ProviderConfig { + provider: AIProvider + isDefault: boolean + isBudibaseAI?: boolean + name: string + active: boolean + baseUrl?: string + apiKey?: string + defaultModel?: string +} export interface AIInnerConfig { - [key: string]: { - provider: AIProvider - isDefault: boolean - name: string - active: boolean - baseUrl?: string - apiKey?: string - defaultModel?: string - } + [key: string]: ProviderConfig } export interface AIConfig extends Config {} diff --git a/packages/types/src/sdk/licensing/license.ts b/packages/types/src/sdk/licensing/license.ts index 105c3680a3..c51fa33203 100644 --- a/packages/types/src/sdk/licensing/license.ts +++ b/packages/types/src/sdk/licensing/license.ts @@ -17,4 +17,5 @@ export interface License { plan: PurchasedPlan billing?: Billing testClockId?: string + tenantId?: string } diff --git a/packages/worker/package.json b/packages/worker/package.json index b67e8d4911..7af2a4cdd6 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -102,6 +102,9 @@ "typescript": "5.7.2", "update-dotenv": "1.1.1" }, + "resolutions": { + "@budibase/pro": "npm:@budibase/pro@latest" + }, "nx": { "targets": { "dev": { diff --git a/yarn.lock b/yarn.lock index 7b855ecf15..2fa83c1deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2795,28 +2795,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "3.4.22" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.22.tgz#943f23cb7056041bc1f433ee60b3d093145e7a4a" - integrity sha512-Du3iZsmRLopfoi2SvxQyY1P2Su3Nw0WbITOrKmZFsVLjZ9MzzTZs0Ph/SJHzrfJpM7rn9+8788BLSf3Z3l9KcQ== - dependencies: - "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "*" - "@budibase/shared-core" "*" - "@budibase/string-templates" "*" - "@budibase/types" "*" - "@koa/router" "13.1.0" - bull "4.10.1" - dd-trace "5.26.0" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - openai "4.59.0" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - "@budibase/vm-browserify@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262"