MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into head-tag-scripts

This commit is contained in:
Andrew Kingston 2025-03-25 10:27:28 +00:00
parent 8ecc0cb51f e39d619049
commit 1b04f0417f
No known key found for this signature in database
26 changed files with 333 additions and 295 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -107,7 +107,6 @@
"@budibase/shared-core": "*", "@budibase/shared-core": "*",
"@budibase/string-templates": "*", "@budibase/string-templates": "*",
"@budibase/types": "*", "@budibase/types": "*",
"@budibase/pro": "npm:@budibase/pro@latest",
"tough-cookie": "4.1.3", "tough-cookie": "4.1.3",
"node-fetch": "2.6.7", "node-fetch": "2.6.7",
"semver": "7.5.3", "semver": "7.5.3",

1
packages/backend-core/src/environment.ts
View File

@ -163,6 +163,7 @@ const environment = {
ACCOUNT_PORTAL_URL: ACCOUNT_PORTAL_URL:
process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app", process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "", ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "",
BUDICLOUD_URL: process.env.BUDICLOUD_URL || "https://budibase.app",
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
SELF_HOSTED: selfHosted, SELF_HOSTED: selfHosted,
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,

8
packages/backend-core/src/users/db.ts
View File

@ -261,9 +261,13 @@ export class UserDB {
} }
} }
const change = dbUser ? 0 : 1 // no change if there is existing user let change = 1
let creatorsChange = 0 let creatorsChange = 0
if (opts.isAccountHolder || dbUser) {
change = 0
creatorsChange = 1
}
if (dbUser) { if (dbUser) {
const [isDbUserCreator, isUserCreator] = await creatorsInList([ const [isDbUserCreator, isUserCreator] = await creatorsInList([
dbUser, dbUser,

25
packages/builder/src/pages/builder/app/[application]/settings/oauth2/OAuth2ConfigModalContent.svelte
View File

@ -15,6 +15,7 @@
import type { InsertOAuth2ConfigRequest } from "@budibase/types" import type { InsertOAuth2ConfigRequest } from "@budibase/types"
import { import {
OAuth2CredentialsMethod, OAuth2CredentialsMethod,
OAuth2GrantType,
PASSWORD_REPLACEMENT, PASSWORD_REPLACEMENT,
} from "@budibase/types" } from "@budibase/types"
import type { ZodType } from "zod" import type { ZodType } from "zod"
@ -27,6 +28,8 @@
$: data = (config as Partial<OAuth2Config>) ?? {} $: data = (config as Partial<OAuth2Config>) ?? {}
$: data.grantType ??= OAuth2GrantType.CLIENT_CREDENTIALS
$: isCreation = !config $: isCreation = !config
$: title = isCreation $: title = isCreation
? "Create new OAuth2 connection" ? "Create new OAuth2 connection"
@ -64,6 +67,9 @@
method: z.nativeEnum(OAuth2CredentialsMethod, { method: z.nativeEnum(OAuth2CredentialsMethod, {
message: "Authentication method is required.", message: "Authentication method is required.",
}), }),
grantType: z.nativeEnum(OAuth2GrantType, {
message: "Grant type is required.",
}),
}) satisfies ZodType<InsertOAuth2ConfigRequest> }) satisfies ZodType<InsertOAuth2ConfigRequest>
const validationResult = validator.safeParse(config) const validationResult = validator.safeParse(config)
@ -97,6 +103,7 @@
clientId: configData.clientId, clientId: configData.clientId,
clientSecret: configData.clientSecret, clientSecret: configData.clientSecret,
method: configData.method, method: configData.method,
grantType: configData.grantType,
}) })
if (!connectionValidation.valid) { if (!connectionValidation.valid) {
let message = "Connection settings could not be validated" let message = "Connection settings could not be validated"
@ -158,6 +165,24 @@
access_token property. access_token property.
</Body> </Body>
</div> </div>
<Select
label="Grant type*"
options={[
{
label: "Client credentials",
value: OAuth2GrantType.CLIENT_CREDENTIALS,
},
]}
bind:value={data.grantType}
error={errors.grantType}
disabled
/>
<div class="field-info">
<Body size="XS" color="var(--spectrum-global-color-gray-700)">
Only client credentials mode is supported currently.
</Body>
</div>
<Input <Input
label="Service URL*" label="Service URL*"
placeholder="E.g. www.google.com" placeholder="E.g. www.google.com"

1
packages/builder/src/stores/builder/oauth2.ts
View File

@ -38,6 +38,7 @@ export class OAuth2Store extends BudiStore<OAuth2StoreState> {
clientId: c.clientId, clientId: c.clientId,
clientSecret: c.clientSecret, clientSecret: c.clientSecret,
method: c.method, method: c.method,
grantType: c.grantType,
lastUsage: c.lastUsage, lastUsage: c.lastUsage,
})), })),
loading: false, loading: false,

2
packages/pro

@ -1 +1 @@
Subproject commit f709bb6a07483785c32ebb6f186709450d735ec3 Subproject commit 761ec71e1543ef04887d6515f99a2c2911999ebf

3
packages/server/package.json
View File

@ -182,6 +182,9 @@
"yargs": "^13.2.4", "yargs": "^13.2.4",
"zod": "^3.23.8" "zod": "^3.23.8"
}, },
"resolutions": {
"@budibase/pro": "npm:@budibase/pro@latest"
},
"nx": { "nx": {
"targets": { "targets": {
"dev": { "dev": {

4
packages/server/src/api/controllers/oauth2.ts
View File

@ -24,6 +24,7 @@ function toFetchOAuth2ConfigsResponse(
clientId: config.clientId, clientId: config.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: config.method, method: config.method,
grantType: config.grantType,
} }
} }
@ -54,6 +55,7 @@ export async function create(
clientId: body.clientId, clientId: body.clientId,
clientSecret: body.clientSecret, clientSecret: body.clientSecret,
method: body.method, method: body.method,
grantType: body.grantType,
} }
const config = await sdk.oauth2.create(newConfig) const config = await sdk.oauth2.create(newConfig)
@ -80,6 +82,7 @@ export async function edit(
clientId: body.clientId, clientId: body.clientId,
clientSecret: body.clientSecret, clientSecret: body.clientSecret,
method: body.method, method: body.method,
grantType: body.grantType,
} }
const config = await sdk.oauth2.update(toUpdate) const config = await sdk.oauth2.update(toUpdate)
@ -104,6 +107,7 @@ export async function validate(
clientId: body.clientId, clientId: body.clientId,
clientSecret: body.clientSecret, clientSecret: body.clientSecret,
method: body.method, method: body.method,
grantType: body.grantType,
} }
if (config.clientSecret === PASSWORD_REPLACEMENT && body._id) { if (config.clientSecret === PASSWORD_REPLACEMENT && body._id) {

9
packages/server/src/api/routes/oauth2.ts
View File

@ -1,5 +1,9 @@
import Router from "@koa/router" import Router from "@koa/router"
import { OAuth2CredentialsMethod, PermissionType } from "@budibase/types" import {
OAuth2CredentialsMethod,
OAuth2GrantType,
PermissionType,
} from "@budibase/types"
import { middleware } from "@budibase/backend-core" import { middleware } from "@budibase/backend-core"
import authorized from "../../middleware/authorized" import authorized from "../../middleware/authorized"
@ -13,6 +17,9 @@ const baseSchema = {
method: Joi.string() method: Joi.string()
.required() .required()
.valid(...Object.values(OAuth2CredentialsMethod)), .valid(...Object.values(OAuth2CredentialsMethod)),
grantType: Joi.string()
.required()
.valid(...Object.values(OAuth2GrantType)),
} }
const insertSchema = Joi.object({ const insertSchema = Joi.object({

8
packages/server/src/api/routes/tests/oauth2.spec.ts
View File

@ -3,6 +3,7 @@ import {
InsertOAuth2ConfigRequest, InsertOAuth2ConfigRequest,
OAuth2ConfigResponse, OAuth2ConfigResponse,
OAuth2CredentialsMethod, OAuth2CredentialsMethod,
OAuth2GrantType,
PASSWORD_REPLACEMENT, PASSWORD_REPLACEMENT,
} from "@budibase/types" } from "@budibase/types"
import * as setup from "./utilities" import * as setup from "./utilities"
@ -19,6 +20,7 @@ describe("/oauth2", () => {
clientId: generator.guid(), clientId: generator.guid(),
clientSecret: generator.hash(), clientSecret: generator.hash(),
method: generator.pickone(Object.values(OAuth2CredentialsMethod)), method: generator.pickone(Object.values(OAuth2CredentialsMethod)),
grantType: generator.pickone(Object.values(OAuth2GrantType)),
} }
} }
@ -58,6 +60,7 @@ describe("/oauth2", () => {
clientId: c.clientId, clientId: c.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: c.method, method: c.method,
grantType: c.grantType,
})) }))
), ),
}) })
@ -80,6 +83,7 @@ describe("/oauth2", () => {
clientId: oauth2Config.clientId, clientId: oauth2Config.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: oauth2Config.method, method: oauth2Config.method,
grantType: oauth2Config.grantType,
}, },
], ],
}) })
@ -102,6 +106,7 @@ describe("/oauth2", () => {
clientId: oauth2Config.clientId, clientId: oauth2Config.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: oauth2Config.method, method: oauth2Config.method,
grantType: oauth2Config.grantType,
}, },
{ {
_id: expectOAuth2ConfigId, _id: expectOAuth2ConfigId,
@ -111,6 +116,7 @@ describe("/oauth2", () => {
clientId: oauth2Config2.clientId, clientId: oauth2Config2.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: oauth2Config2.method, method: oauth2Config2.method,
grantType: oauth2Config2.grantType,
}, },
]) ])
) )
@ -139,6 +145,7 @@ describe("/oauth2", () => {
clientId: oauth2Config.clientId, clientId: oauth2Config.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: oauth2Config.method, method: oauth2Config.method,
grantType: oauth2Config.grantType,
}, },
]) ])
}) })
@ -177,6 +184,7 @@ describe("/oauth2", () => {
clientId: configData.clientId, clientId: configData.clientId,
clientSecret: PASSWORD_REPLACEMENT, clientSecret: PASSWORD_REPLACEMENT,
method: configData.method, method: configData.method,
grantType: configData.grantType,
}, },
]) ])
) )

34
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -16,7 +16,7 @@ const descriptions = datasourceDescribe({
if (descriptions.length) { if (descriptions.length) {
describe.each(descriptions)( describe.each(descriptions)(
"queries ($dbName)", "queries ($dbName)",
({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => { ({ config, dsProvider, isOracle, isMSSQL, isPostgres, isMySQL }) => {
let rawDatasource: Datasource let rawDatasource: Datasource
let datasource: Datasource let datasource: Datasource
let client: Knex let client: Knex
@ -217,6 +217,38 @@ if (descriptions.length) {
expect(res).toBeDefined() expect(res).toBeDefined()
}) })
}) })
isMySQL &&
it("should handle ANSI_QUOTE=off MySQL queries with bindings", async () => {
const query = await createQuery({
fields: {
sql: client(tableName)
.select("*")
.where({
name: client.raw("'{{ name }}'"),
})
.toString(),
},
parameters: [
{
name: "name",
default: "",
},
],
queryVerb: "read",
})
const res = await config.api.query.execute(
query._id!,
{
parameters: { name: "one" },
},
{
status: 200,
}
)
expect(res.data.length).toEqual(1)
expect(res.data[0].name).toEqual("one")
})
}) })
describe("preview", () => { describe("preview", () => {

17
packages/server/src/automations/steps/openai.ts
View File

@ -3,7 +3,7 @@ import { OpenAI } from "openai"
import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types" import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
import { env } from "@budibase/backend-core" import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils" import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro" import { ai } from "@budibase/pro"
/** /**
* Maintains backward compatibility with automation steps created before the introduction * Maintains backward compatibility with automation steps created before the introduction
@ -41,18 +41,9 @@ export async function run({
try { try {
let response let response
const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() const llm = await ai.getLLM(inputs.model)
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() response = llm
? (await llm.prompt(inputs.prompt)).message
let llmWrapper
if (budibaseAIEnabled || customConfigsEnabled) {
llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(
inputs.model
)
}
response = llmWrapper?.llm
? await llmWrapper.run(inputs.prompt)
: await legacyOpenAIPrompt(inputs) : await legacyOpenAIPrompt(inputs)
return { return {

31
packages/server/src/integrations/queries/sql.ts
View File

@ -1,10 +1,33 @@
import { findHBSBlocks } from "@budibase/string-templates" import { findHBSBlocks } from "@budibase/string-templates"
import { DatasourcePlus } from "@budibase/types" import { DatasourcePlus, SourceName } from "@budibase/types"
import sdk from "../../sdk" import sdk from "../../sdk"
const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") const MYSQL_CONST_CHAR_REGEX = new RegExp(`"[^"]*"|'[^']*'`, "g")
const CONST_CHAR_REGEX = new RegExp(`'[^']*'`, "g")
function getConstCharRegex(sourceName: SourceName) {
// MySQL clients support ANSI_QUOTES mode off, this is by default
// but " and ' count as string literals
if (sourceName === SourceName.MYSQL) {
return MYSQL_CONST_CHAR_REGEX
} else {
return CONST_CHAR_REGEX
}
}
function getBindingWithinConstCharRegex(
sourceName: SourceName,
binding: string
) {
if (sourceName === SourceName.MYSQL) {
return new RegExp(`[^']*${binding}[^']*'|"[^"]*${binding}[^"]*"`, "g")
} else {
return new RegExp(`'[^']*${binding}[^']*'`)
}
}
export async function interpolateSQL( export async function interpolateSQL(
sourceName: SourceName,
fields: { sql: string; bindings: any[] }, fields: { sql: string; bindings: any[] },
parameters: { [key: string]: any }, parameters: { [key: string]: any },
integration: DatasourcePlus, integration: DatasourcePlus,
@ -24,10 +47,10 @@ export async function interpolateSQL(
) )
// check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}'
// start by finding all the instances of const character strings // start by finding all the instances of const character strings
const charConstMatch = sql.match(CONST_CHAR_REGEX) || [] const charConstMatch = sql.match(getConstCharRegex(sourceName)) || []
// now look within them to see if a binding is used // now look within them to see if a binding is used
const charConstBindingMatch = charConstMatch.find((string: any) => const charConstBindingMatch = charConstMatch.find((string: any) =>
string.match(new RegExp(`'[^']*${binding}[^']*'`)) string.match(getBindingWithinConstCharRegex(sourceName, binding))
) )
if (charConstBindingMatch) { if (charConstBindingMatch) {
let [part1, part2] = charConstBindingMatch.split(binding) let [part1, part2] = charConstBindingMatch.split(binding)

3
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -6,6 +6,7 @@ import {
BearerRestAuthConfig, BearerRestAuthConfig,
BodyType, BodyType,
OAuth2CredentialsMethod, OAuth2CredentialsMethod,
OAuth2GrantType,
RestAuthType, RestAuthType,
} from "@budibase/types" } from "@budibase/types"
import { Response } from "node-fetch" import { Response } from "node-fetch"
@ -286,6 +287,7 @@ describe("REST Integration", () => {
clientId: generator.guid(), clientId: generator.guid(),
clientSecret: secret, clientSecret: secret,
method: OAuth2CredentialsMethod.HEADER, method: OAuth2CredentialsMethod.HEADER,
grantType: OAuth2GrantType.CLIENT_CREDENTIALS,
}) })
const token = generator.guid() const token = generator.guid()
@ -323,6 +325,7 @@ describe("REST Integration", () => {
clientId: generator.guid(), clientId: generator.guid(),
clientSecret: secret, clientSecret: secret,
method: OAuth2CredentialsMethod.BODY, method: OAuth2CredentialsMethod.BODY,
grantType: OAuth2GrantType.CLIENT_CREDENTIALS,
}) })
const token = generator.guid() const token = generator.guid()

300
packages/server/src/sdk/app/oauth2/tests/utils.spec.ts
View File

@ -6,7 +6,7 @@ import { getToken } from "../utils"
import path from "path" import path from "path"
import { KEYCLOAK_IMAGE } from "../../../../integrations/tests/utils/images" import { KEYCLOAK_IMAGE } from "../../../../integrations/tests/utils/images"
import { startContainer } from "../../../../integrations/tests/utils" import { startContainer } from "../../../../integrations/tests/utils"
import { OAuth2CredentialsMethod } from "@budibase/types" import { OAuth2CredentialsMethod, OAuth2GrantType } from "@budibase/types"
import { cache } from "@budibase/backend-core" import { cache } from "@budibase/backend-core"
import tk from "timekeeper" import tk from "timekeeper"
@ -44,169 +44,175 @@ describe("oauth2 utils", () => {
keycloakUrl = `http://127.0.0.1:${port}` keycloakUrl = `http://127.0.0.1:${port}`
}) })
describe.each(Object.values(OAuth2CredentialsMethod))( describe.each(
"getToken (in %s)", Object.values(OAuth2CredentialsMethod).flatMap(method =>
method => { Object.values(OAuth2GrantType).map<
it("successfully generates tokens", async () => { [OAuth2CredentialsMethod, OAuth2GrantType]
const response = await config.doInContext(config.appId, async () => { >(grantType => [method, grantType])
)
)("generateToken (in %s, grant type %s)", (method, grantType) => {
it("successfully generates tokens", async () => {
const response = await config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "my-secret",
method,
grantType,
})
const response = await getToken(oauthConfig._id)
return response
})
expect(response).toEqual(expect.stringMatching(/^Bearer .+/))
})
it("handles wrong urls", async () => {
await expect(
config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({ const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/wrong/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "my-secret",
method,
grantType,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow("Error fetching oauth2 token: Not Found")
})
it("handles wrong client ids", async () => {
await expect(
config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "wrong-client-id",
clientSecret: "my-secret",
method,
grantType,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow(
"Error fetching oauth2 token: Invalid client or Invalid client credentials"
)
})
it("handles wrong secrets", async () => {
await expect(
config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "wrong-secret",
method,
grantType,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow(
"Error fetching oauth2 token: Invalid client or Invalid client credentials"
)
})
describe("track usages", () => {
beforeAll(() => {
tk.freeze(Date.now())
})
it("tracks usages on generation", async () => {
const oauthConfig = await config.doInContext(config.appId, () =>
sdk.oauth2.create({
name: generator.guid(), name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client", clientId: "my-client",
clientSecret: "my-secret", clientSecret: "my-secret",
method, method,
grantType,
}) })
const response = await getToken(oauthConfig._id)
return response
})
expect(response).toEqual(expect.stringMatching(/^Bearer .+/))
})
it("handles wrong urls", async () => {
await expect(
config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/wrong/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "my-secret",
method,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow("Error fetching oauth2 token: Not Found")
})
it("handles wrong client ids", async () => {
await expect(
config.doInContext(config.appId, async () => {
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "wrong-client-id",
clientSecret: "my-secret",
method,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow(
"Error fetching oauth2 token: Invalid client or Invalid client credentials"
) )
})
it("handles wrong secrets", async () => { await config.doInContext(config.appId, () => getToken(oauthConfig._id))
await expect( await testUtils.queue.processMessages(
config.doInContext(config.appId, async () => { cache.docWritethrough.DocWritethroughProcessor.queue
const oauthConfig = await sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "wrong-secret",
method,
})
await getToken(oauthConfig._id)
})
).rejects.toThrow(
"Error fetching oauth2 token: Invalid client or Invalid client credentials"
) )
const usageLog = await config.doInContext(config.appId, () =>
sdk.oauth2.getLastUsages([oauthConfig._id])
)
expect(usageLog[oauthConfig._id]).toEqual(Date.now())
}) })
describe("track usages", () => { it("does not track on failed usages", async () => {
beforeAll(() => { const oauthConfig = await config.doInContext(config.appId, () =>
tk.freeze(Date.now()) sdk.oauth2.create({
}) name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "wrong-client",
clientSecret: "my-secret",
method,
grantType,
})
)
it("tracks usages on generation", async () => { await expect(
const oauthConfig = await config.doInContext(config.appId, () => config.doInContext(config.appId, () => getToken(oauthConfig._id))
sdk.oauth2.create({ ).rejects.toThrow()
name: generator.guid(), await testUtils.queue.processMessages(
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`, cache.docWritethrough.DocWritethroughProcessor.queue
clientId: "my-client", )
clientSecret: "my-secret",
method,
})
)
await config.doInContext(config.appId, () => const usageLog = await config.doInContext(config.appId, () =>
getToken(oauthConfig._id) sdk.oauth2.getLastUsages([oauthConfig._id])
) )
await testUtils.queue.processMessages(
cache.docWritethrough.DocWritethroughProcessor.queue
)
const usageLog = await config.doInContext(config.appId, () => expect(usageLog[oauthConfig._id]).toBeUndefined()
})
it("tracks usages between prod and dev, keeping always the latest", async () => {
const oauthConfig = await config.doInContext(config.appId, () =>
sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "my-secret",
method,
grantType,
})
)
await config.doInContext(config.appId, () => getToken(oauthConfig._id))
await config.publish()
tk.travel(Date.now() + 100)
await config.doInContext(config.prodAppId, () =>
getToken(oauthConfig._id)
)
await testUtils.queue.processMessages(
cache.docWritethrough.DocWritethroughProcessor.queue
)
for (const appId of [config.appId, config.prodAppId]) {
const usageLog = await config.doInContext(appId, () =>
sdk.oauth2.getLastUsages([oauthConfig._id]) sdk.oauth2.getLastUsages([oauthConfig._id])
) )
expect(usageLog[oauthConfig._id]).toEqual(Date.now()) expect(usageLog).toEqual({
}) [oauthConfig._id]: Date.now(),
})
it("does not track on failed usages", async () => { }
const oauthConfig = await config.doInContext(config.appId, () =>
sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "wrong-client",
clientSecret: "my-secret",
method,
})
)
await expect(
config.doInContext(config.appId, () => getToken(oauthConfig._id))
).rejects.toThrow()
await testUtils.queue.processMessages(
cache.docWritethrough.DocWritethroughProcessor.queue
)
const usageLog = await config.doInContext(config.appId, () =>
sdk.oauth2.getLastUsages([oauthConfig._id])
)
expect(usageLog[oauthConfig._id]).toBeUndefined()
})
it("tracks usages between prod and dev, keeping always the latest", async () => {
const oauthConfig = await config.doInContext(config.appId, () =>
sdk.oauth2.create({
name: generator.guid(),
url: `${keycloakUrl}/realms/myrealm/protocol/openid-connect/token`,
clientId: "my-client",
clientSecret: "my-secret",
method,
})
)
await config.doInContext(config.appId, () =>
getToken(oauthConfig._id)
)
await config.publish()
tk.travel(Date.now() + 100)
await config.doInContext(config.prodAppId, () =>
getToken(oauthConfig._id)
)
await testUtils.queue.processMessages(
cache.docWritethrough.DocWritethroughProcessor.queue
)
for (const appId of [config.appId, config.prodAppId]) {
const usageLog = await config.doInContext(appId, () =>
sdk.oauth2.getLastUsages([oauthConfig._id])
)
expect(usageLog).toEqual({
[oauthConfig._id]: Date.now(),
})
}
})
}) })
} })
) })
}) })

10
packages/server/src/sdk/app/oauth2/utils.ts
View File

@ -1,7 +1,11 @@
import fetch, { RequestInit } from "node-fetch" import fetch, { RequestInit } from "node-fetch"
import { HttpError } from "koa" import { HttpError } from "koa"
import { get } from "../oauth2" import { get } from "../oauth2"
import { Document, OAuth2CredentialsMethod } from "@budibase/types" import {
Document,
OAuth2CredentialsMethod,
OAuth2GrantType,
} from "@budibase/types"
import { cache, context, docIds } from "@budibase/backend-core" import { cache, context, docIds } from "@budibase/backend-core"
interface OAuth2LogDocument extends Document { interface OAuth2LogDocument extends Document {
@ -15,6 +19,7 @@ async function fetchToken(config: {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
}) { }) {
const fetchConfig: RequestInit = { const fetchConfig: RequestInit = {
method: "POST", method: "POST",
@ -37,7 +42,7 @@ async function fetchToken(config: {
} }
} else { } else {
fetchConfig.body = new URLSearchParams({ fetchConfig.body = new URLSearchParams({
grant_type: "client_credentials", grant_type: config.grantType,
client_id: config.clientId, client_id: config.clientId,
client_secret: config.clientSecret, client_secret: config.clientSecret,
}) })
@ -82,6 +87,7 @@ export async function validateConfig(config: {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
}): Promise<{ valid: boolean; message?: string }> { }): Promise<{ valid: boolean; message?: string }> {
try { try {
const resp = await fetchToken(config) const resp = await fetchToken(config)

12
packages/server/src/threads/query.ts
View File

@ -112,9 +112,15 @@ class QueryRunner {
let query: Record<string, any> let query: Record<string, any>
// handle SQL injections by interpolating the variables // handle SQL injections by interpolating the variables
if (isSQL(datasourceClone)) { if (isSQL(datasourceClone)) {
query = await interpolateSQL(fieldsClone, enrichedContext, integration, { query = await interpolateSQL(
nullDefaultSupport, datasource.source,
}) fieldsClone,
enrichedContext,
integration,
{
nullDefaultSupport,
}
)
} else { } else {
query = await sdk.queries.enrichContext(fieldsClone, enrichedContext) query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
} }

76
packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
View File

@ -1,30 +1,11 @@
import { fixAutoColumnSubType, processAIColumns } from "../utils" import { fixAutoColumnSubType } from "../utils"
import { AutoFieldDefaultNames } from "../../../constants" import { AutoFieldDefaultNames } from "../../../constants"
import { import {
AIOperationEnum,
AutoFieldSubType, AutoFieldSubType,
FieldSchema, FieldSchema,
FieldType, FieldType,
INTERNAL_TABLE_SOURCE_ID,
RelationshipType, RelationshipType,
Table,
TableSourceType,
} from "@budibase/types" } from "@budibase/types"
import { generator } from "@budibase/backend-core/tests"
const buildPromptMock = jest.fn()
jest.mock("@budibase/pro", () => ({
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
llm: {},
run: jest.fn(() => "response from LLM"),
buildPromptFromAIOperation: buildPromptMock,
}),
},
},
}))
describe("rowProcessor utility", () => { describe("rowProcessor utility", () => {
describe("fixAutoColumnSubType", () => { describe("fixAutoColumnSubType", () => {
@ -79,59 +60,4 @@ describe("rowProcessor utility", () => {
expect(fixAutoColumnSubType(schema)).toEqual(schema) expect(fixAutoColumnSubType(schema)).toEqual(schema)
}) })
}) })
describe("processAIColumns", () => {
it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
const table: Table = {
_id: generator.guid(),
name: "AITestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
product: {
type: FieldType.STRING,
name: "product",
constraints: {
presence: true,
type: "string",
},
},
aicol: {
type: FieldType.AI,
name: "aicol",
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
},
}
const inputRows = [
{
product: "Car Battery",
},
]
const result = await processAIColumns(table, inputRows, {
contextRows: inputRows,
})
expect(buildPromptMock).toHaveBeenCalledWith({
row: {
product: "Car Battery",
},
schema: {
name: "aicol",
operation: "PROMPT",
prompt: "Translate 'Car Battery' into German",
type: "ai",
},
})
expect(result).toEqual([
{
aicol: "response from LLM",
product: "Car Battery",
},
])
})
})
}) })

17
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -15,7 +15,7 @@ import {
import { OperationFields } from "@budibase/shared-core" import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace" import tracer from "dd-trace"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro" import { ai } from "@budibase/pro"
import { coerce } from "./index" import { coerce } from "./index"
interface FormulaOpts { interface FormulaOpts {
@ -126,10 +126,8 @@ export async function processAIColumns<T extends Row | Row[]>(
const numRows = Array.isArray(inputRows) ? inputRows.length : 1 const numRows = Array.isArray(inputRows) ? inputRows.length : 1
span?.addTags({ table_id: table._id, numRows }) span?.addTags({ table_id: table._id, numRows })
const rows = Array.isArray(inputRows) ? inputRows : [inputRows] const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( const llm = await ai.getLLM()
"gpt-4o-mini" if (rows && llm) {
)
if (rows && llmWrapper.llm) {
// Ensure we have snippet context // Ensure we have snippet context
await context.ensureSnippetContext() await context.ensureSnippetContext()
@ -153,17 +151,12 @@ export async function processAIColumns<T extends Row | Row[]>(
} }
} }
const prompt = llmWrapper.buildPromptFromAIOperation({
schema: aiSchema,
row,
})
return tracer.trace("processAIColumn", {}, async span => { return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column }) span?.addTags({ table_id: table._id, column })
const llmResponse = await llmWrapper.run(prompt) const llmResponse = await llm.operation(aiSchema, row)
return { return {
...row, ...row,
[column]: llmResponse, [column]: llmResponse.message,
} }
}) })
}) })

1
packages/types/src/api/account/license.ts
View File

@ -6,6 +6,7 @@ export interface GetLicenseRequest {
// All fields should be optional to cater for // All fields should be optional to cater for
// historical versions of budibase // historical versions of budibase
quotaUsage?: QuotaUsage quotaUsage?: QuotaUsage
tenantId?: string
install: { install: {
id: string id: string
tenantId: string tenantId: string

6
packages/types/src/api/web/app/oauth2.ts
View File

@ -1,4 +1,4 @@
import { OAuth2CredentialsMethod } from "@budibase/types" import { OAuth2CredentialsMethod, OAuth2GrantType } from "@budibase/types"
export interface OAuth2ConfigResponse { export interface OAuth2ConfigResponse {
_id: string _id: string
@ -8,6 +8,7 @@ export interface OAuth2ConfigResponse {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
} }
export interface FetchOAuth2ConfigsResponse { export interface FetchOAuth2ConfigsResponse {
@ -20,6 +21,7 @@ export interface InsertOAuth2ConfigRequest {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
} }
export interface InsertOAuth2ConfigResponse { export interface InsertOAuth2ConfigResponse {
@ -34,6 +36,7 @@ export interface UpdateOAuth2ConfigRequest {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
} }
export interface UpdateOAuth2ConfigResponse { export interface UpdateOAuth2ConfigResponse {
@ -46,6 +49,7 @@ export interface ValidateConfigRequest {
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
} }
export interface ValidateConfigResponse { export interface ValidateConfigResponse {

5
packages/types/src/documents/app/oauth2.ts
View File

@ -5,10 +5,15 @@ export enum OAuth2CredentialsMethod {
BODY = "BODY", BODY = "BODY",
} }
export enum OAuth2GrantType {
CLIENT_CREDENTIALS = "client_credentials",
}
export interface OAuth2Config extends Document { export interface OAuth2Config extends Document {
name: string name: string
url: string url: string
clientId: string clientId: string
clientSecret: string clientSecret: string
method: OAuth2CredentialsMethod method: OAuth2CredentialsMethod
grantType: OAuth2GrantType
} }

28
packages/types/src/documents/global/config.ts
View File

@ -111,18 +111,26 @@ export interface SCIMInnerConfig {
export interface SCIMConfig extends Config<SCIMInnerConfig> {} export interface SCIMConfig extends Config<SCIMInnerConfig> {}
export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" export type AIProvider =
| "OpenAI"
| "Anthropic"
| "AzureOpenAI"
| "TogetherAI"
| "Custom"
export interface ProviderConfig {
provider: AIProvider
isDefault: boolean
isBudibaseAI?: boolean
name: string
active: boolean
baseUrl?: string
apiKey?: string
defaultModel?: string
}
export interface AIInnerConfig { export interface AIInnerConfig {
[key: string]: { [key: string]: ProviderConfig
provider: AIProvider
isDefault: boolean
name: string
active: boolean
baseUrl?: string
apiKey?: string
defaultModel?: string
}
} }
export interface AIConfig extends Config<AIInnerConfig> {} export interface AIConfig extends Config<AIInnerConfig> {}

1
packages/types/src/sdk/licensing/license.ts
View File

@ -17,4 +17,5 @@ export interface License {
plan: PurchasedPlan plan: PurchasedPlan
billing?: Billing billing?: Billing
testClockId?: string testClockId?: string
tenantId?: string
} }

3
packages/worker/package.json
View File

@ -102,6 +102,9 @@
"typescript": "5.7.2", "typescript": "5.7.2",
"update-dotenv": "1.1.1" "update-dotenv": "1.1.1"
}, },
"resolutions": {
"@budibase/pro": "npm:@budibase/pro@latest"
},
"nx": { "nx": {
"targets": { "targets": {
"dev": { "dev": {

22
yarn.lock
View File

@ -2795,28 +2795,6 @@
pouchdb-promise "^6.0.4" pouchdb-promise "^6.0.4"
through2 "^2.0.0" through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "3.4.22"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.22.tgz#943f23cb7056041bc1f433ee60b3d093145e7a4a"
integrity sha512-Du3iZsmRLopfoi2SvxQyY1P2Su3Nw0WbITOrKmZFsVLjZ9MzzTZs0Ph/SJHzrfJpM7rn9+8788BLSf3Z3l9KcQ==
dependencies:
"@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "*"
"@budibase/shared-core" "*"
"@budibase/string-templates" "*"
"@budibase/types" "*"
"@koa/router" "13.1.0"
bull "4.10.1"
dd-trace "5.26.0"
joi "17.6.0"
jsonwebtoken "9.0.2"
lru-cache "^7.14.1"
memorystream "^0.3.1"
node-fetch "2.6.7"
openai "4.59.0"
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/vm-browserify@^1.1.4": "@budibase/vm-browserify@^1.1.4":
version "1.1.4" version "1.1.4"
resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262"