MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-9127/extract-env-input

This commit is contained in:
Adria Navarro 2025-03-25 10:23:37 +01:00 committed by GitHub
parent be12e7cbbd e39d619049
commit c535d1b594
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 113 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -107,7 +107,6 @@
"@budibase/shared-core": "*",
"@budibase/string-templates": "*",
"@budibase/types": "*",
"@budibase/pro": "npm:@budibase/pro@latest",
"tough-cookie": "4.1.3",
"node-fetch": "2.6.7",
"semver": "7.5.3",

1
packages/backend-core/src/environment.ts
View File

@ -163,6 +163,7 @@ const environment = {
ACCOUNT_PORTAL_URL:
process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "",
BUDICLOUD_URL: process.env.BUDICLOUD_URL || "https://budibase.app",
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
SELF_HOSTED: selfHosted,
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,

8
packages/backend-core/src/users/db.ts
View File

@ -261,9 +261,13 @@ export class UserDB {
}
}
const change = dbUser ? 0 : 1 // no change if there is existing user
let change = 1
let creatorsChange = 0
if (opts.isAccountHolder || dbUser) {
change = 0
creatorsChange = 1
}
if (dbUser) {
const [isDbUserCreator, isUserCreator] = await creatorsInList([
dbUser,

2
packages/pro

@ -1 +1 @@
Subproject commit f709bb6a07483785c32ebb6f186709450d735ec3
Subproject commit 761ec71e1543ef04887d6515f99a2c2911999ebf

3
packages/server/package.json
View File

@ -182,6 +182,9 @@
"yargs": "^13.2.4",
"zod": "^3.23.8"
},
"resolutions": {
"@budibase/pro": "npm:@budibase/pro@latest"
},
"nx": {
"targets": {
"dev": {

34
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -16,7 +16,7 @@ const descriptions = datasourceDescribe({
if (descriptions.length) {
describe.each(descriptions)(
"queries ($dbName)",
({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => {
({ config, dsProvider, isOracle, isMSSQL, isPostgres, isMySQL }) => {
let rawDatasource: Datasource
let datasource: Datasource
let client: Knex
@ -217,6 +217,38 @@ if (descriptions.length) {
expect(res).toBeDefined()
})
})
isMySQL &&
it("should handle ANSI_QUOTE=off MySQL queries with bindings", async () => {
const query = await createQuery({
fields: {
sql: client(tableName)
.select("*")
.where({
name: client.raw("'{{ name }}'"),
})
.toString(),
},
parameters: [
{
name: "name",
default: "",
},
],
queryVerb: "read",
})
const res = await config.api.query.execute(
query._id!,
{
parameters: { name: "one" },
},
{
status: 200,
}
)
expect(res.data.length).toEqual(1)
expect(res.data[0].name).toEqual("one")
})
})
describe("preview", () => {

17
packages/server/src/automations/steps/openai.ts
View File

@ -3,7 +3,7 @@ import { OpenAI } from "openai"
import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro"
import { ai } from "@budibase/pro"
/**
* Maintains backward compatibility with automation steps created before the introduction
@ -41,18 +41,9 @@ export async function run({
try {
let response
const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
let llmWrapper
if (budibaseAIEnabled || customConfigsEnabled) {
llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(
inputs.model
)
}
response = llmWrapper?.llm
? await llmWrapper.run(inputs.prompt)
const llm = await ai.getLLM(inputs.model)
response = llm
? (await llm.prompt(inputs.prompt)).message
: await legacyOpenAIPrompt(inputs)
return {

31
packages/server/src/integrations/queries/sql.ts
View File

@ -1,10 +1,33 @@
import { findHBSBlocks } from "@budibase/string-templates"
import { DatasourcePlus } from "@budibase/types"
import { DatasourcePlus, SourceName } from "@budibase/types"
import sdk from "../../sdk"
const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
const MYSQL_CONST_CHAR_REGEX = new RegExp(`"[^"]*"|'[^']*'`, "g")
const CONST_CHAR_REGEX = new RegExp(`'[^']*'`, "g")
function getConstCharRegex(sourceName: SourceName) {
// MySQL clients support ANSI_QUOTES mode off, this is by default
// but " and ' count as string literals
if (sourceName === SourceName.MYSQL) {
return MYSQL_CONST_CHAR_REGEX
} else {
return CONST_CHAR_REGEX
}
}
function getBindingWithinConstCharRegex(
sourceName: SourceName,
binding: string
) {
if (sourceName === SourceName.MYSQL) {
return new RegExp(`[^']*${binding}[^']*'|"[^"]*${binding}[^"]*"`, "g")
} else {
return new RegExp(`'[^']*${binding}[^']*'`)
}
}
export async function interpolateSQL(
sourceName: SourceName,
fields: { sql: string; bindings: any[] },
parameters: { [key: string]: any },
integration: DatasourcePlus,
@ -24,10 +47,10 @@ export async function interpolateSQL(
)
// check if the variable was used as part of a string concat e.g. 'Hello {{binding}}'
// start by finding all the instances of const character strings
const charConstMatch = sql.match(CONST_CHAR_REGEX) || []
const charConstMatch = sql.match(getConstCharRegex(sourceName)) || []
// now look within them to see if a binding is used
const charConstBindingMatch = charConstMatch.find((string: any) =>
string.match(new RegExp(`'[^']*${binding}[^']*'`))
string.match(getBindingWithinConstCharRegex(sourceName, binding))
)
if (charConstBindingMatch) {
let [part1, part2] = charConstBindingMatch.split(binding)

12
packages/server/src/threads/query.ts
View File

@ -112,9 +112,15 @@ class QueryRunner {
let query: Record<string, any>
// handle SQL injections by interpolating the variables
if (isSQL(datasourceClone)) {
query = await interpolateSQL(fieldsClone, enrichedContext, integration, {
nullDefaultSupport,
})
query = await interpolateSQL(
datasource.source,
fieldsClone,
enrichedContext,
integration,
{
nullDefaultSupport,
}
)
} else {
query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
}

76
packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
View File

@ -1,30 +1,11 @@
import { fixAutoColumnSubType, processAIColumns } from "../utils"
import { fixAutoColumnSubType } from "../utils"
import { AutoFieldDefaultNames } from "../../../constants"
import {
AIOperationEnum,
AutoFieldSubType,
FieldSchema,
FieldType,
INTERNAL_TABLE_SOURCE_ID,
RelationshipType,
Table,
TableSourceType,
} from "@budibase/types"
import { generator } from "@budibase/backend-core/tests"
const buildPromptMock = jest.fn()
jest.mock("@budibase/pro", () => ({
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
llm: {},
run: jest.fn(() => "response from LLM"),
buildPromptFromAIOperation: buildPromptMock,
}),
},
},
}))
describe("rowProcessor utility", () => {
describe("fixAutoColumnSubType", () => {
@ -79,59 +60,4 @@ describe("rowProcessor utility", () => {
expect(fixAutoColumnSubType(schema)).toEqual(schema)
})
})
describe("processAIColumns", () => {
it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
const table: Table = {
_id: generator.guid(),
name: "AITestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
product: {
type: FieldType.STRING,
name: "product",
constraints: {
presence: true,
type: "string",
},
},
aicol: {
type: FieldType.AI,
name: "aicol",
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
},
}
const inputRows = [
{
product: "Car Battery",
},
]
const result = await processAIColumns(table, inputRows, {
contextRows: inputRows,
})
expect(buildPromptMock).toHaveBeenCalledWith({
row: {
product: "Car Battery",
},
schema: {
name: "aicol",
operation: "PROMPT",
prompt: "Translate 'Car Battery' into German",
type: "ai",
},
})
expect(result).toEqual([
{
aicol: "response from LLM",
product: "Car Battery",
},
])
})
})
})

17
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -15,7 +15,7 @@ import {
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
import { ai } from "@budibase/pro"
import { coerce } from "./index"
interface FormulaOpts {
@ -126,10 +126,8 @@ export async function processAIColumns<T extends Row | Row[]>(
const numRows = Array.isArray(inputRows) ? inputRows.length : 1
span?.addTags({ table_id: table._id, numRows })
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(
"gpt-4o-mini"
)
if (rows && llmWrapper.llm) {
const llm = await ai.getLLM()
if (rows && llm) {
// Ensure we have snippet context
await context.ensureSnippetContext()
@ -153,17 +151,12 @@ export async function processAIColumns<T extends Row | Row[]>(
}
}
const prompt = llmWrapper.buildPromptFromAIOperation({
schema: aiSchema,
row,
})
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })
const llmResponse = await llmWrapper.run(prompt)
const llmResponse = await llm.operation(aiSchema, row)
return {
...row,
[column]: llmResponse,
[column]: llmResponse.message,
}
})
})

1
packages/types/src/api/account/license.ts
View File

@ -6,6 +6,7 @@ export interface GetLicenseRequest {
// All fields should be optional to cater for
// historical versions of budibase
quotaUsage?: QuotaUsage
tenantId?: string
install: {
id: string
tenantId: string

28
packages/types/src/documents/global/config.ts
View File

@ -111,18 +111,26 @@ export interface SCIMInnerConfig {
export interface SCIMConfig extends Config<SCIMInnerConfig> {}
export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom"
export type AIProvider =
| "OpenAI"
| "Anthropic"
| "AzureOpenAI"
| "TogetherAI"
| "Custom"
export interface ProviderConfig {
provider: AIProvider
isDefault: boolean
isBudibaseAI?: boolean
name: string
active: boolean
baseUrl?: string
apiKey?: string
defaultModel?: string
}
export interface AIInnerConfig {
[key: string]: {
provider: AIProvider
isDefault: boolean
name: string
active: boolean
baseUrl?: string
apiKey?: string
defaultModel?: string
}
[key: string]: ProviderConfig
}
export interface AIConfig extends Config<AIInnerConfig> {}

1
packages/types/src/sdk/licensing/license.ts
View File

@ -17,4 +17,5 @@ export interface License {
plan: PurchasedPlan
billing?: Billing
testClockId?: string
tenantId?: string
}

3
packages/worker/package.json
View File

@ -102,6 +102,9 @@
"typescript": "5.7.2",
"update-dotenv": "1.1.1"
},
"resolutions": {
"@budibase/pro": "npm:@budibase/pro@latest"
},
"nx": {
"targets": {
"dev": {

22
yarn.lock
View File

@ -2795,28 +2795,6 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "3.4.22"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.22.tgz#943f23cb7056041bc1f433ee60b3d093145e7a4a"
integrity sha512-Du3iZsmRLopfoi2SvxQyY1P2Su3Nw0WbITOrKmZFsVLjZ9MzzTZs0Ph/SJHzrfJpM7rn9+8788BLSf3Z3l9KcQ==
dependencies:
"@anthropic-ai/sdk" "^0.27.3"
"@budibase/backend-core" "*"
"@budibase/shared-core" "*"
"@budibase/string-templates" "*"
"@budibase/types" "*"
"@koa/router" "13.1.0"
bull "4.10.1"
dd-trace "5.26.0"
joi "17.6.0"
jsonwebtoken "9.0.2"
lru-cache "^7.14.1"
memorystream "^0.3.1"
node-fetch "2.6.7"
openai "4.59.0"
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/vm-browserify@^1.1.4":
version "1.1.4"
resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262"