diff --git a/package.json b/package.json index d083dbad90..0bf3c29b2b 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,6 @@ "@budibase/shared-core": "*", "@budibase/string-templates": "*", "@budibase/types": "*", - "@budibase/pro": "npm:@budibase/pro@latest", "tough-cookie": "4.1.3", "node-fetch": "2.6.7", "semver": "7.5.3", diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 9a46758062..0a5fcc0707 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -163,6 +163,7 @@ const environment = { ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app", ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "", + BUDICLOUD_URL: process.env.BUDICLOUD_URL || "https://budibase.app", DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL, SELF_HOSTED: selfHosted, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 677feed678..ac3c9e30ab 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -261,9 +261,13 @@ export class UserDB { } } - const change = dbUser ? 0 : 1 // no change if there is existing user - + let change = 1 let creatorsChange = 0 + if (opts.isAccountHolder || dbUser) { + change = 0 + creatorsChange = 1 + } + if (dbUser) { const [isDbUserCreator, isUserCreator] = await creatorsInList([ dbUser, diff --git a/packages/pro b/packages/pro index f709bb6a07..761ec71e15 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f709bb6a07483785c32ebb6f186709450d735ec3 +Subproject commit 761ec71e1543ef04887d6515f99a2c2911999ebf diff --git a/packages/server/package.json b/packages/server/package.json index 38dd746c4f..ee89218c52 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -182,6 +182,9 @@ "yargs": "^13.2.4", "zod": "^3.23.8" }, + "resolutions": { + "@budibase/pro": "npm:@budibase/pro@latest" + }, "nx": { "targets": { "dev": { diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 4a545b253e..59a9b98ff9 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -16,7 +16,7 @@ const descriptions = datasourceDescribe({ if (descriptions.length) { describe.each(descriptions)( "queries ($dbName)", - ({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => { + ({ config, dsProvider, isOracle, isMSSQL, isPostgres, isMySQL }) => { let rawDatasource: Datasource let datasource: Datasource let client: Knex @@ -217,6 +217,38 @@ if (descriptions.length) { expect(res).toBeDefined() }) }) + + isMySQL && + it("should handle ANSI_QUOTE=off MySQL queries with bindings", async () => { + const query = await createQuery({ + fields: { + sql: client(tableName) + .select("*") + .where({ + name: client.raw("'{{ name }}'"), + }) + .toString(), + }, + parameters: [ + { + name: "name", + default: "", + }, + ], + queryVerb: "read", + }) + const res = await config.api.query.execute( + query._id!, + { + parameters: { name: "one" }, + }, + { + status: 200, + } + ) + expect(res.data.length).toEqual(1) + expect(res.data[0].name).toEqual("one") + }) }) describe("preview", () => { diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 55f57aff7c..fc61af92df 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -3,7 +3,7 @@ import { OpenAI } from "openai" import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" -import * as pro from "@budibase/pro" +import { ai } from "@budibase/pro" /** * Maintains backward compatibility with automation steps created before the introduction @@ -41,18 +41,9 @@ export async function run({ try { let response - const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - - let llmWrapper - if (budibaseAIEnabled || customConfigsEnabled) { - llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( - inputs.model - ) - } - - response = llmWrapper?.llm - ? await llmWrapper.run(inputs.prompt) + const llm = await ai.getLLM(inputs.model) + response = llm + ? (await llm.prompt(inputs.prompt)).message : await legacyOpenAIPrompt(inputs) return { diff --git a/packages/server/src/integrations/queries/sql.ts b/packages/server/src/integrations/queries/sql.ts index f6b0d68e7f..e7419ea403 100644 --- a/packages/server/src/integrations/queries/sql.ts +++ b/packages/server/src/integrations/queries/sql.ts @@ -1,10 +1,33 @@ import { findHBSBlocks } from "@budibase/string-templates" -import { DatasourcePlus } from "@budibase/types" +import { DatasourcePlus, SourceName } from "@budibase/types" import sdk from "../../sdk" -const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") +const MYSQL_CONST_CHAR_REGEX = new RegExp(`"[^"]*"|'[^']*'`, "g") +const CONST_CHAR_REGEX = new RegExp(`'[^']*'`, "g") + +function getConstCharRegex(sourceName: SourceName) { + // MySQL clients support ANSI_QUOTES mode off, this is by default + // but " and ' count as string literals + if (sourceName === SourceName.MYSQL) { + return MYSQL_CONST_CHAR_REGEX + } else { + return CONST_CHAR_REGEX + } +} + +function getBindingWithinConstCharRegex( + sourceName: SourceName, + binding: string +) { + if (sourceName === SourceName.MYSQL) { + return new RegExp(`[^']*${binding}[^']*'|"[^"]*${binding}[^"]*"`, "g") + } else { + return new RegExp(`'[^']*${binding}[^']*'`) + } +} export async function interpolateSQL( + sourceName: SourceName, fields: { sql: string; bindings: any[] }, parameters: { [key: string]: any }, integration: DatasourcePlus, @@ -24,10 +47,10 @@ export async function interpolateSQL( ) // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}' // start by finding all the instances of const character strings - const charConstMatch = sql.match(CONST_CHAR_REGEX) || [] + const charConstMatch = sql.match(getConstCharRegex(sourceName)) || [] // now look within them to see if a binding is used const charConstBindingMatch = charConstMatch.find((string: any) => - string.match(new RegExp(`'[^']*${binding}[^']*'`)) + string.match(getBindingWithinConstCharRegex(sourceName, binding)) ) if (charConstBindingMatch) { let [part1, part2] = charConstBindingMatch.split(binding) diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 3ba4995b2c..59c4b6dc89 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -112,9 +112,15 @@ class QueryRunner { let query: Record // handle SQL injections by interpolating the variables if (isSQL(datasourceClone)) { - query = await interpolateSQL(fieldsClone, enrichedContext, integration, { - nullDefaultSupport, - }) + query = await interpolateSQL( + datasource.source, + fieldsClone, + enrichedContext, + integration, + { + nullDefaultSupport, + } + ) } else { query = await sdk.queries.enrichContext(fieldsClone, enrichedContext) } diff --git a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts index fa674fcc52..49dd05910d 100644 --- a/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/utils.spec.ts @@ -1,30 +1,11 @@ -import { fixAutoColumnSubType, processAIColumns } from "../utils" +import { fixAutoColumnSubType } from "../utils" import { AutoFieldDefaultNames } from "../../../constants" import { - AIOperationEnum, AutoFieldSubType, FieldSchema, FieldType, - INTERNAL_TABLE_SOURCE_ID, RelationshipType, - Table, - TableSourceType, } from "@budibase/types" -import { generator } from "@budibase/backend-core/tests" - -const buildPromptMock = jest.fn() - -jest.mock("@budibase/pro", () => ({ - ai: { - LargeLanguageModel: { - forCurrentTenant: async () => ({ - llm: {}, - run: jest.fn(() => "response from LLM"), - buildPromptFromAIOperation: buildPromptMock, - }), - }, - }, -})) describe("rowProcessor utility", () => { describe("fixAutoColumnSubType", () => { @@ -79,59 +60,4 @@ describe("rowProcessor utility", () => { expect(fixAutoColumnSubType(schema)).toEqual(schema) }) }) - - describe("processAIColumns", () => { - it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => { - const table: Table = { - _id: generator.guid(), - name: "AITestTable", - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - product: { - type: FieldType.STRING, - name: "product", - constraints: { - presence: true, - type: "string", - }, - }, - aicol: { - type: FieldType.AI, - name: "aicol", - operation: AIOperationEnum.PROMPT, - prompt: "Translate '{{ product }}' into German", - }, - }, - } - - const inputRows = [ - { - product: "Car Battery", - }, - ] - - const result = await processAIColumns(table, inputRows, { - contextRows: inputRows, - }) - expect(buildPromptMock).toHaveBeenCalledWith({ - row: { - product: "Car Battery", - }, - schema: { - name: "aicol", - operation: "PROMPT", - prompt: "Translate 'Car Battery' into German", - type: "ai", - }, - }) - expect(result).toEqual([ - { - aicol: "response from LLM", - product: "Car Battery", - }, - ]) - }) - }) }) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 7d2f8b49f4..8344ca5ce1 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -15,7 +15,7 @@ import { import { OperationFields } from "@budibase/shared-core" import tracer from "dd-trace" import { context } from "@budibase/backend-core" -import * as pro from "@budibase/pro" +import { ai } from "@budibase/pro" import { coerce } from "./index" interface FormulaOpts { @@ -126,10 +126,8 @@ export async function processAIColumns( const numRows = Array.isArray(inputRows) ? inputRows.length : 1 span?.addTags({ table_id: table._id, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] - const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant( - "gpt-4o-mini" - ) - if (rows && llmWrapper.llm) { + const llm = await ai.getLLM() + if (rows && llm) { // Ensure we have snippet context await context.ensureSnippetContext() @@ -153,17 +151,12 @@ export async function processAIColumns( } } - const prompt = llmWrapper.buildPromptFromAIOperation({ - schema: aiSchema, - row, - }) - return tracer.trace("processAIColumn", {}, async span => { span?.addTags({ table_id: table._id, column }) - const llmResponse = await llmWrapper.run(prompt) + const llmResponse = await llm.operation(aiSchema, row) return { ...row, - [column]: llmResponse, + [column]: llmResponse.message, } }) }) diff --git a/packages/types/src/api/account/license.ts b/packages/types/src/api/account/license.ts index 342e4882e6..9f3a96c6bd 100644 --- a/packages/types/src/api/account/license.ts +++ b/packages/types/src/api/account/license.ts @@ -6,6 +6,7 @@ export interface GetLicenseRequest { // All fields should be optional to cater for // historical versions of budibase quotaUsage?: QuotaUsage + tenantId?: string install: { id: string tenantId: string diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index c5190354d8..bd0340595c 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,18 +111,26 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" +export type AIProvider = + | "OpenAI" + | "Anthropic" + | "AzureOpenAI" + | "TogetherAI" + | "Custom" + +export interface ProviderConfig { + provider: AIProvider + isDefault: boolean + isBudibaseAI?: boolean + name: string + active: boolean + baseUrl?: string + apiKey?: string + defaultModel?: string +} export interface AIInnerConfig { - [key: string]: { - provider: AIProvider - isDefault: boolean - name: string - active: boolean - baseUrl?: string - apiKey?: string - defaultModel?: string - } + [key: string]: ProviderConfig } export interface AIConfig extends Config {} diff --git a/packages/types/src/sdk/licensing/license.ts b/packages/types/src/sdk/licensing/license.ts index 105c3680a3..c51fa33203 100644 --- a/packages/types/src/sdk/licensing/license.ts +++ b/packages/types/src/sdk/licensing/license.ts @@ -17,4 +17,5 @@ export interface License { plan: PurchasedPlan billing?: Billing testClockId?: string + tenantId?: string } diff --git a/packages/worker/package.json b/packages/worker/package.json index b67e8d4911..7af2a4cdd6 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -102,6 +102,9 @@ "typescript": "5.7.2", "update-dotenv": "1.1.1" }, + "resolutions": { + "@budibase/pro": "npm:@budibase/pro@latest" + }, "nx": { "targets": { "dev": { diff --git a/yarn.lock b/yarn.lock index 7b855ecf15..2fa83c1deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2795,28 +2795,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "3.4.22" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.22.tgz#943f23cb7056041bc1f433ee60b3d093145e7a4a" - integrity sha512-Du3iZsmRLopfoi2SvxQyY1P2Su3Nw0WbITOrKmZFsVLjZ9MzzTZs0Ph/SJHzrfJpM7rn9+8788BLSf3Z3l9KcQ== - dependencies: - "@anthropic-ai/sdk" "^0.27.3" - "@budibase/backend-core" "*" - "@budibase/shared-core" "*" - "@budibase/string-templates" "*" - "@budibase/types" "*" - "@koa/router" "13.1.0" - bull "4.10.1" - dd-trace "5.26.0" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - openai "4.59.0" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - "@budibase/vm-browserify@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262"