Merge branch 'master' into BUDI-9127/extract-env-input

package.json

@@ -107,7 +107,6 @@
     "@budibase/shared-core": "*",
     "@budibase/string-templates": "*",
     "@budibase/types": "*",
-    "@budibase/pro": "npm:@budibase/pro@latest",
     "tough-cookie": "4.1.3",
     "node-fetch": "2.6.7",
     "semver": "7.5.3",

packages/backend-core/src/environment.ts

@@ -163,6 +163,7 @@ const environment = {
   ACCOUNT_PORTAL_URL:
     process.env.ACCOUNT_PORTAL_URL || "https://account.budibase.app",
   ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY || "",
+  BUDICLOUD_URL: process.env.BUDICLOUD_URL || "https://budibase.app",
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SELF_HOSTED: selfHosted,
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,

packages/backend-core/src/users/db.ts

@@ -261,9 +261,13 @@ export class UserDB {
       }
     }
 
-    const change = dbUser ? 0 : 1 // no change if there is existing user
+    let change = 1
-
     let creatorsChange = 0
+    if (opts.isAccountHolder || dbUser) {
+      change = 0
+      creatorsChange = 1
+    }
+
     if (dbUser) {
       const [isDbUserCreator, isUserCreator] = await creatorsInList([
         dbUser,

packages/pro

@@ -1 +1 @@
-Subproject commit f709bb6a07483785c32ebb6f186709450d735ec3
+Subproject commit 761ec71e1543ef04887d6515f99a2c2911999ebf

packages/server/package.json

@@ -182,6 +182,9 @@
     "yargs": "^13.2.4",
     "zod": "^3.23.8"
   },
+  "resolutions": {
+    "@budibase/pro": "npm:@budibase/pro@latest"
+  },
   "nx": {
     "targets": {
       "dev": {

packages/server/src/api/routes/tests/queries/generic-sql.spec.ts

@@ -16,7 +16,7 @@ const descriptions = datasourceDescribe({
 if (descriptions.length) {
   describe.each(descriptions)(
     "queries ($dbName)",
-    ({ config, dsProvider, isOracle, isMSSQL, isPostgres }) => {
+    ({ config, dsProvider, isOracle, isMSSQL, isPostgres, isMySQL }) => {
       let rawDatasource: Datasource
       let datasource: Datasource
       let client: Knex
@@ -217,6 +217,38 @@ if (descriptions.length) {
               expect(res).toBeDefined()
             })
         })
+
+        isMySQL &&
+          it("should handle ANSI_QUOTE=off MySQL queries with bindings", async () => {
+            const query = await createQuery({
+              fields: {
+                sql: client(tableName)
+                  .select("*")
+                  .where({
+                    name: client.raw("'{{ name }}'"),
+                  })
+                  .toString(),
+              },
+              parameters: [
+                {
+                  name: "name",
+                  default: "",
+                },
+              ],
+              queryVerb: "read",
+            })
+            const res = await config.api.query.execute(
+              query._id!,
+              {
+                parameters: { name: "one" },
+              },
+              {
+                status: 200,
+              }
+            )
+            expect(res.data.length).toEqual(1)
+            expect(res.data[0].name).toEqual("one")
+          })
       })
 
       describe("preview", () => {

packages/server/src/automations/steps/openai.ts

@@ -3,7 +3,7 @@ import { OpenAI } from "openai"
 import { OpenAIStepInputs, OpenAIStepOutputs } from "@budibase/types"
 import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
-import * as pro from "@budibase/pro"
+import { ai } from "@budibase/pro"
 
 /**
  * Maintains backward compatibility with automation steps created before the introduction
@@ -41,18 +41,9 @@ export async function run({
 
   try {
     let response
-    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
+    const llm = await ai.getLLM(inputs.model)
-    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
+    response = llm
-
+      ? (await llm.prompt(inputs.prompt)).message
-    let llmWrapper
-    if (budibaseAIEnabled || customConfigsEnabled) {
-      llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(
-        inputs.model
-      )
-    }
-
-    response = llmWrapper?.llm
-      ? await llmWrapper.run(inputs.prompt)
       : await legacyOpenAIPrompt(inputs)
 
     return {

packages/server/src/integrations/queries/sql.ts

@@ -1,10 +1,33 @@
 import { findHBSBlocks } from "@budibase/string-templates"
-import { DatasourcePlus } from "@budibase/types"
+import { DatasourcePlus, SourceName } from "@budibase/types"
 import sdk from "../../sdk"
 
-const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g")
+const MYSQL_CONST_CHAR_REGEX = new RegExp(`"[^"]*"|'[^']*'`, "g")
+const CONST_CHAR_REGEX = new RegExp(`'[^']*'`, "g")
+
+function getConstCharRegex(sourceName: SourceName) {
+  // MySQL clients support ANSI_QUOTES mode off, this is by default
+  // but " and ' count as string literals
+  if (sourceName === SourceName.MYSQL) {
+    return MYSQL_CONST_CHAR_REGEX
+  } else {
+    return CONST_CHAR_REGEX
+  }
+}
+
+function getBindingWithinConstCharRegex(
+  sourceName: SourceName,
+  binding: string
+) {
+  if (sourceName === SourceName.MYSQL) {
+    return new RegExp(`[^']*${binding}[^']*'|"[^"]*${binding}[^"]*"`, "g")
+  } else {
+    return new RegExp(`'[^']*${binding}[^']*'`)
+  }
+}
 
 export async function interpolateSQL(
+  sourceName: SourceName,
   fields: { sql: string; bindings: any[] },
   parameters: { [key: string]: any },
   integration: DatasourcePlus,
@@ -24,10 +47,10 @@ export async function interpolateSQL(
     )
     // check if the variable was used as part of a string concat e.g. 'Hello {{binding}}'
     // start by finding all the instances of const character strings
-    const charConstMatch = sql.match(CONST_CHAR_REGEX) || []
+    const charConstMatch = sql.match(getConstCharRegex(sourceName)) || []
     // now look within them to see if a binding is used
     const charConstBindingMatch = charConstMatch.find((string: any) =>
-      string.match(new RegExp(`'[^']*${binding}[^']*'`))
+      string.match(getBindingWithinConstCharRegex(sourceName, binding))
     )
     if (charConstBindingMatch) {
       let [part1, part2] = charConstBindingMatch.split(binding)

packages/server/src/threads/query.ts

@@ -112,9 +112,15 @@ class QueryRunner {
     let query: Record<string, any>
     // handle SQL injections by interpolating the variables
     if (isSQL(datasourceClone)) {
-      query = await interpolateSQL(fieldsClone, enrichedContext, integration, {
+      query = await interpolateSQL(
-        nullDefaultSupport,
+        datasource.source,
-      })
+        fieldsClone,
+        enrichedContext,
+        integration,
+        {
+          nullDefaultSupport,
+        }
+      )
     } else {
       query = await sdk.queries.enrichContext(fieldsClone, enrichedContext)
     }

packages/server/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -1,30 +1,11 @@
-import { fixAutoColumnSubType, processAIColumns } from "../utils"
+import { fixAutoColumnSubType } from "../utils"
 import { AutoFieldDefaultNames } from "../../../constants"
 import {
-  AIOperationEnum,
   AutoFieldSubType,
   FieldSchema,
   FieldType,
-  INTERNAL_TABLE_SOURCE_ID,
   RelationshipType,
-  Table,
-  TableSourceType,
 } from "@budibase/types"
-import { generator } from "@budibase/backend-core/tests"
-
-const buildPromptMock = jest.fn()
-
-jest.mock("@budibase/pro", () => ({
-  ai: {
-    LargeLanguageModel: {
-      forCurrentTenant: async () => ({
-        llm: {},
-        run: jest.fn(() => "response from LLM"),
-        buildPromptFromAIOperation: buildPromptMock,
-      }),
-    },
-  },
-}))
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
@@ -79,59 +60,4 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema)).toEqual(schema)
     })
   })
-
-  describe("processAIColumns", () => {
-    it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
-      const table: Table = {
-        _id: generator.guid(),
-        name: "AITestTable",
-        type: "table",
-        sourceId: INTERNAL_TABLE_SOURCE_ID,
-        sourceType: TableSourceType.INTERNAL,
-        schema: {
-          product: {
-            type: FieldType.STRING,
-            name: "product",
-            constraints: {
-              presence: true,
-              type: "string",
-            },
-          },
-          aicol: {
-            type: FieldType.AI,
-            name: "aicol",
-            operation: AIOperationEnum.PROMPT,
-            prompt: "Translate '{{ product }}' into German",
-          },
-        },
-      }
-
-      const inputRows = [
-        {
-          product: "Car Battery",
-        },
-      ]
-
-      const result = await processAIColumns(table, inputRows, {
-        contextRows: inputRows,
-      })
-      expect(buildPromptMock).toHaveBeenCalledWith({
-        row: {
-          product: "Car Battery",
-        },
-        schema: {
-          name: "aicol",
-          operation: "PROMPT",
-          prompt: "Translate 'Car Battery' into German",
-          type: "ai",
-        },
-      })
-      expect(result).toEqual([
-        {
-          aicol: "response from LLM",
-          product: "Car Battery",
-        },
-      ])
-    })
-  })
 })

packages/server/src/utilities/rowProcessor/utils.ts

@@ -15,7 +15,7 @@ import {
 import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
-import * as pro from "@budibase/pro"
+import { ai } from "@budibase/pro"
 import { coerce } from "./index"
 
 interface FormulaOpts {
@@ -126,10 +126,8 @@ export async function processAIColumns<T extends Row | Row[]>(
     const numRows = Array.isArray(inputRows) ? inputRows.length : 1
     span?.addTags({ table_id: table._id, numRows })
     const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
-    const llmWrapper = await pro.ai.LargeLanguageModel.forCurrentTenant(
+    const llm = await ai.getLLM()
-      "gpt-4o-mini"
+    if (rows && llm) {
-    )
-    if (rows && llmWrapper.llm) {
       // Ensure we have snippet context
       await context.ensureSnippetContext()
 
@@ -153,17 +151,12 @@ export async function processAIColumns<T extends Row | Row[]>(
             }
           }
 
-          const prompt = llmWrapper.buildPromptFromAIOperation({
-            schema: aiSchema,
-            row,
-          })
-
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })
-            const llmResponse = await llmWrapper.run(prompt)
+            const llmResponse = await llm.operation(aiSchema, row)
             return {
               ...row,
-              [column]: llmResponse,
+              [column]: llmResponse.message,
             }
           })
         })

packages/types/src/api/account/license.ts

@@ -6,6 +6,7 @@ export interface GetLicenseRequest {
   // All fields should be optional to cater for
   // historical versions of budibase
   quotaUsage?: QuotaUsage
+  tenantId?: string
   install: {
     id: string
     tenantId: string

packages/types/src/documents/global/config.ts

@@ -111,18 +111,26 @@ export interface SCIMInnerConfig {
 
 export interface SCIMConfig extends Config<SCIMInnerConfig> {}
 
-export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom"
+export type AIProvider =
+  | "OpenAI"
+  | "Anthropic"
+  | "AzureOpenAI"
+  | "TogetherAI"
+  | "Custom"
+
+export interface ProviderConfig {
+  provider: AIProvider
+  isDefault: boolean
+  isBudibaseAI?: boolean
+  name: string
+  active: boolean
+  baseUrl?: string
+  apiKey?: string
+  defaultModel?: string
+}
 
 export interface AIInnerConfig {
-  [key: string]: {
+  [key: string]: ProviderConfig
-    provider: AIProvider
-    isDefault: boolean
-    name: string
-    active: boolean
-    baseUrl?: string
-    apiKey?: string
-    defaultModel?: string
-  }
 }
 
 export interface AIConfig extends Config<AIInnerConfig> {}

packages/types/src/sdk/licensing/license.ts

@@ -17,4 +17,5 @@ export interface License {
   plan: PurchasedPlan
   billing?: Billing
   testClockId?: string
+  tenantId?: string
 }

packages/worker/package.json

@@ -102,6 +102,9 @@
     "typescript": "5.7.2",
     "update-dotenv": "1.1.1"
   },
+  "resolutions": {
+    "@budibase/pro": "npm:@budibase/pro@latest"
+  },
   "nx": {
     "targets": {
       "dev": {

yarn.lock

@@ -2795,28 +2795,6 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@npm:@budibase/pro@latest":
-  version "3.4.22"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-3.4.22.tgz#943f23cb7056041bc1f433ee60b3d093145e7a4a"
-  integrity sha512-Du3iZsmRLopfoi2SvxQyY1P2Su3Nw0WbITOrKmZFsVLjZ9MzzTZs0Ph/SJHzrfJpM7rn9+8788BLSf3Z3l9KcQ==
-  dependencies:
-    "@anthropic-ai/sdk" "^0.27.3"
-    "@budibase/backend-core" "*"
-    "@budibase/shared-core" "*"
-    "@budibase/string-templates" "*"
-    "@budibase/types" "*"
-    "@koa/router" "13.1.0"
-    bull "4.10.1"
-    dd-trace "5.26.0"
-    joi "17.6.0"
-    jsonwebtoken "9.0.2"
-    lru-cache "^7.14.1"
-    memorystream "^0.3.1"
-    node-fetch "2.6.7"
-    openai "4.59.0"
-    scim-patch "^0.8.1"
-    scim2-parse-filter "^0.2.8"
-
 "@budibase/vm-browserify@^1.1.4":
   version "1.1.4"
   resolved "https://registry.yarnpkg.com/@budibase/vm-browserify/-/vm-browserify-1.1.4.tgz#eecb001bd9521cb7647e26fb4d2d29d0a4dce262"