MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/mysql-correlated-queries

This commit is contained in:
Michael Drury 2024-09-23 18:41:43 +01:00 committed by GitHub
parent 680c68a35b c491050baa
commit 617b7df07e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 427 additions and 192 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/nextjs-api-sales/package.json
View File

@ -10,7 +10,7 @@
},
"dependencies": {
"bulma": "^0.9.3",
"next": "14.1.1",
"next": "14.2.10",
"node-fetch": "^3.2.10",
"sass": "^1.52.3",
"react": "17.0.2",

124
examples/nextjs-api-sales/yarn.lock
View File

@ -46,10 +46,10 @@
resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
"@next/env@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac"
integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA==
"@next/env@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c"
integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw==
"@next/eslint-plugin-next@12.1.0":
version "12.1.0"
@ -58,50 +58,50 @@
dependencies:
glob "7.1.7"
"@next/swc-darwin-arm64@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64"
integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==
"@next/swc-darwin-arm64@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80"
integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ==
"@next/swc-darwin-x64@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b"
integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==
"@next/swc-darwin-x64@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8"
integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA==
"@next/swc-linux-arm64-gnu@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa"
integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==
"@next/swc-linux-arm64-gnu@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20"
integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA==
"@next/swc-linux-arm64-musl@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a"
integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==
"@next/swc-linux-arm64-musl@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47"
integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ==
"@next/swc-linux-x64-gnu@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528"
integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==
"@next/swc-linux-x64-gnu@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082"
integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg==
"@next/swc-linux-x64-musl@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25"
integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==
"@next/swc-linux-x64-musl@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3"
integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA==
"@next/swc-win32-arm64-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9"
integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==
"@next/swc-win32-arm64-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209"
integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ==
"@next/swc-win32-ia32-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2"
integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==
"@next/swc-win32-ia32-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041"
integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg==
"@next/swc-win32-x64-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52"
integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==
"@next/swc-win32-x64-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455"
integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
@ -129,11 +129,17 @@
resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323"
integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A==
"@swc/helpers@0.5.2":
version "0.5.2"
resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d"
integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==
"@swc/counter@^0.1.3":
version "0.1.3"
resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9"
integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==
"@swc/helpers@0.5.5":
version "0.5.5"
resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0"
integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==
dependencies:
"@swc/counter" "^0.1.3"
tslib "^2.4.0"
"@types/json5@^0.0.29":
@ -1245,28 +1251,28 @@ natural-compare@^1.4.0:
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
next@14.1.1:
version "14.1.1"
resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171"
integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==
next@14.2.10:
version "14.2.10"
resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6"
integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww==
dependencies:
"@next/env" "14.1.1"
"@swc/helpers" "0.5.2"
"@next/env" "14.2.10"
"@swc/helpers" "0.5.5"
busboy "1.6.0"
caniuse-lite "^1.0.30001579"
graceful-fs "^4.2.11"
postcss "8.4.31"
styled-jsx "5.1.1"
optionalDependencies:
"@next/swc-darwin-arm64" "14.1.1"
"@next/swc-darwin-x64" "14.1.1"
"@next/swc-linux-arm64-gnu" "14.1.1"
"@next/swc-linux-arm64-musl" "14.1.1"
"@next/swc-linux-x64-gnu" "14.1.1"
"@next/swc-linux-x64-musl" "14.1.1"
"@next/swc-win32-arm64-msvc" "14.1.1"
"@next/swc-win32-ia32-msvc" "14.1.1"
"@next/swc-win32-x64-msvc" "14.1.1"
"@next/swc-darwin-arm64" "14.2.10"
"@next/swc-darwin-x64" "14.2.10"
"@next/swc-linux-arm64-gnu" "14.2.10"
"@next/swc-linux-arm64-musl" "14.2.10"
"@next/swc-linux-x64-gnu" "14.2.10"
"@next/swc-linux-x64-musl" "14.2.10"
"@next/swc-win32-arm64-msvc" "14.2.10"
"@next/swc-win32-ia32-msvc" "14.2.10"
"@next/swc-win32-x64-msvc" "14.2.10"
node-domexception@^1.0.0:
version "1.0.0"

7
packages/backend-core/src/configs/configs.ts
View File

@ -1,4 +1,5 @@
import {
AIConfig,
Config,
ConfigType,
GoogleConfig,
@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
return config?.config
}
// AI
export async function getAIConfig(): Promise<AIConfig | undefined> {
return getConfig<AIConfig>(ConfigType.AI)
}

5
packages/builder/src/pages/builder/portal/settings/ai/index.svelte
View File

@ -56,10 +56,13 @@
} else {
// We don't store the default BB AI config in the DB
delete fullAIConfig.config.budibase_ai
// unset the default value from other configs if default is set
if (editingAIConfig.isDefault) {
for (let key in fullAIConfig.config) {
fullAIConfig.config[key].isDefault = false
if (key !== id) {
fullAIConfig.config[key].isDefault = false
}
}
}
// Add new or update existing custom AI Config

2
packages/pro

@ -1 +1 @@
Subproject commit 922431260e90d558a1ca55398475412e75088057
Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733

2
packages/server/package.json
View File

@ -101,7 +101,7 @@
"mysql2": "3.9.8",
"node-fetch": "2.6.7",
"object-sizeof": "2.6.1",
"openai": "^4.52.1",
"openai": "4.59.0",
"openapi-types": "9.3.1",
"oracledb": "6.5.1",
"pg": "8.10.0",

61
packages/server/src/automations/steps/openai.ts
View File

@ -10,6 +10,7 @@ import {
} from "@budibase/types"
import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro"
enum Model {
GPT_35_TURBO = "gpt-3.5-turbo",
@ -62,19 +63,33 @@ export const definition: AutomationStepDefinition = {
},
}
/**
* Maintains backward compatibility with automation steps created before the introduction
* of custom configurations and Budibase AI
* @param inputs - automation inputs from the OpenAI automation step.
*/
async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
return completion?.choices[0]?.message?.content
}
export async function run({
inputs,
}: {
inputs: OpenAIStepInputs
}): Promise<OpenAIStepOutputs> {
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
if (inputs.prompt == null) {
return {
success: false,
@ -83,20 +98,24 @@ export async function run({
}
try {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
let response
const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
const response = completion?.choices[0]?.message?.content
if (budibaseAIEnabled || customConfigsEnabled) {
const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model)
response = await llm.run(inputs.prompt)
} else {
// fallback to the default that uses the environment variable for backwards compat
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
response = await legacyOpenAIPrompt(inputs)
}
return {
response,

39
packages/server/src/automations/tests/openai.spec.ts
View File

@ -4,6 +4,7 @@ import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import * as pro from "@budibase/pro"
jest.mock("openai", () => ({
OpenAI: jest.fn().mockImplementation(() => ({
@ -22,7 +23,23 @@ jest.mock("openai", () => ({
},
})),
}))
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: jest.fn().mockImplementation(() => ({
init: jest.fn(),
run: jest.fn(),
})),
},
},
features: {
isAICustomConfigsEnabled: jest.fn(),
isBudibaseAIEnabled: jest.fn(),
},
}))
const mockedPro = jest.mocked(pro)
const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
const OPENAI_PROMPT = "What is the meaning of life?"
@ -41,6 +58,7 @@ describe("test the openai action", () => {
afterEach(() => {
resetEnv()
jest.clearAllMocks()
})
afterAll(_afterAll)
@ -94,4 +112,25 @@ describe("test the openai action", () => {
)
expect(res.success).toBeFalsy()
})
it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => {
jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true)
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?"
await runStep("OPENAI", {
model: "gpt-4o-mini",
prompt,
})
expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith(
"gpt-4o-mini"
)
const llmInstance =
mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value
// init does not appear to be called currently
// expect(llmInstance.init).toHaveBeenCalled()
expect(llmInstance.run).toHaveBeenCalledWith(prompt)
})
})

11
packages/server/src/integrations/googlesheets.ts
View File

@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
rows = await sheet.getRows()
}
if (hasFilters && query.paginate) {
rows = rows.slice(offset, offset + limit)
}
const headerValues = sheet.headerValues
let response = rows.map(row =>
this.buildRowObject(headerValues, row.toObject(), row.rowNumber)
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber)
)
response = dataFilters.runQuery(response, query.filters || {})
if (hasFilters && query.paginate) {
response = response.slice(offset, offset + limit)
}
if (query.sort) {
if (Object.keys(query.sort).length !== 1) {
console.warn("Googlesheets does not support multiple sorting", {

190
packages/server/src/integrations/tests/googlesheets.spec.ts
View File

@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
import {
Datasource,
FieldType,
Row,
SourceName,
Table,
TableSourceType,
@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => {
)
})
})
describe("search", () => {
let table: Table
beforeEach(async () => {
table = await config.api.table.save({
name: "Test Table",
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
type: "string",
},
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Foo",
},
{
name: "Bar",
},
{
name: "Baz",
},
],
})
})
it("should be able to find rows with equals filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
equal: {
name: "Foo",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].name).toEqual("Foo")
})
it("should be able to find rows with not equals filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
notEqual: {
name: "Foo",
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Bar")
expect(response.rows[1].name).toEqual("Baz")
})
it("should be able to find rows with empty filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
empty: {
name: null,
},
},
})
expect(response.rows).toHaveLength(0)
})
it("should be able to find rows with not empty filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
notEmpty: {
name: null,
},
},
})
expect(response.rows).toHaveLength(3)
})
it("should be able to find rows with one of filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
oneOf: {
name: ["Foo", "Bar"],
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Foo")
expect(response.rows[1].name).toEqual("Bar")
})
it("should be able to find rows with fuzzy filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
fuzzy: {
name: "oo",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].name).toEqual("Foo")
})
it("should be able to find rows with range filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
range: {
name: {
low: "A",
high: "C",
},
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Bar")
expect(response.rows[1].name).toEqual("Baz")
})
it("should paginate correctly", async () => {
await config.api.row.bulkImport(table._id!, {
rows: Array.from({ length: 50 }, () => ({
name: `Unique value!`,
})),
})
await config.api.row.bulkImport(table._id!, {
rows: Array.from({ length: 50 }, () => ({
name: `Non-unique value!`,
})),
})
let response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: { equal: { name: "Unique value!" } },
paginate: true,
limit: 10,
})
let rows: Row[] = response.rows
while (response.hasNextPage) {
response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: { equal: { name: "Unique value!" } },
paginate: true,
limit: 10,
bookmark: response.bookmark,
})
expect(response.rows.length).toBeLessThanOrEqual(10)
rows = rows.concat(response.rows)
}
// Make sure we only get rows matching the query.
expect(rows.length).toEqual(50)
expect(rows.map(row => row.name)).toEqual(
expect.arrayContaining(
Array.from({ length: 50 }, () => "Unique value!")
)
)
// Make sure all of the rows have a unique ID.
const ids = Object.keys(
rows.reduce((acc, row) => {
acc[row._id!] = true
return acc
}, {})
)
expect(ids.length).toEqual(50)
})
})
})

2
packages/server/src/integrations/tests/utils/googlesheets.ts
View File

@ -440,6 +440,8 @@ export class GoogleSheetsMock {
endColumnIndex: 0,
})
sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length
return {
spreadsheetId: this.spreadsheet.spreadsheetId,
tableRange: range,

2
packages/types/src/documents/global/config.ts
View File

@ -111,7 +111,7 @@ export interface SCIMInnerConfig {
export interface SCIMConfig extends Config<SCIMInnerConfig> {}
type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom"
export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom"
export interface AIInnerConfig {
[key: string]: {

35
packages/worker/src/api/controllers/global/configs.ts
View File

@ -253,6 +253,7 @@ export async function save(ctx: UserCtx<Config>) {
if (existingConfig) {
await verifyAIConfig(config, existingConfig)
}
await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length)
break
}
} catch (err: any) {
@ -334,32 +335,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
)
}
async function enrichAIConfig(aiConfig: AIConfig) {
// Strip out the API Keys from the response so they don't show in the UI
for (const key in aiConfig.config) {
if (aiConfig.config[key].apiKey) {
aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
}
}
// Return the Budibase AI data source as part of the response if licensing allows
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
const defaultConfigExists = Object.keys(aiConfig.config).some(
key => aiConfig.config[key].isDefault
)
if (budibaseAIEnabled) {
aiConfig.config["budibase_ai"] = {
provider: "OpenAI",
active: true,
isDefault: !defaultConfigExists,
defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "",
name: "Budibase AI",
}
}
return aiConfig
}
export async function find(ctx: UserCtx) {
try {
// Find the config with the most granular scope based on context
@ -372,7 +347,13 @@ export async function find(ctx: UserCtx) {
}
if (type === ConfigType.AI) {
await enrichAIConfig(scopedConfig)
await pro.sdk.ai.enrichAIConfig(scopedConfig)
// Strip out the API Keys from the response so they don't show in the UI
for (const key in scopedConfig.config) {
if (scopedConfig.config[key].apiKey) {
scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT
}
}
}
ctx.body = scopedConfig
} else {

50
packages/worker/src/api/controllers/global/tests/configs.spec.ts
View File

@ -1,4 +1,3 @@
import * as pro from "@budibase/pro"
import { verifyAIConfig } from "../configs"
import { TestConfiguration, structures } from "../../../../tests"
import { AIInnerConfig } from "@budibase/types"
@ -35,55 +34,6 @@ describe("Global configs controller", () => {
})
})
it("Should return the default BB AI config when the feature is turned on", async () => {
jest
.spyOn(pro.features, "isBudibaseAIEnabled")
.mockImplementation(() => Promise.resolve(true))
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)
const response = await config.api.configs.getAIConfig()
expect(response.body.config).toEqual({
budibase_ai: {
provider: "OpenAI",
active: true,
isDefault: true,
name: "Budibase AI",
defaultModel: "",
},
ai: {
active: true,
apiKey: "--secret-value--",
baseUrl: "https://api.example.com",
defaultModel: "gpt4",
isDefault: false,
name: "Test",
provider: "OpenAI",
},
})
})
it("Should not not return the default Budibase AI config when on self host", async () => {
jest
.spyOn(pro.features, "isBudibaseAIEnabled")
.mockImplementation(() => Promise.resolve(false))
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)
const response = await config.api.configs.getAIConfig()
expect(response.body.config).toEqual({
ai: {
active: true,
apiKey: "--secret-value--",
baseUrl: "https://api.example.com",
defaultModel: "gpt4",
isDefault: false,
name: "Test",
provider: "OpenAI",
},
})
})
it("Should not update existing secrets when updating an existing AI Config", async () => {
const data = structures.configs.ai()
await config.api.configs.saveConfig(data)

87
yarn.lock
View File

@ -33,6 +33,19 @@
"@jridgewell/gen-mapping" "^0.3.5"
"@jridgewell/trace-mapping" "^0.3.24"
"@anthropic-ai/sdk@^0.27.3":
version "0.27.3"
resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475"
integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
abort-controller "^3.0.0"
agentkeepalive "^4.2.1"
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"
"@apidevtools/json-schema-ref-parser@^9.0.6":
version "9.1.2"
resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8"
@ -2053,7 +2066,7 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.32.6":
"@budibase/backend-core@2.32.5":
version "0.0.0"
dependencies:
"@budibase/nano" "10.1.5"
@ -2134,14 +2147,14 @@
through2 "^2.0.0"
"@budibase/pro@npm:@budibase/pro@latest":
version "2.32.6"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f"
integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ==
version "2.32.5"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea"
integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA==
dependencies:
"@budibase/backend-core" "2.32.6"
"@budibase/shared-core" "2.32.6"
"@budibase/string-templates" "2.32.6"
"@budibase/types" "2.32.6"
"@budibase/backend-core" "2.32.5"
"@budibase/shared-core" "2.32.5"
"@budibase/string-templates" "2.32.5"
"@budibase/types" "2.32.5"
"@koa/router" "8.0.8"
bull "4.10.1"
dd-trace "5.2.0"
@ -2153,13 +2166,13 @@
scim-patch "^0.8.1"
scim2-parse-filter "^0.2.8"
"@budibase/shared-core@2.32.6":
"@budibase/shared-core@2.32.5":
version "0.0.0"
dependencies:
"@budibase/types" "0.0.0"
cron-validate "1.4.5"
"@budibase/string-templates@2.32.6":
"@budibase/string-templates@2.32.5":
version "0.0.0"
dependencies:
"@budibase/handlebars-helpers" "^0.13.2"
@ -2167,7 +2180,7 @@
handlebars "^4.7.8"
lodash.clonedeep "^4.5.0"
"@budibase/types@2.32.6":
"@budibase/types@2.32.5":
version "0.0.0"
dependencies:
scim-patch "^0.8.1"
@ -6117,6 +6130,11 @@
resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb"
integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==
"@types/qs@^6.9.15":
version "6.9.16"
resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794"
integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==
"@types/range-parser@*":
version "1.2.4"
resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
@ -17097,19 +17115,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
is-docker "^2.1.1"
is-wsl "^2.2.0"
openai@^4.52.1:
version "4.52.1"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
openai@4.59.0:
version "4.59.0"
resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08"
integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==
dependencies:
"@types/node" "^18.11.18"
"@types/node-fetch" "^2.6.4"
"@types/qs" "^6.9.15"
abort-controller "^3.0.0"
agentkeepalive "^4.2.1"
form-data-encoder "1.7.2"
formdata-node "^4.3.2"
node-fetch "^2.6.7"
web-streams-polyfill "^3.2.1"
qs "^6.10.3"
openapi-response-validator@^9.2.0:
version "9.3.1"
@ -20767,7 +20786,16 @@ string-similarity@^4.0.4:
resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"
"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@ -20858,7 +20886,7 @@ stringify-object@^3.2.1:
is-obj "^1.0.1"
is-regexp "^1.0.0"
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@ -20872,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
dependencies:
ansi-regex "^4.1.0"
strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"
strip-ansi@^7.0.1:
version "7.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@ -22599,11 +22634,6 @@ web-streams-polyfill@4.0.0-beta.3:
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
web-streams-polyfill@^3.2.1:
version "3.3.3"
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b"
integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==
web-vitals@^4.0.1:
version "4.2.3"
resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7"
@ -22832,7 +22862,7 @@ worker-farm@1.7.0:
dependencies:
errno "~0.1.7"
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@ -22850,6 +22880,15 @@ wrap-ansi@^5.1.0:
string-width "^3.0.0"
strip-ansi "^5.0.0"
wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"
wrap-ansi@^8.1.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"