From c85bc88bf9099d2e4f9082e5bb2baff00fedeaa9 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 16 Sep 2024 11:54:04 +0100 Subject: [PATCH 01/37] draft of openai impl --- packages/backend-core/src/configs/configs.ts | 9 ++++ .../server/src/automations/steps/openai.ts | 51 ++++++++++++++----- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..00ccee8c4e 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,6 @@ import { + AIConfig, + AIInnerConfig, Config, ConfigType, GoogleConfig, @@ -254,3 +256,10 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + +// AI +export async function getAIConfig(): Promise { + const config = await getConfig(ConfigType.AI) + return config?.config +} + diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 1c148b2e73..d8017d0ceb 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -10,6 +10,7 @@ import { } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" +import * as pro from "@budibase/pro" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", @@ -60,6 +61,23 @@ export const definition: AutomationStepDefinition = { }, } +async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { + const openai = new OpenAI({ + apiKey: env.OPENAI_API_KEY, + }) + + const completion = await openai.chat.completions.create({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt, + }, + ], + }) + return completion?.choices[0]?.message?.content +} + export async function run({ inputs, }: { @@ -81,20 +99,27 @@ export async function run({ } try { - const openai = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }) + let response + const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() + const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const completion = await openai.chat.completions.create({ - model: inputs.model, - messages: [ - { - role: "user", - content: inputs.prompt, - }, - ], - }) - const response = completion?.choices[0]?.message?.content + if (budibaseAIEnabled || customConfigsEnabled) { + // Enterprise has custom configs + // if custom configs are enabled full stop + // Don't use their budibase AI credits, unless it uses the budibase AI configuration + // TODO: grab the config from the database (maybe wrap this in the pro AI module) + // TODO: pass it into the model to execute the prompt + + // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config + // Make sure it uses their credits + // Should be handled in the LLM wrapper in pro + const llm = new pro.ai.LLMWrapper() + await llm.init() + response = await llm.run(inputs.prompt) + } else { + // fallback to the default that uses the environment variable for backwards compat + response = await legacyOpenAIPrompt(inputs) + } return { response, From 224be10b6c21e7d69c9c1d3dc63cabca75236f0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 16:08:59 +0100 Subject: [PATCH 02/37] support for overriding models --- packages/backend-core/src/configs/configs.ts | 6 +- packages/pro | 2 +- packages/server/package.json | 2 +- .../server/src/automations/steps/openai.ts | 2 +- .../src/api/controllers/global/configs.ts | 34 +-- yarn.lock | 223 ++++++++++++++---- 6 files changed, 189 insertions(+), 80 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 00ccee8c4e..379c1de7a4 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -258,8 +258,8 @@ export async function getSCIMConfig(): Promise { } // AI -export async function getAIConfig(): Promise { - const config = await getConfig(ConfigType.AI) - return config?.config + +export async function getAIConfig(): Promise { + return getConfig(ConfigType.AI) } diff --git a/packages/pro b/packages/pro index 922431260e..ff141defc6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 922431260e90d558a1ca55398475412e75088057 +Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f diff --git a/packages/server/package.json b/packages/server/package.json index 6dfd528963..41ce10c135 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -101,7 +101,7 @@ "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", - "openai": "^4.52.1", + "openai": "4.59.0", "openapi-types": "9.3.1", "oracledb": "6.5.1", "pg": "8.10.0", diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 1dffc37d1f..13b0008654 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -115,7 +115,7 @@ export async function run({ // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config // Make sure it uses their credits // Should be handled in the LLM wrapper in pro - const llm = new pro.ai.LLMWrapper() + const llm = new pro.ai.LargeLanguageModel(inputs.model) await llm.init() response = await llm.run(inputs.prompt) } else { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 70b2279f6c..d7a18713a2 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -334,32 +334,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { ) } -async function enrichAIConfig(aiConfig: AIConfig) { - // Strip out the API Keys from the response so they don't show in the UI - for (const key in aiConfig.config) { - if (aiConfig.config[key].apiKey) { - aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT - } - } - - // Return the Budibase AI data source as part of the response if licensing allows - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const defaultConfigExists = Object.keys(aiConfig.config).some( - key => aiConfig.config[key].isDefault - ) - if (budibaseAIEnabled) { - aiConfig.config["budibase_ai"] = { - provider: "OpenAI", - active: true, - isDefault: !defaultConfigExists, - defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "", - name: "Budibase AI", - } - } - - return aiConfig -} - export async function find(ctx: UserCtx) { try { // Find the config with the most granular scope based on context @@ -372,7 +346,13 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await enrichAIConfig(scopedConfig) + await pro.ai.getAIConfig(scopedConfig) + // Strip out the API Keys from the response so they don't show in the UI + for (const key in scopedConfig.config) { + if (scopedConfig.config[key].apiKey) { + scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT + } + } } ctx.body = scopedConfig } else { diff --git a/yarn.lock b/yarn.lock index 110cbd7a15..69c3978aaf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,6 +33,19 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.24" +"@anthropic-ai/sdk@^0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475" + integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.1.2" resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8" @@ -759,20 +772,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@4.2.1", "@azure/identity@^3.4.1": - version "4.2.1" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" - integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== +"@azure/identity@^3.4.1": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" + integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.3.0" + "@azure/core-util" "^1.6.1" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.11.1" - "@azure/msal-node" "^2.9.2" + "@azure/msal-browser" "^3.5.0" + "@azure/msal-node" "^2.5.1" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -803,7 +816,7 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.11.1": +"@azure/msal-browser@^3.5.0": version "3.23.0" resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== @@ -815,7 +828,7 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== -"@azure/msal-node@^2.9.2": +"@azure/msal-node@^2.5.1": version "2.13.1" resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== @@ -6040,6 +6053,11 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== +"@types/qs@^6.9.15": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -7448,7 +7466,30 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11290,7 +11331,14 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5: + version "4.2.5" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" + integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== + dependencies: + strnum "^1.0.5" + +fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11544,6 +11592,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0, follow-redirects@^1.14.4: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12637,7 +12690,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13162,7 +13220,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@~1.1.6: +is-buffer@^1.1.5, is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -13648,11 +13706,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14562,7 +14615,14 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@^3.0.2, kind-of@^3.1.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== + dependencies: + is-buffer "^1.1.5" + +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -16205,7 +16265,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16399,13 +16459,25 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17020,19 +17092,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^4.52.1: - version "4.52.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" - integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== +openai@4.59.0: + version "4.59.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08" + integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" + "@types/qs" "^6.9.15" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" - web-streams-polyfill "^3.2.1" + qs "^6.10.3" openapi-response-validator@^9.2.0: version "9.3.1" @@ -17556,7 +17629,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18862,7 +18943,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19934,6 +20015,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20006,13 +20092,33 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21575,7 +21681,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21585,6 +21691,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22113,14 +22227,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22522,11 +22628,6 @@ web-streams-polyfill@4.0.0-beta.3: resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== -web-streams-polyfill@^3.2.1: - version "3.3.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" - integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== - web-vitals@^4.0.1: version "4.2.3" resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7" @@ -22900,10 +23001,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22913,6 +23037,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From e40f397c25a96e9cdab5f8ad134637a850c72a0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 16:29:44 +0100 Subject: [PATCH 03/37] move check for OpenAI config variable to legacy path --- packages/pro | 2 +- .../server/src/automations/steps/openai.ts | 30 ++++++++----------- .../src/api/controllers/global/configs.ts | 2 +- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/pro b/packages/pro index ff141defc6..72e9846146 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ff141defc6eb744f7edac788eaaaa046423cfa7f +Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 13b0008654..79b6664674 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -63,6 +63,11 @@ export const definition: AutomationStepDefinition = { }, } +/** + * Maintains backward compatibility with automation steps created before the introduction + * of custom configurations and Budibase AI + * @param inputs - automation inputs from the OpenAI automation step. + */ async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY, @@ -85,14 +90,6 @@ export async function run({ }: { inputs: OpenAIStepInputs }): Promise { - if (!env.OPENAI_API_KEY) { - return { - success: false, - response: - "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", - } - } - if (inputs.prompt == null) { return { success: false, @@ -106,20 +103,19 @@ export async function run({ const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() if (budibaseAIEnabled || customConfigsEnabled) { - // Enterprise has custom configs - // if custom configs are enabled full stop - // Don't use their budibase AI credits, unless it uses the budibase AI configuration - // TODO: grab the config from the database (maybe wrap this in the pro AI module) - // TODO: pass it into the model to execute the prompt - - // TODO: if in cloud and budibaseAI is enabled, use the standard budibase AI config - // Make sure it uses their credits - // Should be handled in the LLM wrapper in pro const llm = new pro.ai.LargeLanguageModel(inputs.model) await llm.init() response = await llm.run(inputs.prompt) } else { // fallback to the default that uses the environment variable for backwards compat + if (!env.OPENAI_API_KEY) { + return { + success: false, + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + response = await legacyOpenAIPrompt(inputs) } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index d7a18713a2..f53868d7df 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await pro.ai.getAIConfig(scopedConfig) + await pro.ai.enrichAIConfig(scopedConfig) // Strip out the API Keys from the response so they don't show in the UI for (const key in scopedConfig.config) { if (scopedConfig.config[key].apiKey) { From 8e2d21c85fb85d38a217422d914d3bf86efd6589 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 17 Sep 2024 18:27:49 +0100 Subject: [PATCH 04/37] call right sdk method on ai module for enriching AI config --- packages/pro | 2 +- packages/worker/src/api/controllers/global/configs.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 72e9846146..69633aa64b 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 72e9846146b46efc7cbdc69daea17761cb45f1d3 +Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f53868d7df..f01c7c8869 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -346,7 +346,7 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await pro.ai.enrichAIConfig(scopedConfig) + await pro.sdk.ai.enrichAIConfig(scopedConfig) // Strip out the API Keys from the response so they don't show in the UI for (const key in scopedConfig.config) { if (scopedConfig.config[key].apiKey) { From d7334f100b263e358492e7f79456dd2f0ca72462 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 01:21:13 +0000 Subject: [PATCH 05/37] Bump next from 14.1.1 to 14.2.10 in /examples/nextjs-api-sales Bumps [next](https://github.com/vercel/next.js) from 14.1.1 to 14.2.10. - [Release notes](https://github.com/vercel/next.js/releases) - [Changelog](https://github.com/vercel/next.js/blob/canary/release.js) - [Commits](https://github.com/vercel/next.js/compare/v14.1.1...v14.2.10) --- updated-dependencies: - dependency-name: next dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- examples/nextjs-api-sales/package.json | 2 +- examples/nextjs-api-sales/yarn.lock | 124 +++++++++++++------------ 2 files changed, 66 insertions(+), 60 deletions(-) diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 7ecf264add..f1ef4843a1 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "14.1.1", + "next": "14.2.10", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index a44956ba21..9acbdfdeb6 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" - integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== +"@next/env@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c" + integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,50 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" - integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== +"@next/swc-darwin-arm64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80" + integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ== -"@next/swc-darwin-x64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" - integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== +"@next/swc-darwin-x64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8" + integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA== -"@next/swc-linux-arm64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" - integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== +"@next/swc-linux-arm64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20" + integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA== -"@next/swc-linux-arm64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" - integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== +"@next/swc-linux-arm64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47" + integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ== -"@next/swc-linux-x64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" - integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== +"@next/swc-linux-x64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082" + integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg== -"@next/swc-linux-x64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" - integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== +"@next/swc-linux-x64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3" + integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA== -"@next/swc-win32-arm64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" - integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== +"@next/swc-win32-arm64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209" + integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ== -"@next/swc-win32-ia32-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" - integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== +"@next/swc-win32-ia32-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041" + integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg== -"@next/swc-win32-x64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" - integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== +"@next/swc-win32-x64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455" + integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -129,11 +129,17 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@types/json5@^0.0.29": @@ -1245,28 +1251,28 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" - integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== +next@14.2.10: + version "14.2.10" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6" + integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww== dependencies: - "@next/env" "14.1.1" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.10" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.1" - "@next/swc-darwin-x64" "14.1.1" - "@next/swc-linux-arm64-gnu" "14.1.1" - "@next/swc-linux-arm64-musl" "14.1.1" - "@next/swc-linux-x64-gnu" "14.1.1" - "@next/swc-linux-x64-musl" "14.1.1" - "@next/swc-win32-arm64-msvc" "14.1.1" - "@next/swc-win32-ia32-msvc" "14.1.1" - "@next/swc-win32-x64-msvc" "14.1.1" + "@next/swc-darwin-arm64" "14.2.10" + "@next/swc-darwin-x64" "14.2.10" + "@next/swc-linux-arm64-gnu" "14.2.10" + "@next/swc-linux-arm64-musl" "14.2.10" + "@next/swc-linux-x64-gnu" "14.2.10" + "@next/swc-linux-x64-musl" "14.2.10" + "@next/swc-win32-arm64-msvc" "14.2.10" + "@next/swc-win32-ia32-msvc" "14.2.10" + "@next/swc-win32-x64-msvc" "14.2.10" node-domexception@^1.0.0: version "1.0.0" From 2cf07a40f6467aaac25416509c1e6f528bb42c95 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 09:44:33 +0100 Subject: [PATCH 06/37] fix for setting default configs --- .../src/pages/builder/portal/settings/ai/index.svelte | 5 ++++- packages/pro | 2 +- packages/server/src/automations/steps/openai.ts | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 2ac1609e7c..b60ea24dbc 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -56,10 +56,13 @@ } else { // We don't store the default BB AI config in the DB delete fullAIConfig.config.budibase_ai + // unset the default value from other configs if default is set if (editingAIConfig.isDefault) { for (let key in fullAIConfig.config) { - fullAIConfig.config[key].isDefault = false + if (key !== id) { + fullAIConfig.config[key].isDefault = false + } } } // Add new or update existing custom AI Config diff --git a/packages/pro b/packages/pro index 69633aa64b..5daf17e325 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 69633aa64b962ac4bad01360a421fe941a15fb2c +Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 79b6664674..5eff12db47 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -115,7 +115,6 @@ export async function run({ "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", } } - response = await legacyOpenAIPrompt(inputs) } From cb8d0984b1fa484e4482dd7a2009842c71d293d0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 10:52:52 +0100 Subject: [PATCH 07/37] update Providers --- packages/types/src/documents/global/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 8d64b49ee9..33f7e10584 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,7 +111,7 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom" +export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" export interface AIInnerConfig { [key: string]: { From 9cbaa02f5f305542453493d0e9520cf0ca7681bc Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 14:01:54 +0100 Subject: [PATCH 08/37] add test for automation run step when LLM feature flags are on --- packages/pro | 2 +- .../src/automations/tests/openai.spec.ts | 35 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/pro b/packages/pro index 5daf17e325..320f8ecf8a 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5daf17e32595e539f1f4a92b59a2ea2854d9dbd4 +Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11 diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3a5a57475a..0a64411a12 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -4,6 +4,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import * as pro from "@budibase/pro" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -23,6 +24,20 @@ jest.mock("openai", () => ({ })), })) +jest.mock("@budibase/pro", () => ({ + ...jest.requireActual("@budibase/pro"), + ai: { + LargeLanguageModel: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, + features: { + isAICustomConfigsEnabled: jest.fn(), + isBudibaseAIEnabled: jest.fn(), + } +})) + const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -41,6 +56,7 @@ describe("test the openai action", () => { afterEach(() => { resetEnv() + jest.clearAllMocks() }) afterAll(_afterAll) @@ -86,7 +102,7 @@ describe("test the openai action", () => { ) const res = await runStep("OPENAI", { - prompt: OPENAI_PROMPT, + prompt: OPENAI_PROMPT,jj }) expect(res.response).toEqual( @@ -94,4 +110,21 @@ describe("test the openai action", () => { ) expect(res.success).toBeFalsy() }) + + it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true) + jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) + + const prompt = "What is the meaning of life?" + await runStep("OPENAI", { + model: "gpt-4o-mini", + prompt + }) + + expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + + const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value + expect(llmInstance.init).toHaveBeenCalled() + expect(llmInstance.run).toHaveBeenCalledWith(prompt) + }) }) From 69b2bf9bdb80aad9350ecb6ebf4b4f82555a7d5f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 14:21:10 +0100 Subject: [PATCH 09/37] lint --- packages/backend-core/src/configs/configs.ts | 1 - packages/server/src/automations/tests/openai.spec.ts | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 379c1de7a4..6c2f2fe586 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -262,4 +262,3 @@ export async function getSCIMConfig(): Promise { export async function getAIConfig(): Promise { return getConfig(ConfigType.AI) } - diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 0a64411a12..498d15b93c 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -35,7 +35,7 @@ jest.mock("@budibase/pro", () => ({ features: { isAICustomConfigsEnabled: jest.fn(), isBudibaseAIEnabled: jest.fn(), - } + }, })) const mockedOpenAI = OpenAI as jest.MockedClass @@ -102,7 +102,7 @@ describe("test the openai action", () => { ) const res = await runStep("OPENAI", { - prompt: OPENAI_PROMPT,jj + prompt: OPENAI_PROMPT, }) expect(res.response).toEqual( @@ -118,7 +118,7 @@ describe("test the openai action", () => { const prompt = "What is the meaning of life?" await runStep("OPENAI", { model: "gpt-4o-mini", - prompt + prompt, }) expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") From 8a399bfbdfd33a8b117ee08ad0612befeb05f24c Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 17:42:55 +0100 Subject: [PATCH 10/37] set custom configs quotas to right number --- packages/pro | 2 +- packages/worker/src/api/controllers/global/configs.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 320f8ecf8a..4300b75bde 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 320f8ecf8ae769995590ddc4e3679ef7c110bc11 +Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71 diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f01c7c8869..f3262dc0dd 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,6 +253,8 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } + const numConfigs = Object.keys(config).length + await pro.quotas.updateCustomAIConfigCount(numConfigs) break } } catch (err: any) { From 950bd45b2c1abd58a0bc46f1248dd7cdac76b2c0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 18 Sep 2024 18:04:13 +0100 Subject: [PATCH 11/37] add ts ignore for mock --- packages/server/src/automations/tests/openai.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 498d15b93c..342288a6a1 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -123,6 +123,7 @@ describe("test the openai action", () => { expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + // @ts-ignore const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) From e0fb44f63fbde2997d1c1c4765409ccf6f7fc3a2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:21:49 +0100 Subject: [PATCH 12/37] lint --- packages/backend-core/src/configs/configs.ts | 1 - .../src/api/controllers/global/configs.ts | 3 +- yarn.lock | 266 ++++++++---------- 3 files changed, 115 insertions(+), 155 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 6c2f2fe586..e4f4a874a5 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,6 +1,5 @@ import { AIConfig, - AIInnerConfig, Config, ConfigType, GoogleConfig, diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index f3262dc0dd..e6e80ff3a5 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,8 +253,7 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } - const numConfigs = Object.keys(config).length - await pro.quotas.updateCustomAIConfigCount(numConfigs) + await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length) break } } catch (err: any) { diff --git a/yarn.lock b/yarn.lock index 69c3978aaf..84d64637e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -772,20 +772,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@^3.4.1": - version "3.4.2" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" - integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== +"@azure/identity@4.2.1", "@azure/identity@^3.4.1": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" + integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.6.1" + "@azure/core-util" "^1.3.0" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.5.0" - "@azure/msal-node" "^2.5.1" + "@azure/msal-browser" "^3.11.1" + "@azure/msal-node" "^2.9.2" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -816,7 +816,7 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.5.0": +"@azure/msal-browser@^3.11.1": version "3.23.0" resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== @@ -828,7 +828,7 @@ resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== -"@azure/msal-node@^2.5.1": +"@azure/msal-node@^2.9.2": version "2.13.1" resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== @@ -2066,6 +2066,44 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/backend-core@2.32.5": + version "0.0.0" + dependencies: + "@budibase/nano" "10.1.5" + "@budibase/pouchdb-replication-stream" "1.2.11" + "@budibase/shared-core" "0.0.0" + "@budibase/types" "0.0.0" + aws-cloudfront-sign "3.0.2" + aws-sdk "2.1030.0" + bcrypt "5.1.0" + bcryptjs "2.4.3" + bull "4.10.1" + correlation-id "4.0.0" + dd-trace "5.2.0" + dotenv "16.0.1" + ioredis "5.3.2" + joi "17.6.0" + jsonwebtoken "9.0.2" + knex "2.4.2" + koa-passport "^6.0.0" + koa-pino-logger "4.0.0" + lodash "4.17.21" + node-fetch "2.6.7" + passport-google-oauth "2.0.0" + passport-local "1.0.0" + passport-oauth2-refresh "^2.1.0" + pino "8.11.0" + pino-http "8.3.3" + posthog-node "4.0.1" + pouchdb "7.3.0" + pouchdb-find "7.2.2" + redlock "4.2.0" + rotating-file-stream "3.1.0" + sanitize-s3-objectkey "0.0.1" + semver "^7.5.4" + tar-fs "2.1.1" + uuid "^8.3.2" + "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2108,6 +2146,45 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "2.32.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea" + integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA== + dependencies: + "@budibase/backend-core" "2.32.5" + "@budibase/shared-core" "2.32.5" + "@budibase/string-templates" "2.32.5" + "@budibase/types" "2.32.5" + "@koa/router" "8.0.8" + bull "4.10.1" + dd-trace "5.2.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + +"@budibase/shared-core@2.32.5": + version "0.0.0" + dependencies: + "@budibase/types" "0.0.0" + cron-validate "1.4.5" + +"@budibase/string-templates@2.32.5": + version "0.0.0" + dependencies: + "@budibase/handlebars-helpers" "^0.13.2" + dayjs "^1.10.8" + handlebars "^4.7.8" + lodash.clonedeep "^4.5.0" + +"@budibase/types@2.32.5": + version "0.0.0" + dependencies: + scim-patch "^0.8.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7466,30 +7543,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0: - version "0.24.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" - integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== - dependencies: - follow-redirects "^1.14.4" - -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11331,14 +11385,7 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5: - version "4.2.5" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" - integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== - dependencies: - strnum "^1.0.5" - -fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: +fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== @@ -11592,11 +11639,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.14.4: - version "1.15.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" - integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== - follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12690,12 +12732,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13220,7 +13257,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -13706,6 +13743,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14615,14 +14657,7 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@^3.0.2, kind-of@^3.1.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -16265,7 +16300,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16459,25 +16494,13 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9, node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17629,15 +17652,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18943,7 +18958,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -20015,11 +20030,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20092,33 +20102,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21681,7 +21671,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21691,14 +21681,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22227,6 +22209,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -23001,33 +22991,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -23037,11 +23004,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 062732b29d3514010c1b17743d74c0e673d414a8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:36:00 +0100 Subject: [PATCH 13/37] lint --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 4300b75bde..5929fbb3c6 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 4300b75bde89e37bb0bace927a0e8d204367ae71 +Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e From 6a5f001977a4cfeda3f9d1842d2716165262df4a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 10:54:25 +0100 Subject: [PATCH 14/37] remove tests that are no longer relevant in worker --- .../controllers/global/tests/configs.spec.ts | 49 ------------------- 1 file changed, 49 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 3ff6a5298c..ad2e2aa974 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -35,55 +35,6 @@ describe("Global configs controller", () => { }) }) - it("Should return the default BB AI config when the feature is turned on", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(true)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - budibase_ai: { - provider: "OpenAI", - active: true, - isDefault: true, - name: "Budibase AI", - defaultModel: "", - }, - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - - it("Should not not return the default Budibase AI config when on self host", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(false)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - it("Should not update existing secrets when updating an existing AI Config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data) From b45017dd5af5984009e399105a1c6859c5c43d0e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 19 Sep 2024 17:06:09 +0100 Subject: [PATCH 15/37] lint --- packages/worker/src/api/controllers/global/tests/configs.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index ad2e2aa974..9091f29247 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,4 +1,3 @@ -import * as pro from "@budibase/pro" import { verifyAIConfig } from "../configs" import { TestConfiguration, structures } from "../../../../tests" import { AIInnerConfig } from "@budibase/types" From af26b915b18e105f07d7a3bf9b525b08a3f246df Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 10:17:45 +0100 Subject: [PATCH 16/37] Add Google Sheets search test. --- .../server/src/integrations/googlesheets.ts | 11 +- .../integrations/tests/googlesheets.spec.ts | 190 ++++++++++++++++++ .../integrations/tests/utils/googlesheets.ts | 2 + 3 files changed, 197 insertions(+), 6 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6012ff7789..831528f84d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rows = await sheet.getRows() } - if (hasFilters && query.paginate) { - rows = rows.slice(offset, offset + limit) - } - const headerValues = sheet.headerValues - let response = rows.map(row => - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber) ) response = dataFilters.runQuery(response, query.filters || {}) + if (hasFilters && query.paginate) { + response = response.slice(offset, offset + limit) + } + if (query.sort) { if (Object.keys(query.sort).length !== 1) { console.warn("Googlesheets does not support multiple sorting", { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index dcf4a61b50..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => { ) }) }) + + describe("search", () => { + let table: Table + + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Foo", + }, + { + name: "Bar", + }, + { + name: "Baz", + }, + ], + }) + }) + + it("should be able to find rows with equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + equal: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with not equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEqual: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should be able to find rows with empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + empty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(0) + }) + + it("should be able to find rows with not empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEmpty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(3) + }) + + it("should be able to find rows with one of filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + oneOf: { + name: ["Foo", "Bar"], + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Foo") + expect(response.rows[1].name).toEqual("Bar") + }) + + it("should be able to find rows with fuzzy filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + fuzzy: { + name: "oo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with range filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + range: { + name: { + low: "A", + high: "C", + }, + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Unique value!`, + })), + }) + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Non-unique value!`, + })), + }) + + let response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + }) + let rows: Row[] = response.rows + + while (response.hasNextPage) { + response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + bookmark: response.bookmark, + }) + + expect(response.rows.length).toBeLessThanOrEqual(10) + rows = rows.concat(response.rows) + } + + // Make sure we only get rows matching the query. + expect(rows.length).toEqual(50) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining( + Array.from({ length: 50 }, () => "Unique value!") + ) + ) + + // Make sure all of the rows have a unique ID. + const ids = Object.keys( + rows.reduce((acc, row) => { + acc[row._id!] = true + return acc + }, {}) + ) + expect(ids.length).toEqual(50) + }) + }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 4747f5f9bf..4b9445ebca 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -440,6 +440,8 @@ export class GoogleSheetsMock { endColumnIndex: 0, }) + sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length + return { spreadsheetId: this.spreadsheet.spreadsheetId, tableRange: range, From 7d4aa252449dad10f75247125a49b542820cb281 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 20 Sep 2024 11:53:48 +0100 Subject: [PATCH 17/37] fix issue with loop bindings showing for non loop steps --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index af67ae8d22..927b8588b3 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -752,13 +752,21 @@ : allSteps[idx].icon if (wasLoopBlock) { - loopBlockCount++ schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) }) } + + if ( + allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP && + allSteps + .slice(0, blockIdx) + .some(step => step.stepId === ActionStepID.LOOP) + ) { + bindings = bindings.filter(x => !x.readableBinding.includes("loop")) + } return bindings } From 876d837c3301c03eb09fe3cad2692ea3acf47105 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 14:58:10 +0100 Subject: [PATCH 18/37] adjust instantiation of llm class after static initialiser change --- packages/pro | 2 +- packages/server/src/automations/steps/openai.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/pro b/packages/pro index 5929fbb3c6..922431260e 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 5929fbb3c69b46c525faef4a5a805e0a0015bd8e +Subproject commit 922431260e90d558a1ca55398475412e75088057 diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 5eff12db47..b1dfa3df5b 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -103,8 +103,7 @@ export async function run({ const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() if (budibaseAIEnabled || customConfigsEnabled) { - const llm = new pro.ai.LargeLanguageModel(inputs.model) - await llm.init() + const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) response = await llm.run(inputs.prompt) } else { // fallback to the default that uses the environment variable for backwards compat From 4fec5fcddd3e49ebdaebfe536ef7685ce3240fbd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 17:48:56 +0100 Subject: [PATCH 19/37] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 922431260e..f088614945 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 922431260e90d558a1ca55398475412e75088057 +Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98 From efdfbe7229b94027077da5d2bc4320f3e01f4c45 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 20 Sep 2024 17:58:31 +0100 Subject: [PATCH 20/37] Fixing an issue that was stopping the limit from being applied to MySQL, it needs to wrap the query the same as all other DBs, however it needs to apply the where statement in a slightly different manner. --- packages/backend-core/src/sql/sql.ts | 55 +++++++++++++++++----------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 55f71d76b0..a9400b1839 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -930,7 +930,8 @@ class InternalBuilder { } const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, - fromAlias = aliases?.[fromTable] || fromTable + fromAlias = aliases?.[fromTable] || fromTable, + throughAlias = (throughTable && aliases?.[throughTable]) || throughTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, @@ -961,30 +962,38 @@ class InternalBuilder { // add sorting to get consistent order .orderBy(primaryKey) - // many-to-many relationship with junction table - if (throughTable && toPrimary && fromPrimary) { - const throughAlias = aliases?.[throughTable] || throughTable + const addCorrelatedWhere = ( + query: Knex.QueryBuilder, + column1: string, + column2: string + ) => { + return query.where( + column1, + "=", + knex.raw(this.quotedIdentifier(column2)) + ) + } + + const isManyToMany = throughTable && toPrimary && fromPrimary + let correlatedTo = isManyToMany + ? `${throughAlias}.${fromKey}` + : `${toAlias}.${toKey}`, + correlatedFrom = isManyToMany + ? `${fromAlias}.${fromPrimary}` + : `${fromAlias}.${fromKey}` + // many-to-many relationship needs junction table join + if (isManyToMany) { let throughTableWithSchema = this.tableNameWithSchema(throughTable, { alias: throughAlias, schema: endpoint.schema, }) - subQuery = subQuery - .join(throughTableWithSchema, function () { - this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) - }) - .where( - `${throughAlias}.${fromKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) - ) + subQuery = subQuery.join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) } - // one-to-many relationship with foreign key - else { - subQuery = subQuery.where( - `${toAlias}.${toKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) - ) + // my-sql needs the where statement to be part of main query, not sub-query + if (sqlClient !== SqlClient.MY_SQL) { + subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) } const standardWrap = (select: string): Knex.QueryBuilder => { @@ -1009,8 +1018,10 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = subQuery.select( - knex.raw(`json_arrayagg(json_object(${fieldList}))`) + wrapperQuery = addCorrelatedWhere( + standardWrap(`json_arrayagg(json_object(${fieldList}))`), + isManyToMany ? fromKey! : toKey!, + correlatedFrom ) break case SqlClient.ORACLE: From 3b984629ac9318899a642a824dfaecf609e999c7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 18:35:34 +0100 Subject: [PATCH 21/37] pro reference --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index f088614945..1da5c27fae 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit f088614945e4d3da8aafb4149abe0f1bfc62df98 +Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea From dc6c6ad9c80e515aeb93508e85a5118fb03c9793 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 20 Sep 2024 18:42:50 +0100 Subject: [PATCH 22/37] ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 1da5c27fae..e2fe0f9cc8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 1da5c27fae01692b901114351911fe7f2d41aeea +Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 From b0252469ed3a19461452d019093f4fb6df3b0a4d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 12:33:25 +0100 Subject: [PATCH 23/37] Removing wrap for MySQL. --- packages/backend-core/src/sql/sql.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a9400b1839..91b4e124cc 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -991,10 +991,8 @@ class InternalBuilder { this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) }) } - // my-sql needs the where statement to be part of main query, not sub-query - if (sqlClient !== SqlClient.MY_SQL) { - subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) - } + + subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) const standardWrap = (select: string): Knex.QueryBuilder => { subQuery = subQuery.select(`${toAlias}.*`) @@ -1018,11 +1016,7 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = addCorrelatedWhere( - standardWrap(`json_arrayagg(json_object(${fieldList}))`), - isManyToMany ? fromKey! : toKey!, - correlatedFrom - ) + wrapperQuery = knex.raw(`json_arrayagg(json_object(${fieldList}))`) break case SqlClient.ORACLE: wrapperQuery = standardWrap( From c8d1956c73baed7c025fa15b2144cdd7bc309de4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 12:59:20 +0100 Subject: [PATCH 24/37] Fixing AI test cases. --- .../src/automations/tests/openai.spec.ts | 23 ++++++----- yarn.lock | 39 +++++++++++++++---- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 342288a6a1..6b5e1fbb6a 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -23,14 +23,15 @@ jest.mock("openai", () => ({ }, })), })) - jest.mock("@budibase/pro", () => ({ ...jest.requireActual("@budibase/pro"), ai: { - LargeLanguageModel: jest.fn().mockImplementation(() => ({ - init: jest.fn(), - run: jest.fn(), - })), + LargeLanguageModel: { + forCurrentTenant: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, }, features: { isAICustomConfigsEnabled: jest.fn(), @@ -38,6 +39,7 @@ jest.mock("@budibase/pro", () => ({ }, })) +const mockedPro = jest.mocked(pro) const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -121,11 +123,14 @@ describe("test the openai action", () => { prompt, }) - expect(pro.ai.LargeLanguageModel).toHaveBeenCalledWith("gpt-4o-mini") + expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( + "gpt-4o-mini" + ) - // @ts-ignore - const llmInstance = pro.ai.LargeLanguageModel.mock.results[0].value - expect(llmInstance.init).toHaveBeenCalled() + const llmInstance = + mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() expect(llmInstance.run).toHaveBeenCalledWith(prompt) }) }) diff --git a/yarn.lock b/yarn.lock index 84d64637e4..cd850e833d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12372,10 +12372,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad" - integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w== +"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f" + integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ== dependencies: axios "^1.4.0" lodash "^4.17.21" @@ -20786,7 +20786,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20877,7 +20886,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20891,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22846,7 +22862,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22864,6 +22880,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From c083fe3bd7fa99d52a26201e65f0b81ae254e498 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 14:43:46 +0100 Subject: [PATCH 25/37] fix issue with js bindings not using quotes --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 927b8588b3..aceb980786 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -643,8 +643,8 @@ runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName - ? `stepsByName[${bindingName}].${name}` - : `steps[${idx - loopBlockCount}].${name}` + ? `stepsByName["${bindingName}"].${name}` + : `steps["${idx - loopBlockCount}"].${name}` } else { runtimeName = hasUserDefinedName ? `stepsByName.${bindingName}.${name}` From 189caa6235b14404f0ccff4a66797ad1bc7497aa Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 14:59:41 +0100 Subject: [PATCH 26/37] fix issue where you could have multiple steps with the same name --- .../FlowChart/FlowItemHeader.svelte | 61 +++++++++++-------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 5533572511..52f3f49511 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -16,9 +16,11 @@ export let enableNaming = true let validRegex = /^[A-Za-z0-9_\s]+$/ let typing = false + let editing = false const dispatch = createEventDispatcher() $: stepNames = $selectedAutomation?.definition.stepNames + $: allSteps = $selectedAutomation?.definition.steps || [] $: automationName = stepNames?.[block.id] || block?.name || "" $: automationNameError = getAutomationNameError(automationName) $: status = updateStatus(testResult) @@ -56,9 +58,17 @@ } } const getAutomationNameError = name => { - if (stepNames) { + if (stepNames && editing) { + // Check against stepNames for (const [key, value] of Object.entries(stepNames)) { - if (name === value && key !== block.id) { + if (name !== block.name && name === value && key !== block.id) { + return "This name already exists, please enter a unique name" + } + } + + // Check against other block names + for (const step of allSteps) { + if (step.id !== block.id && name === step.name) { return "This name already exists, please enter a unique name" } } @@ -67,11 +77,11 @@ if (name !== block.name && name?.length > 0) { let invalidRoleName = !validRegex.test(name) if (invalidRoleName) { - return "Please enter a role name consisting of only alphanumeric symbols and underscores" + return "Please enter a name consisting of only alphanumeric symbols and underscores" } - - return null } + + return null } const startTyping = async () => { @@ -89,13 +99,28 @@ await automationStore.actions.saveAutomationName(block.id, automationName) } } + + const startEditing = () => { + editing = true + typing = true + } + + const stopEditing = async () => { + editing = false + typing = false + if (automationNameError) { + automationName = stepNames[block.id] || block?.name + } else { + await saveName() + } + }
dispatch("toggle")} > @@ -132,7 +157,7 @@ { e.stopPropagation() - startTyping() + startEditing() }} on:keydown={async e => { if (e.key === "Enter") { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() - } - } - }} - on:blur={async () => { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() + await stopEditing() } }} + on:blur={stopEditing} /> {:else}
@@ -222,7 +235,7 @@ /> {/if}
- {#if automationNameError} + {#if automationNameError && editing}
From 335240718c94ee4e433d4f2c1706f17c86f6524b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:02:37 +0100 Subject: [PATCH 27/37] tidy up --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 52f3f49511..f85496ec81 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -59,14 +59,12 @@ } const getAutomationNameError = name => { if (stepNames && editing) { - // Check against stepNames for (const [key, value] of Object.entries(stepNames)) { if (name !== block.name && name === value && key !== block.id) { return "This name already exists, please enter a unique name" } } - // Check against other block names for (const step of allSteps) { if (step.id !== block.id && name === step.name) { return "This name already exists, please enter a unique name" From 63651b21e6f43f827ee6db38b5b90c47b4fa9f15 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:17:44 +0100 Subject: [PATCH 28/37] lint --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index f85496ec81..361164cfe5 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -82,10 +82,6 @@ return null } - const startTyping = async () => { - typing = true - } - const saveName = async () => { if (automationNameError || block.name === automationName) { return From 89354f640bf30534924ddb1814f2fb4eb681b120 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Mon, 23 Sep 2024 15:29:27 +0100 Subject: [PATCH 29/37] dupe string --- .../AutomationBuilder/FlowChart/FlowItemHeader.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 361164cfe5..a98c597142 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -58,16 +58,18 @@ } } const getAutomationNameError = name => { + const duplicateError = + "This name already exists, please enter a unique name" if (stepNames && editing) { for (const [key, value] of Object.entries(stepNames)) { if (name !== block.name && name === value && key !== block.id) { - return "This name already exists, please enter a unique name" + return duplicateError } } for (const step of allSteps) { if (step.id !== block.id && name === step.name) { - return "This name already exists, please enter a unique name" + return duplicateError } } } From 676058bbbdf47ce79c4b699a3638ea1ec4604bd3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 17:16:34 +0100 Subject: [PATCH 30/37] Updates to limit the response of JSON_ARRAYAGG in mysql/mariaDB - rather than using a limited sub-query which is dis-allowed in MySQL/MariaDB due to the nature of the correlated sub-query. --- packages/backend-core/src/sql/sql.ts | 29 +++++++++++++--------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 91b4e124cc..db4ddf180b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -958,22 +958,9 @@ class InternalBuilder { const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) - .limit(getRelationshipLimit()) // add sorting to get consistent order .orderBy(primaryKey) - const addCorrelatedWhere = ( - query: Knex.QueryBuilder, - column1: string, - column2: string - ) => { - return query.where( - column1, - "=", - knex.raw(this.quotedIdentifier(column2)) - ) - } - const isManyToMany = throughTable && toPrimary && fromPrimary let correlatedTo = isManyToMany ? `${throughAlias}.${fromKey}` @@ -992,10 +979,15 @@ class InternalBuilder { }) } - subQuery = addCorrelatedWhere(subQuery, correlatedTo, correlatedFrom) + // add the correlation to the overall query + subQuery = subQuery.where( + correlatedTo, + "=", + knex.raw(this.quotedIdentifier(correlatedFrom)) + ) const standardWrap = (select: string): Knex.QueryBuilder => { - subQuery = subQuery.select(`${toAlias}.*`) + subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, @@ -1016,7 +1008,12 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: - wrapperQuery = knex.raw(`json_arrayagg(json_object(${fieldList}))`) + // can't use the standard wrap due to correlated sub-query limitations in MariaDB + wrapperQuery = subQuery.select( + knex.raw( + `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})` + ) + ) break case SqlClient.ORACLE: wrapperQuery = standardWrap( From 680c68a35b07559b496552c630af41e2c45599aa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 23 Sep 2024 18:41:23 +0100 Subject: [PATCH 31/37] Adding test case. --- .../src/api/routes/tests/search.spec.ts | 94 ++++++++++++++----- .../src/sdk/app/tables/external/index.ts | 6 +- .../src/sdk/app/tables/external/utils.ts | 6 +- 3 files changed, 78 insertions(+), 28 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 0b0802bab2..4a695edc06 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -42,6 +42,7 @@ import { Knex } from "knex" import { structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" import { generateRowIdField } from "../../../integrations/utils" +import { cloneDeep } from "lodash/fp" describe.each([ ["in-memory", undefined], @@ -66,6 +67,35 @@ describe.each([ let table: Table let rows: Row[] + async function basicRelationshipTables(type: RelationshipType) { + const relatedTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: type, + name: "productCat", + fieldName: "product", + tableId: relatedTable._id!, + constraints: { + type: "array", + }, + }, + }, + "product" + ) + return { + relatedTable: await config.api.table.get(relatedTable._id!), + table, + } + } + beforeAll(async () => { await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init()) if (isLucene) { @@ -201,6 +231,7 @@ describe.each([ // rows returned by the query will also cause the assertion to fail. async toMatchExactly(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -211,7 +242,7 @@ describe.each([ expect.objectContaining(this.popRow(expectedRow, foundRows)) ) ) - return response + return cloned } // Asserts that the query returns rows matching exactly the set of rows @@ -219,6 +250,7 @@ describe.each([ // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -231,7 +263,7 @@ describe.each([ ) ) ) - return response + return cloned } // Asserts that the query returns some property values - this cannot be used @@ -239,6 +271,7 @@ describe.each([ // typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...) async toMatch(properties: Record) { const response = await this.performSearch() + const cloned = cloneDeep(response) const keys = Object.keys(properties) as Array> for (let key of keys) { // eslint-disable-next-line jest/no-standalone-expect @@ -248,17 +281,18 @@ describe.each([ expect(response[key]).toEqual(properties[key]) } } - return response + return cloned } // Asserts that the query doesn't return a property, e.g. pagination parameters. async toNotHaveProperty(properties: (keyof SearchResponse)[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) for (let property of properties) { // eslint-disable-next-line jest/no-standalone-expect expect(response[property]).toBeUndefined() } - return response + return cloned } // Asserts that the query returns rows matching the set of rows passed in. @@ -266,6 +300,7 @@ describe.each([ // assertion to fail. async toContain(expectedRows: any[]) { const response = await this.performSearch() + const cloned = cloneDeep(response) const foundRows = response.rows // eslint-disable-next-line jest/no-standalone-expect @@ -276,7 +311,7 @@ describe.each([ ) ) ) - return response + return cloned } async toFindNothing() { @@ -2196,28 +2231,10 @@ describe.each([ let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - productCategoryTable = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - }, - "productCategory" - ) - table = await createTable( - { - name: { name: "name", type: FieldType.STRING }, - productCat: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "productCat", - fieldName: "product", - tableId: productCategoryTable._id!, - constraints: { - type: "array", - }, - }, - }, - "product" + const { relatedTable } = await basicRelationshipTables( + RelationshipType.ONE_TO_MANY ) + productCategoryTable = relatedTable productCatRows = await Promise.all([ config.api.row.save(productCategoryTable._id!, { name: "foo" }), @@ -2262,6 +2279,31 @@ describe.each([ }).toContainExactly([{ name: "baz", productCat: undefined }]) }) }) + + isSql && + describe("big relations", () => { + beforeAll(async () => { + const { relatedTable } = await basicRelationshipTables( + RelationshipType.MANY_TO_ONE + ) + const mainRow = await config.api.row.save(table._id!, { + name: "foo", + }) + for (let i = 0; i < 11; i++) { + await config.api.row.save(relatedTable._id!, { + name: i, + product: [mainRow._id!], + }) + } + }) + + it("can only pull 500 related rows", async () => { + await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(10) + }) + }) + }) ;(isSqs || isLucene) && describe("relations to same table", () => { let relatedTable: Table, relatedRows: Row[] diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 842b6b5648..913eae6d1f 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -198,6 +198,7 @@ export async function save( } } generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) + tables[relatedTable.name] = relatedTable schema.main = true } @@ -231,7 +232,10 @@ export async function save( // remove the rename prop delete tableToSave._rename - datasource.entities[tableToSave.name] = tableToSave + datasource.entities = { + ...datasource.entities, + ...tables, + } // store it into couch now for budibase reference await db.put(populateExternalTableSchemas(datasource)) diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index 321bd990f5..f27c59dc5a 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -22,12 +22,16 @@ export function cleanupRelationships( tables: Record, oldTable?: Table ) { + if (!oldTable) { + return + } const tableToIterate = oldTable ? oldTable : table // clean up relationships in couch table schemas for (let [key, schema] of Object.entries(tableToIterate.schema)) { if ( schema.type === FieldType.LINK && - (!oldTable || table.schema[key] == null) + oldTable.schema[key] != null && + table.schema[key] == null ) { const schemaTableId = schema.tableId const relatedTable = Object.values(tables).find( From 956df101e84891aeef9e7c49a22772e03ae33ef9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 11:16:42 +0100 Subject: [PATCH 32/37] PR comments and type improvements. --- .../src/api/routes/tests/search.spec.ts | 8 +++++++- .../src/sdk/app/tables/external/index.ts | 6 ++++-- .../src/sdk/app/tables/external/utils.ts | 20 +++++++++++++------ .../server/src/tests/utilities/structures.ts | 10 ++++------ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 4a695edc06..4125d7bf5b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -77,6 +77,7 @@ describe.each([ table = await createTable( { name: { name: "name", type: FieldType.STRING }, + //@ts-ignore - API accepts this structure, will build out rest of definition productCat: { type: FieldType.LINK, relationshipType: type, @@ -2297,12 +2298,17 @@ describe.each([ } }) - it("can only pull 500 related rows", async () => { + it("can only pull 10 related rows", async () => { await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => { const response = await expectQuery({}).toContain([{ name: "foo" }]) expect(response.rows[0].productCat).toBeArrayOfSize(10) }) }) + + it("can pull max rows when env not set (defaults to 500)", async () => { + const response = await expectQuery({}).toContain([{ name: "foo" }]) + expect(response.rows[0].productCat).toBeArrayOfSize(11) + }) }) ;(isSqs || isLucene) && describe("relations to same table", () => { diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index 913eae6d1f..e374e70c87 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -204,7 +204,9 @@ export async function save( // add in the new table for relationship purposes tables[tableToSave.name] = tableToSave - cleanupRelationships(tableToSave, tables, oldTable) + if (oldTable) { + cleanupRelationships(tableToSave, tables, { oldTable }) + } const operation = tableId ? Operation.UPDATE_TABLE : Operation.CREATE_TABLE await makeTableRequest( @@ -259,7 +261,7 @@ export async function destroy(datasourceId: string, table: Table) { const operation = Operation.DELETE_TABLE if (tables) { await makeTableRequest(datasource, operation, table, tables) - cleanupRelationships(table, tables) + cleanupRelationships(table, tables, { deleting: true }) delete tables[table.name] datasource.entities = tables } diff --git a/packages/server/src/sdk/app/tables/external/utils.ts b/packages/server/src/sdk/app/tables/external/utils.ts index f27c59dc5a..21ffa21053 100644 --- a/packages/server/src/sdk/app/tables/external/utils.ts +++ b/packages/server/src/sdk/app/tables/external/utils.ts @@ -20,17 +20,25 @@ import { cloneDeep } from "lodash/fp" export function cleanupRelationships( table: Table, tables: Record, - oldTable?: Table -) { - if (!oldTable) { - return - } + opts: { oldTable: Table } +): void +export function cleanupRelationships( + table: Table, + tables: Record, + opts: { deleting: boolean } +): void +export function cleanupRelationships( + table: Table, + tables: Record, + opts?: { oldTable?: Table; deleting?: boolean } +): void { + const oldTable = opts?.oldTable const tableToIterate = oldTable ? oldTable : table // clean up relationships in couch table schemas for (let [key, schema] of Object.entries(tableToIterate.schema)) { if ( schema.type === FieldType.LINK && - oldTable.schema[key] != null && + (opts?.deleting || oldTable?.schema[key] != null) && table.schema[key] == null ) { const schemaTableId = schema.tableId diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 2e501932b8..72cd31e383 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -600,10 +600,10 @@ export function fullSchemaWithoutLinks({ allRequired, }: { allRequired?: boolean -}) { - const schema: { - [type in Exclude]: FieldSchema & { type: type } - } = { +}): { + [type in Exclude]: FieldSchema & { type: type } +} { + return { [FieldType.STRING]: { name: "string", type: FieldType.STRING, @@ -741,8 +741,6 @@ export function fullSchemaWithoutLinks({ }, }, } - - return schema } export function basicAttachment() { return { From 6a7959e93c4c6ece9e62ce0402e80ab522a923ac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 11:36:14 +0100 Subject: [PATCH 33/37] Fixing test case. --- .../src/api/routes/tests/search.spec.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 4125d7bf5b..706151ba2c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -39,20 +39,20 @@ import tk from "timekeeper" import { encodeJSBinding } from "@budibase/string-templates" import { dataFilters } from "@budibase/shared-core" import { Knex } from "knex" -import { structures } from "@budibase/backend-core/tests" +import { generator, structures } from "@budibase/backend-core/tests" import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default" import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - ["in-memory", undefined], - ["lucene", undefined], - ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // ["in-memory", undefined], + // ["lucene", undefined], + // ["sqs", undefined], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" @@ -72,7 +72,7 @@ describe.each([ { name: { name: "name", type: FieldType.STRING }, }, - "productCategory" + generator.guid().substring(0, 10) ) table = await createTable( { @@ -89,7 +89,7 @@ describe.each([ }, }, }, - "product" + generator.guid().substring(0, 10) ) return { relatedTable: await config.api.table.get(relatedTable._id!), @@ -2268,7 +2268,7 @@ describe.each([ it("should be able to filter by relationship using table name", async () => { await expectQuery({ - equal: { ["productCategory.name"]: "foo" }, + equal: { [`${productCategoryTable.name}.name`]: "foo" }, }).toContainExactly([ { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, ]) From 464f973f122154c2c3169abbd4484417b53c355a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 12:01:47 +0100 Subject: [PATCH 34/37] Adding a separation for MariaDB and MySQL, mariaDB is the core of the problem, this solves for it by separating them and allowing us to use the special json_arrayagg for mariaDB, but use a correlated sub-query for MySQL. --- packages/backend-core/src/sql/sql.ts | 20 +++++++++++++------ packages/backend-core/src/sql/sqlTable.ts | 13 +++++++++++- .../src/api/routes/tests/search.spec.ts | 14 ++++++------- packages/server/src/integrations/mysql.ts | 10 ++++++++++ packages/types/src/sdk/search.ts | 1 + 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index db4ddf180b..4bdec363a4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -150,6 +150,7 @@ class InternalBuilder { return `"${str}"` case SqlClient.MS_SQL: return `[${str}]` + case SqlClient.MARIADB: case SqlClient.MY_SQL: return `\`${str}\`` } @@ -559,7 +560,10 @@ class InternalBuilder { )}${wrap}, FALSE)` ) }) - } else if (this.client === SqlClient.MY_SQL) { + } else if ( + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB + ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (q, key, value) => { return q[rawFnc]( @@ -1007,7 +1011,7 @@ class InternalBuilder { `json_agg(json_build_object(${fieldList}))` ) break - case SqlClient.MY_SQL: + case SqlClient.MARIADB: // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( knex.raw( @@ -1015,6 +1019,7 @@ class InternalBuilder { ) ) break + case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -1181,7 +1186,8 @@ class InternalBuilder { if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || - this.client === SqlClient.MY_SQL + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB ) { const primary = this.table.primary if (!primary) { @@ -1328,12 +1334,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { - client: sqlClient, + client: this.getBaseSqlClient(), } if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } - const client = knex(config) let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient, client, json) @@ -1442,7 +1447,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { let id if (sqlClient === SqlClient.MS_SQL) { id = results?.[0].id - } else if (sqlClient === SqlClient.MY_SQL) { + } else if ( + sqlClient === SqlClient.MY_SQL || + sqlClient === SqlClient.MARIADB + ) { id = results?.insertId } row = processFn( diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 35d7978449..f5b02cc4e4 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -210,16 +210,27 @@ function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder { class SqlTableQueryBuilder { private readonly sqlClient: SqlClient + private extendedSqlClient: SqlClient | undefined // pass through client to get flavour of SQL constructor(client: SqlClient) { this.sqlClient = client } - getSqlClient(): SqlClient { + getBaseSqlClient(): SqlClient { return this.sqlClient } + getSqlClient(): SqlClient { + return this.extendedSqlClient || this.sqlClient + } + + // if working in a database like MySQL with many variants (MariaDB) + // we can set another client which overrides the base one + setExtendedSqlClient(client: SqlClient) { + this.extendedSqlClient = client + } + /** * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 706151ba2c..c770c4e460 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils" import { cloneDeep } from "lodash/fp" describe.each([ - // ["in-memory", undefined], - // ["lucene", undefined], - // ["sqs", undefined], - // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + ["in-memory", undefined], + ["lucene", undefined], + ["sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("search (%s)", (name, dsProvider) => { const isSqs = name === "sqs" const isLucene = name === "lucene" diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index f5b575adb8..8b1ada4184 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -241,6 +241,16 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async connect() { this.client = await mysql.createConnection(this.config) + const res = await this.internalQuery( + { + sql: "SELECT VERSION();", + }, + { connect: false } + ) + const version = res?.[0]?.["VERSION()"] + if (version?.toLowerCase().includes("mariadb")) { + this.setExtendedSqlClient(SqlClient.MARIADB) + } } async disconnect() { diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 7d61aebdfb..7c691beb71 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -195,6 +195,7 @@ export enum SqlClient { MS_SQL = "mssql", POSTGRES = "pg", MY_SQL = "mysql2", + MARIADB = "mariadb", ORACLE = "oracledb", SQL_LITE = "sqlite3", } From 5f5b38d9a440b9e593e61638de43e48feb7ff27d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 24 Sep 2024 12:31:56 +0100 Subject: [PATCH 35/37] Update the version of PostHog used in the cli package. --- packages/cli/package.json | 2 +- packages/cli/src/analytics/Client.ts | 2 +- yarn.lock | 101 ++------------------------- 3 files changed, 7 insertions(+), 98 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index c9ff373142..8efdcc7816 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -28,7 +28,7 @@ "inquirer": "8.0.0", "lookpath": "1.1.0", "node-fetch": "2.6.7", - "posthog-node": "1.3.0", + "posthog-node": "4.0.1", "pouchdb": "7.3.0", "@budibase/pouchdb-replication-stream": "1.2.11", "randomstring": "1.1.5", diff --git a/packages/cli/src/analytics/Client.ts b/packages/cli/src/analytics/Client.ts index 19b171026d..0d7f7fea8f 100644 --- a/packages/cli/src/analytics/Client.ts +++ b/packages/cli/src/analytics/Client.ts @@ -1,4 +1,4 @@ -import PostHog from "posthog-node" +import { PostHog } from "posthog-node" import { POSTHOG_TOKEN, AnalyticsEvent } from "../constants" import { ConfigManager } from "../structures/ConfigManager" diff --git a/yarn.lock b/yarn.lock index cd850e833d..0fe11b1f76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7535,15 +7535,7 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -axios-retry@^3.1.9: - version "3.4.0" - resolved "https://registry.yarnpkg.com/axios-retry/-/axios-retry-3.4.0.tgz#f464dbe9408e5aa78fa319afd38bb69b533d8854" - integrity sha512-VdgaP+gHH4iQYCCNUWF2pcqeciVOdGrBBAYUfTY+wPcO5Ltvp/37MLFNCmJKo7Gj3SHvCSdL8ouI1qLYJN3liA== - dependencies: - "@babel/runtime" "^7.15.4" - is-retry-allowed "^2.2.0" - -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -8396,11 +8388,6 @@ chardet@^0.7.0: resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== -charenc@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" - integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== - cheap-watch@^1.0.2, cheap-watch@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/cheap-watch/-/cheap-watch-1.0.4.tgz#0bcb4a3a8fbd9d5327936493f6b56baa668d8fef" @@ -8787,11 +8774,6 @@ component-emitter@^1.3.0: resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== -component-type@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9" - integrity sha512-Kgy+2+Uwr75vAi6ChWXgHuLvd+QLD7ssgpaRq2zCvt80ptvAfMc/hijcJxXkBa2wMlEZcJvC2H8Ubo+A9ATHIg== - compress-commons@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/compress-commons/-/compress-commons-4.1.2.tgz#6542e59cb63e1f46a8b21b0e06f9a32e4c8b06df" @@ -9210,11 +9192,6 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" -crypt@0.0.2: - version "0.0.2" - resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" - integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== - crypto-browserify@^3.11.0: version "3.12.0" resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" @@ -13257,11 +13234,6 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@~1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== - is-builtin-module@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-3.2.1.tgz#f03271717d8654cfcaf07ab0463faa3571581169" @@ -13546,11 +13518,6 @@ is-retry-allowed@^1.1.0: resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4" integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg== -is-retry-allowed@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz#88f34cbd236e043e71b6932d09b0c65fb7b4d71d" - integrity sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg== - is-self-closing@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/is-self-closing/-/is-self-closing-1.0.1.tgz#5f406b527c7b12610176320338af0fa3896416e4" @@ -14307,11 +14274,6 @@ joi@^17.13.1: "@sideway/formula" "^3.0.1" "@sideway/pinpoint" "^2.0.0" -join-component@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/join-component/-/join-component-1.1.0.tgz#b8417b750661a392bee2c2537c68b2a9d4977cd5" - integrity sha512-bF7vcQxbODoGK1imE2P9GS9aw4zD0Sd+Hni68IMZLj7zRnquH7dXUmMw9hDI5S/Jzt7q+IyTXN0rSg2GI0IKhQ== - joycon@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/joycon/-/joycon-3.1.1.tgz#bce8596d6ae808f8b68168f5fc69280996894f03" @@ -15801,15 +15763,6 @@ md5.js@^1.3.4: inherits "^2.0.1" safe-buffer "^5.1.2" -md5@^2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" - integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== - dependencies: - charenc "0.0.2" - crypt "0.0.2" - is-buffer "~1.1.6" - mdn-data@2.0.14: version "2.0.14" resolved "https://registry.yarnpkg.com/mdn-data/-/mdn-data-2.0.14.tgz#7113fc4281917d63ce29b43446f701e68c25ba50" @@ -16271,7 +16224,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -18440,20 +18393,6 @@ posthog-js@^1.13.4: preact "^10.19.3" web-vitals "^4.0.1" -posthog-node@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-1.3.0.tgz#804ed2f213a2f05253f798bf9569d55a9cad94f7" - integrity sha512-2+VhqiY/rKIqKIXyvemBFHbeijHE25sP7eKltnqcFqAssUE6+sX6vusN9A4luzToOqHQkUZexiCKxvuGagh7JA== - dependencies: - axios "0.24.0" - axios-retry "^3.1.9" - component-type "^1.2.1" - join-component "^1.1.0" - md5 "^2.3.0" - ms "^2.1.3" - remove-trailing-slash "^0.1.1" - uuid "^8.3.2" - posthog-node@4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/posthog-node/-/posthog-node-4.0.1.tgz#eb8b6cdf68c3fdd0dc2b75e8aab2e0ec3727fb2a" @@ -19494,11 +19433,6 @@ remixicon@2.5.0: resolved "https://registry.yarnpkg.com/remixicon/-/remixicon-2.5.0.tgz#b5e245894a1550aa23793f95daceadbf96ad1a41" integrity sha512-q54ra2QutYDZpuSnFjmeagmEiN9IMo56/zz5dDNitzKD23oFRw77cWo4TsrAdmdkPiEn8mxlrTqxnkujDbEGww== -remove-trailing-slash@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz#be2285a59f39c74d1bce4f825950061915e3780d" - integrity sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA== - request@^2.88.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" @@ -20786,16 +20720,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20886,7 +20811,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20900,13 +20825,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22862,7 +22780,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22880,15 +22798,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From c643c82654e4feb9987ff70e8facdd735d849c16 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 13:44:26 +0100 Subject: [PATCH 36/37] Fix for SQL server. --- packages/backend-core/src/sql/sql.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4bdec363a4..2b20938981 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1031,7 +1031,9 @@ class InternalBuilder { .select(`${fromAlias}.*`) // @ts-ignore - from alias syntax not TS supported .from({ - [fromAlias]: subQuery.select(`${toAlias}.*`), + [fromAlias]: subQuery + .select(`${toAlias}.*`) + .limit(getRelationshipLimit()), })} FOR JSON PATH))` ) break From 2337a4b0b3e6916c8e7c3ce4a41f79b3507c3f68 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 24 Sep 2024 13:29:21 +0000 Subject: [PATCH 37/37] Bump version to 2.32.7 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index d695869907..272a1dd0c6 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.6", + "version": "2.32.7", "npmClient": "yarn", "packages": [ "packages/*",