diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 4c5cc94d2b..4b9ebf1e5d 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -108,7 +108,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.3.3 & + docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 & docker pull redis & wait $(jobs -p) @@ -179,7 +179,7 @@ jobs: docker pull minio/minio & docker pull redis & docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb:v3.3.3 & + docker pull budibase/couchdb:v3.3.3-sqs-v2.1.1 & wait $(jobs -p) diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index dde912410c..2c1525bd90 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -641,7 +641,7 @@ couchdb: # @ignore repository: budibase/couchdb # @ignore - tag: v3.3.3 + tag: v3.3.3-sqs-v2.1.1 # @ignore pullPolicy: Always diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 7ecf264add..f1ef4843a1 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "14.1.1", + "next": "14.2.10", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index a44956ba21..9acbdfdeb6 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" - integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== +"@next/env@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c" + integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,50 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" - integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== +"@next/swc-darwin-arm64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80" + integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ== -"@next/swc-darwin-x64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" - integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== +"@next/swc-darwin-x64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8" + integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA== -"@next/swc-linux-arm64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" - integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== +"@next/swc-linux-arm64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20" + integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA== -"@next/swc-linux-arm64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" - integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== +"@next/swc-linux-arm64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47" + integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ== -"@next/swc-linux-x64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" - integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== +"@next/swc-linux-x64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082" + integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg== -"@next/swc-linux-x64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" - integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== +"@next/swc-linux-x64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3" + integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA== -"@next/swc-win32-arm64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" - integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== +"@next/swc-win32-arm64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209" + integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ== -"@next/swc-win32-ia32-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" - integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== +"@next/swc-win32-ia32-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041" + integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg== -"@next/swc-win32-x64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" - integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== +"@next/swc-win32-x64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455" + integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -129,11 +129,17 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@types/json5@^0.0.29": @@ -1245,28 +1251,28 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" - integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== +next@14.2.10: + version "14.2.10" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6" + integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww== dependencies: - "@next/env" "14.1.1" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.10" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.1" - "@next/swc-darwin-x64" "14.1.1" - "@next/swc-linux-arm64-gnu" "14.1.1" - "@next/swc-linux-arm64-musl" "14.1.1" - "@next/swc-linux-x64-gnu" "14.1.1" - "@next/swc-linux-x64-musl" "14.1.1" - "@next/swc-win32-arm64-msvc" "14.1.1" - "@next/swc-win32-ia32-msvc" "14.1.1" - "@next/swc-win32-x64-msvc" "14.1.1" + "@next/swc-darwin-arm64" "14.2.10" + "@next/swc-darwin-x64" "14.2.10" + "@next/swc-linux-arm64-gnu" "14.2.10" + "@next/swc-linux-arm64-musl" "14.2.10" + "@next/swc-linux-x64-gnu" "14.2.10" + "@next/swc-linux-x64-musl" "14.2.10" + "@next/swc-win32-arm64-msvc" "14.2.10" + "@next/swc-win32-ia32-msvc" "14.2.10" + "@next/swc-win32-x64-msvc" "14.2.10" node-domexception@^1.0.0: version "1.0.0" diff --git a/globalSetup.ts b/globalSetup.ts index aa1cb00fe1..5d8b0381c0 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -46,7 +46,7 @@ export default async function setup() { await killContainers(containers) try { - const couchdb = new GenericContainer("budibase/couchdb:v3.3.3") + const couchdb = new GenericContainer("budibase/couchdb:v3.3.3-sqs-v2.1.1") .withExposedPorts(5984, 4984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index dfcfe566bd..ded0bc17dc 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -1,4 +1,4 @@ -ARG BASEIMG=budibase/couchdb:v3.3.3 +ARG BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 FROM node:20-slim as build # install node-gyp dependencies diff --git a/lerna.json b/lerna.json index 4a7cfc95cc..d695869907 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.0", + "version": "2.32.6", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 582e35180e..fc7e202e3d 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,10 @@ "axios": "1.6.3", "xml2js": "0.6.2", "unset-value": "2.0.1", - "passport": "0.6.0" + "passport": "0.6.0", + "fast-xml-parser": "4.4.1", + "@azure/identity": "4.2.1", + "kind-of": "6.0.3" }, "engines": { "node": ">=20.0.0 <21.0.0" diff --git a/packages/backend-core/src/cache/user.ts b/packages/backend-core/src/cache/user.ts index d319c5dcb6..9de09a431d 100644 --- a/packages/backend-core/src/cache/user.ts +++ b/packages/backend-core/src/cache/user.ts @@ -63,14 +63,25 @@ async function populateUsersFromDB( * If not present fallback to loading the user directly and re-caching. * @param userId the id of the user to get * @param tenantId the tenant of the user to get + * @param email the email of the user to populate from account if needed * @param populateUser function to provide the user for re-caching. default to couch db * @returns */ -export async function getUser( - userId: string, - tenantId?: string, - populateUser?: (userId: string, tenantId: string) => Promise -) { +export async function getUser({ + userId, + tenantId, + email, + populateUser, +}: { + userId: string + email?: string + tenantId?: string + populateUser?: ( + userId: string, + tenantId: string, + email?: string + ) => Promise +}) { if (!populateUser) { populateUser = populateFromDB } @@ -85,7 +96,7 @@ export async function getUser( // try cache let user: User = await client.get(userId) if (!user) { - user = await populateUser(userId, tenantId) + user = await populateUser(userId, tenantId, email) await client.store(userId, user, EXPIRY_SECONDS) } if (user && !user.tenantId && tenantId) { diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..e4f4a874a5 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,5 @@ import { + AIConfig, Config, ConfigType, GoogleConfig, @@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + +// AI + +export async function getAIConfig(): Promise { + return getConfig(ConfigType.AI) +} diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6bef6efeb3..2ab8c550cc 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -143,6 +143,7 @@ const environment = { POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN, POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com", + POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN, diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 5b6ea4ca92..0765d09036 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -6,7 +6,12 @@ import tracer from "dd-trace" let posthog: PostHog | undefined export function init(opts?: PostHogOptions) { - if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) { + if ( + env.POSTHOG_TOKEN && + env.POSTHOG_API_HOST && + !env.SELF_HOSTED && + env.POSTHOG_FEATURE_FLAGS_ENABLED + ) { console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index d092585cc6..01c9bfa3c6 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -148,6 +148,7 @@ describe("feature flags", () => { const env: Partial = { TENANT_FEATURE_FLAGS: environmentFlags, SELF_HOSTED: false, + POSTHOG_FEATURE_FLAGS_ENABLED: "true", } if (posthogFlags) { diff --git a/packages/backend-core/src/middleware/authenticated.ts b/packages/backend-core/src/middleware/authenticated.ts index 51cd4ec2af..85aa2293ef 100644 --- a/packages/backend-core/src/middleware/authenticated.ts +++ b/packages/backend-core/src/middleware/authenticated.ts @@ -43,7 +43,11 @@ function finalise(ctx: any, opts: FinaliseOpts = {}) { async function checkApiKey( apiKey: string, - populateUser?: (userId: string, tenantId: string) => Promise + populateUser?: ( + userId: string, + tenantId: string, + email?: string + ) => Promise ) { // check both the primary and the fallback internal api keys // this allows for rotation @@ -70,7 +74,11 @@ async function checkApiKey( if (userId) { return { valid: true, - user: await getUser(userId, tenantId, populateUser), + user: await getUser({ + userId, + tenantId, + populateUser, + }), } } else { throw new InvalidAPIKeyError() @@ -123,13 +131,18 @@ export default function ( // getting session handles error checking (if session exists etc) session = await getSession(userId, sessionId) if (opts && opts.populateUser) { - user = await getUser( + user = await getUser({ userId, - session.tenantId, - opts.populateUser(ctx) - ) + tenantId: session.tenantId, + email: session.email, + populateUser: opts.populateUser(ctx), + }) } else { - user = await getUser(userId, session.tenantId) + user = await getUser({ + userId, + tenantId: session.tenantId, + email: session.email, + }) } // @ts-ignore user.csrfToken = session.csrfToken @@ -148,7 +161,11 @@ export default function ( } // this is an internal request, no user made it if (!authenticated && apiKey) { - const populateUser = opts.populateUser ? opts.populateUser(ctx) : null + const populateUser: ( + userId: string, + tenantId: string, + email?: string + ) => Promise = opts.populateUser ? opts.populateUser(ctx) : null const { valid, user: foundUser } = await checkApiKey( apiKey, populateUser diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index a64be6b319..65339832cf 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -7,8 +7,9 @@ import { doWithDB, } from "../db" import { getAppDB } from "../context" -import { Screen, Role as RoleDoc } from "@budibase/types" +import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" +import { RoleColor } from "@budibase/shared-core" export const BUILTIN_ROLE_IDS = { ADMIN: "ADMIN", @@ -44,11 +45,18 @@ export class Role implements RoleDoc { permissionId: string inherits?: string version?: string - permissions = {} + permissions: Record = {} + uiMetadata?: RoleUIMetadata - constructor(id: string, name: string, permissionId: string) { + constructor( + id: string, + name: string, + permissionId: string, + uiMetadata?: RoleUIMetadata + ) { this._id = id this.name = name + this.uiMetadata = uiMetadata this.permissionId = permissionId // version for managing the ID - removing the role_ when responding this.version = RoleIDVersion.NAME @@ -63,21 +71,54 @@ export class Role implements RoleDoc { const BUILTIN_ROLES = { ADMIN: new Role( BUILTIN_IDS.ADMIN, - "Admin", - BuiltinPermissionID.ADMIN + BUILTIN_IDS.ADMIN, + BuiltinPermissionID.ADMIN, + { + displayName: "App admin", + description: "Can do everything", + color: RoleColor.ADMIN, + } ).addInheritance(BUILTIN_IDS.POWER), POWER: new Role( BUILTIN_IDS.POWER, - "Power", - BuiltinPermissionID.POWER + BUILTIN_IDS.POWER, + BuiltinPermissionID.POWER, + { + displayName: "App power user", + description: "An app user with more access", + color: RoleColor.POWER, + } ).addInheritance(BUILTIN_IDS.BASIC), BASIC: new Role( BUILTIN_IDS.BASIC, - "Basic", - BuiltinPermissionID.WRITE + BUILTIN_IDS.BASIC, + BuiltinPermissionID.WRITE, + { + displayName: "App user", + description: "Any logged in user", + color: RoleColor.BASIC, + } ).addInheritance(BUILTIN_IDS.PUBLIC), - PUBLIC: new Role(BUILTIN_IDS.PUBLIC, "Public", BuiltinPermissionID.PUBLIC), - BUILDER: new Role(BUILTIN_IDS.BUILDER, "Builder", BuiltinPermissionID.ADMIN), + PUBLIC: new Role( + BUILTIN_IDS.PUBLIC, + BUILTIN_IDS.PUBLIC, + BuiltinPermissionID.PUBLIC, + { + displayName: "Public user", + description: "Accessible to anyone", + color: RoleColor.PUBLIC, + } + ), + BUILDER: new Role( + BUILTIN_IDS.BUILDER, + BUILTIN_IDS.BUILDER, + BuiltinPermissionID.ADMIN, + { + displayName: "Builder user", + description: "Users that can edit this app", + color: RoleColor.BUILDER, + } + ), } export function getBuiltinRoles(): { [key: string]: RoleDoc } { @@ -244,9 +285,9 @@ export async function getUserRoleHierarchy( // some templates/older apps will use a simple string instead of array for roles // convert the string to an array using the theory that write is higher than read export function checkForRoleResourceArray( - rolePerms: { [key: string]: string[] }, + rolePerms: Record, resourceId: string -) { +): Record { if (rolePerms && !Array.isArray(rolePerms[resourceId])) { const permLevel = rolePerms[resourceId] as any rolePerms[resourceId] = [permLevel] diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b5243f856..55f71d76b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any -const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -56,6 +55,20 @@ function getRelationshipLimit() { return envLimit || 500 } +function prioritisedArraySort(toSort: string[], priorities: string[]) { + return toSort.sort((a, b) => { + const aPriority = priorities.find(field => field && a.endsWith(field)) + const bPriority = priorities.find(field => field && b.endsWith(field)) + if (aPriority && !bPriority) { + return -1 + } + if (!aPriority && bPriority) { + return 1 + } + return a.localeCompare(b) + }) +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -877,6 +890,22 @@ class InternalBuilder { return `'${unaliased}'${separator}${tableField}` } + maxFunctionParameters() { + // functions like say json_build_object() in SQL have a limit as to how many can be performed + // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as + // for JSON column building as part of relationships. We also have a default limit to avoid very complex + // functions being built - it is likely this is not necessary or the best way to do it. + switch (this.client) { + case SqlClient.SQL_LITE: + return 127 + case SqlClient.POSTGRES: + return 100 + // other DBs don't have a limit, but set some sort of limit + default: + return 200 + } + } + addJsonRelationships( query: Knex.QueryBuilder, fromTable: string, @@ -884,7 +913,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint } = this.query + const { resource, tableAliases: aliases, endpoint, meta } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -899,21 +928,27 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } + const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let relationshipFields = fields.filter( - field => field.split(".")[0] === toAlias + const requiredFields = [ + ...(relatedTable?.primary || []), + relatedTable?.primaryDisplay, + ].filter(field => field) as string[] + // sort the required fields to first in the list, so they don't get sliced out + let relationshipFields = prioritisedArraySort( + fields.filter(field => field.split(".")[0] === toAlias), + requiredFields + ) + + relationshipFields = relationshipFields.slice( + 0, + Math.floor(this.maxFunctionParameters() / 2) ) - if (this.client === SqlClient.SQL_LITE) { - relationshipFields = relationshipFields.slice( - 0, - MAX_SQS_RELATIONSHIP_FIELDS - ) - } const fieldList: string = relationshipFields .map(field => this.buildJsonField(field)) .join(",") diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index ec7d2af794..bc9a3b635c 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -102,14 +102,6 @@ export const useAppBuilders = () => { return useFeature(Feature.APP_BUILDERS) } -export const useViewPermissions = () => { - return useFeature(Feature.VIEW_PERMISSIONS) -} - -export const useViewReadonlyColumns = () => { - return useFeature(Feature.VIEW_READONLY_COLUMNS) -} - // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6c4865b539..af67ae8d22 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -595,9 +595,13 @@ let loopBlockCount = 0 const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => { if (!name) return - const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock) + const runtimeBinding = determineRuntimeBinding( + name, + idx, + isLoopBlock, + bindingName + ) const categoryName = determineCategoryName(idx, isLoopBlock, bindingName) - bindings.push( createBindingObject( name, @@ -613,7 +617,7 @@ ) } - const determineRuntimeBinding = (name, idx, isLoopBlock) => { + const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => { let runtimeName /* Begin special cases for generating custom schemas based on triggers */ @@ -634,12 +638,17 @@ } /* End special cases for generating custom schemas based on triggers */ + let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id] if (isLoopBlock) { runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { - runtimeName = `steps[${idx - loopBlockCount}].${name}` + runtimeName = hasUserDefinedName + ? `stepsByName[${bindingName}].${name}` + : `steps[${idx - loopBlockCount}].${name}` } else { - runtimeName = `steps.${idx - loopBlockCount}.${name}` + runtimeName = hasUserDefinedName + ? `stepsByName.${bindingName}.${name}` + : `steps.${idx - loopBlockCount}.${name}` } return idx === 0 ? `trigger.${name}` : runtimeName } @@ -666,11 +675,11 @@ const field = Object.values(FIELDS).find( field => field.type === value.type && field.subtype === value.subtype ) - return { - readableBinding: bindingName - ? `${bindingName}.${name}` - : runtimeBinding, + readableBinding: + bindingName && !isLoopBlock + ? `steps.${bindingName}.${name}` + : runtimeBinding, runtimeBinding, type: value.type, description: value.description, @@ -690,8 +699,12 @@ allSteps[idx]?.stepId === ActionStepID.LOOP && allSteps.some(x => x.blockToLoop === block.id) let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {} + if (allSteps[idx]?.name.includes("Looping")) { + isLoopBlock = true + loopBlockCount++ + } let bindingName = - automation.stepNames?.[allSteps[idx - loopBlockCount].id] + automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name if (isLoopBlock) { schema = { @@ -740,13 +753,12 @@ if (wasLoopBlock) { loopBlockCount++ - continue + schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } - Object.entries(schema).forEach(([name, value]) => + Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) - ) + }) } - return bindings } diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte index b56c5f6568..90e5e216f3 100644 --- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte +++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte @@ -1,6 +1,6 @@ {#if fancySelect} @@ -134,9 +143,12 @@ getOptionValue={role => role._id} getOptionColour={getColor} getOptionIcon={getIcon} - isOptionEnabled={option => - option._id !== Constants.Roles.CREATOR || - $licensing.perAppBuildersEnabled} + isOptionEnabled={option => { + if (option._id === Constants.Roles.CREATOR) { + return isEnterprisePlan(subType) + } + return true + }} {placeholder} {error} /> @@ -154,10 +166,12 @@ getOptionValue={role => role._id} getOptionColour={getColor} getOptionIcon={getIcon} - isOptionEnabled={option => - (option._id !== Constants.Roles.CREATOR || - $licensing.perAppBuildersEnabled) && - option.enabled !== false} + isOptionEnabled={option => { + if (option._id === Constants.Roles.CREATOR) { + return isEnterprisePlan(subType) + } + return option.enabled !== false + }} {placeholder} {error} /> diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js new file mode 100644 index 0000000000..ad18213ab0 --- /dev/null +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -0,0 +1,117 @@ +import { AutomationActionStepId } from "@budibase/types" + +export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => { + if (typeof inputs === "string") { + return inputs + .replace( + new RegExp(`stepsByName\\.${oldName}\\.`, "g"), + `stepsByName.${newName}.` + ) + .replace( + new RegExp(`steps\\.${stepIndex}\\.`, "g"), + `stepsByName.${newName}.` + ) + } + + if (Array.isArray(inputs)) { + return inputs.map(item => + updateBindingsInInputs(item, oldName, newName, stepIndex) + ) + } + + if (typeof inputs === "object" && inputs !== null) { + const updatedInputs = {} + for (const [key, value] of Object.entries(inputs)) { + const updatedKey = updateBindingsInInputs( + key, + oldName, + newName, + stepIndex + ) + updatedInputs[updatedKey] = updateBindingsInInputs( + value, + oldName, + newName, + stepIndex + ) + } + return updatedInputs + } + return inputs +} + +export const updateBindingsInSteps = ( + steps, + oldName, + newName, + changedStepIndex +) => { + return steps.map(step => { + const updatedStep = { + ...step, + inputs: updateBindingsInInputs( + step.inputs, + oldName, + newName, + changedStepIndex + ), + } + + if ("branches" in updatedStep.inputs) { + updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({ + ...branch, + condition: updateBindingsInInputs( + branch.condition, + oldName, + newName, + changedStepIndex + ), + })) + + if (updatedStep.inputs.children) { + for (const [key, childSteps] of Object.entries( + updatedStep.inputs.children + )) { + updatedStep.inputs.children[key] = updateBindingsInSteps( + childSteps, + oldName, + newName, + changedStepIndex + ) + } + } + } + + return updatedStep + }) +} +export const getNewStepName = (automation, step) => { + const baseName = step.name + + const countExistingSteps = steps => { + return steps.reduce((count, currentStep) => { + if (currentStep.name && currentStep.name.startsWith(baseName)) { + count++ + } + if ( + currentStep.stepId === AutomationActionStepId.BRANCH && + currentStep.inputs && + currentStep.inputs.children + ) { + Object.values(currentStep.inputs.children).forEach(branchSteps => { + count += countExistingSteps(branchSteps) + }) + } + return count + }, 0) + } + let existingCount = 0 + if (automation?.definition) { + existingCount = countExistingSteps(automation.definition.steps) + } + if (existingCount === 0) { + return baseName + } + + return `${baseName} ${existingCount + 1}` +} diff --git a/packages/builder/src/helpers/tests/nameHelpers.spec.js b/packages/builder/src/helpers/tests/nameHelpers.spec.js new file mode 100644 index 0000000000..1ce2d1987a --- /dev/null +++ b/packages/builder/src/helpers/tests/nameHelpers.spec.js @@ -0,0 +1,177 @@ +import { cloneDeep } from "lodash" +import { + updateBindingsInInputs, + updateBindingsInSteps, +} from "../automations/nameHelpers" +describe("Automation Binding Update Functions", () => { + const sampleAutomation = { + definition: { + steps: [ + { + name: "First Step", + inputs: { + text: "Starting automation", + }, + id: "step1", + }, + { + name: "Second Step", + inputs: { + text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}", + }, + id: "step2", + }, + { + name: "Branch", + inputs: { + branches: [ + { + name: "branch1", + condition: { + equal: { + "steps.1.success": true, + }, + }, + }, + ], + children: { + branch1: [ + { + name: "Nested Step", + inputs: { + text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}", + }, + id: "nestedStep", + }, + ], + }, + }, + id: "branchStep", + }, + ], + stepNames: { + step1: "First Step", + step2: "Second Step", + branchStep: "Branch", + }, + }, + } + + it("updateBindingsInInputs updates string bindings correctly", () => { + const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}" + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toBe( + "{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}" + ) + }) + + it("updateBindingsInInputs handles nested objects", () => { + const input = { + text: "{{ stepsByName.oldName.success }}", + nested: { + value: "{{ steps.1.message }}", + }, + } + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toEqual({ + text: "{{ stepsByName.newName.success }}", + nested: { + value: "{{ stepsByName.newName.message }}", + }, + }) + }) + + it("updateBindingsInSteps updates bindings in all steps", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].name).toBe("Second Step") + + expect(result[2].inputs.branches[0].condition.equal).toEqual({ + "stepsByName.Renamed Step.success": true, + }) + + const nestedStepText = result[2].inputs.children.branch1[0].inputs.text + expect(nestedStepText).toBe( + "{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}" + ) + }) + + it("updateBindingsInSteps handles steps with no bindings", () => { + const steps = [ + { + name: "No Binding Step", + inputs: { + text: "Plain text", + }, + id: "noBindingStep", + }, + ] + const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0) + expect(result).toEqual(steps) + }) + + it("updateBindingsInSteps updates bindings in deeply nested branches", () => { + const deeplyNestedStep = { + name: "Deep Branch", + inputs: { + branches: [ + { + name: "deepBranch", + condition: { + equal: { + "stepsByName.Second Step.success": true, + }, + }, + }, + ], + children: { + deepBranch: [ + { + name: "Deep Log", + inputs: { + text: "{{ steps.1.message }}", + }, + }, + ], + }, + }, + } + + const steps = [...sampleAutomation.definition.steps, deeplyNestedStep] + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect( + result[3].inputs.branches[0].condition.equal[ + "stepsByName.Renamed Step.success" + ] + ).toBe(true) + expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe( + "{{ stepsByName.Renamed Step.message }}" + ) + }) + + it("updateBindingsInSteps does not affect unrelated bindings", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].inputs.text).toBe( + "{{ steps.0.success }} and {{ stepsByName.First Step.message }}" + ) + }) +}) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 2ac1609e7c..b60ea24dbc 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -56,10 +56,13 @@ } else { // We don't store the default BB AI config in the DB delete fullAIConfig.config.budibase_ai + // unset the default value from other configs if default is set if (editingAIConfig.isDefault) { for (let key in fullAIConfig.config) { - fullAIConfig.config[key].isDefault = false + if (key !== id) { + fullAIConfig.config[key].isDefault = false + } } } // Add new or update existing custom AI Config diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index fdb0991911..6627c67080 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -6,6 +6,10 @@ import { createHistoryStore } from "stores/builder/history" import { notifications } from "@budibase/bbui" import { updateReferencesInObject } from "dataBinding" import { AutomationTriggerStepId } from "@budibase/types" +import { + updateBindingsInSteps, + getNewStepName, +} from "helpers/automations/nameHelpers" const initialAutomationState = { automations: [], @@ -275,13 +279,17 @@ const automationActions = store => ({ await store.actions.save(newAutomation) }, constructBlock(type, stepId, blockDefinition) { - return { + let newName + const newStep = { ...blockDefinition, inputs: blockDefinition.inputs || {}, stepId, type, id: generate(), } + newName = getNewStepName(get(selectedAutomation), newStep) + newStep.name = newName + return newStep }, addBlockToAutomation: async (block, blockIdx) => { const automation = get(selectedAutomation) @@ -301,15 +309,34 @@ const automationActions = store => ({ saveAutomationName: async (blockId, name) => { const automation = get(selectedAutomation) let newAutomation = cloneDeep(automation) - if (!automation) { + if (!newAutomation) { return } - newAutomation.definition.stepNames = { - ...newAutomation.definition.stepNames, - [blockId]: name.trim(), - } - await store.actions.save(newAutomation) + const stepIndex = newAutomation.definition.steps.findIndex( + step => step.id === blockId + ) + + if (stepIndex !== -1) { + const oldName = newAutomation.definition.steps[stepIndex].name + const newName = name.trim() + + newAutomation.definition.stepNames = { + ...newAutomation.definition.stepNames, + [blockId]: newName, + } + + newAutomation.definition.steps[stepIndex].name = newName + + newAutomation.definition.steps = updateBindingsInSteps( + newAutomation.definition.steps, + oldName, + newName, + stepIndex + ) + + await store.actions.save(newAutomation) + } }, deleteAutomationName: async blockId => { const automation = get(selectedAutomation) diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index d48377207c..f0ce7046f2 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -140,10 +140,6 @@ export const createLicensingStore = () => { Constants.Features.VIEW_PERMISSIONS ) - const isViewReadonlyColumnsEnabled = license.features.includes( - Constants.Features.VIEW_READONLY_COLUMNS - ) - const budibaseAIEnabled = license.features.includes( Constants.Features.BUDIBASE_AI ) @@ -173,7 +169,6 @@ export const createLicensingStore = () => { triggerAutomationRunEnabled, isViewPermissionsEnabled, perAppBuildersEnabled, - isViewReadonlyColumnsEnabled, } }) }, diff --git a/packages/client/src/utils/conditions.js b/packages/client/src/utils/conditions.js index a23d879e45..b68afa4ab3 100644 --- a/packages/client/src/utils/conditions.js +++ b/packages/client/src/utils/conditions.js @@ -1,4 +1,5 @@ import { QueryUtils } from "@budibase/frontend-core" +import { EmptyFilterOption } from "@budibase/types" export const getActiveConditions = conditions => { if (!conditions?.length) { @@ -33,7 +34,8 @@ export const getActiveConditions = conditions => { value: condition.referenceValue, } - const query = QueryUtils.buildQuery([luceneCondition]) + let query = QueryUtils.buildQuery([luceneCondition]) + query.onEmptyFilter = EmptyFilterOption.RETURN_NONE const result = QueryUtils.runQuery([luceneCondition], query) return result.length > 0 }) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte index b4940c8903..2a3ca139fc 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte @@ -4,16 +4,13 @@ import ColumnsSettingContent from "./ColumnsSettingContent.svelte" import { FieldPermissions } from "../../../constants" - export let allowViewReadonlyColumns = false - const { columns, datasource } = getContext("grid") let open = false let anchor $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length - $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns" - + $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns" $: permissions = $datasource.type === "viewV2" ? [ @@ -22,9 +19,6 @@ FieldPermissions.HIDDEN, ] : [FieldPermissions.WRITABLE, FieldPermissions.HIDDEN] - $: disabledPermissions = allowViewReadonlyColumns - ? [] - : [FieldPermissions.READONLY]
@@ -41,9 +35,5 @@
- + diff --git a/packages/frontend-core/src/components/grid/layout/Grid.svelte b/packages/frontend-core/src/components/grid/layout/Grid.svelte index f24ff0ae10..f2aeffb9f4 100644 --- a/packages/frontend-core/src/components/grid/layout/Grid.svelte +++ b/packages/frontend-core/src/components/grid/layout/Grid.svelte @@ -58,7 +58,6 @@ export let buttons = null export let darkMode export let isCloud = null - export let allowViewReadonlyColumns = false export let rowConditions = null // Unique identifier for DOM nodes inside this instance @@ -115,7 +114,6 @@ buttons, darkMode, isCloud, - allowViewReadonlyColumns, rowConditions, }) @@ -157,7 +155,7 @@
- +
diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index d2f25b351a..105928c66b 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -85,6 +85,7 @@ export const PlanType = { TEAM: "team", PRO: "pro", BUSINESS: "business", + PREMIUM: "premium", ENTERPRISE: "enterprise", ENTERPRISE_BASIC_TRIAL: "enterprise_basic_trial", } diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 30507fc827..2206d49d9b 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -1,4 +1,4 @@ -FROM node:20-slim +FROM node:20-alpine LABEL com.centurylinklabs.watchtower.lifecycle.pre-check="scripts/watchtower-hooks/pre-check.sh" LABEL com.centurylinklabs.watchtower.lifecycle.pre-update="scripts/watchtower-hooks/pre-update.sh" @@ -15,37 +15,35 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU ENV ACCOUNT_PORTAL_URL=https://account.budibase.app ENV TOP_LEVEL_PATH=/ -# handle node-gyp -RUN apt-get update \ - && apt-get install -y --no-install-recommends g++ make python3 jq -RUN yarn global add pm2 +# handle node-gyp and install postgres client for pg_dump utils +RUN apk add --no-cache \ + g++ \ + make \ + python3 \ + jq \ + bash \ + postgresql-client \ + git -# Install postgres client for pg_dump utils -RUN apt update && apt upgrade -y \ - && apt install software-properties-common apt-transport-https curl gpg -y \ - && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ - && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ - && apt update -y \ - && apt install postgresql-client-15 -y \ - && apt remove software-properties-common apt-transport-https curl gpg -y +RUN yarn global add pm2 WORKDIR / COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh RUN chmod +x ./scripts/removeWorkspaceDependencies.sh - WORKDIR /app COPY packages/server/package.json . COPY packages/server/dist/yarn.lock . COPY scripts/removeWorkspaceDependencies.sh scripts/removeWorkspaceDependencies.sh RUN chmod +x ./scripts/removeWorkspaceDependencies.sh -RUN ./scripts/removeWorkspaceDependencies.sh package.json +RUN ./scripts/removeWorkspaceDependencies.sh package.json +# Install yarn packages with caching RUN --mount=type=cache,target=/root/.yarn YARN_CACHE_FOLDER=/root/.yarn yarn install --production=true --network-timeout 1000000 \ - # Remove unneeded data from file system to reduce image size - && yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python jq \ + && yarn cache clean \ + && apk del g++ make python3 jq \ && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp COPY packages/server/dist/ dist/ @@ -69,7 +67,7 @@ EXPOSE 4001 # due to this causing yarn to stop installing dev dependencies # which are actually needed to get this environment up and running ENV NODE_ENV=production -# this is required for isolated-vm to work on Node 20+ +# This is required for isolated-vm to work on Node 20+ ENV NODE_OPTIONS="--no-node-snapshot" ENV CLUSTER_MODE=${CLUSTER_MODE} ENV TOP_LEVEL_PATH=/app diff --git a/packages/server/package.json b/packages/server/package.json index df0ece7bb6..76dd03b5a8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -63,7 +63,7 @@ "@koa/router": "8.0.8", "@socket.io/redis-adapter": "^8.2.1", "@types/xml2js": "^0.4.14", - "airtable": "0.10.1", + "airtable": "0.12.2", "arangojs": "7.2.0", "archiver": "7.0.1", "aws-sdk": "2.1030.0", @@ -80,7 +80,7 @@ "dotenv": "8.2.0", "form-data": "4.0.0", "global-agent": "3.0.0", - "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.3", + "google-spreadsheet": "npm:@budibase/google-spreadsheet@4.1.5", "ioredis": "5.3.2", "isolated-vm": "^4.7.2", "jimp": "0.22.12", @@ -101,7 +101,7 @@ "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", - "openai": "^4.52.1", + "openai": "4.59.0", "openapi-types": "9.3.1", "oracledb": "6.5.1", "pg": "8.10.0", diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index cdfa6d8b1c..66a3254348 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -1,4 +1,4 @@ -import { permissions, roles, context, HTTPError } from "@budibase/backend-core" +import { permissions, roles, context } from "@budibase/backend-core" import { UserCtx, Database, @@ -45,18 +45,6 @@ async function updatePermissionOnRole( }: { roleId: string; resourceId: string; level: PermissionLevel }, updateType: PermissionUpdateType ) { - const allowedAction = await sdk.permissions.resourceActionAllowed({ - resourceId, - level, - }) - - if (!allowedAction.allowed) { - throw new HTTPError( - `You are not allowed to '${allowedAction.level}' the resource type '${allowedAction.resourceType}'`, - 403 - ) - } - const db = context.getAppDB() const remove = updateType === PermissionUpdateType.REMOVE const isABuiltin = roles.isBuiltin(roleId) @@ -75,7 +63,9 @@ async function updatePermissionOnRole( // resource from another role and then adding to the new role for (let role of dbRoles) { let updated = false - const rolePermissions = role.permissions ? role.permissions : {} + const rolePermissions: Record = role.permissions + ? role.permissions + : {} // make sure its an array, also handle migrating if ( !rolePermissions[resourceId] || @@ -83,7 +73,7 @@ async function updatePermissionOnRole( ) { rolePermissions[resourceId] = typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId] as unknown as string] + ? [rolePermissions[resourceId] as unknown as PermissionLevel] : [] } // handle the removal/updating the role which has this permission first @@ -182,9 +172,6 @@ export async function getResourcePerms( }, {} as Record ), - requiresPlanToModify: ( - await sdk.permissions.allowsExplicitPermissions(resourceId) - ).minPlan, } } diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 3398c8102c..b6b9ac1a29 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -17,9 +17,9 @@ import { SaveRoleResponse, UserCtx, UserMetadata, - UserRoles, + DocumentType, } from "@budibase/types" -import { sdk as sharedSdk } from "@budibase/shared-core" +import { RoleColor, sdk as sharedSdk } from "@budibase/shared-core" import sdk from "../../sdk" const UpdateRolesOptions = { @@ -62,7 +62,8 @@ export async function find(ctx: UserCtx) { export async function save(ctx: UserCtx) { const db = context.getAppDB() - let { _id, name, inherits, permissionId, version } = ctx.request.body + let { _id, name, inherits, permissionId, version, uiMetadata } = + ctx.request.body let isCreate = false const isNewVersion = version === roles.RoleIDVersion.NAME @@ -80,17 +81,25 @@ export async function save(ctx: UserCtx) { _id = dbCore.prefixRoleID(_id) } - let dbRole - if (!isCreate) { - dbRole = await db.get(_id) + let dbRole: Role | undefined + if (!isCreate && _id?.startsWith(DocumentType.ROLE)) { + dbRole = await db.get(_id) } if (dbRole && dbRole.name !== name && isNewVersion) { ctx.throw(400, "Cannot change custom role name") } - const role = new roles.Role(_id, name, permissionId).addInheritance(inherits) - if (ctx.request.body._rev) { - role._rev = ctx.request.body._rev + const role = new roles.Role(_id, name, permissionId, { + displayName: uiMetadata?.displayName || name, + description: uiMetadata?.description || "Custom role", + color: uiMetadata?.color || RoleColor.DEFAULT_CUSTOM, + }).addInheritance(inherits) + if (dbRole?.permissions && !role.permissions) { + role.permissions = dbRole.permissions + } + const foundRev = ctx.request.body._rev || dbRole?._rev + if (foundRev) { + role._rev = foundRev } const result = await db.put(role) if (isCreate) { diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index ac2d1e8c39..9c9bd0b284 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -12,6 +12,7 @@ import { OneToManyRelationshipFieldMetadata, Operation, PaginationJson, + QueryJson, RelationshipFieldMetadata, Row, SearchFilters, @@ -161,7 +162,6 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} - private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -170,7 +170,6 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } - this.tableList = Object.values(this.tables) } private prepareFilters( @@ -301,7 +300,6 @@ export class ExternalRequest { throw "No tables found, fetch tables before query." } this.tables = this.datasource.entities - this.tableList = Object.values(this.tables) } return { tables: this.tables, datasource: this.datasource } } @@ -463,7 +461,7 @@ export class ExternalRequest { breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] - if (!lookupField || !row[lookupField]) { + if (!lookupField || !row?.[lookupField] == null) { continue } const endpoint = getEndpoint(relatedTableId, Operation.READ) @@ -631,7 +629,8 @@ export class ExternalRequest { const { datasource: ds } = await this.retrieveMetadata(datasourceId) datasource = ds } - const table = this.tables[tableName] + const tables = this.tables + const table = tables[tableName] let isSql = isSQL(datasource) if (!table) { throw new Error( @@ -686,7 +685,7 @@ export class ExternalRequest { ) { throw "Deletion must be filtered" } - let json = { + let json: QueryJson = { endpoint: { datasourceId: datasourceId!, entityId: tableName, @@ -715,7 +714,7 @@ export class ExternalRequest { }, meta: { table, - id: config.id, + tables: tables, }, } diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cd85f57982..2e5785157d 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx) { const { tableId } = utils.getSourceId(ctx) const processedRows = request.rows.map(row => { - let processedRow: Row = typeof row == "string" ? { _id: row } : row + let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row return !processedRow._rev ? addRev(fixRow(processedRow, ctx.params), tableId) : fixRow(processedRow, ctx.params) diff --git a/packages/server/src/api/routes/tests/permissions.spec.ts b/packages/server/src/api/routes/tests/permissions.spec.ts index 838e1aca0b..0f059998ae 100644 --- a/packages/server/src/api/routes/tests/permissions.spec.ts +++ b/packages/server/src/api/routes/tests/permissions.spec.ts @@ -1,20 +1,5 @@ -const mockedSdk = sdk.permissions as jest.Mocked -jest.mock("../../../sdk/app/permissions", () => ({ - ...jest.requireActual("../../../sdk/app/permissions"), - resourceActionAllowed: jest.fn(), -})) - -import sdk from "../../../sdk" - import { roles } from "@budibase/backend-core" -import { - Document, - DocumentType, - PermissionLevel, - Row, - Table, - ViewV2, -} from "@budibase/types" +import { Document, PermissionLevel, Row, Table, ViewV2 } from "@budibase/types" import * as setup from "./utilities" import { generator, mocks } from "@budibase/backend-core/tests" @@ -40,7 +25,6 @@ describe("/permission", () => { beforeEach(async () => { mocks.licenses.useCloudFree() - mockedSdk.resourceActionAllowed.mockResolvedValue({ allowed: true }) table = (await config.createTable()) as typeof table row = await config.createRow() @@ -112,29 +96,6 @@ describe("/permission", () => { expect(allRes.body[table._id]["read"]).toEqual(STD_ROLE_ID) expect(allRes.body[table._id]["write"]).toEqual(HIGHER_ROLE_ID) }) - - it("throw forbidden if the action is not allowed for the resource", async () => { - mockedSdk.resourceActionAllowed.mockResolvedValue({ - allowed: false, - resourceType: DocumentType.DATASOURCE, - level: PermissionLevel.READ, - }) - - await config.api.permission.add( - { - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.EXECUTE, - }, - { - status: 403, - body: { - message: - "You are not allowed to 'read' the resource type 'datasource'", - }, - } - ) - }) }) describe("remove", () => { @@ -148,29 +109,6 @@ describe("/permission", () => { const permsRes = await config.api.permission.get(table._id) expect(permsRes.permissions[STD_ROLE_ID]).toBeUndefined() }) - - it("throw forbidden if the action is not allowed for the resource", async () => { - mockedSdk.resourceActionAllowed.mockResolvedValue({ - allowed: false, - resourceType: DocumentType.DATASOURCE, - level: PermissionLevel.READ, - }) - - await config.api.permission.revoke( - { - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.EXECUTE, - }, - { - status: 403, - body: { - message: - "You are not allowed to 'read' the resource type 'datasource'", - }, - } - ) - }) }) describe("check public user allowed", () => { @@ -206,27 +144,7 @@ describe("/permission", () => { await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 }) }) - it("should ignore the view permissions if the flag is not on", async () => { - await config.api.permission.add({ - roleId: STD_ROLE_ID, - resourceId: view.id, - level: PermissionLevel.READ, - }) - await config.api.permission.revoke({ - roleId: STD_ROLE_ID, - resourceId: table._id, - level: PermissionLevel.READ, - }) - // replicate changes before checking permissions - await config.publish() - - await config.api.viewV2.publicSearch(view.id, undefined, { - status: 401, - }) - }) - - it("should use the view permissions if the flag is on", async () => { - mocks.licenses.useViewPermissions() + it("should use the view permissions", async () => { await config.api.permission.add({ roleId: STD_ROLE_ID, resourceId: view.id, diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9790703806..dc03a21d6d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1138,6 +1138,18 @@ describe.each([ await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) }) + it("should be able to delete a row with ID only", async () => { + const createdRow = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const res = await config.api.row.bulkDelete(table._id!, { + rows: [createdRow._id!], + }) + expect(res[0]._id).toEqual(createdRow._id) + expect(res[0].tableId).toEqual(table._id!) + await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) + }) + it("should be able to bulk delete rows, including a row that doesn't exist", async () => { const createdRow = await config.api.row.save(table._id!, {}) const createdRow2 = await config.api.row.save(table._id!, {}) diff --git a/packages/server/src/api/routes/tests/rowAction.spec.ts b/packages/server/src/api/routes/tests/rowAction.spec.ts index 3f4447c50d..ef7d2afbba 100644 --- a/packages/server/src/api/routes/tests/rowAction.spec.ts +++ b/packages/server/src/api/routes/tests/rowAction.spec.ts @@ -763,10 +763,6 @@ describe("/rowsActions", () => { }) describe("role permission checks", () => { - beforeAll(() => { - mocks.licenses.useViewPermissions() - }) - afterAll(() => { mocks.licenses.useCloudFree() }) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c9e7f4c3c5..0b0802bab2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3080,4 +3080,46 @@ describe.each([ }).toHaveLength(4) }) }) + + isSql && + describe("max related columns", () => { + let relatedRows: Row[] + + beforeAll(async () => { + const relatedSchema: TableSchema = {} + const row: Row = {} + for (let i = 0; i < 100; i++) { + const name = `column${i}` + relatedSchema[name] = { name, type: FieldType.NUMBER } + row[name] = i + } + const relatedTable = await createTable(relatedSchema) + table = await createTable({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable._id!, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable._id!, row), + ]) + await config.api.row.save(table._id!, { + name: "foo", + related1: [relatedRows[0]._id], + }) + }) + + it("retrieve the row with relationships", async () => { + await expectQuery({}).toContainExactly([ + { + name: "foo", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) + }) }) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 3ca28f31aa..c4a39ae8a9 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -309,10 +309,6 @@ describe.each([ }) describe("readonly fields", () => { - beforeEach(() => { - mocks.licenses.useViewReadonlyColumns() - }) - it("readonly fields are persisted", async () => { const table = await config.api.table.save( saveTableRequest({ @@ -436,7 +432,7 @@ describe.each([ }) }) - it("readonly fields cannot be used on free license", async () => { + it("readonly fields can be used on free license", async () => { mocks.licenses.useCloudFree() const table = await config.api.table.save( saveTableRequest({ @@ -466,11 +462,7 @@ describe.each([ } await config.api.viewV2.create(newView, { - status: 400, - body: { - message: "Readonly fields are not enabled", - status: 400, - }, + status: 201, }) }) }) @@ -513,7 +505,6 @@ describe.each([ }) it("display fields can be readonly", async () => { - mocks.licenses.useViewReadonlyColumns() const table = await config.api.table.save( saveTableRequest({ schema: { @@ -588,7 +579,6 @@ describe.each([ }) it("can update all fields", async () => { - mocks.licenses.useViewReadonlyColumns() const tableId = table._id! const updatedData: Required = { @@ -802,71 +792,6 @@ describe.each([ ) }) - it("cannot update views with readonly on on free license", async () => { - mocks.licenses.useViewReadonlyColumns() - - view = await config.api.viewV2.update({ - ...view, - schema: { - id: { visible: true }, - Price: { - visible: true, - readonly: true, - }, - }, - }) - - mocks.licenses.useCloudFree() - await config.api.viewV2.update(view, { - status: 400, - body: { - message: "Readonly fields are not enabled", - }, - }) - }) - - it("can remove readonly config after license downgrade", async () => { - mocks.licenses.useViewReadonlyColumns() - - view = await config.api.viewV2.update({ - ...view, - schema: { - id: { visible: true }, - Price: { - visible: true, - readonly: true, - }, - Category: { - visible: true, - readonly: true, - }, - }, - }) - mocks.licenses.useCloudFree() - const res = await config.api.viewV2.update({ - ...view, - schema: { - id: { visible: true }, - Price: { - visible: true, - readonly: false, - }, - }, - }) - expect(res).toEqual( - expect.objectContaining({ - ...view, - schema: { - id: { visible: true }, - Price: { - visible: true, - readonly: false, - }, - }, - }) - ) - }) - isInternal && it("updating schema will only validate modified field", async () => { let view = await config.api.viewV2.create({ @@ -1046,7 +971,6 @@ describe.each([ }) it("should be able to fetch readonly config after downgrades", async () => { - mocks.licenses.useViewReadonlyColumns() const res = await config.api.viewV2.create({ name: generator.name(), tableId: table._id!, @@ -1112,8 +1036,6 @@ describe.each([ }) it("rejects if field is readonly in any view", async () => { - mocks.licenses.useViewReadonlyColumns() - await config.api.viewV2.create({ name: "view a", tableId: table._id!, @@ -1538,7 +1460,6 @@ describe.each([ }) it("can't persist readonly columns", async () => { - mocks.licenses.useViewReadonlyColumns() const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -1607,7 +1528,6 @@ describe.each([ }) it("can't update readonly columns", async () => { - mocks.licenses.useViewReadonlyColumns() const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), @@ -2297,7 +2217,6 @@ describe.each([ describe("permissions", () => { beforeEach(async () => { - mocks.licenses.useViewPermissions() await Promise.all( Array.from({ length: 10 }, () => config.api.row.save(table._id!, {})) ) diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index 5e2a585b4a..b589d44b31 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -200,7 +200,7 @@ export function webhookValidator() { export function roleValidator() { const permLevelArray = Object.values(permissions.PermissionLevel) - + const permissionString = Joi.string().valid(...permLevelArray) return auth.joiValidator.body( Joi.object({ _id: OPTIONAL_STRING, @@ -208,12 +208,23 @@ export function roleValidator() { name: Joi.string() .regex(/^[a-zA-Z0-9_]*$/) .required(), + uiMetadata: Joi.object({ + displayName: OPTIONAL_STRING, + color: OPTIONAL_STRING, + description: OPTIONAL_STRING, + }).optional(), // this is the base permission ID (for now a built in) permissionId: Joi.string() .valid(...Object.values(permissions.BuiltinPermissionID)) .required(), permissions: Joi.object() - .pattern(/.*/, [Joi.string().valid(...permLevelArray)]) + .pattern( + /.*/, + Joi.alternatives().try( + Joi.array().items(permissionString), + permissionString + ) + ) .optional(), inherits: OPTIONAL_STRING, }).unknown(true) diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index 1c148b2e73..b1dfa3df5b 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -10,11 +10,14 @@ import { } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" +import * as pro from "@budibase/pro" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API GPT_4 = "gpt-4", + GPT_4O = "gpt-4o", + GPT_4O_MINI = "gpt-4o-mini", } export const definition: AutomationStepDefinition = { @@ -60,19 +63,33 @@ export const definition: AutomationStepDefinition = { }, } +/** + * Maintains backward compatibility with automation steps created before the introduction + * of custom configurations and Budibase AI + * @param inputs - automation inputs from the OpenAI automation step. + */ +async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { + const openai = new OpenAI({ + apiKey: env.OPENAI_API_KEY, + }) + + const completion = await openai.chat.completions.create({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt, + }, + ], + }) + return completion?.choices[0]?.message?.content +} + export async function run({ inputs, }: { inputs: OpenAIStepInputs }): Promise { - if (!env.OPENAI_API_KEY) { - return { - success: false, - response: - "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", - } - } - if (inputs.prompt == null) { return { success: false, @@ -81,20 +98,24 @@ export async function run({ } try { - const openai = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }) + let response + const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() + const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const completion = await openai.chat.completions.create({ - model: inputs.model, - messages: [ - { - role: "user", - content: inputs.prompt, - }, - ], - }) - const response = completion?.choices[0]?.message?.content + if (budibaseAIEnabled || customConfigsEnabled) { + const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) + response = await llm.run(inputs.prompt) + } else { + // fallback to the default that uses the environment variable for backwards compat + if (!env.OPENAI_API_KEY) { + return { + success: false, + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + response = await legacyOpenAIPrompt(inputs) + } return { response, diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3a5a57475a..6b5e1fbb6a 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -4,6 +4,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import * as pro from "@budibase/pro" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -22,7 +23,23 @@ jest.mock("openai", () => ({ }, })), })) +jest.mock("@budibase/pro", () => ({ + ...jest.requireActual("@budibase/pro"), + ai: { + LargeLanguageModel: { + forCurrentTenant: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, + }, + features: { + isAICustomConfigsEnabled: jest.fn(), + isBudibaseAIEnabled: jest.fn(), + }, +})) +const mockedPro = jest.mocked(pro) const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -41,6 +58,7 @@ describe("test the openai action", () => { afterEach(() => { resetEnv() + jest.clearAllMocks() }) afterAll(_afterAll) @@ -94,4 +112,25 @@ describe("test the openai action", () => { ) expect(res.success).toBeFalsy() }) + + it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true) + jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) + + const prompt = "What is the meaning of life?" + await runStep("OPENAI", { + model: "gpt-4o-mini", + prompt, + }) + + expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( + "gpt-4o-mini" + ) + + const llmInstance = + mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() + expect(llmInstance.run).toHaveBeenCalledWith(prompt) + }) }) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9f7ff04156..05d82ad7ba 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -244,6 +244,33 @@ describe("Loop automations", () => { expect(results.steps[3].outputs.message).toContain("- 3") }) + it("should use automation names to loop with", async () => { + const builder = createAutomationBuilder({ + name: "Test Trigger with Loop and Create Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .loop( + { + option: LoopStepType.ARRAY, + binding: [1, 2, 3], + }, + { stepName: "FirstLoopStep" } + ) + .serverLog( + { text: "Message {{loop.currentItem}}" }, + { stepName: "FirstLoopLog" } + ) + .serverLog( + { text: "{{steps.FirstLoopLog.iterations}}" }, + { stepName: "FirstLoopIterationLog" } + ) + .run() + + expect(results.steps[1].outputs.message).toContain("- 3") + }) + it("should run an automation with a loop and update row step", async () => { const table = await config.createTable({ name: "TestTable", diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 12b8392ffc..b54b7c0e0e 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -50,151 +50,188 @@ describe("Automation Scenarios", () => { }, }) }) - }) - it("should trigger an automation which querys the database", async () => { - const table = await config.createTable() - const row = { - name: "Test Row", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + it("should trigger an automation which querys the database", async () => { + const table = await config.createTable() + const row = { + name: "Test Row", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(1) + expect(results.steps[0].outputs.rows).toHaveLength(2) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, + it("should trigger an automation which querys the database then deletes a row", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", }) - .run() - expect(results.steps).toHaveLength(1) - expect(results.steps[0].outputs.rows).toHaveLength(2) - }) + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.1.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() - it("should trigger an automation which querys the database then deletes a row", async () => { - const table = await config.createTable() - const row = { - name: "DFN", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, - }) - .deleteRow({ - tableId: table._id!, - id: "{{ steps.1.rows.0._id }}", - }) - .queryRows({ - tableId: table._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - expect(results.steps[1].outputs.success).toBeTruthy() - expect(results.steps[2].outputs.rows).toHaveLength(1) - }) - - it("should query an external database for some data then insert than into an internal table", async () => { - const { datasource, client } = await setup.setupTestDatasource( - config, - DatabaseName.MYSQL - ) - - const newTable = await config.createTable({ - name: "table", - type: "table", - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, - constraints: { - presence: true, - }, - }, - }, - }) - - const tableName = await setup.createTestTable(client, { - name: { type: "string" }, - age: { type: "number" }, - }) - - const rows = [ - { name: "Joe", age: 20 }, - { name: "Bob", age: 25 }, - { name: "Paul", age: 30 }, - ] - - await setup.insertTestData(client, tableName, rows) - - const query = await setup.saveTestQuery( - config, - client, - tableName, - datasource - ) - - const builder = createAutomationBuilder({ - name: "Test external query and save", - }) - - const results = await builder - .appAction({ - fields: {}, - }) - .executeQuery({ - query: { - queryId: query._id!, - }, - }) - .loop({ - option: LoopStepType.ARRAY, - binding: "{{ steps.1.response }}", - }) - .createRow({ - row: { - name: "{{ loop.currentItem.name }}", - age: "{{ loop.currentItem.age }}", - tableId: newTable._id!, - }, - }) - .queryRows({ - tableId: newTable._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - - expect(results.steps[1].outputs.iterations).toBe(3) - expect(results.steps[1].outputs.items).toHaveLength(3) - - expect(results.steps[2].outputs.rows).toHaveLength(3) - - rows.forEach(expectedRow => { - expect(results.steps[2].outputs.rows).toEqual( - expect.arrayContaining([expect.objectContaining(expectedRow)]) + it("should query an external database for some data then insert than into an internal table", async () => { + const { datasource, client } = await setup.setupTestDatasource( + config, + DatabaseName.MYSQL ) + + const newTable = await config.createTable({ + name: "table", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const tableName = await setup.createTestTable(client, { + name: { type: "string" }, + age: { type: "number" }, + }) + + const rows = [ + { name: "Joe", age: 20 }, + { name: "Bob", age: 25 }, + { name: "Paul", age: 30 }, + ] + + await setup.insertTestData(client, tableName, rows) + + const query = await setup.saveTestQuery( + config, + client, + tableName, + datasource + ) + + const builder = createAutomationBuilder({ + name: "Test external query and save", + }) + + const results = await builder + .appAction({ + fields: {}, + }) + .executeQuery({ + query: { + queryId: query._id!, + }, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.response }}", + }) + .createRow({ + row: { + name: "{{ loop.currentItem.name }}", + age: "{{ loop.currentItem.age }}", + tableId: newTable._id!, + }, + }) + .queryRows({ + tableId: newTable._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[1].outputs.iterations).toBe(3) + expect(results.steps[1].outputs.items).toHaveLength(3) + + expect(results.steps[2].outputs.rows).toHaveLength(3) + + rows.forEach(expectedRow => { + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining([expect.objectContaining(expectedRow)]) + ) + }) + }) + }) + + describe("Name Based Automations", () => { + it("should fetch and delete a rpw using automation naming", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Query and Delete Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + }, + { stepName: "InitialQueryStep" } + ) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.InitialQueryStep.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) }) describe("Automations with filter", () => { diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 6c89bd1f60..2269f075b2 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -58,21 +58,27 @@ type BranchConfig = { class BaseStepBuilder { protected steps: AutomationStep[] = [] + protected stepNames: { [key: string]: string } = {} protected step( stepId: TStep, stepSchema: Omit, - inputs: AutomationStepInputs + inputs: AutomationStepInputs, + stepName?: string ): this { + const id = uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, - id: uuidv4(), + id, stepId, + name: stepName || stepSchema.name, }) + if (stepName) { + this.stepNames[id] = stepName + } return this } - protected addBranchStep(branchConfig: BranchConfig): void { const branchStepInputs: BranchStepInputs = { branches: [] as Branch[], @@ -100,66 +106,80 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs): this { + createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, - inputs + inputs, + opts?.stepName ) } - updateRow(inputs: UpdateRowStepInputs): this { + updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, - inputs + inputs, + opts?.stepName ) } - deleteRow(inputs: DeleteRowStepInputs): this { + deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, - inputs + inputs, + opts?.stepName ) } - sendSmtpEmail(inputs: SmtpEmailStepInputs): this { + sendSmtpEmail( + inputs: SmtpEmailStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, - inputs + inputs, + opts?.stepName ) } - executeQuery(inputs: ExecuteQueryStepInputs): this { + executeQuery( + inputs: ExecuteQueryStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, - inputs + inputs, + opts?.stepName ) } - queryRows(inputs: QueryRowsStepInputs): this { + queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, - inputs + inputs, + opts?.stepName ) } - loop(inputs: LoopStepInputs): this { + loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, - inputs + inputs, + opts?.stepName ) } - serverLog(input: ServerLogStepInputs): this { + serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, - input + input, + opts?.stepName ) } @@ -195,6 +215,7 @@ class AutomationBuilder extends BaseStepBuilder { definition: { steps: [], trigger: {} as AutomationTrigger, + stepNames: {}, }, type: "automation", appId: options.appId ?? setup.getConfig().getAppId(), @@ -277,6 +298,7 @@ class AutomationBuilder extends BaseStepBuilder { build(): Automation { this.automationConfig.definition.steps = this.steps + this.automationConfig.definition.stepNames = this.stepNames return this.automationConfig } diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 18b033cdcf..110ccfa37a 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -18,6 +18,7 @@ import { SearchFilters, AutomationStoppedReason, AutomationStatus, + AutomationRowEvent, } from "@budibase/types" import { executeInThread } from "../threads/automation" import { dataFilters, sdk } from "@budibase/shared-core" @@ -28,6 +29,7 @@ const JOB_OPTS = { removeOnFail: true, } import * as automationUtils from "../automations/automationUtils" +import { doesTableExist } from "../sdk/app/tables/getters" async function getAllAutomations() { const db = context.getAppDB() @@ -38,25 +40,35 @@ async function getAllAutomations() { } async function queueRelevantRowAutomations( - event: { appId: string; row: Row; oldRow: Row }, - eventType: string + event: AutomationRowEvent, + eventType: AutomationEventType ) { + const tableId = event.row.tableId if (event.appId == null) { throw `No appId specified for ${eventType} - check event emitters.` } + // make sure table exists and is valid before proceeding + if (!tableId || !(await doesTableExist(tableId))) { + return + } + await context.doInAppContext(event.appId, async () => { let automations = await getAllAutomations() // filter down to the correct event type and enabled automations + // make sure it is the correct table ID as well automations = automations.filter(automation => { const trigger = automation.definition.trigger - return trigger && trigger.event === eventType && !automation.disabled + return ( + trigger && + trigger.event === eventType && + !automation.disabled && + trigger?.inputs?.tableId === event.row.tableId + ) }) for (const automation of automations) { - const automationDef = automation.definition - const automationTrigger = automationDef?.trigger // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) @@ -72,11 +84,7 @@ async function queueRelevantRowAutomations( row: event.row, oldRow: event.oldRow, }) - if ( - automationTrigger?.inputs && - automationTrigger.inputs.tableId === event.row.tableId && - shouldTrigger - ) { + if (shouldTrigger) { try { await automationQueue.add({ automation, event }, JOB_OPTS) } catch (e) { @@ -87,6 +95,17 @@ async function queueRelevantRowAutomations( }) } +async function queueRowAutomations( + event: AutomationRowEvent, + type: AutomationEventType +) { + try { + await queueRelevantRowAutomations(event, type) + } catch (err: any) { + logging.logWarn("Unable to process row event", err) + } +} + emitter.on( AutomationEventType.ROW_SAVE, async function (event: UpdatedRowEventEmitter) { @@ -94,7 +113,7 @@ emitter.on( if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_SAVE) + await queueRowAutomations(event, AutomationEventType.ROW_SAVE) } ) @@ -103,7 +122,7 @@ emitter.on(AutomationEventType.ROW_UPDATE, async function (event) { if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_UPDATE) + await queueRowAutomations(event, AutomationEventType.ROW_UPDATE) }) emitter.on(AutomationEventType.ROW_DELETE, async function (event) { @@ -111,7 +130,7 @@ emitter.on(AutomationEventType.ROW_DELETE, async function (event) { if (!event || !event.row || !event.row.tableId) { return } - await queueRelevantRowAutomations(event, AutomationEventType.ROW_DELETE) + await queueRowAutomations(event, AutomationEventType.ROW_DELETE) }) function rowPassesFilters(row: Row, filters: SearchFilters) { diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index e84eecea51..6488e604e9 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,6 +15,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + stepsByName?: Record env?: Record trigger: any } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 0d766ac1ef..831528f84d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -330,15 +330,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return { tables: {}, errors: {} } } await this.connect() + const sheets = this.client.sheetsByIndex const tables: Record = {} let errors: Record = {} + await utils.parallelForeach( sheets, async sheet => { - // must fetch rows to determine schema try { - await sheet.getRows() + await sheet.getRows({ limit: 1 }) } catch (err) { // We expect this to always be an Error so if it's not, rethrow it to // make sure we don't fail quietly. @@ -346,26 +347,34 @@ export class GoogleSheetsIntegration implements DatasourcePlus { throw err } - if (err.message.startsWith("No values in the header row")) { - errors[sheet.title] = err.message - } else { - // If we get an error we don't expect, rethrow to avoid failing - // quietly. - throw err + if ( + err.message.startsWith("No values in the header row") || + err.message.startsWith("All your header cells are blank") + ) { + errors[ + sheet.title + ] = `Failed to find a header row in sheet "${sheet.title}", is the first row blank?` + return } - return - } - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - datasourceId, - id - ) + // If we get an error we don't expect, rethrow to avoid failing + // quietly. + throw err + } }, 10 ) + + for (const sheet of sheets) { + const id = buildExternalTableId(datasourceId, sheet.title) + tables[sheet.title] = this.getTableSchema( + sheet.title, + sheet.headerValues, + datasourceId, + id + ) + } + let externalTables = finaliseExternalTables(tables, entities) errors = { ...errors, ...checkExternalTables(externalTables) } return { tables: externalTables, errors } @@ -551,11 +560,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus { await this.connect() const hasFilters = dataFilters.hasFilters(query.filters) const limit = query.paginate?.limit || 100 - const page: number = - typeof query.paginate?.page === "number" - ? query.paginate.page - : parseInt(query.paginate?.page || "1") - const offset = (page - 1) * limit + let offset = query.paginate?.offset || 0 + + let page = query.paginate?.page + if (typeof page === "string") { + page = parseInt(page) + } + if (page !== undefined) { + offset = page * limit + } + const sheet = this.client.sheetsByTitle[query.sheet] let rows: GoogleSpreadsheetRow[] = [] if (query.paginate && !hasFilters) { @@ -567,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rows = await sheet.getRows() } - if (hasFilters && query.paginate) { - rows = rows.slice(offset, offset + limit) - } - const headerValues = sheet.headerValues - let response = rows.map(row => - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber) ) response = dataFilters.runQuery(response, query.filters || {}) + if (hasFilters && query.paginate) { + response = response.slice(offset, offset + limit) + } + if (query.sort) { if (Object.keys(query.sort).length !== 1) { console.warn("Googlesheets does not support multiple sorting", { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 9781b97972..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -208,6 +209,56 @@ describe("Google Sheets Integration", () => { expect(row2.name).toEqual("Test Contact 2") expect(row2.description).toEqual("original description 2") }) + + it("can paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 248 }, (_, i) => ({ + name: `${i}`, + description: "", + })), + }) + + let resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + }) + let rows = resp.rows + + while (resp.hasNextPage) { + resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + bookmark: resp.bookmark, + }) + rows = rows.concat(resp.rows) + if (rows.length > 250) { + throw new Error("Too many rows returned") + } + } + + expect(rows.length).toEqual(250) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`)) + ) + }) + + it("can export rows", async () => { + const resp = await config.api.row.exportRows(table._id!, {}) + const parsed = JSON.parse(resp) + expect(parsed.length).toEqual(2) + expect(parsed[0]).toMatchObject({ + name: "Test Contact 1", + description: "original description 1", + }) + expect(parsed[1]).toMatchObject({ + name: "Test Contact 2", + description: "original description 2", + }) + }) }) describe("update", () => { @@ -299,5 +350,442 @@ describe("Google Sheets Integration", () => { expect(mock.cell("A2")).toEqual("Test Contact Updated") expect(mock.cell("B2")).toEqual("original description updated") }) + + it("should be able to rename a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const { name, ...otherColumns } = table.schema + const renamedTable = await config.api.table.save({ + ...table, + schema: { + ...otherColumns, + renamed: { + ...table.schema.name, + }, + }, + _rename: { + old: "name", + updated: "renamed", + }, + }) + + expect(renamedTable.schema.name).not.toBeDefined() + expect(renamedTable.schema.renamed).toBeDefined() + + expect(mock.cell("A1")).toEqual("renamed") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const renamedRow = await config.api.row.get(table._id!, row._id!) + expect(renamedRow.renamed).toEqual("Test Contact") + expect(renamedRow.description).toEqual("original description") + expect(renamedRow.name).not.toBeDefined() + }) + + // TODO: this gets the error "Sheet is not large enough to fit 27 columns. Resize the sheet first." + // eslint-disable-next-line jest/no-commented-out-tests + // it("should be able to add a new column", async () => { + // const updatedTable = await config.api.table.save({ + // ...table, + // schema: { + // ...table.schema, + // newColumn: { + // name: "newColumn", + // type: FieldType.STRING, + // }, + // }, + // }) + + // expect(updatedTable.schema.newColumn).toBeDefined() + + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual("description") + // expect(mock.cell("C1")).toEqual("newColumn") + // }) + + it("should be able to delete a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const updatedTable = await config.api.table.save({ + ...table, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + + expect(updatedTable.schema.name).toBeDefined() + expect(updatedTable.schema.description).not.toBeDefined() + + // TODO: we don't delete data in deleted columns yet, should we? + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual(null) + + const updatedRow = await config.api.row.get(table._id!, row._id!) + expect(updatedRow.name).toEqual("Test Contact") + expect(updatedRow.description).not.toBeDefined() + }) + }) + + describe("delete", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + }) + + it("can delete a table", async () => { + expect(mock.sheet(table.name)).toBeDefined() + await config.api.table.destroy(table._id!, table._rev!) + expect(mock.sheet(table.name)).toBeUndefined() + }) + + it("can delete a row", async () => { + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(2) + + // Because row IDs in Google Sheets are sequential and determined by the + // actual row in the sheet, deleting a row will shift the row IDs down by + // one. This is why we reverse the rows before deleting them. + for (const row of rows.reverse()) { + await config.api.row.delete(table._id!, { _id: row._id! }) + } + + expect(mock.cell("A1")).toEqual("name") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual(null) + expect(mock.cell("B2")).toEqual(null) + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const emptyRows = await config.api.row.fetch(table._id!) + expect(emptyRows.length).toEqual(0) + }) + }) + + describe("fetch schema", () => { + it("should fail to import a completely blank sheet", async () => { + mock.createSheet({ title: "Sheet1" }) + await config.api.datasource.fetchSchema( + { + datasourceId: datasource._id!, + tablesFilter: ["Sheet1"], + }, + { + status: 200, + body: { + errors: { + Sheet1: + 'Failed to find a header row in sheet "Sheet1", is the first row blank?', + }, + }, + } + ) + }) + + it("should fail to import multiple sheets with blank headers", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2"], + }, + { + status: 200, + body: { + errors: { + Sheet1: + 'Failed to find a header row in sheet "Sheet1", is the first row blank?', + Sheet2: + 'Failed to find a header row in sheet "Sheet2", is the first row blank?', + }, + }, + } + ) + }) + + it("should only fail the sheet with missing headers", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + mock.createSheet({ title: "Sheet3" }) + + mock.set("Sheet1!A1", "name") + mock.set("Sheet1!B1", "dob") + mock.set("Sheet2!A1", "name") + mock.set("Sheet2!B1", "dob") + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2", "Sheet3"], + }, + { + status: 200, + body: { + errors: { + Sheet3: + 'Failed to find a header row in sheet "Sheet3", is the first row blank?', + }, + }, + } + ) + }) + + it("should only succeed if sheet with missing headers is not being imported", async () => { + mock.createSheet({ title: "Sheet1" }) + mock.createSheet({ title: "Sheet2" }) + mock.createSheet({ title: "Sheet3" }) + + mock.set("Sheet1!A1", "name") + mock.set("Sheet1!B1", "dob") + mock.set("Sheet2!A1", "name") + mock.set("Sheet2!B1", "dob") + + await config.api.datasource.fetchSchema( + { + datasourceId: datasource!._id!, + tablesFilter: ["Sheet1", "Sheet2"], + }, + { + status: 200, + body: { errors: {} }, + } + ) + }) + }) + + describe("search", () => { + let table: Table + + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Foo", + }, + { + name: "Bar", + }, + { + name: "Baz", + }, + ], + }) + }) + + it("should be able to find rows with equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + equal: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with not equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEqual: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should be able to find rows with empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + empty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(0) + }) + + it("should be able to find rows with not empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEmpty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(3) + }) + + it("should be able to find rows with one of filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + oneOf: { + name: ["Foo", "Bar"], + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Foo") + expect(response.rows[1].name).toEqual("Bar") + }) + + it("should be able to find rows with fuzzy filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + fuzzy: { + name: "oo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with range filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + range: { + name: { + low: "A", + high: "C", + }, + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Unique value!`, + })), + }) + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Non-unique value!`, + })), + }) + + let response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + }) + let rows: Row[] = response.rows + + while (response.hasNextPage) { + response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + bookmark: response.bookmark, + }) + + expect(response.rows.length).toBeLessThanOrEqual(10) + rows = rows.concat(response.rows) + } + + // Make sure we only get rows matching the query. + expect(rows.length).toEqual(50) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining( + Array.from({ length: 50 }, () => "Unique value!") + ) + ) + + // Make sure all of the rows have a unique ID. + const ids = Object.keys( + rows.reduce((acc, row) => { + acc[row._id!] = true + return acc + }, {}) + ) + expect(ids.length).toEqual(50) + }) }) }) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index cf5b88b0fd..5a505fbb40 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -162,7 +162,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) @@ -170,7 +170,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9548499c65..fc5af4238c 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -63,7 +63,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` ) ), }) @@ -95,7 +95,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -113,7 +113,7 @@ describe("Captures of real examples", () => { bindings: [...filters, relationshipLimit, relationshipLimit], sql: multiline( `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) - select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc` ), diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index c58066bee5..4b9445ebca 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -21,6 +21,8 @@ import type { CellFormat, CellPadding, Color, + GridRange, + DataSourceSheetProperties, } from "google-spreadsheet/src/lib/types/sheets-types" const BLACK: Color = { red: 0, green: 0, blue: 0 } @@ -88,11 +90,38 @@ interface UpdateValuesResponse { updatedData: ValueRange } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest +interface AddSheetRequest { + properties: Partial +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse interface AddSheetResponse { properties: WorksheetProperties } +interface DeleteRangeRequest { + range: GridRange + shiftDimension: WorksheetDimension +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#DeleteSheetRequest +interface DeleteSheetRequest { + sheetId: number +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request +interface BatchUpdateRequest { + requests: { + addSheet?: AddSheetRequest + deleteRange?: DeleteRangeRequest + deleteSheet?: DeleteSheetRequest + }[] + includeSpreadsheetInResponse: boolean + responseRanges: string[] + responseIncludeGridData: boolean +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response interface BatchUpdateResponse { spreadsheetId: string @@ -102,23 +131,6 @@ interface BatchUpdateResponse { updatedSpreadsheet: Spreadsheet } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest -interface AddSheetRequest { - properties: WorksheetProperties -} - -interface Request { - addSheet?: AddSheetRequest -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request -interface BatchUpdateRequest { - requests: Request[] - includeSpreadsheetInResponse: boolean - responseRanges: string[] - responseIncludeGridData: boolean -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData interface RowData { values: CellData[] @@ -225,6 +237,38 @@ export class GoogleSheetsMock { this.mockAPI() } + public cell(cell: string): Value | undefined { + const cellData = this.cellData(cell) + if (!cellData) { + return undefined + } + return this.cellValue(cellData) + } + + public set(cell: string, value: Value): void { + const cellData = this.cellData(cell) + if (!cellData) { + throw new Error(`Cell ${cell} not found`) + } + cellData.userEnteredValue = this.createValue(value) + } + + public sheet(name: string | number): Sheet | undefined { + if (typeof name === "number") { + return this.getSheetById(name) + } + return this.getSheetByName(name) + } + + public createSheet(opts: Partial): Sheet { + const properties = this.defaultWorksheetProperties(opts) + if (this.getSheetByName(properties.title)) { + throw new Error(`Sheet ${properties.title} already exists`) + } + const resp = this.handleAddSheet({ properties }) + return this.getSheetById(resp.properties.sheetId)! + } + private route( method: "get" | "put" | "post", path: string | RegExp, @@ -369,13 +413,17 @@ export class GoogleSheetsMock { private handleValueAppend(request: AppendRequest): AppendResponse { const { range, params, body } = request - const { sheet, bottomRight } = this.parseA1Notation(range) + const { sheetId, endRowIndex } = this.parseA1Notation(range) + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${sheetId} not found`) + } const newRows = body.values.map(v => this.valuesToRowData(v)) const toDelete = params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0 - sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows) - sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, { + sheet.data[0].rowData.splice(endRowIndex + 1, toDelete, ...newRows) + sheet.data[0].rowMetadata.splice(endRowIndex + 1, toDelete, { hiddenByUser: false, hiddenByFilter: false, pixelSize: 100, @@ -384,17 +432,15 @@ export class GoogleSheetsMock { // It's important to give back a correct updated range because the API // library we use makes use of it to assign the correct row IDs to rows. - const updatedRange = this.createA1FromRanges( - sheet, - { - row: bottomRight.row + 1, - column: 0, - }, - { - row: bottomRight.row + newRows.length, - column: 0, - } - ) + const updatedRange = this.createA1({ + sheetId, + startRowIndex: endRowIndex + 1, + startColumnIndex: 0, + endRowIndex: endRowIndex + newRows.length, + endColumnIndex: 0, + }) + + sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length return { spreadsheetId: this.spreadsheet.spreadsheetId, @@ -438,40 +484,69 @@ export class GoogleSheetsMock { addSheet: this.handleAddSheet(request.addSheet), }) } + if (request.deleteRange) { + this.handleDeleteRange(request.deleteRange) + response.replies.push({}) + } + if (request.deleteSheet) { + this.handleDeleteSheet(request.deleteSheet) + response.replies.push({}) + } } return response } - private handleAddSheet(request: AddSheetRequest): AddSheetResponse { - const properties: Omit = { + private defaultWorksheetProperties( + opts: Partial + ): WorksheetProperties { + return { index: this.spreadsheet.sheets.length, hidden: false, rightToLeft: false, tabColor: BLACK, tabColorStyle: { rgbColor: BLACK }, sheetType: "GRID", - title: request.properties.title, + title: "Sheet", sheetId: this.spreadsheet.sheets.length, gridProperties: { rowCount: 100, columnCount: 26, - frozenRowCount: 0, - frozenColumnCount: 0, - hideGridlines: false, - rowGroupControlAfter: false, - columnGroupControlAfter: false, }, + dataSourceSheetProperties: {} as DataSourceSheetProperties, + ...opts, + } + } + + private handleAddSheet(request: AddSheetRequest): AddSheetResponse { + const properties = this.defaultWorksheetProperties(request.properties) + this.spreadsheet.sheets.push({ + properties, + data: [ + this.createEmptyGrid( + properties.gridProperties.rowCount, + properties.gridProperties.columnCount + ), + ], + }) + return { properties } + } + + private handleDeleteRange(request: DeleteRangeRequest) { + const { range, shiftDimension } = request + + if (shiftDimension !== "ROWS") { + throw new Error("Only row-based deletes are supported") } - this.spreadsheet.sheets.push({ - properties: properties as WorksheetProperties, - data: [this.createEmptyGrid(100, 26)], + this.iterateRange(range, cell => { + cell.userEnteredValue = this.createValue(null) }) + } - // dataSourceSheetProperties is only returned by the API if the sheet type is - // DATA_SOURCE, which we aren't using, so sadly we need to cast here. - return { properties: properties as WorksheetProperties } + private handleDeleteSheet(request: DeleteSheetRequest) { + const { sheetId } = request + this.spreadsheet.sheets.splice(sheetId, 1) } private handleGetSpreadsheet(): Spreadsheet { @@ -479,7 +554,7 @@ export class GoogleSheetsMock { } private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { - this.iterateCells(valueRange, (cell, value) => { + this.iterateValueRange(valueRange, (cell, value) => { cell.userEnteredValue = this.createValue(value) }) @@ -494,7 +569,27 @@ export class GoogleSheetsMock { return response } - private iterateCells( + private iterateRange(range: GridRange, cb: (cell: CellData) => void) { + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = this.ensureGridRange(range) + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) + if (!cell) { + throw new Error("Cell not found") + } + cb(cell) + } + } + } + + private iterateValueRange( valueRange: ValueRange, cb: (cell: CellData, value: Value) => void ) { @@ -502,33 +597,46 @@ export class GoogleSheetsMock { throw new Error("Only row-major updates are supported") } - const { sheet, topLeft, bottomRight } = this.parseA1Notation( - valueRange.range - ) - for (let row = topLeft.row; row <= bottomRight.row; row++) { - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = this.parseA1Notation(valueRange.range) + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } - const value = valueRange.values[row - topLeft.row][col - topLeft.column] + const value = + valueRange.values[row - startRowIndex][col - startColumnIndex] cb(cell, value) } } } private getValueRange(range: string): ValueRange { - const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = this.parseA1Notation(range) + const valueRange: ValueRange = { range, majorDimension: "ROWS", values: [], } - for (let row = topLeft.row; row <= bottomRight.row; row++) { + for (let row = startRowIndex; row <= endRowIndex; row++) { const values: Value[] = [] - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } @@ -693,26 +801,24 @@ export class GoogleSheetsMock { } private cellData(cell: string): CellData | undefined { - const { - sheet, - topLeft: { row, column }, - } = this.parseA1Notation(cell) - return this.getCellNumericIndexes(sheet, row, column) - } - - cell(cell: string): Value | undefined { - const cellData = this.cellData(cell) - if (!cellData) { - return undefined - } - return this.cellValue(cellData) + const { sheetId, startColumnIndex, startRowIndex } = + this.parseA1Notation(cell) + return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex) } private getCellNumericIndexes( - sheet: Sheet, + sheet: Sheet | number, row: number, column: number ): CellData | undefined { + if (typeof sheet === "number") { + const foundSheet = this.getSheetById(sheet) + if (!foundSheet) { + return undefined + } + sheet = foundSheet + } + const data = sheet.data[0] const rowData = data.rowData[row] if (!rowData) { @@ -751,11 +857,7 @@ export class GoogleSheetsMock { // "Sheet1!A:B" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } } // "Sheet1!1:1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } } // "Sheet1!1:2" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } } - private parseA1Notation(range: string): { - sheet: Sheet - topLeft: Range - bottomRight: Range - } { + private parseA1Notation(range: string): Required { let sheet: Sheet let rest: string if (!range.includes("!")) { @@ -793,35 +895,54 @@ export class GoogleSheetsMock { parsedBottomRight = parsedTopLeft } - if (parsedTopLeft && parsedTopLeft.row === undefined) { - parsedTopLeft.row = 0 - } - if (parsedTopLeft && parsedTopLeft.column === undefined) { - parsedTopLeft.column = 0 - } - if (parsedBottomRight && parsedBottomRight.row === undefined) { - parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 - } - if (parsedBottomRight && parsedBottomRight.column === undefined) { - parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 + return this.ensureGridRange({ + sheetId: sheet.properties.sheetId, + startRowIndex: parsedTopLeft.row, + endRowIndex: parsedBottomRight.row, + startColumnIndex: parsedTopLeft.column, + endColumnIndex: parsedBottomRight.column, + }) + } + + private ensureGridRange(range: GridRange): Required { + const sheet = this.getSheetById(range.sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) } return { - sheet, - topLeft: parsedTopLeft as Range, - bottomRight: parsedBottomRight as Range, + sheetId: range.sheetId, + startRowIndex: range.startRowIndex ?? 0, + endRowIndex: + range.endRowIndex ?? sheet.properties.gridProperties.rowCount - 1, + startColumnIndex: range.startColumnIndex ?? 0, + endColumnIndex: + range.endColumnIndex ?? sheet.properties.gridProperties.columnCount - 1, } } - private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) { + private createA1(range: Required) { + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = range + + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) + } + let title = sheet.properties.title if (title.includes(" ")) { title = `'${title}'` } - const topLeftLetter = this.numberToLetter(topLeft.column) - const bottomRightLetter = this.numberToLetter(bottomRight.column) - const topLeftRow = topLeft.row + 1 - const bottomRightRow = bottomRight.row + 1 + const topLeftLetter = this.numberToLetter(startColumnIndex) + const bottomRightLetter = this.numberToLetter(endColumnIndex) + const topLeftRow = startRowIndex + 1 + const bottomRightRow = endRowIndex + 1 return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}` } @@ -860,4 +981,10 @@ export class GoogleSheetsMock { sheet => sheet.properties.title === name ) } + + private getSheetById(id: number): Sheet | undefined { + return this.spreadsheet.sheets.find( + sheet => sheet.properties.sheetId === id + ) + } } diff --git a/packages/server/src/sdk/app/permissions/index.ts b/packages/server/src/sdk/app/permissions/index.ts index 18a376aaf0..a6e81652ee 100644 --- a/packages/server/src/sdk/app/permissions/index.ts +++ b/packages/server/src/sdk/app/permissions/index.ts @@ -1,10 +1,7 @@ import { db, roles } from "@budibase/backend-core" -import { features } from "@budibase/pro" import { - DocumentType, PermissionLevel, PermissionSource, - PlanType, VirtualDocumentType, } from "@budibase/types" import { extractViewInfoFromID, isViewID } from "../../../db/utils" @@ -15,36 +12,6 @@ import { import sdk from "../../../sdk" import { isV2 } from "../views" -type ResourceActionAllowedResult = - | { allowed: true } - | { - allowed: false - level: PermissionLevel - resourceType: DocumentType | VirtualDocumentType - } - -export async function resourceActionAllowed({ - resourceId, - level, -}: { - resourceId: string - level: PermissionLevel -}): Promise { - if (!isViewID(resourceId)) { - return { allowed: true } - } - - if (await features.isViewPermissionEnabled()) { - return { allowed: true } - } - - return { - allowed: false, - level, - resourceType: VirtualDocumentType.VIEW, - } -} - type ResourcePermissions = Record< string, { role: string; type: PermissionSource } @@ -58,20 +25,6 @@ export async function getInheritablePermissions( } } -export async function allowsExplicitPermissions(resourceId: string) { - if (isViewID(resourceId)) { - const allowed = await features.isViewPermissionEnabled() - const minPlan = !allowed ? PlanType.PREMIUM_PLUS : undefined - - return { - allowed, - minPlan, - } - } - - return { allowed: true } -} - export async function getResourcePerms( resourceId: string ): Promise { @@ -81,16 +34,14 @@ export async function getResourcePerms( const permsToInherit = await getInheritablePermissions(resourceId) - const allowsExplicitPerm = (await allowsExplicitPermissions(resourceId)) - .allowed - for (let level of CURRENTLY_SUPPORTED_LEVELS) { // update the various roleIds in the resource permissions for (let role of rolesList) { - const rolePerms = allowsExplicitPerm - ? roles.checkForRoleResourceArray(role.permissions || {}, resourceId) - : {} - if (rolePerms[resourceId]?.indexOf(level) > -1) { + const rolePerms = roles.checkForRoleResourceArray( + role.permissions || {}, + resourceId + ) + if (rolePerms[resourceId]?.indexOf(level as PermissionLevel) > -1) { permissions[level] = { role: roles.getExternalRoleID(role._id!, role.version), type: PermissionSource.EXPLICIT, diff --git a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts b/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts deleted file mode 100644 index 4c233e68fa..0000000000 --- a/packages/server/src/sdk/app/permissions/tests/permissions.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { PermissionLevel } from "@budibase/types" -import { mocks, structures } from "@budibase/backend-core/tests" -import { resourceActionAllowed } from ".." -import { generateViewID } from "../../../../db/utils" -import { initProMocks } from "../../../../tests/utilities/mocks/pro" - -initProMocks() - -describe("permissions sdk", () => { - beforeEach(() => { - mocks.licenses.useCloudFree() - }) - - describe("resourceActionAllowed", () => { - it("non view resources actions are always allowed", async () => { - const resourceId = structures.users.user()._id! - - const result = await resourceActionAllowed({ - resourceId, - level: PermissionLevel.READ, - }) - - expect(result).toEqual({ allowed: true }) - }) - - it("view resources actions allowed if the feature flag is enabled", async () => { - mocks.licenses.useViewPermissions() - const resourceId = generateViewID(structures.generator.guid()) - - const result = await resourceActionAllowed({ - resourceId, - level: PermissionLevel.READ, - }) - - expect(result).toEqual({ allowed: true }) - }) - - it("view resources actions allowed if the feature flag is disabled", async () => { - const resourceId = generateViewID(structures.generator.guid()) - - const result = await resourceActionAllowed({ - resourceId, - level: PermissionLevel.READ, - }) - - expect(result).toEqual({ - allowed: false, - level: "read", - resourceType: "view", - }) - }) - }) -}) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 27e9962a1a..5ff000fe12 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -101,6 +101,15 @@ export async function getTable(tableId: string): Promise { return await processTable(output) } +export async function doesTableExist(tableId: string): Promise { + try { + const table = await getTable(tableId) + return !!table + } catch (err) { + return false + } +} + export async function getAllTables() { const [internal, external] = await Promise.all([ getAllInternalTables(), diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index d7e05abf2f..269158e61e 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -5,13 +5,11 @@ import { Table, TableSchema, View, - ViewFieldMetadata, ViewV2, ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" import { HTTPError } from "@budibase/backend-core" -import { features } from "@budibase/pro" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -59,13 +57,6 @@ async function guardViewSchema( } if (viewSchema[field].readonly) { - if ( - !(await features.isViewReadonlyColumnsEnabled()) && - !(tableSchemaField as ViewFieldMetadata).readonly - ) { - throw new HTTPError(`Readonly fields are not enabled`, 400) - } - if (!viewSchema[field].visible) { throw new HTTPError( `Field "${field}" must be visible if you want to make it readonly`, diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index d5ed2ce906..214c9498d6 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -333,6 +333,7 @@ export default class TestConfiguration { sessionId: this.sessionIdForUser(_id), tenantId: this.getTenantId(), csrfToken: this.csrfToken, + email, }) const resp = await db.put(user) await cache.user.invalidateUser(_id) @@ -396,16 +397,17 @@ export default class TestConfiguration { } // make sure the user exists in the global DB if (roleId !== roles.BUILTIN_ROLE_IDS.PUBLIC) { - await this.globalUser({ + const user = await this.globalUser({ _id: userId, builder: { global: builder }, roles: { [appId]: roleId || roles.BUILTIN_ROLE_IDS.BASIC }, }) + await sessions.createASession(userId, { + sessionId: this.sessionIdForUser(userId), + tenantId: this.getTenantId(), + email: user.email, + }) } - await sessions.createASession(userId, { - sessionId: this.sessionIdForUser(userId), - tenantId: this.getTenantId(), - }) // have to fake this const authObj = { userId, diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 64ff392a8f..a59935c6aa 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -89,7 +89,12 @@ class Orchestrator { delete triggerOutput.appId delete triggerOutput.metadata // step zero is never used as the template string is zero indexed for customer facing - this.context = { steps: [{}], trigger: triggerOutput } + this.context = { + steps: [{}], + stepsByName: {}, + trigger: triggerOutput, + } + this.automation = automation // create an emitter which has the chain count for this automation run in it, so it can block // excessive chaining if required @@ -451,6 +456,9 @@ class Orchestrator { outputs: tempOutput, inputs: steps[stepToLoopIndex].inputs, }) + + const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id + this.context.stepsByName![stepName] = tempOutput this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") @@ -555,8 +563,13 @@ class Orchestrator { loopIteration ) } + const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject(originalStepInput, this.context) + let inputs = await this.addContextAndProcess( + originalStepInput, + this.context + ) + inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) const outputs = await stepFn({ @@ -583,6 +596,18 @@ class Orchestrator { return null } + private async addContextAndProcess(inputs: any, context: any) { + const processContext = { + ...context, + steps: { + ...context.steps, + ...context.stepsByName, + }, + } + + return processObject(inputs, processContext) + } + private handleStepOutput( step: AutomationStep, outputs: any, @@ -600,6 +625,8 @@ class Orchestrator { } else { this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) this.context.steps[this.context.steps.length] = outputs + const stepName = step.name || step.id + this.context.stepsByName![stepName] = outputs } } } diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index ba451a3325..c7e28d3bf4 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -247,7 +247,9 @@ class QueryRunner { if (!resp.err) { const globalUserId = getGlobalIDFromUserMetadataID(_id) await auth.updateUserOAuth(globalUserId, resp) - this.ctx.user = await cache.user.getUser(globalUserId) + this.ctx.user = await cache.user.getUser({ + userId: globalUserId, + }) } else { // In this event the user may have oAuth issues that // could require re-authenticating with their provider. diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index bbb84c1882..c1fcf35634 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -77,7 +77,9 @@ export async function getCachedSelf( ): Promise { // this has to be tenant aware, can't depend on the context to find it out // running some middlewares before the tenancy causes context to break - const user = await cache.user.getUser(ctx.user?._id!) + const user = await cache.user.getUser({ + userId: ctx.user?._id!, + }) return processUser(user, { appId }) } diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 874113f6f1..b02ea2ff60 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -35,7 +35,9 @@ export async function processInputBBReference( } try { - await cache.user.getUser(id) + await cache.user.getUser({ + userId: id, + }) return id } catch (e: any) { if (e.statusCode === 404) { @@ -125,7 +127,9 @@ export async function processOutputBBReference( case BBReferenceFieldSubType.USER: { let user try { - user = await cache.user.getUser(value as string) + user = await cache.user.getUser({ + userId: value as string, + }) } catch (err: any) { if (err.statusCode !== 404) { throw err diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index c2a63fd674..f6cf44d6d6 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -110,7 +110,9 @@ async function processDefaultValues(table: Table, row: Row) { const identity = context.getIdentity() if (identity?._id && identity.type === IdentityType.USER) { - const user = await cache.user.getUser(identity._id) + const user = await cache.user.getUser({ + userId: identity._id, + }) delete user.password ctx["Current User"] = user diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index c5a7b7fc84..28c16dad71 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -74,7 +74,9 @@ describe("bbReferenceProcessor", () => { expect(result).toEqual(userId) expect(cacheGetUserSpy).toHaveBeenCalledTimes(1) - expect(cacheGetUserSpy).toHaveBeenCalledWith(userId) + expect(cacheGetUserSpy).toHaveBeenCalledWith({ + userId, + }) }) it("throws an error given an invalid id", async () => { @@ -98,7 +100,9 @@ describe("bbReferenceProcessor", () => { expect(result).toEqual(userId) expect(cacheGetUserSpy).toHaveBeenCalledTimes(1) - expect(cacheGetUserSpy).toHaveBeenCalledWith(userId) + expect(cacheGetUserSpy).toHaveBeenCalledWith({ + userId, + }) }) it("empty strings will return null", async () => { @@ -243,7 +247,9 @@ describe("bbReferenceProcessor", () => { lastName: user.lastName, }) expect(cacheGetUserSpy).toHaveBeenCalledTimes(1) - expect(cacheGetUserSpy).toHaveBeenCalledWith(userId) + expect(cacheGetUserSpy).toHaveBeenCalledWith({ + userId, + }) }) it("returns undefined given an unexisting user", async () => { @@ -255,7 +261,9 @@ describe("bbReferenceProcessor", () => { expect(result).toBeUndefined() expect(cacheGetUserSpy).toHaveBeenCalledTimes(1) - expect(cacheGetUserSpy).toHaveBeenCalledWith(userId) + expect(cacheGetUserSpy).toHaveBeenCalledWith({ + userId, + }) }) }) }) diff --git a/packages/shared-core/src/constants/colors.ts b/packages/shared-core/src/constants/colors.ts new file mode 100644 index 0000000000..0abff46c3a --- /dev/null +++ b/packages/shared-core/src/constants/colors.ts @@ -0,0 +1,8 @@ +export enum RoleColor { + ADMIN = "var(--spectrum-global-color-static-red-400)", + POWER = "var(--spectrum-global-color-static-orange-400)", + BASIC = "var(--spectrum-global-color-static-green-400)", + PUBLIC = "var(--spectrum-global-color-static-blue-400)", + BUILDER = "var(--spectrum-global-color-static-magenta-600)", + DEFAULT_CUSTOM = "var(--spectrum-global-color-static-magenta-400)", +} diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index 78984aafa4..5a42fc5677 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,6 +1,7 @@ export * from "./api" export * from "./fields" export * from "./rows" +export * from "./colors" export const OperatorOptions = { Equals: { diff --git a/packages/types/src/api/web/app/permission.ts b/packages/types/src/api/web/app/permission.ts index 88ff4e9d2f..bead2a4279 100644 --- a/packages/types/src/api/web/app/permission.ts +++ b/packages/types/src/api/web/app/permission.ts @@ -1,4 +1,4 @@ -import { PermissionLevel, PlanType } from "../../../sdk" +import { PermissionLevel } from "../../../sdk" export interface ResourcePermissionInfo { role: string @@ -8,7 +8,6 @@ export interface ResourcePermissionInfo { export interface GetResourcePermsResponse { permissions: Record - requiresPlanToModify?: PlanType } export interface GetDependantResourcesResponse { diff --git a/packages/types/src/api/web/role.ts b/packages/types/src/api/web/role.ts index c37dee60e0..644222b8f9 100644 --- a/packages/types/src/api/web/role.ts +++ b/packages/types/src/api/web/role.ts @@ -1,4 +1,4 @@ -import { Role } from "../../documents" +import { Role, RoleUIMetadata } from "../../documents" export interface SaveRoleRequest { _id?: string @@ -7,6 +7,7 @@ export interface SaveRoleRequest { inherits: string permissionId: string version: string + uiMetadata?: RoleUIMetadata } export interface SaveRoleResponse extends Role {} diff --git a/packages/types/src/documents/app/automation/StepInputsOutputs.ts b/packages/types/src/documents/app/automation/StepInputsOutputs.ts index 983a67daf5..9ba7b6cc1d 100644 --- a/packages/types/src/documents/app/automation/StepInputsOutputs.ts +++ b/packages/types/src/documents/app/automation/StepInputsOutputs.ts @@ -140,6 +140,8 @@ enum Model { GPT_35_TURBO = "gpt-3.5-turbo", // will only work with api keys that have access to the GPT4 API GPT_4 = "gpt-4", + GPT_4O = "gpt-4o", + GPT_4O_MINI = "gpt-4o-mini", } export type OpenAIStepOutputs = Omit & { diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 72f8a1aa7c..effe99a328 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -124,6 +124,8 @@ export interface Automation extends Document { definition: { steps: AutomationStep[] trigger: AutomationTrigger + // stepNames is used to lookup step names from their correspnding step ID. + stepNames?: Record } screenId?: string uiTree?: any diff --git a/packages/types/src/documents/app/role.ts b/packages/types/src/documents/app/role.ts index f32ba810b0..6557b7e19d 100644 --- a/packages/types/src/documents/app/role.ts +++ b/packages/types/src/documents/app/role.ts @@ -1,9 +1,17 @@ import { Document } from "../document" +import { PermissionLevel } from "../../sdk" + +export interface RoleUIMetadata { + displayName?: string + color?: string + description?: string +} export interface Role extends Document { permissionId: string inherits?: string - permissions: { [key: string]: string[] } + permissions: Record version?: string name: string + uiMetadata?: RoleUIMetadata } diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 8d64b49ee9..33f7e10584 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,7 +111,7 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom" +export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" export interface AIInnerConfig { [key: string]: { diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index af5c11374d..99e01fedf0 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -74,9 +74,8 @@ export enum UserStatus { INACTIVE = "inactive", } -export interface UserRoles { - [key: string]: string -} +// specifies a map of app ID to role ID +export type UserRoles = Record // UTILITY TYPES diff --git a/packages/types/src/sdk/auth.ts b/packages/types/src/sdk/auth.ts index edcaf197e2..5bc75e8377 100644 --- a/packages/types/src/sdk/auth.ts +++ b/packages/types/src/sdk/auth.ts @@ -10,6 +10,7 @@ export interface AuthToken { export interface CreateSession { sessionId: string tenantId: string + email: string csrfToken?: string hosting?: Hosting } diff --git a/packages/types/src/sdk/automations/index.ts b/packages/types/src/sdk/automations/index.ts index 5ea22148a5..d04f126c32 100644 --- a/packages/types/src/sdk/automations/index.ts +++ b/packages/types/src/sdk/automations/index.ts @@ -15,4 +15,10 @@ export interface AutomationData { automation: Automation } +export interface AutomationRowEvent { + appId: string + row: Row + oldRow: Row +} + export type AutomationJob = Job diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index 9d09f1d14f..286faaa218 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -13,6 +13,7 @@ export enum Feature { APP_BUILDERS = "appBuilders", OFFLINE = "offline", EXPANDED_PUBLIC_API = "expandedPublicApi", + // deprecated - no longer licensed VIEW_PERMISSIONS = "viewPermissions", VIEW_READONLY_COLUMNS = "viewReadonlyColumns", BUDIBASE_AI = "budibaseAI", diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 70b2279f6c..e6e80ff3a5 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,6 +253,7 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } + await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length) break } } catch (err: any) { @@ -334,32 +335,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { ) } -async function enrichAIConfig(aiConfig: AIConfig) { - // Strip out the API Keys from the response so they don't show in the UI - for (const key in aiConfig.config) { - if (aiConfig.config[key].apiKey) { - aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT - } - } - - // Return the Budibase AI data source as part of the response if licensing allows - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const defaultConfigExists = Object.keys(aiConfig.config).some( - key => aiConfig.config[key].isDefault - ) - if (budibaseAIEnabled) { - aiConfig.config["budibase_ai"] = { - provider: "OpenAI", - active: true, - isDefault: !defaultConfigExists, - defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "", - name: "Budibase AI", - } - } - - return aiConfig -} - export async function find(ctx: UserCtx) { try { // Find the config with the most granular scope based on context @@ -372,7 +347,13 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await enrichAIConfig(scopedConfig) + await pro.sdk.ai.enrichAIConfig(scopedConfig) + // Strip out the API Keys from the response so they don't show in the UI + for (const key in scopedConfig.config) { + if (scopedConfig.config[key].apiKey) { + scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT + } + } } ctx.body = scopedConfig } else { diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 3ff6a5298c..9091f29247 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,4 +1,3 @@ -import * as pro from "@budibase/pro" import { verifyAIConfig } from "../configs" import { TestConfiguration, structures } from "../../../../tests" import { AIInnerConfig } from "@budibase/types" @@ -35,55 +34,6 @@ describe("Global configs controller", () => { }) }) - it("Should return the default BB AI config when the feature is turned on", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(true)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - budibase_ai: { - provider: "OpenAI", - active: true, - isDefault: true, - name: "Budibase AI", - defaultModel: "", - }, - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - - it("Should not not return the default Budibase AI config when on self host", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(false)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - it("Should not update existing secrets when updating an existing AI Config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data) diff --git a/packages/worker/src/api/routes/global/tests/roles.spec.ts b/packages/worker/src/api/routes/global/tests/roles.spec.ts index 0c3a1c60a9..11de06328e 100644 --- a/packages/worker/src/api/routes/global/tests/roles.spec.ts +++ b/packages/worker/src/api/routes/global/tests/roles.spec.ts @@ -35,8 +35,9 @@ describe("/api/global/roles", () => { const role = new roles.Role( db.generateRoleID(ROLE_NAME), - roles.BUILTIN_ROLE_IDS.BASIC, - permissions.BuiltinPermissionID.READ_ONLY + ROLE_NAME, + permissions.BuiltinPermissionID.READ_ONLY, + { displayName: roles.BUILTIN_ROLE_IDS.BASIC } ) beforeAll(async () => { diff --git a/packages/worker/src/sdk/auth/auth.ts b/packages/worker/src/sdk/auth/auth.ts index 3f24de440a..078ac441e9 100644 --- a/packages/worker/src/sdk/auth/auth.ts +++ b/packages/worker/src/sdk/auth/auth.ts @@ -19,12 +19,17 @@ import { EmailTemplatePurpose } from "../../constants" export async function loginUser(user: User) { const sessionId = coreUtils.newid() const tenantId = tenancy.getTenantId() - await sessions.createASession(user._id!, { sessionId, tenantId }) + await sessions.createASession(user._id!, { + sessionId, + tenantId, + email: user.email, + }) return jwt.sign( { userId: user._id, sessionId, tenantId, + email: user.email, }, coreEnv.JWT_SECRET! ) diff --git a/packages/worker/src/tests/TestConfiguration.ts b/packages/worker/src/tests/TestConfiguration.ts index c1c8aa8b19..c2cf005308 100644 --- a/packages/worker/src/tests/TestConfiguration.ts +++ b/packages/worker/src/tests/TestConfiguration.ts @@ -170,19 +170,26 @@ class TestConfiguration { async _createSession({ userId, tenantId, + email, }: { userId: string tenantId: string + email: string }) { await sessions.createASession(userId!, { sessionId: "sessionid", - tenantId: tenantId, + tenantId, csrfToken: CSRF_TOKEN, + email, }) } async createSession(user: User) { - return this._createSession({ userId: user._id!, tenantId: user.tenantId }) + return this._createSession({ + userId: user._id!, + tenantId: user.tenantId, + email: user.email, + }) } cookieHeader(cookies: any) { diff --git a/scripts/build-single-image-sqs.sh b/scripts/build-single-image-sqs.sh index 502ba5fa14..40b97013a1 100644 --- a/scripts/build-single-image-sqs.sh +++ b/scripts/build-single-image-sqs.sh @@ -2,4 +2,4 @@ yarn build:apps version=$(./scripts/getCurrentVersion.sh) -docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs . +docker build -f hosting/single/Dockerfile -t budibase:sqs --build-arg BUDIBASE_VERSION=$version --build-arg TARGETBUILD=single --build-arg BASEIMG=budibase/couchdb:v3.3.3-sqs-v2.1.1 . diff --git a/yarn.lock b/yarn.lock index 146fa74d5b..cd850e833d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,6 +33,19 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.24" +"@anthropic-ai/sdk@^0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475" + integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.1.2" resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8" @@ -759,20 +772,20 @@ "@azure/abort-controller" "^1.0.0" tslib "^2.2.0" -"@azure/identity@^3.4.1": - version "3.4.2" - resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-3.4.2.tgz#6b01724c9caac7cadab6b63c76584345bda8e2de" - integrity sha512-0q5DL4uyR0EZ4RXQKD8MadGH6zTIcloUoS/RVbCpNpej4pwte0xpqYxk8K97Py2RiuUvI7F4GXpoT4046VfufA== +"@azure/identity@4.2.1", "@azure/identity@^3.4.1": + version "4.2.1" + resolved "https://registry.yarnpkg.com/@azure/identity/-/identity-4.2.1.tgz#22b366201e989b7b41c0e1690e103bd579c31e4c" + integrity sha512-U8hsyC9YPcEIzoaObJlRDvp7KiF0MGS7xcWbyJSVvXRkC/HXo1f0oYeBYmEvVgRfacw7GHf6D6yAoh9JHz6A5Q== dependencies: "@azure/abort-controller" "^1.0.0" "@azure/core-auth" "^1.5.0" "@azure/core-client" "^1.4.0" "@azure/core-rest-pipeline" "^1.1.0" "@azure/core-tracing" "^1.0.0" - "@azure/core-util" "^1.6.1" + "@azure/core-util" "^1.3.0" "@azure/logger" "^1.0.0" - "@azure/msal-browser" "^3.5.0" - "@azure/msal-node" "^2.5.1" + "@azure/msal-browser" "^3.11.1" + "@azure/msal-node" "^2.9.2" events "^3.0.0" jws "^4.0.0" open "^8.0.0" @@ -803,24 +816,24 @@ dependencies: tslib "^2.2.0" -"@azure/msal-browser@^3.5.0": - version "3.18.0" - resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.18.0.tgz#dabbde2c53195a2e0ec8404f61f337c82c159b71" - integrity sha512-jvK5bDUWbpOaJt2Io/rjcaOVcUzkqkrCme/WntdV1SMUc67AiTcEdKuY6G/nMQ7N5Cfsk9SfpugflQwDku53yg== +"@azure/msal-browser@^3.11.1": + version "3.23.0" + resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b" + integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw== dependencies: - "@azure/msal-common" "14.13.0" + "@azure/msal-common" "14.14.2" -"@azure/msal-common@14.13.0": - version "14.13.0" - resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.13.0.tgz#7377b4909a46d19ea91dadd24af7705e6aa947af" - integrity sha512-b4M/tqRzJ4jGU91BiwCsLTqChveUEyFK3qY2wGfZ0zBswIBZjAxopx5CYt5wzZFKuN15HqRDYXQbztttuIC3nA== +"@azure/msal-common@14.14.2": + version "14.14.2" + resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4" + integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog== -"@azure/msal-node@^2.5.1": - version "2.10.0" - resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.10.0.tgz#0b893ab05dbef5c963aba080c88a0330393c4973" - integrity sha512-JxsSE0464a8IA/+q5EHKmchwNyUFJHtCH00tSXsLaOddwLjG6yVvTH6lGgPcWMhO7YWUXj/XVgVgeE9kZtsPUQ== +"@azure/msal-node@^2.9.2": + version "2.13.1" + resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47" + integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw== dependencies: - "@azure/msal-common" "14.13.0" + "@azure/msal-common" "14.14.2" jsonwebtoken "^9.0.0" uuid "^8.3.0" @@ -2053,7 +2066,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.31.8": +"@budibase/backend-core@2.32.5": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2134,14 +2147,14 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "2.31.8" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.31.8.tgz#92b27f99f815f5d20bf58bfae916760b14a036da" - integrity sha512-nmNKVoMdUVqEIq6xqoBq0gVBCLkoPMszmn0Zu0SJ/Dc2SpsXhPz9S3n9xXfAA+FHUg9LgUAS+eKPCKPWZXtDHQ== + version "2.32.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea" + integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA== dependencies: - "@budibase/backend-core" "2.31.8" - "@budibase/shared-core" "2.31.8" - "@budibase/string-templates" "2.31.8" - "@budibase/types" "2.31.8" + "@budibase/backend-core" "2.32.5" + "@budibase/shared-core" "2.32.5" + "@budibase/string-templates" "2.32.5" + "@budibase/types" "2.32.5" "@koa/router" "8.0.8" bull "4.10.1" dd-trace "5.2.0" @@ -2153,13 +2166,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.31.8": +"@budibase/shared-core@2.32.5": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@2.31.8": +"@budibase/string-templates@2.32.5": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2167,7 +2180,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@2.31.8": +"@budibase/types@2.32.5": version "0.0.0" dependencies: scim-patch "^0.8.1" @@ -6117,6 +6130,11 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== +"@types/qs@^6.9.15": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -6967,16 +6985,16 @@ aggregate-error@^3.0.0: clean-stack "^2.0.0" indent-string "^4.0.0" -airtable@0.10.1: - version "0.10.1" - resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.10.1.tgz#0b311002bb44b39f19bf7c4bd2d47d75c733bf87" - integrity sha512-obFW+R3ly2mKtCj0D/xto0ggUvYwdM0RJT3VJ9wvgqoxDkzqg2mNtkuTNfYjF6wWQA0GvoHG9guqzgBBqFjItw== +airtable@0.12.2: + version "0.12.2" + resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.12.2.tgz#e53e66db86744f9bc684faa58881d6c9c12f0e6f" + integrity sha512-HS3VytUBTKj8A0vPl7DDr5p/w3IOGv6RXL0fv7eczOWAtj9Xe8ri4TAiZRXoOyo+Z/COADCj+oARFenbxhmkIg== dependencies: "@types/node" ">=8.0.0 <15" abort-controller "^3.0.0" abortcontroller-polyfill "^1.4.0" - lodash "^4.17.19" - node-fetch "^2.6.1" + lodash "^4.17.21" + node-fetch "^2.6.7" ajv-formats@^2.0.2: version "2.1.1" @@ -11367,27 +11385,13 @@ fast-url-parser@^1.1.3: dependencies: punycode "^1.3.2" -fast-xml-parser@4.2.5: - version "4.2.5" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" - integrity "sha1-pnR6CSlqbLNPKuY0AZvxc487Qh8= sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==" - dependencies: - strnum "^1.0.5" - -fast-xml-parser@^4.1.3: +fast-xml-parser@4.2.5, fast-xml-parser@4.4.1, fast-xml-parser@^4.1.3, fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: version "4.4.1" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz#86dbf3f18edf8739326447bcaac31b4ae7f6514f" integrity sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw== dependencies: strnum "^1.0.5" -fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5: - version "4.4.0" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz#341cc98de71e9ba9e651a67f41f1752d1441a501" - integrity sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg== - dependencies: - strnum "^1.0.5" - fastq@^1.6.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -12368,10 +12372,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.3.tgz#bcee7bd9d90f82c54b16a9aca963b87aceb050ad" - integrity sha512-03VX3/K5NXIh6+XAIDZgcHPmR76xwd8vIDL7RedMpvM2IcXK0Iq/KU7FmLY0t/mKqORAGC7+0rajd0jLFezC4w== +"google-spreadsheet@npm:@budibase/google-spreadsheet@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@budibase/google-spreadsheet/-/google-spreadsheet-4.1.5.tgz#c89ffcbfcb1a3538e910d9275f73efc1d7deb85f" + integrity sha512-t1uBjuRSkNLnZ89DYtYQ2GW33xVU84qOyOPbGi+M0w7cAJofs95PwlBLhVol6Pv5VbeL0I1J7M4XyVqp0nSZtQ== dependencies: axios "^1.4.0" lodash "^4.17.21" @@ -13253,7 +13257,7 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-buffer@^1.1.5, is-buffer@~1.1.6: +is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== @@ -14653,14 +14657,7 @@ kill-port@^1.6.1: get-them-args "1.3.2" shell-exec "1.0.2" -kind-of@^3.0.2, kind-of@^3.1.0: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ== - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: +kind-of@6.0.3, kind-of@^3.0.2, kind-of@^3.1.0, kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== @@ -15485,7 +15482,7 @@ lodash.xor@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6" integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ== -lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: +lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.20, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== @@ -17118,19 +17115,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^4.52.1: - version "4.52.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" - integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== +openai@4.59.0: + version "4.59.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08" + integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" + "@types/qs" "^6.9.15" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" - web-streams-polyfill "^3.2.1" + qs "^6.10.3" openapi-response-validator@^9.2.0: version "9.3.1" @@ -17798,11 +17796,21 @@ periscopic@^3.1.0: estree-walker "^3.0.0" is-reference "^3.0.0" +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== + pg-connection-string@2.5.0, pg-connection-string@^2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34" integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ== +pg-connection-string@^2.6.4: + version "2.6.4" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.6.4.tgz#f543862adfa49fa4e14bc8a8892d2a84d754246d" + integrity sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -17813,11 +17821,21 @@ pg-pool@^3.6.0: resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.0.tgz#3190df3e4747a0d23e5e9e8045bcd99bda0a712e" integrity sha512-clFRf2ksqd+F497kWFyM21tMjeikn60oGDmqMT8UBrynEwVEX/5R5xd2sdvdo1cZCFlguORNpVuqxIj+aK4cfQ== +pg-pool@^3.6.2: + version "3.6.2" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.6.2.tgz#3a592370b8ae3f02a7c8130d245bc02fa2c5f3f2" + integrity sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg== + pg-protocol@*, pg-protocol@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.0.tgz#4c91613c0315349363af2084608db843502f8833" integrity sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q== +pg-protocol@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" + integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== + pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" @@ -17842,6 +17860,19 @@ pg@8.10.0: pg-types "^2.1.0" pgpass "1.x" +pg@^8.12.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.12.0.tgz#9341724db571022490b657908f65aee8db91df79" + integrity sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ== + dependencies: + pg-connection-string "^2.6.4" + pg-pool "^3.6.2" + pg-protocol "^1.6.1" + pg-types "^2.1.0" + pgpass "1.x" + optionalDependencies: + pg-cloudflare "^1.1.1" + pgpass@1.x: version "1.0.5" resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" @@ -20755,7 +20786,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20846,7 +20886,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20860,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22587,11 +22634,6 @@ web-streams-polyfill@4.0.0-beta.3: resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== -web-streams-polyfill@^3.2.1: - version "3.3.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" - integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== - web-vitals@^4.0.1: version "4.2.3" resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7" @@ -22820,7 +22862,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22838,6 +22880,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"