diff --git a/.tool-versions b/.tool-versions index 946d5198ce..cf78481d93 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ nodejs 20.10.0 python 3.10.0 -yarn 1.22.19 +yarn 1.22.22 diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 7ecf264add..f1ef4843a1 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "14.1.1", + "next": "14.2.10", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index a44956ba21..9acbdfdeb6 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" - integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== +"@next/env@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c" + integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,50 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" - integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== +"@next/swc-darwin-arm64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80" + integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ== -"@next/swc-darwin-x64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" - integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== +"@next/swc-darwin-x64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8" + integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA== -"@next/swc-linux-arm64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" - integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== +"@next/swc-linux-arm64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20" + integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA== -"@next/swc-linux-arm64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" - integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== +"@next/swc-linux-arm64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47" + integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ== -"@next/swc-linux-x64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" - integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== +"@next/swc-linux-x64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082" + integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg== -"@next/swc-linux-x64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" - integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== +"@next/swc-linux-x64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3" + integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA== -"@next/swc-win32-arm64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" - integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== +"@next/swc-win32-arm64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209" + integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ== -"@next/swc-win32-ia32-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" - integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== +"@next/swc-win32-ia32-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041" + integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg== -"@next/swc-win32-x64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" - integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== +"@next/swc-win32-x64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455" + integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -129,11 +129,17 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@types/json5@^0.0.29": @@ -1245,28 +1251,28 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" - integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== +next@14.2.10: + version "14.2.10" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6" + integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww== dependencies: - "@next/env" "14.1.1" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.10" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.1" - "@next/swc-darwin-x64" "14.1.1" - "@next/swc-linux-arm64-gnu" "14.1.1" - "@next/swc-linux-arm64-musl" "14.1.1" - "@next/swc-linux-x64-gnu" "14.1.1" - "@next/swc-linux-x64-musl" "14.1.1" - "@next/swc-win32-arm64-msvc" "14.1.1" - "@next/swc-win32-ia32-msvc" "14.1.1" - "@next/swc-win32-x64-msvc" "14.1.1" + "@next/swc-darwin-arm64" "14.2.10" + "@next/swc-darwin-x64" "14.2.10" + "@next/swc-linux-arm64-gnu" "14.2.10" + "@next/swc-linux-arm64-musl" "14.2.10" + "@next/swc-linux-x64-gnu" "14.2.10" + "@next/swc-linux-x64-musl" "14.2.10" + "@next/swc-win32-arm64-msvc" "14.2.10" + "@next/swc-win32-ia32-msvc" "14.2.10" + "@next/swc-win32-x64-msvc" "14.2.10" node-domexception@^1.0.0: version "1.0.0" diff --git a/lerna.json b/lerna.json index d695869907..092e9a133e 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.6", + "version": "2.32.10", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index d442511fb8..144836029f 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -11,6 +11,7 @@ export interface DeletedApp { } const EXPIRY_SECONDS = 3600 +const INVALID_EXPIRY_SECONDS = 60 /** * The default populate app metadata function @@ -48,9 +49,8 @@ export async function getAppMetadata(appId: string): Promise { // app DB left around, but no metadata, it is invalid if (err && err.status === 404) { metadata = { state: AppState.INVALID } - // don't expire the reference to an invalid app, it'll only be - // updated if a metadata doc actually gets stored (app is remade/reverted) - expiry = undefined + // expire invalid apps regularly, in-case it was only briefly invalid + expiry = INVALID_EXPIRY_SECONDS } else { throw err } diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..e4f4a874a5 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,5 @@ import { + AIConfig, Config, ConfigType, GoogleConfig, @@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + +// AI + +export async function getAIConfig(): Promise { + return getConfig(ConfigType.AI) +} diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index a52a17dd53..25b273e51c 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -10,7 +10,7 @@ import { StaticDatabases, DEFAULT_TENANT_ID, } from "../constants" -import { Database, IdentityContext, Snippet, App } from "@budibase/types" +import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types" import { ContextMap } from "./types" let TEST_APP_ID: string | null = null @@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record) { context.featureFlagCache ??= {} context.featureFlagCache[key] = value } + +export function getTableForView(viewId: string): Table | undefined { + const context = getCurrentContext() + if (!context) { + return + } + return context.viewToTableCache?.[viewId] +} + +export function setTableForView(viewId: string, table: Table) { + const context = getCurrentContext() + if (!context) { + return + } + context.viewToTableCache ??= {} + context.viewToTableCache[viewId] = table +} diff --git a/packages/backend-core/src/context/types.ts b/packages/backend-core/src/context/types.ts index fe6072e85c..ee84b49459 100644 --- a/packages/backend-core/src/context/types.ts +++ b/packages/backend-core/src/context/types.ts @@ -1,4 +1,4 @@ -import { IdentityContext, Snippet, VM } from "@budibase/types" +import { IdentityContext, Snippet, Table, VM } from "@budibase/types" import { OAuth2Client } from "google-auth-library" import { GoogleSpreadsheet } from "google-spreadsheet" @@ -21,4 +21,5 @@ export type ContextMap = { featureFlagCache?: { [key: string]: Record } + viewToTableCache?: Record } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index aa4656bf64..274c1b9e93 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,6 +43,9 @@ function buildNano(couchInfo: { url: string; cookie: string }) { } type DBCall = () => Promise +type DBCallback = ( + db: Nano.DocumentScope +) => Promise> | DBCall class CouchDBError extends Error implements DBError { status: number @@ -171,8 +174,8 @@ export class DatabaseImpl implements Database { } // this function fetches the DB and handles if DB creation is needed - private async performCall( - call: (db: Nano.DocumentScope) => Promise> | DBCall + private async performCallWithDBCreation( + call: DBCallback ): Promise { const db = this.getDb() const fnc = await call(db) @@ -181,13 +184,24 @@ export class DatabaseImpl implements Database { } catch (err: any) { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() - return await this.performCall(call) + return await this.performCallWithDBCreation(call) } // stripping the error down the props which are safe/useful, drop everything else throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } + private async performCall(call: DBCallback): Promise { + const db = this.getDb() + const fnc = await call(db) + try { + return await fnc() + } catch (err: any) { + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, err) + } + } + async get(id?: string): Promise { return this.performCall(db => { if (!id) { @@ -227,6 +241,7 @@ export class DatabaseImpl implements Database { } async remove(idOrDoc: string | Document, rev?: string) { + // not a read call - but don't create a DB to delete a document return this.performCall(db => { let _id: string let _rev: string @@ -286,7 +301,7 @@ export class DatabaseImpl implements Database { if (!document._id) { throw new Error("Cannot store document without _id field.") } - return this.performCall(async db => { + return this.performCallWithDBCreation(async db => { if (!document.createdAt) { document.createdAt = new Date().toISOString() } @@ -309,7 +324,7 @@ export class DatabaseImpl implements Database { async bulkDocs(documents: AnyDocument[]) { const now = new Date().toISOString() - return this.performCall(db => { + return this.performCallWithDBCreation(db => { return () => db.bulk({ docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })), @@ -321,7 +336,21 @@ export class DatabaseImpl implements Database { params: DatabaseQueryOpts ): Promise> { return this.performCall(db => { - return () => db.list(params) + return async () => { + try { + return (await db.list(params)) as AllDocsResponse + } catch (err: any) { + if (err.reason === DATABASE_NOT_FOUND) { + return { + offset: 0, + total_rows: 0, + rows: [], + } + } else { + throw err + } + } + } }) } diff --git a/packages/backend-core/src/db/lucene.ts b/packages/backend-core/src/db/lucene.ts index f5ad7e6433..0206bb2140 100644 --- a/packages/backend-core/src/db/lucene.ts +++ b/packages/backend-core/src/db/lucene.ts @@ -612,7 +612,6 @@ async function runQuery( * limit {number} The number of results to fetch * bookmark {string|null} Current bookmark in the recursive search * rows {array|null} Current results in the recursive search - * @returns {Promise<*[]|*>} */ async function recursiveSearch( dbName: string, diff --git a/packages/backend-core/src/docIds/params.ts b/packages/backend-core/src/docIds/params.ts index 093724b55e..016604b69b 100644 --- a/packages/backend-core/src/docIds/params.ts +++ b/packages/backend-core/src/docIds/params.ts @@ -6,7 +6,7 @@ import { ViewName, } from "../constants" import { getProdAppID } from "./conversions" -import { DatabaseQueryOpts } from "@budibase/types" +import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types" /** * If creating DB allDocs/query params with only a single top level ID this can be used, this @@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) { /** * Check if a given ID is that of a table. - * @returns {boolean} */ -export const isTableId = (id: string) => { +export const isTableId = (id: string): boolean => { // this includes datasource plus tables return ( !!id && @@ -77,13 +76,16 @@ export const isTableId = (id: string) => { ) } +export function isViewId(id: string): boolean { + return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`) +} + /** * Check if a given ID is that of a datasource or datasource plus. - * @returns {boolean} */ -export const isDatasourceId = (id: string) => { +export const isDatasourceId = (id: string): boolean => { // this covers both datasources and datasource plus - return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) + return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`) } /** diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 0765d09036..2b915e5689 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -3,6 +3,7 @@ import * as context from "../context" import { PostHog, PostHogOptions } from "posthog-node" import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types" import tracer from "dd-trace" +import { Duration } from "../utils" let posthog: PostHog | undefined export function init(opts?: PostHogOptions) { @@ -16,6 +17,7 @@ export function init(opts?: PostHogOptions) { posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, personalApiKey: env.POSTHOG_PERSONAL_TOKEN, + featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(), ...opts, }) } else { @@ -275,5 +277,5 @@ export const flags = new FlagSet({ AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), SQS: Flag.boolean(env.isDev()), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), - [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(false), + [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), }) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 55f71d76b0..627be039ca 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -11,10 +11,12 @@ import { } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { + Aggregation, AnySearchFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, + CalculationType, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -69,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) { }) } -function getTableName(table?: Table): string | undefined { - // SQS uses the table ID rather than the table name - if ( - table?.sourceType === TableSourceType.INTERNAL || - table?.sourceId === INTERNAL_TABLE_SOURCE_ID - ) { - return table?._id - } else { - return table?.name - } -} - function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { if (Array.isArray(query)) { return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery) @@ -97,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] { return query } +function isSqs(table: Table): boolean { + return ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -150,6 +147,7 @@ class InternalBuilder { return `"${str}"` case SqlClient.MS_SQL: return `[${str}]` + case SqlClient.MARIADB: case SqlClient.MY_SQL: return `\`${str}\`` } @@ -177,15 +175,13 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { meta, endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } - const alias = tableAliases?.[endpoint.entityId] - ? tableAliases?.[endpoint.entityId] - : endpoint.entityId + const alias = this.getTableName(endpoint.entityId) const schema = meta.table.schema if (!this.isFullSelectStatementRequired()) { return [this.knex.raw(`${this.quote(alias)}.*`)] @@ -559,7 +555,10 @@ class InternalBuilder { )}${wrap}, FALSE)` ) }) - } else if (this.client === SqlClient.MY_SQL) { + } else if ( + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB + ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" iterate(mode, (q, key, value) => { return q[rawFnc]( @@ -807,26 +806,88 @@ class InternalBuilder { return query } + isSqs(): boolean { + return isSqs(this.table) + } + + getTableName(tableOrName?: Table | string): string { + let table: Table + if (typeof tableOrName === "string") { + const name = tableOrName + if (this.query.table?.name === name) { + table = this.query.table + } else if (this.query.meta.table?.name === name) { + table = this.query.meta.table + } else if (!this.query.meta.tables?.[name]) { + // This can legitimately happen in custom queries, where the user is + // querying against a table that may not have been imported into + // Budibase. + return name + } else { + table = this.query.meta.tables[name] + } + } else if (tableOrName) { + table = tableOrName + } else { + table = this.table + } + + let name = table.name + if (isSqs(table) && table._id) { + // SQS uses the table ID rather than the table name + name = table._id + } + const aliases = this.query.tableAliases || {} + return aliases[name] ? aliases[name] : name + } + addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder { - const primary = this.table.primary - const aliases = this.query.tableAliases - const aliased = - this.table.name && aliases?.[this.table.name] - ? aliases[this.table.name] - : this.table.name - if (!primary) { + if (!this.table.primary) { throw new Error("SQL counting requires primary key to be supplied") } - return query.countDistinct(`${aliased}.${primary[0]} as total`) + return query.countDistinct( + `${this.getTableName()}.${this.table.primary[0]} as __bb_total` + ) + } + + addAggregations( + query: Knex.QueryBuilder, + aggregations: Aggregation[] + ): Knex.QueryBuilder { + const fields = this.query.resource?.fields || [] + const tableName = this.getTableName() + if (fields.length > 0) { + query = query.groupBy(fields.map(field => `${tableName}.${field}`)) + query = query.select(fields.map(field => `${tableName}.${field}`)) + } + for (const aggregation of aggregations) { + const op = aggregation.calculationType + const field = `${tableName}.${aggregation.field} as ${aggregation.name}` + switch (op) { + case CalculationType.COUNT: + query = query.count(field) + break + case CalculationType.SUM: + query = query.sum(field) + break + case CalculationType.AVG: + query = query.avg(field) + break + case CalculationType.MIN: + query = query.min(field) + break + case CalculationType.MAX: + query = query.max(field) + break + } + } + return query } addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { - let { sort } = this.query + let { sort, resource } = this.query const primaryKey = this.table.primary - const tableName = getTableName(this.table) - const aliases = this.query.tableAliases - const aliased = - tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name + const aliased = this.getTableName() if (!Array.isArray(primaryKey)) { throw new Error("Sorting requires primary key to be specified for table") } @@ -858,7 +919,8 @@ class InternalBuilder { // add sorting by the primary key if the result isn't already sorted by it, // to make sure result is deterministic - if (!sort || sort[primaryKey[0]] === undefined) { + const hasAggregations = (resource?.aggregations?.length ?? 0) > 0 + if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) { query = query.orderBy(`${aliased}.${primaryKey[0]}`) } return query @@ -930,7 +992,8 @@ class InternalBuilder { } const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, - fromAlias = aliases?.[fromTable] || fromTable + fromAlias = aliases?.[fromTable] || fromTable, + throughAlias = (throughTable && aliases?.[throughTable]) || throughTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, @@ -957,38 +1020,36 @@ class InternalBuilder { const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) - .limit(getRelationshipLimit()) // add sorting to get consistent order .orderBy(primaryKey) - // many-to-many relationship with junction table - if (throughTable && toPrimary && fromPrimary) { - const throughAlias = aliases?.[throughTable] || throughTable + const isManyToMany = throughTable && toPrimary && fromPrimary + let correlatedTo = isManyToMany + ? `${throughAlias}.${fromKey}` + : `${toAlias}.${toKey}`, + correlatedFrom = isManyToMany + ? `${fromAlias}.${fromPrimary}` + : `${fromAlias}.${fromKey}` + // many-to-many relationship needs junction table join + if (isManyToMany) { let throughTableWithSchema = this.tableNameWithSchema(throughTable, { alias: throughAlias, schema: endpoint.schema, }) - subQuery = subQuery - .join(throughTableWithSchema, function () { - this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) - }) - .where( - `${throughAlias}.${fromKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) - ) - } - // one-to-many relationship with foreign key - else { - subQuery = subQuery.where( - `${toAlias}.${toKey}`, - "=", - knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) - ) + subQuery = subQuery.join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) } + // add the correlation to the overall query + subQuery = subQuery.where( + correlatedTo, + "=", + knex.raw(this.quotedIdentifier(correlatedFrom)) + ) + const standardWrap = (select: string): Knex.QueryBuilder => { - subQuery = subQuery.select(`${toAlias}.*`) + subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit()) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, @@ -1008,11 +1069,15 @@ class InternalBuilder { `json_agg(json_build_object(${fieldList}))` ) break - case SqlClient.MY_SQL: + case SqlClient.MARIADB: + // can't use the standard wrap due to correlated sub-query limitations in MariaDB wrapperQuery = subQuery.select( - knex.raw(`json_arrayagg(json_object(${fieldList}))`) + knex.raw( + `json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})` + ) ) break + case SqlClient.MY_SQL: case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -1024,7 +1089,9 @@ class InternalBuilder { .select(`${fromAlias}.*`) // @ts-ignore - from alias syntax not TS supported .from({ - [fromAlias]: subQuery.select(`${toAlias}.*`), + [fromAlias]: subQuery + .select(`${toAlias}.*`) + .limit(getRelationshipLimit()), })} FOR JSON PATH))` ) break @@ -1179,7 +1246,8 @@ class InternalBuilder { if ( this.client === SqlClient.POSTGRES || this.client === SqlClient.SQL_LITE || - this.client === SqlClient.MY_SQL + this.client === SqlClient.MY_SQL || + this.client === SqlClient.MARIADB ) { const primary = this.table.primary if (!primary) { @@ -1236,10 +1304,15 @@ class InternalBuilder { } } - // if counting, use distinct count, else select - query = !counting - ? query.select(this.generateSelectStatement()) - : this.addDistinctCount(query) + const aggregations = this.query.resource?.aggregations || [] + if (counting) { + query = this.addDistinctCount(query) + } else if (aggregations.length > 0) { + query = this.addAggregations(query, aggregations) + } else { + query = query.select(this.generateSelectStatement()) + } + // have to add after as well (this breaks MS-SQL) if (!counting) { query = this.addSorting(query) @@ -1326,12 +1399,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { _query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] { const sqlClient = this.getSqlClient() const config: Knex.Config = { - client: sqlClient, + client: this.getBaseSqlClient(), } if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) { config.useNullAsDefault = true } - const client = knex(config) let query: Knex.QueryBuilder const builder = new InternalBuilder(sqlClient, client, json) @@ -1440,7 +1512,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { let id if (sqlClient === SqlClient.MS_SQL) { id = results?.[0].id - } else if (sqlClient === SqlClient.MY_SQL) { + } else if ( + sqlClient === SqlClient.MY_SQL || + sqlClient === SqlClient.MARIADB + ) { id = results?.insertId } row = processFn( @@ -1456,23 +1531,40 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return results.length ? results : [{ [operation.toLowerCase()]: true }] } + private getTableName( + table: Table, + aliases?: Record + ): string | undefined { + let name = table.name + if ( + table.sourceType === TableSourceType.INTERNAL || + table.sourceId === INTERNAL_TABLE_SOURCE_ID + ) { + if (!table._id) { + return + } + // SQS uses the table ID rather than the table name + name = table._id + } + return aliases?.[name] || name + } + convertJsonStringColumns>( table: Table, results: T[], aliases?: Record ): T[] { - const tableName = getTableName(table) + const tableName = this.getTableName(table, aliases) for (const [name, field] of Object.entries(table.schema)) { if (!this._isJsonColumn(field)) { continue } - const aliasedTableName = (tableName && aliases?.[tableName]) || tableName - const fullName = `${aliasedTableName}.${name}` + const fullName = `${tableName}.${name}` as keyof T for (let row of results) { - if (typeof row[fullName as keyof T] === "string") { - row[fullName as keyof T] = JSON.parse(row[fullName]) + if (typeof row[fullName] === "string") { + row[fullName] = JSON.parse(row[fullName]) } - if (typeof row[name as keyof T] === "string") { + if (typeof row[name] === "string") { row[name as keyof T] = JSON.parse(row[name]) } } diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 35d7978449..f5b02cc4e4 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -210,16 +210,27 @@ function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder { class SqlTableQueryBuilder { private readonly sqlClient: SqlClient + private extendedSqlClient: SqlClient | undefined // pass through client to get flavour of SQL constructor(client: SqlClient) { this.sqlClient = client } - getSqlClient(): SqlClient { + getBaseSqlClient(): SqlClient { return this.sqlClient } + getSqlClient(): SqlClient { + return this.extendedSqlClient || this.sqlClient + } + + // if working in a database like MySQL with many variants (MariaDB) + // we can set another client which overrides the base one + setExtendedSqlClient(client: SqlClient) { + this.extendedSqlClient = client + } + /** * @param json the input JSON structure from which an SQL query will be built. * @return the operation that was found in the JSON. diff --git a/packages/bbui/src/Form/Core/Dropzone.svelte b/packages/bbui/src/Form/Core/Dropzone.svelte index c69bf0d6bb..2922d88e7a 100644 --- a/packages/bbui/src/Form/Core/Dropzone.svelte +++ b/packages/bbui/src/Form/Core/Dropzone.svelte @@ -396,6 +396,11 @@ padding: 6px 10px; margin-bottom: 8px; } + + .compact .placeholder { + height: fit-content; + } + .title { display: flex; flex-direction: row; diff --git a/packages/builder/assets/MagicWand.svelte b/packages/builder/assets/MagicWand.svelte new file mode 100644 index 0000000000..14a06ebaec --- /dev/null +++ b/packages/builder/assets/MagicWand.svelte @@ -0,0 +1,29 @@ + + + + + + + + diff --git a/packages/builder/package.json b/packages/builder/package.json index f9e6becbab..aec0b509f0 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -67,6 +67,7 @@ "@spectrum-css/vars": "^3.0.1", "@zerodevx/svelte-json-view": "^1.0.7", "codemirror": "^5.65.16", + "cron-parser": "^4.9.0", "dayjs": "^1.10.8", "downloadjs": "1.4.7", "fast-json-patch": "^3.1.1", diff --git a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte index 5533572511..a98c597142 100644 --- a/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte +++ b/packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte @@ -16,9 +16,11 @@ export let enableNaming = true let validRegex = /^[A-Za-z0-9_\s]+$/ let typing = false + let editing = false const dispatch = createEventDispatcher() $: stepNames = $selectedAutomation?.definition.stepNames + $: allSteps = $selectedAutomation?.definition.steps || [] $: automationName = stepNames?.[block.id] || block?.name || "" $: automationNameError = getAutomationNameError(automationName) $: status = updateStatus(testResult) @@ -56,10 +58,18 @@ } } const getAutomationNameError = name => { - if (stepNames) { + const duplicateError = + "This name already exists, please enter a unique name" + if (stepNames && editing) { for (const [key, value] of Object.entries(stepNames)) { - if (name === value && key !== block.id) { - return "This name already exists, please enter a unique name" + if (name !== block.name && name === value && key !== block.id) { + return duplicateError + } + } + + for (const step of allSteps) { + if (step.id !== block.id && name === step.name) { + return duplicateError } } } @@ -67,15 +77,11 @@ if (name !== block.name && name?.length > 0) { let invalidRoleName = !validRegex.test(name) if (invalidRoleName) { - return "Please enter a role name consisting of only alphanumeric symbols and underscores" + return "Please enter a name consisting of only alphanumeric symbols and underscores" } - - return null } - } - const startTyping = async () => { - typing = true + return null } const saveName = async () => { @@ -89,13 +95,28 @@ await automationStore.actions.saveAutomationName(block.id, automationName) } } + + const startEditing = () => { + editing = true + typing = true + } + + const stopEditing = async () => { + editing = false + typing = false + if (automationNameError) { + automationName = stepNames[block.id] || block?.name + } else { + await saveName() + } + }
dispatch("toggle")} > @@ -132,7 +153,7 @@ { e.stopPropagation() - startTyping() + startEditing() }} on:keydown={async e => { if (e.key === "Enter") { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() - } - } - }} - on:blur={async () => { - typing = false - if (automationNameError) { - automationName = stepNames[block.id] || block?.name - } else { - await saveName() + await stopEditing() } }} + on:blur={stopEditing} /> {:else}
@@ -222,7 +231,7 @@ /> {/if}
- {#if automationNameError} + {#if automationNameError && editing}
diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 251d4a04e6..efa40ce929 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -648,16 +648,18 @@ let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id] if (isLoopBlock) { runtimeName = `loop.${name}` + } else if (idx === 0) { + runtimeName = `trigger.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName - ? `stepsByName[${bindingName}].${name}` - : `steps[${idx - loopBlockCount}].${name}` + ? `stepsByName["${bindingName}"].${name}` + : `steps["${idx - loopBlockCount}"].${name}` } else { runtimeName = hasUserDefinedName ? `stepsByName.${bindingName}.${name}` : `steps.${idx - loopBlockCount}.${name}` } - return idx === 0 ? `trigger.${name}` : runtimeName + return runtimeName } const determineCategoryName = (idx, isLoopBlock, bindingName) => { @@ -684,7 +686,7 @@ ) return { readableBinding: - bindingName && !isLoopBlock + bindingName && !isLoopBlock && idx !== 0 ? `steps.${bindingName}.${name}` : runtimeBinding, runtimeBinding, @@ -759,13 +761,21 @@ : allSteps[idx].icon if (wasLoopBlock) { - loopBlockCount++ schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) }) } + + if ( + allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP && + allSteps + .slice(0, blockIdx) + .some(step => step.stepId === ActionStepID.LOOP) + ) { + bindings = bindings.filter(x => !x.readableBinding.includes("loop")) + } return bindings } @@ -1052,7 +1062,7 @@ {:else if value.customType === "cron"} onChange({ [key]: e.detail })} - value={inputData[key]} + cronExpression={inputData[key]} /> {:else if value.customType === "automationFields"} - import { Button, Select, Input, Label } from "@budibase/bbui" + import { + Select, + InlineAlert, + Input, + Label, + Layout, + notifications, + } from "@budibase/bbui" import { onMount, createEventDispatcher } from "svelte" import { flags } from "stores/builder" + import { licensing } from "stores/portal" + import { API } from "api" + import MagicWand from "../../../../assets/MagicWand.svelte" + import { helpers, REBOOT_CRON } from "@budibase/shared-core" const dispatch = createEventDispatcher() - export let value + export let cronExpression + let error + let nextExecutions + // AI prompt + let aiCronPrompt = "" + let loadingAICronExpression = false + + $: aiEnabled = + $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: { - const exists = CRON_EXPRESSIONS.some(cron => cron.value === value) - const customIndex = CRON_EXPRESSIONS.findIndex( - cron => cron.label === "Custom" - ) - - if (!exists && customIndex === -1) { - CRON_EXPRESSIONS[0] = { label: "Custom", value: value } - } else if (exists && customIndex !== -1) { - CRON_EXPRESSIONS.splice(customIndex, 1) + if (cronExpression) { + try { + nextExecutions = helpers.cron + .getNextExecutionDates(cronExpression) + .join("\n") + } catch (err) { + nextExecutions = null + } } } const onChange = e => { - if (value !== REBOOT_CRON) { + if (e.detail !== REBOOT_CRON) { error = helpers.cron.validate(e.detail).err } - if (e.detail === value || error) { + if (e.detail === cronExpression || error) { return } - value = e.detail + cronExpression = e.detail dispatch("change", e.detail) } + const updatePreset = e => { + aiCronPrompt = "" + onChange(e) + } + + const updateCronExpression = e => { + aiCronPrompt = "" + cronExpression = null + nextExecutions = null + onChange(e) + } + let touched = false - let presets = false const CRON_EXPRESSIONS = [ { @@ -64,45 +93,130 @@ }) } }) + + async function generateAICronExpression() { + loadingAICronExpression = true + try { + const response = await API.generateCronExpression({ + prompt: aiCronPrompt, + }) + cronExpression = response.message + dispatch("change", response.message) + } catch (err) { + notifications.error(err.message) + } finally { + loadingAICronExpression = false + } + } -
+ + + + + {#if aiCronPrompt} +
+ +
+ {/if} +
+ {/if} (touched = true)} updateOnChange={false} /> - {#if touched && !value} + {#if touched && !cronExpression} {/if} -
- - {#if presets} -