MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v3-ui' of github.com:Budibase/budibase into default-values-options

This commit is contained in:
Andrew Kingston 2024-10-02 10:19:37 +01:00
parent baa088049a 82006c4088
commit f6d937fda5
No known key found for this signature in database
129 changed files with 3593 additions and 1566 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.tool-versions
View File

@ -1,3 +1,3 @@
nodejs 20.10.0
python 3.10.0
yarn 1.22.19
yarn 1.22.22

2
examples/nextjs-api-sales/package.json
View File

@ -10,7 +10,7 @@
},
"dependencies": {
"bulma": "^0.9.3",
"next": "14.1.1",
"next": "14.2.10",
"node-fetch": "^3.2.10",
"sass": "^1.52.3",
"react": "17.0.2",

124
examples/nextjs-api-sales/yarn.lock
View File

@ -46,10 +46,10 @@
resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45"
integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==
"@next/env@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac"
integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA==
"@next/env@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c"
integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw==
"@next/eslint-plugin-next@12.1.0":
version "12.1.0"
@ -58,50 +58,50 @@
dependencies:
glob "7.1.7"
"@next/swc-darwin-arm64@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64"
integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ==
"@next/swc-darwin-arm64@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80"
integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ==
"@next/swc-darwin-x64@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b"
integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw==
"@next/swc-darwin-x64@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8"
integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA==
"@next/swc-linux-arm64-gnu@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa"
integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg==
"@next/swc-linux-arm64-gnu@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20"
integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA==
"@next/swc-linux-arm64-musl@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a"
integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ==
"@next/swc-linux-arm64-musl@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47"
integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ==
"@next/swc-linux-x64-gnu@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528"
integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ==
"@next/swc-linux-x64-gnu@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082"
integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg==
"@next/swc-linux-x64-musl@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25"
integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og==
"@next/swc-linux-x64-musl@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3"
integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA==
"@next/swc-win32-arm64-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9"
integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A==
"@next/swc-win32-arm64-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209"
integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ==
"@next/swc-win32-ia32-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2"
integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw==
"@next/swc-win32-ia32-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041"
integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg==
"@next/swc-win32-x64-msvc@14.1.1":
version "14.1.1"
resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52"
integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A==
"@next/swc-win32-x64-msvc@14.2.10":
version "14.2.10"
resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455"
integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ==
"@nodelib/fs.scandir@2.1.5":
version "2.1.5"
@ -129,11 +129,17 @@
resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323"
integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A==
"@swc/helpers@0.5.2":
version "0.5.2"
resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d"
integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==
"@swc/counter@^0.1.3":
version "0.1.3"
resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9"
integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==
"@swc/helpers@0.5.5":
version "0.5.5"
resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0"
integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==
dependencies:
"@swc/counter" "^0.1.3"
tslib "^2.4.0"
"@types/json5@^0.0.29":
@ -1245,28 +1251,28 @@ natural-compare@^1.4.0:
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
next@14.1.1:
version "14.1.1"
resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171"
integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww==
next@14.2.10:
version "14.2.10"
resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6"
integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww==
dependencies:
"@next/env" "14.1.1"
"@swc/helpers" "0.5.2"
"@next/env" "14.2.10"
"@swc/helpers" "0.5.5"
busboy "1.6.0"
caniuse-lite "^1.0.30001579"
graceful-fs "^4.2.11"
postcss "8.4.31"
styled-jsx "5.1.1"
optionalDependencies:
"@next/swc-darwin-arm64" "14.1.1"
"@next/swc-darwin-x64" "14.1.1"
"@next/swc-linux-arm64-gnu" "14.1.1"
"@next/swc-linux-arm64-musl" "14.1.1"
"@next/swc-linux-x64-gnu" "14.1.1"
"@next/swc-linux-x64-musl" "14.1.1"
"@next/swc-win32-arm64-msvc" "14.1.1"
"@next/swc-win32-ia32-msvc" "14.1.1"
"@next/swc-win32-x64-msvc" "14.1.1"
"@next/swc-darwin-arm64" "14.2.10"
"@next/swc-darwin-x64" "14.2.10"
"@next/swc-linux-arm64-gnu" "14.2.10"
"@next/swc-linux-arm64-musl" "14.2.10"
"@next/swc-linux-x64-gnu" "14.2.10"
"@next/swc-linux-x64-musl" "14.2.10"
"@next/swc-win32-arm64-msvc" "14.2.10"
"@next/swc-win32-ia32-msvc" "14.2.10"
"@next/swc-win32-x64-msvc" "14.2.10"
node-domexception@^1.0.0:
version "1.0.0"

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.6",
"version": "2.32.10",
"npmClient": "yarn",
"packages": [
"packages/*",

6
packages/backend-core/src/cache/appMetadata.ts vendored
View File

@ -11,6 +11,7 @@ export interface DeletedApp {
}
const EXPIRY_SECONDS = 3600
const INVALID_EXPIRY_SECONDS = 60
/**
* The default populate app metadata function
@ -48,9 +49,8 @@ export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
// app DB left around, but no metadata, it is invalid
if (err && err.status === 404) {
metadata = { state: AppState.INVALID }
// don't expire the reference to an invalid app, it'll only be
// updated if a metadata doc actually gets stored (app is remade/reverted)
expiry = undefined
// expire invalid apps regularly, in-case it was only briefly invalid
expiry = INVALID_EXPIRY_SECONDS
} else {
throw err
}

7
packages/backend-core/src/configs/configs.ts
View File

@ -1,4 +1,5 @@
import {
AIConfig,
Config,
ConfigType,
GoogleConfig,
@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
return config?.config
}
// AI
export async function getAIConfig(): Promise<AIConfig | undefined> {
return getConfig<AIConfig>(ConfigType.AI)
}

19
packages/backend-core/src/context/mainContext.ts
View File

@ -10,7 +10,7 @@ import {
StaticDatabases,
DEFAULT_TENANT_ID,
} from "../constants"
import { Database, IdentityContext, Snippet, App } from "@budibase/types"
import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types"
import { ContextMap } from "./types"
let TEST_APP_ID: string | null = null
@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record<string, any>) {
context.featureFlagCache ??= {}
context.featureFlagCache[key] = value
}
export function getTableForView(viewId: string): Table | undefined {
const context = getCurrentContext()
if (!context) {
return
}
return context.viewToTableCache?.[viewId]
}
export function setTableForView(viewId: string, table: Table) {
const context = getCurrentContext()
if (!context) {
return
}
context.viewToTableCache ??= {}
context.viewToTableCache[viewId] = table
}

3
packages/backend-core/src/context/types.ts
View File

@ -1,4 +1,4 @@
import { IdentityContext, Snippet, VM } from "@budibase/types"
import { IdentityContext, Snippet, Table, VM } from "@budibase/types"
import { OAuth2Client } from "google-auth-library"
import { GoogleSpreadsheet } from "google-spreadsheet"
@ -21,4 +21,5 @@ export type ContextMap = {
featureFlagCache?: {
[key: string]: Record<string, any>
}
viewToTableCache?: Record<string, Table>
}

41
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -43,6 +43,9 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
}
type DBCall<T> = () => Promise<T>
type DBCallback<T> = (
db: Nano.DocumentScope<any>
) => Promise<DBCall<T>> | DBCall<T>
class CouchDBError extends Error implements DBError {
status: number
@ -171,8 +174,8 @@ export class DatabaseImpl implements Database {
}
// this function fetches the DB and handles if DB creation is needed
private async performCall<T>(
call: (db: Nano.DocumentScope<any>) => Promise<DBCall<T>> | DBCall<T>
private async performCallWithDBCreation<T>(
call: DBCallback<T>
): Promise<any> {
const db = this.getDb()
const fnc = await call(db)
@ -181,13 +184,24 @@ export class DatabaseImpl implements Database {
} catch (err: any) {
if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
await this.checkAndCreateDb()
return await this.performCall(call)
return await this.performCallWithDBCreation(call)
}
// stripping the error down the props which are safe/useful, drop everything else
throw new CouchDBError(`CouchDB error: ${err.message}`, err)
}
}
private async performCall<T>(call: DBCallback<T>): Promise<any> {
const db = this.getDb()
const fnc = await call(db)
try {
return await fnc()
} catch (err: any) {
// stripping the error down the props which are safe/useful, drop everything else
throw new CouchDBError(`CouchDB error: ${err.message}`, err)
}
}
async get<T extends Document>(id?: string): Promise<T> {
return this.performCall(db => {
if (!id) {
@ -227,6 +241,7 @@ export class DatabaseImpl implements Database {
}
async remove(idOrDoc: string | Document, rev?: string) {
// not a read call - but don't create a DB to delete a document
return this.performCall(db => {
let _id: string
let _rev: string
@ -286,7 +301,7 @@ export class DatabaseImpl implements Database {
if (!document._id) {
throw new Error("Cannot store document without _id field.")
}
return this.performCall(async db => {
return this.performCallWithDBCreation(async db => {
if (!document.createdAt) {
document.createdAt = new Date().toISOString()
}
@ -309,7 +324,7 @@ export class DatabaseImpl implements Database {
async bulkDocs(documents: AnyDocument[]) {
const now = new Date().toISOString()
return this.performCall(db => {
return this.performCallWithDBCreation(db => {
return () =>
db.bulk({
docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })),
@ -321,7 +336,21 @@ export class DatabaseImpl implements Database {
params: DatabaseQueryOpts
): Promise<AllDocsResponse<T>> {
return this.performCall(db => {
return () => db.list(params)
return async () => {
try {
return (await db.list(params)) as AllDocsResponse<T>
} catch (err: any) {
if (err.reason === DATABASE_NOT_FOUND) {
return {
offset: 0,
total_rows: 0,
rows: [],
}
} else {
throw err
}
}
}
})
}

1
packages/backend-core/src/db/lucene.ts
View File

@ -612,7 +612,6 @@ async function runQuery<T>(
* limit {number} The number of results to fetch
* bookmark {string|null} Current bookmark in the recursive search
* rows {array|null} Current results in the recursive search
* @returns {Promise<*[]|*>}
*/
async function recursiveSearch<T>(
dbName: string,

14
packages/backend-core/src/docIds/params.ts
View File

@ -6,7 +6,7 @@ import {
ViewName,
} from "../constants"
import { getProdAppID } from "./conversions"
import { DatabaseQueryOpts } from "@budibase/types"
import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types"
/**
* If creating DB allDocs/query params with only a single top level ID this can be used, this
@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) {
/**
* Check if a given ID is that of a table.
* @returns {boolean}
*/
export const isTableId = (id: string) => {
export const isTableId = (id: string): boolean => {
// this includes datasource plus tables
return (
!!id &&
@ -77,13 +76,16 @@ export const isTableId = (id: string) => {
)
}
export function isViewId(id: string): boolean {
return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`)
}
/**
* Check if a given ID is that of a datasource or datasource plus.
* @returns {boolean}
*/
export const isDatasourceId = (id: string) => {
export const isDatasourceId = (id: string): boolean => {
// this covers both datasources and datasource plus
return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
}
/**

4
packages/backend-core/src/features/index.ts
View File

@ -3,6 +3,7 @@ import * as context from "../context"
import { PostHog, PostHogOptions } from "posthog-node"
import { FeatureFlag, IdentityType, UserCtx } from "@budibase/types"
import tracer from "dd-trace"
import { Duration } from "../utils"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
@ -16,6 +17,7 @@ export function init(opts?: PostHogOptions) {
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,
personalApiKey: env.POSTHOG_PERSONAL_TOKEN,
featureFlagsPollingInterval: Duration.fromMinutes(3).toMs(),
...opts,
})
} else {
@ -275,5 +277,5 @@ export const flags = new FlagSet({
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(false),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
})

234
packages/backend-core/src/sql/sql.ts
View File

@ -11,10 +11,12 @@ import {
} from "./utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
Aggregation,
AnySearchFilter,
ArrayOperator,
BasicOperator,
BBReferenceFieldMetadata,
CalculationType,
FieldSchema,
FieldType,
INTERNAL_TABLE_SOURCE_ID,
@ -69,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) {
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
return table?._id
} else {
return table?.name
}
}
function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
if (Array.isArray(query)) {
return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
@ -97,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
return query
}
function isSqs(table: Table): boolean {
return (
table.sourceType === TableSourceType.INTERNAL ||
table.sourceId === INTERNAL_TABLE_SOURCE_ID
)
}
class InternalBuilder {
private readonly client: SqlClient
private readonly query: QueryJson
@ -150,6 +147,7 @@ class InternalBuilder {
return `"${str}"`
case SqlClient.MS_SQL:
return `[${str}]`
case SqlClient.MARIADB:
case SqlClient.MY_SQL:
return `\`${str}\``
}
@ -177,15 +175,13 @@ class InternalBuilder {
}
private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
const { meta, endpoint, resource, tableAliases } = this.query
const { meta, endpoint, resource } = this.query
if (!resource || !resource.fields || resource.fields.length === 0) {
return "*"
}
const alias = tableAliases?.[endpoint.entityId]
? tableAliases?.[endpoint.entityId]
: endpoint.entityId
const alias = this.getTableName(endpoint.entityId)
const schema = meta.table.schema
if (!this.isFullSelectStatementRequired()) {
return [this.knex.raw(`${this.quote(alias)}.*`)]
@ -559,7 +555,10 @@ class InternalBuilder {
)}${wrap}, FALSE)`
)
})
} else if (this.client === SqlClient.MY_SQL) {
} else if (
this.client === SqlClient.MY_SQL ||
this.client === SqlClient.MARIADB
) {
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
iterate(mode, (q, key, value) => {
return q[rawFnc](
@ -807,26 +806,88 @@ class InternalBuilder {
return query
}
isSqs(): boolean {
return isSqs(this.table)
}
getTableName(tableOrName?: Table | string): string {
let table: Table
if (typeof tableOrName === "string") {
const name = tableOrName
if (this.query.table?.name === name) {
table = this.query.table
} else if (this.query.meta.table?.name === name) {
table = this.query.meta.table
} else if (!this.query.meta.tables?.[name]) {
// This can legitimately happen in custom queries, where the user is
// querying against a table that may not have been imported into
// Budibase.
return name
} else {
table = this.query.meta.tables[name]
}
} else if (tableOrName) {
table = tableOrName
} else {
table = this.table
}
let name = table.name
if (isSqs(table) && table._id) {
// SQS uses the table ID rather than the table name
name = table._id
}
const aliases = this.query.tableAliases || {}
return aliases[name] ? aliases[name] : name
}
addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder {
const primary = this.table.primary
const aliases = this.query.tableAliases
const aliased =
this.table.name && aliases?.[this.table.name]
? aliases[this.table.name]
: this.table.name
if (!primary) {
if (!this.table.primary) {
throw new Error("SQL counting requires primary key to be supplied")
}
return query.countDistinct(`${aliased}.${primary[0]} as total`)
return query.countDistinct(
`${this.getTableName()}.${this.table.primary[0]} as __bb_total`
)
}
addAggregations(
query: Knex.QueryBuilder,
aggregations: Aggregation[]
): Knex.QueryBuilder {
const fields = this.query.resource?.fields || []
const tableName = this.getTableName()
if (fields.length > 0) {
query = query.groupBy(fields.map(field => `${tableName}.${field}`))
query = query.select(fields.map(field => `${tableName}.${field}`))
}
for (const aggregation of aggregations) {
const op = aggregation.calculationType
const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
switch (op) {
case CalculationType.COUNT:
query = query.count(field)
break
case CalculationType.SUM:
query = query.sum(field)
break
case CalculationType.AVG:
query = query.avg(field)
break
case CalculationType.MIN:
query = query.min(field)
break
case CalculationType.MAX:
query = query.max(field)
break
}
}
return query
}
addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
let { sort } = this.query
let { sort, resource } = this.query
const primaryKey = this.table.primary
const tableName = getTableName(this.table)
const aliases = this.query.tableAliases
const aliased =
tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name
const aliased = this.getTableName()
if (!Array.isArray(primaryKey)) {
throw new Error("Sorting requires primary key to be specified for table")
}
@ -858,7 +919,8 @@ class InternalBuilder {
// add sorting by the primary key if the result isn't already sorted by it,
// to make sure result is deterministic
if (!sort || sort[primaryKey[0]] === undefined) {
const hasAggregations = (resource?.aggregations?.length ?? 0) > 0
if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) {
query = query.orderBy(`${aliased}.${primaryKey[0]}`)
}
return query
@ -930,7 +992,8 @@ class InternalBuilder {
}
const relatedTable = meta.tables?.[toTable]
const toAlias = aliases?.[toTable] || toTable,
fromAlias = aliases?.[fromTable] || fromTable
fromAlias = aliases?.[fromTable] || fromTable,
throughAlias = (throughTable && aliases?.[throughTable]) || throughTable
let toTableWithSchema = this.tableNameWithSchema(toTable, {
alias: toAlias,
schema: endpoint.schema,
@ -957,38 +1020,36 @@ class InternalBuilder {
const primaryKey = `${toAlias}.${toPrimary || toKey}`
let subQuery: Knex.QueryBuilder = knex
.from(toTableWithSchema)
.limit(getRelationshipLimit())
// add sorting to get consistent order
.orderBy(primaryKey)
// many-to-many relationship with junction table
if (throughTable && toPrimary && fromPrimary) {
const throughAlias = aliases?.[throughTable] || throughTable
const isManyToMany = throughTable && toPrimary && fromPrimary
let correlatedTo = isManyToMany
? `${throughAlias}.${fromKey}`
: `${toAlias}.${toKey}`,
correlatedFrom = isManyToMany
? `${fromAlias}.${fromPrimary}`
: `${fromAlias}.${fromKey}`
// many-to-many relationship needs junction table join
if (isManyToMany) {
let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
alias: throughAlias,
schema: endpoint.schema,
})
subQuery = subQuery
.join(throughTableWithSchema, function () {
this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
})
.where(
`${throughAlias}.${fromKey}`,
"=",
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
)
}
// one-to-many relationship with foreign key
else {
subQuery = subQuery.where(
`${toAlias}.${toKey}`,
"=",
knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
)
subQuery = subQuery.join(throughTableWithSchema, function () {
this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
})
}
// add the correlation to the overall query
subQuery = subQuery.where(
correlatedTo,
"=",
knex.raw(this.quotedIdentifier(correlatedFrom))
)
const standardWrap = (select: string): Knex.QueryBuilder => {
subQuery = subQuery.select(`${toAlias}.*`)
subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit())
// @ts-ignore - the from alias syntax isn't in Knex typing
return knex.select(knex.raw(select)).from({
[toAlias]: subQuery,
@ -1008,11 +1069,15 @@ class InternalBuilder {
`json_agg(json_build_object(${fieldList}))`
)
break
case SqlClient.MY_SQL:
case SqlClient.MARIADB:
// can't use the standard wrap due to correlated sub-query limitations in MariaDB
wrapperQuery = subQuery.select(
knex.raw(`json_arrayagg(json_object(${fieldList}))`)
knex.raw(
`json_arrayagg(json_object(${fieldList}) LIMIT ${getRelationshipLimit()})`
)
)
break
case SqlClient.MY_SQL:
case SqlClient.ORACLE:
wrapperQuery = standardWrap(
`json_arrayagg(json_object(${fieldList}))`
@ -1024,7 +1089,9 @@ class InternalBuilder {
.select(`${fromAlias}.*`)
// @ts-ignore - from alias syntax not TS supported
.from({
[fromAlias]: subQuery.select(`${toAlias}.*`),
[fromAlias]: subQuery
.select(`${toAlias}.*`)
.limit(getRelationshipLimit()),
})} FOR JSON PATH))`
)
break
@ -1179,7 +1246,8 @@ class InternalBuilder {
if (
this.client === SqlClient.POSTGRES ||
this.client === SqlClient.SQL_LITE ||
this.client === SqlClient.MY_SQL
this.client === SqlClient.MY_SQL ||
this.client === SqlClient.MARIADB
) {
const primary = this.table.primary
if (!primary) {
@ -1236,10 +1304,15 @@ class InternalBuilder {
}
}
// if counting, use distinct count, else select
query = !counting
? query.select(this.generateSelectStatement())
: this.addDistinctCount(query)
const aggregations = this.query.resource?.aggregations || []
if (counting) {
query = this.addDistinctCount(query)
} else if (aggregations.length > 0) {
query = this.addAggregations(query, aggregations)
} else {
query = query.select(this.generateSelectStatement())
}
// have to add after as well (this breaks MS-SQL)
if (!counting) {
query = this.addSorting(query)
@ -1326,12 +1399,11 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
_query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
const sqlClient = this.getSqlClient()
const config: Knex.Config = {
client: sqlClient,
client: this.getBaseSqlClient(),
}
if (sqlClient === SqlClient.SQL_LITE || sqlClient === SqlClient.ORACLE) {
config.useNullAsDefault = true
}
const client = knex(config)
let query: Knex.QueryBuilder
const builder = new InternalBuilder(sqlClient, client, json)
@ -1440,7 +1512,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
let id
if (sqlClient === SqlClient.MS_SQL) {
id = results?.[0].id
} else if (sqlClient === SqlClient.MY_SQL) {
} else if (
sqlClient === SqlClient.MY_SQL ||
sqlClient === SqlClient.MARIADB
) {
id = results?.insertId
}
row = processFn(
@ -1456,23 +1531,40 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
return results.length ? results : [{ [operation.toLowerCase()]: true }]
}
private getTableName(
table: Table,
aliases?: Record<string, string>
): string | undefined {
let name = table.name
if (
table.sourceType === TableSourceType.INTERNAL ||
table.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
if (!table._id) {
return
}
// SQS uses the table ID rather than the table name
name = table._id
}
return aliases?.[name] || name
}
convertJsonStringColumns<T extends Record<string, any>>(
table: Table,
results: T[],
aliases?: Record<string, string>
): T[] {
const tableName = getTableName(table)
const tableName = this.getTableName(table, aliases)
for (const [name, field] of Object.entries(table.schema)) {
if (!this._isJsonColumn(field)) {
continue
}
const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
const fullName = `${aliasedTableName}.${name}`
const fullName = `${tableName}.${name}` as keyof T
for (let row of results) {
if (typeof row[fullName as keyof T] === "string") {
row[fullName as keyof T] = JSON.parse(row[fullName])
if (typeof row[fullName] === "string") {
row[fullName] = JSON.parse(row[fullName])
}
if (typeof row[name as keyof T] === "string") {
if (typeof row[name] === "string") {
row[name as keyof T] = JSON.parse(row[name])
}
}

13
packages/backend-core/src/sql/sqlTable.ts
View File

@ -210,16 +210,27 @@ function buildDeleteTable(knex: SchemaBuilder, table: Table): SchemaBuilder {
class SqlTableQueryBuilder {
private readonly sqlClient: SqlClient
private extendedSqlClient: SqlClient | undefined
// pass through client to get flavour of SQL
constructor(client: SqlClient) {
this.sqlClient = client
}
getSqlClient(): SqlClient {
getBaseSqlClient(): SqlClient {
return this.sqlClient
}
getSqlClient(): SqlClient {
return this.extendedSqlClient || this.sqlClient
}
// if working in a database like MySQL with many variants (MariaDB)
// we can set another client which overrides the base one
setExtendedSqlClient(client: SqlClient) {
this.extendedSqlClient = client
}
/**
* @param json the input JSON structure from which an SQL query will be built.
* @return the operation that was found in the JSON.

5
packages/bbui/src/Form/Core/Dropzone.svelte
View File

@ -396,6 +396,11 @@
padding: 6px 10px;
margin-bottom: 8px;
}
.compact .placeholder {
height: fit-content;
}
.title {
display: flex;
flex-direction: row;

29
packages/builder/assets/MagicWand.svelte Normal file
View File

@ -0,0 +1,29 @@
<script>
export let width
export let height
</script>
<svg
{width}
{height}
viewBox="0 0 13 12"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
<path
d="M9.4179 4.13222C9.4179 3.73121 9.26166 3.35428 8.97913 3.07175C8.41342 2.50538 7.4239 2.50408 6.85753 3.07175L5.64342 4.28586C5.6291 4.30018 5.61543 4.3158 5.60305 4.33143C5.58678 4.3438 5.5718 4.35747 5.55683 4.37244L0.491426 9.43785C0.208245 9.72103 0.052002 10.098 0.052002 10.4983C0.052002 10.8987 0.208245 11.2756 0.491426 11.5588C0.774607 11.842 1.15153 11.9982 1.5519 11.9982C1.95227 11.9982 2.32919 11.842 2.61238 11.5588L8.97848 5.1927C9.26166 4.90952 9.4179 4.53259 9.4179 4.13222ZM1.90539 10.8518C1.7166 11.0406 1.3872 11.0406 1.1984 10.8518C1.10401 10.7574 1.05193 10.6318 1.05193 10.4983C1.05193 10.3649 1.104 10.2392 1.1984 10.1448L5.99821 5.34503L6.70845 6.04875L1.90539 10.8518ZM8.2715 4.48571L7.41544 5.34178L6.7052 4.63805L7.56452 3.77873C7.7533 3.58995 8.08271 3.58929 8.2715 3.77939C8.36589 3.87313 8.41798 3.99877 8.41798 4.13223C8.41798 4.26569 8.3659 4.39132 8.2715 4.48571Z"
fill="#C8C8C8"
/>
<path
d="M11.8552 6.55146L11.0144 6.21913L10.879 5.32449C10.8356 5.03919 10.3737 4.98776 10.2686 5.255L9.93606 6.09642L9.04143 6.23085C8.89951 6.25216 8.78884 6.36658 8.77257 6.50947C8.75629 6.65253 8.83783 6.78826 8.97193 6.84148L9.81335 7.17464L9.94794 8.06862C9.9691 8.21053 10.0835 8.32121 10.2266 8.33748C10.3695 8.35375 10.5052 8.27221 10.5586 8.13811L10.8914 7.29751L11.7855 7.1621C11.9283 7.1403 12.0381 7.02637 12.0544 6.88348C12.0707 6.74058 11.9887 6.60403 11.8552 6.55146Z"
fill="#F9634C"
/>
<path
d="M8.94215 1.76145L9.78356 2.0946L9.91815 2.9885C9.93931 3.13049 10.0539 3.24117 10.1968 3.25744C10.3398 3.27371 10.4756 3.19218 10.5288 3.05807L10.8618 2.21739L11.7559 2.08207C11.8985 2.06034 12.0085 1.94633 12.0248 1.80344C12.0411 1.66054 11.959 1.524 11.8254 1.47143L10.9847 1.13909L10.8494 0.244456C10.806 -0.0409246 10.3439 -0.0922745 10.2388 0.174881L9.90643 1.0163L9.0118 1.15089C8.86972 1.17213 8.75905 1.28654 8.74278 1.42952C8.72651 1.57249 8.80804 1.70823 8.94215 1.76145Z"
fill="#8488FD"
/>
<path
d="M3.2379 2.46066L3.92063 2.73091L4.02984 3.45637C4.04709 3.57151 4.14002 3.66135 4.25606 3.67453C4.37194 3.6878 4.48212 3.62163 4.52541 3.51276L4.79557 2.83059L5.52094 2.72074C5.63682 2.70316 5.72601 2.61072 5.73936 2.49468C5.75254 2.37864 5.68597 2.26797 5.57758 2.22533L4.89533 1.95565L4.78548 1.22963C4.75016 0.998038 4.37535 0.956375 4.29007 1.17315L4.0204 1.85597L3.29437 1.96517C3.17915 1.98235 3.08931 2.07527 3.07613 2.19131C3.06294 2.30727 3.12902 2.41737 3.2379 2.46066Z"
fill="#F7D804"
/>
</svg>

1
packages/builder/package.json
View File

@ -67,6 +67,7 @@
"@spectrum-css/vars": "^3.0.1",
"@zerodevx/svelte-json-view": "^1.0.7",
"codemirror": "^5.65.16",
"cron-parser": "^4.9.0",
"dayjs": "^1.10.8",
"downloadjs": "1.4.7",
"fast-json-patch": "^3.1.1",

65
packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItemHeader.svelte
View File

@ -16,9 +16,11 @@
export let enableNaming = true
let validRegex = /^[A-Za-z0-9_\s]+$/
let typing = false
let editing = false
const dispatch = createEventDispatcher()
$: stepNames = $selectedAutomation?.definition.stepNames
$: allSteps = $selectedAutomation?.definition.steps || []
$: automationName = stepNames?.[block.id] || block?.name || ""
$: automationNameError = getAutomationNameError(automationName)
$: status = updateStatus(testResult)
@ -56,10 +58,18 @@
}
}
const getAutomationNameError = name => {
if (stepNames) {
const duplicateError =
"This name already exists, please enter a unique name"
if (stepNames && editing) {
for (const [key, value] of Object.entries(stepNames)) {
if (name === value && key !== block.id) {
return "This name already exists, please enter a unique name"
if (name !== block.name && name === value && key !== block.id) {
return duplicateError
}
}
for (const step of allSteps) {
if (step.id !== block.id && name === step.name) {
return duplicateError
}
}
}
@ -67,15 +77,11 @@
if (name !== block.name && name?.length > 0) {
let invalidRoleName = !validRegex.test(name)
if (invalidRoleName) {
return "Please enter a role name consisting of only alphanumeric symbols and underscores"
return "Please enter a name consisting of only alphanumeric symbols and underscores"
}
return null
}
}
const startTyping = async () => {
typing = true
return null
}
const saveName = async () => {
@ -89,13 +95,28 @@
await automationStore.actions.saveAutomationName(block.id, automationName)
}
}
const startEditing = () => {
editing = true
typing = true
}
const stopEditing = async () => {
editing = false
typing = false
if (automationNameError) {
automationName = stepNames[block.id] || block?.name
} else {
await saveName()
}
}
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div
class:typing={typing && !automationNameError}
class:typing-error={automationNameError}
class:typing={typing && !automationNameError && editing}
class:typing-error={automationNameError && editing}
class="blockSection"
on:click={() => dispatch("toggle")}
>
@ -132,7 +153,7 @@
<input
class="input-text"
disabled={!enableNaming}
placeholder="Enter some text"
placeholder="Enter step name"
name="name"
autocomplete="off"
value={automationName}
@ -141,26 +162,14 @@
}}
on:click={e => {
e.stopPropagation()
startTyping()
startEditing()
}}
on:keydown={async e => {
if (e.key === "Enter") {
typing = false
if (automationNameError) {
automationName = stepNames[block.id] || block?.name
} else {
await saveName()
}
}
}}
on:blur={async () => {
typing = false
if (automationNameError) {
automationName = stepNames[block.id] || block?.name
} else {
await saveName()
await stopEditing()
}
}}
on:blur={stopEditing}
/>
{:else}
<div class="input-text">
@ -222,7 +231,7 @@
/>
{/if}
</div>
{#if automationNameError}
{#if automationNameError && editing}
<div class="error-container">
<AbsTooltip type="negative" text={automationNameError}>
<div class="error-icon">

22
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -648,16 +648,18 @@
let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
if (isLoopBlock) {
runtimeName = `loop.${name}`
} else if (idx === 0) {
runtimeName = `trigger.${name}`
} else if (block.name.startsWith("JS")) {
runtimeName = hasUserDefinedName
? `stepsByName[${bindingName}].${name}`
: `steps[${idx - loopBlockCount}].${name}`
? `stepsByName["${bindingName}"].${name}`
: `steps["${idx - loopBlockCount}"].${name}`
} else {
runtimeName = hasUserDefinedName
? `stepsByName.${bindingName}.${name}`
: `steps.${idx - loopBlockCount}.${name}`
}
return idx === 0 ? `trigger.${name}` : runtimeName
return runtimeName
}
const determineCategoryName = (idx, isLoopBlock, bindingName) => {
@ -684,7 +686,7 @@
)
return {
readableBinding:
bindingName && !isLoopBlock
bindingName && !isLoopBlock && idx !== 0
? `steps.${bindingName}.${name}`
: runtimeBinding,
runtimeBinding,
@ -759,13 +761,21 @@
: allSteps[idx].icon
if (wasLoopBlock) {
loopBlockCount++
schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
}
Object.entries(schema).forEach(([name, value]) => {
addBinding(name, value, icon, idx, isLoopBlock, bindingName)
})
}
if (
allSteps[blockIdx - 1]?.stepId !== ActionStepID.LOOP &&
allSteps
.slice(0, blockIdx)
.some(step => step.stepId === ActionStepID.LOOP)
) {
bindings = bindings.filter(x => !x.readableBinding.includes("loop"))
}
return bindings
}
@ -1052,7 +1062,7 @@
{:else if value.customType === "cron"}
<CronBuilder
on:change={e => onChange({ [key]: e.detail })}
value={inputData[key]}
cronExpression={inputData[key]}
/>
{:else if value.customType === "automationFields"}
<AutomationSelector

192
packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte
View File

@ -1,41 +1,70 @@
<script>
import { Button, Select, Input, Label } from "@budibase/bbui"
import {
Select,
InlineAlert,
Input,
Label,
Layout,
notifications,
} from "@budibase/bbui"
import { onMount, createEventDispatcher } from "svelte"
import { flags } from "stores/builder"
import { licensing } from "stores/portal"
import { API } from "api"
import MagicWand from "../../../../assets/MagicWand.svelte"
import { helpers, REBOOT_CRON } from "@budibase/shared-core"
const dispatch = createEventDispatcher()
export let value
export let cronExpression
let error
let nextExecutions
// AI prompt
let aiCronPrompt = ""
let loadingAICronExpression = false
$: aiEnabled =
$licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
$: {
const exists = CRON_EXPRESSIONS.some(cron => cron.value === value)
const customIndex = CRON_EXPRESSIONS.findIndex(
cron => cron.label === "Custom"
)
if (!exists && customIndex === -1) {
CRON_EXPRESSIONS[0] = { label: "Custom", value: value }
} else if (exists && customIndex !== -1) {
CRON_EXPRESSIONS.splice(customIndex, 1)
if (cronExpression) {
try {
nextExecutions = helpers.cron
.getNextExecutionDates(cronExpression)
.join("\n")
} catch (err) {
nextExecutions = null
}
}
}
const onChange = e => {
if (value !== REBOOT_CRON) {
if (e.detail !== REBOOT_CRON) {
error = helpers.cron.validate(e.detail).err
}
if (e.detail === value || error) {
if (e.detail === cronExpression || error) {
return
}
value = e.detail
cronExpression = e.detail
dispatch("change", e.detail)
}
const updatePreset = e => {
aiCronPrompt = ""
onChange(e)
}
const updateCronExpression = e => {
aiCronPrompt = ""
cronExpression = null
nextExecutions = null
onChange(e)
}
let touched = false
let presets = false
const CRON_EXPRESSIONS = [
{
@ -64,45 +93,130 @@
})
}
})
async function generateAICronExpression() {
loadingAICronExpression = true
try {
const response = await API.generateCronExpression({
prompt: aiCronPrompt,
})
cronExpression = response.message
dispatch("change", response.message)
} catch (err) {
notifications.error(err.message)
} finally {
loadingAICronExpression = false
}
}
</script>
<div class="block-field">
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<Layout noPadding gap="S">
<Select
on:change={updatePreset}
value={cronExpression || "Custom"}
secondary
extraThin
label="Use a Preset (Optional)"
options={CRON_EXPRESSIONS}
/>
{#if aiEnabled}
<div class="cron-ai-generator">
<Input
bind:value={aiCronPrompt}
label="Generate Cron Expression with AI"
size="S"
placeholder="Run every hour between 1pm to 4pm everyday of the week"
/>
{#if aiCronPrompt}
<div
class="icon"
class:pulsing-text={loadingAICronExpression}
on:click={generateAICronExpression}
>
<MagicWand height="17" width="17" />
</div>
{/if}
</div>
{/if}
<Input
label="Cron Expression"
{error}
on:change={onChange}
{value}
on:change={updateCronExpression}
value={cronExpression}
on:blur={() => (touched = true)}
updateOnChange={false}
/>
{#if touched && !value}
{#if touched && !cronExpression}
<Label><div class="error">Please specify a CRON expression</div></Label>
{/if}
<div class="presets">
<Button on:click={() => (presets = !presets)}
>{presets ? "Hide" : "Show"} Presets</Button
>
{#if presets}
<Select
on:change={onChange}
value={value || "Custom"}
secondary
extraThin
label="Presets"
options={CRON_EXPRESSIONS}
/>
{/if}
</div>
</div>
{#if nextExecutions}
<InlineAlert
type="info"
header="Next Executions"
message={nextExecutions}
/>
{/if}
</Layout>
<style>
.presets {
margin-top: var(--spacing-m);
.cron-ai-generator {
flex: 1;
position: relative;
}
.block-field {
padding-top: var(--spacing-s);
.icon {
right: 1px;
bottom: 1px;
position: absolute;
justify-content: center;
align-items: center;
display: flex;
flex-direction: row;
box-sizing: border-box;
border-left: 1px solid var(--spectrum-alias-border-color);
border-top-right-radius: var(--spectrum-alias-border-radius-regular);
border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
width: 31px;
color: var(--spectrum-alias-text-color);
background-color: var(--spectrum-global-color-gray-75);
transition: background-color
var(--spectrum-global-animation-duration-100, 130ms),
box-shadow var(--spectrum-global-animation-duration-100, 130ms),
border-color var(--spectrum-global-animation-duration-100, 130ms);
height: calc(var(--spectrum-alias-item-height-m) - 2px);
}
.icon:hover {
cursor: pointer;
color: var(--spectrum-alias-text-color-hover);
background-color: var(--spectrum-global-color-gray-50);
border-color: var(--spectrum-alias-border-color-hover);
}
.error {
padding-top: var(--spacing-xs);
color: var(--spectrum-global-color-red-500);
}
.pulsing-text {
font-size: 24px;
font-weight: bold;
animation: pulse 1.5s infinite;
}
@keyframes pulse {
0% {
opacity: 0.3;
transform: scale(1);
}
50% {
opacity: 1;
transform: scale(1.05);
}
100% {
opacity: 0.3;
transform: scale(1);
}
}
</style>

4
packages/builder/src/components/backend/DataTable/Table.svelte
View File

@ -6,7 +6,7 @@
import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
import RoleCell from "./cells/RoleCell.svelte"
import { createEventDispatcher } from "svelte"
import { canBeSortColumn } from "@budibase/shared-core"
import { canBeSortColumn } from "@budibase/frontend-core"
export let schema = {}
export let data = []
@ -31,7 +31,7 @@
acc[key] =
typeof schema[key] === "string" ? { type: schema[key] } : schema[key]
if (!canBeSortColumn(acc[key].type)) {
if (!canBeSortColumn(acc[key])) {
acc[key].sortable = false
}
return acc

2
packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte
View File

@ -121,8 +121,10 @@
label: name,
schema: {
type: column.type,
subtype: column.subtype,
visible: column.visible,
readonly: column.readonly,
constraints: column.constraints, // This is needed to properly display "users" column
},
}
})

10
packages/builder/src/components/backend/DataTable/buttons/grid/GridColumnsSettingButton.svelte
View File

@ -13,12 +13,14 @@
import { isEnabled } from "helpers/featureFlags"
import { FeatureFlag } from "@budibase/types"
const { columns, datasource } = getContext("grid")
const { tableColumns, datasource } = getContext("grid")
let open = false
let anchor
$: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
$: anyRestricted = $tableColumns.filter(
col => !col.visible || col.readonly
).length
$: text = anyRestricted ? `Columns: ${anyRestricted} restricted` : "Columns"
$: permissions =
$datasource.type === "viewV2"
@ -37,7 +39,7 @@
size="M"
on:click={() => (open = !open)}
selected={open || anyRestricted}
disabled={!$columns.length}
disabled={!$tableColumns.length}
accentColor="#674D00"
>
{text}
@ -46,7 +48,7 @@
<Popover bind:open {anchor} align="left">
<ColumnsSettingContent
columns={$columns}
columns={$tableColumns}
canSetRelationshipSchemas={isEnabled(FeatureFlag.ENRICHED_RELATIONSHIPS)}
{permissions}
/>

5
packages/builder/src/components/backend/DataTable/buttons/grid/GridSortButton.svelte
View File

@ -1,7 +1,7 @@
<script>
import { getContext } from "svelte"
import { ActionButton, Popover, Select } from "@budibase/bbui"
import { canBeSortColumn } from "@budibase/shared-core"
import { canBeSortColumn } from "@budibase/frontend-core"
const { sort, columns } = getContext("grid")
@ -13,8 +13,9 @@
label: col.label || col.name,
value: col.name,
type: col.schema?.type,
related: col.related,
}))
.filter(col => canBeSortColumn(col.type))
.filter(col => canBeSortColumn(col))
$: orderOptions = getOrderOptions($sort.column, columnOptions)
const getOrderOptions = (column, columnOptions) => {

34
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -4,6 +4,7 @@
Button,
Label,
Select,
Multiselect,
Toggle,
Icon,
DatePicker,
@ -19,9 +20,9 @@
helpers,
PROTECTED_INTERNAL_COLUMNS,
PROTECTED_EXTERNAL_COLUMNS,
canBeDisplayColumn,
canHaveDefaultColumn,
} from "@budibase/shared-core"
import { makePropSafe } from "@budibase/string-templates"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
@ -43,10 +44,11 @@
SourceName,
} from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core"
import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
import OptionsEditor from "./OptionsEditor.svelte"
import { isEnabled } from "helpers/featureFlags"
import { getUserBindings } from "dataBinding"
const AUTO_TYPE = FieldType.AUTO
const FORMULA_TYPE = FieldType.FORMULA
@ -167,7 +169,7 @@
: availableAutoColumns
// used to select what different options can be displayed for column type
$: canBeDisplay =
canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
$: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
$: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
$: canBeRequired =
@ -192,7 +194,19 @@
fieldId: makeFieldId(t.type, t.subtype),
...t,
}))
$: bindings = getBindings({ table })
$: defaultValueBindings = [
{
type: "context",
runtimeBinding: `${makePropSafe("now")}`,
readableBinding: `Date`,
category: "Date",
icon: "Date",
display: {
name: "Server date",
},
},
...getUserBindings(),
]
const fieldDefinitions = Object.values(FIELDS).reduce(
// Storing the fields by complex field id
@ -801,6 +815,15 @@
on:change={e => (editableColumn.default = e.detail)}
placeholder="None"
/>
{:else if editableColumn.type === FieldType.ARRAY}
<Multiselect
disabled={!canHaveDefault}
options={editableColumn.constraints?.inclusion || []}
label="Default value"
value={editableColumn.default}
on:change={e => (editableColumn.default = e.detail)}
placeholder="None"
/>
{:else}
<ModalBindableInput
disabled={!canHaveDefault}
@ -810,9 +833,8 @@
placeholder="None"
value={editableColumn.default}
on:change={e => (editableColumn.default = e.detail)}
{bindings}
bindings={defaultValueBindings}
allowJS
context={rowGoldenSample}
/>
{/if}
{/if}

9
packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte
View File

@ -2,7 +2,12 @@
import { getContext } from "svelte"
import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
const { datasource } = getContext("grid")
const { datasource, rows } = getContext("grid")
const onUpdate = async () => {
await datasource.actions.refreshDefinition()
await rows.actions.refreshData()
}
</script>
<CreateEditColumn on:updatecolumns={datasource.actions.refreshDefinition} />
<CreateEditColumn on:updatecolumns={onUpdate} />

7
packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
View File

@ -1,7 +1,8 @@
<script>
import { Select, Icon } from "@budibase/bbui"
import { FIELDS } from "constants/backend"
import { canBeDisplayColumn, utils } from "@budibase/shared-core"
import { utils } from "@budibase/shared-core"
import { canBeDisplayColumn } from "@budibase/frontend-core"
import { API } from "api"
import { parseFile } from "./utils"
@ -100,10 +101,10 @@
let rawRows = []
$: displayColumnOptions = Object.keys(schema || {}).filter(column => {
return validation[column] && canBeDisplayColumn(schema[column].type)
return validation[column] && canBeDisplayColumn(schema[column])
})
$: if (displayColumn && !canBeDisplayColumn(schema[displayColumn].type)) {
$: if (displayColumn && !canBeDisplayColumn(schema[displayColumn])) {
displayColumn = null
}

4
packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
View File

@ -1,4 +1,5 @@
<script>
import { enrichSchemaWithRelColumns } from "@budibase/frontend-core"
import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
import { selectedScreen, componentStore } from "stores/builder"
import DraggableList from "../DraggableList/DraggableList.svelte"
@ -28,7 +29,8 @@
delete schema._rev
}
return schema
const result = enrichSchemaWithRelColumns(schema)
return result
}
$: datasource = getDatasourceForProvider($selectedScreen, componentInstance)

2
packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
View File

@ -82,7 +82,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
active: column.active,
field: column.field,
label: column.label,
columnType: schema[column.field].type,
columnType: column.columnType || schema[column.field].type,
width: column.width,
conditions: column.conditions,
},

4
packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
View File

@ -3,7 +3,7 @@
import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
import { selectedScreen } from "stores/builder"
import { createEventDispatcher } from "svelte"
import { canBeSortColumn } from "@budibase/shared-core"
import { canBeSortColumn } from "@budibase/frontend-core"
export let componentInstance = {}
export let value = ""
@ -17,7 +17,7 @@
const getSortableFields = schema => {
return Object.entries(schema || {})
.filter(entry => canBeSortColumn(entry[1].type))
.filter(entry => canBeSortColumn(entry[1]))
.map(entry => entry[0])
}

10
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
View File

@ -1,6 +1,6 @@
<script>
import { viewsV2, rowActions } from "stores/builder"
import { admin } from "stores/portal"
import { admin, themeStore } from "stores/portal"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@ -23,6 +23,11 @@
}
$: buttons = makeRowActionButtons($rowActions[id])
$: rowActions.refreshRowActions(id)
$: currentTheme = $themeStore?.theme
$: darkMode = !currentTheme.includes("light")
$: currentTheme = $themeStore?.theme
$: darkMode = !currentTheme.includes("light")
const makeRowActionButtons = actions => {
return (actions || []).map(action => ({
@ -40,13 +45,14 @@
<Grid
{API}
{darkMode}
{datasource}
{buttons}
allowAddRows
allowDeleteRows
showAvatars={false}
on:updatedatasource={handleGridViewUpdate}
isCloud={$admin.cloud}
{buttons}
buttonsCollapsed
>
<svelte:fragment slot="controls">

8
packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
View File

@ -4,6 +4,7 @@
import OpenAILogo from "./logos/OpenAI.svelte"
import AnthropicLogo from "./logos/Anthropic.svelte"
import TogetherAILogo from "./logos/TogetherAI.svelte"
import AzureOpenAILogo from "./logos/AzureOpenAI.svelte"
import { Providers } from "./constants"
const logos = {
@ -11,6 +12,7 @@
[Providers.OpenAI.name]: OpenAILogo,
[Providers.Anthropic.name]: AnthropicLogo,
[Providers.TogetherAI.name]: TogetherAILogo,
[Providers.AzureOpenAI.name]: AzureOpenAILogo,
}
export let config
@ -26,8 +28,8 @@
<div class="icon">
<svelte:component
this={logos[config.name || config.provider]}
height="30"
width="30"
height="18"
width="18"
/>
</div>
<div class="header">
@ -110,7 +112,7 @@
.tag {
display: flex;
color: var(--spectrum-body-m-text-color);
color: #ffffff;
padding: 4px 8px;
justify-content: center;
align-items: center;

38
packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
View File

@ -1,6 +1,6 @@
import { it, expect, describe, vi } from "vitest"
import AISettings from "./index.svelte"
import { render } from "@testing-library/svelte"
import { render, fireEvent } from "@testing-library/svelte"
import { admin, licensing } from "stores/portal"
import { notifications } from "@budibase/bbui"
@ -55,39 +55,43 @@ describe("AISettings", () => {
expect(enterpriseTag).toBeInTheDocument()
})
it("should show the premium label on cloud when Budibase AI isn't enabled", async () => {
setupEnv(Hosting.Cloud)
instance = render(AISettings, {})
const premiumTag = instance.queryByText("Premium")
expect(premiumTag).toBeInTheDocument()
})
it("should not show the add configuration button if the user doesn't have the correct license on cloud", async () => {
it("the add configuration button should not do anything the user doesn't have the correct license on cloud", async () => {
let addConfigurationButton
let configModal
setupEnv(Hosting.Cloud)
instance = render(AISettings)
addConfigurationButton = instance.queryByText("Add configuration")
expect(addConfigurationButton).not.toBeInTheDocument()
expect(addConfigurationButton).toBeInTheDocument()
await fireEvent.click(addConfigurationButton)
configModal = instance.queryByText("Custom AI Configuration")
expect(configModal).not.toBeInTheDocument()
})
it("the add configuration button should open the config modal if the user has the correct license on cloud", async () => {
let addConfigurationButton
let configModal
setupEnv(Hosting.Cloud, { customAIConfigsEnabled: true })
instance = render(AISettings)
addConfigurationButton = instance.queryByText("Add configuration")
expect(addConfigurationButton).toBeInTheDocument()
await fireEvent.click(addConfigurationButton)
configModal = instance.queryByText("Custom AI Configuration")
expect(configModal).toBeInTheDocument()
})
it("should not show the add configuration button if the user doesn't have the correct license on self host", async () => {
it("the add configuration button should open the config modal if the user has the correct license on self host", async () => {
let addConfigurationButton
setupEnv(Hosting.Self)
instance = render(AISettings)
addConfigurationButton = instance.queryByText("Add configuration")
expect(addConfigurationButton).not.toBeInTheDocument()
let configModal
setupEnv(Hosting.Self, { customAIConfigsEnabled: true })
instance = render(AISettings, {})
instance = render(AISettings)
addConfigurationButton = instance.queryByText("Add configuration")
expect(addConfigurationButton).toBeInTheDocument()
await fireEvent.click(addConfigurationButton)
configModal = instance.queryByText("Custom AI Configuration")
expect(configModal).toBeInTheDocument()
})
})
})

6
packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
View File

@ -84,8 +84,10 @@
<Label size="M">API Key</Label>
<Input type="password" bind:value={config.apiKey} />
</div>
<Toggle text="Active" bind:value={config.active} />
<Toggle text="Set as default" bind:value={config.isDefault} />
<div class="form-row">
<Toggle text="Active" bind:value={config.active} />
<Toggle text="Set as default" bind:value={config.isDefault} />
</div>
</ModalContent>
<style>

2
packages/builder/src/pages/builder/portal/settings/ai/constants.js
View File

@ -23,7 +23,7 @@ export const Providers = {
models: [{ label: "Llama 3 8B", value: "meta-llama/Meta-Llama-3-8B" }],
},
AzureOpenAI: {
name: "Azure Open AI",
name: "Azure OpenAI",
models: [
{ label: "GPT 4o Mini", value: "gpt-4o-mini" },
{ label: "GPT 4o", value: "gpt-4o" },

76
packages/builder/src/pages/builder/portal/settings/ai/index.svelte
View File

@ -27,7 +27,6 @@
let editingUuid
$: isCloud = $admin.cloud
$: budibaseAIEnabled = $licensing.budibaseAIEnabled
$: customAIConfigsEnabled = $licensing.customAIConfigsEnabled
async function fetchAIConfig() {
@ -56,10 +55,13 @@
} else {
// We don't store the default BB AI config in the DB
delete fullAIConfig.config.budibase_ai
// unset the default value from other configs if default is set
if (editingAIConfig.isDefault) {
for (let key in fullAIConfig.config) {
fullAIConfig.config[key].isDefault = false
if (key !== id) {
fullAIConfig.config[key].isDefault = false
}
}
}
// Add new or update existing custom AI Config
@ -124,18 +126,8 @@
</Modal>
<Layout noPadding>
<Layout gap="XS" noPadding>
<Heading size="M">AI</Heading>
{#if isCloud && !budibaseAIEnabled}
<Tags>
<Tag icon="LockClosed">Premium</Tag>
</Tags>
{/if}
<Body>Configure your AI settings within this section:</Body>
</Layout>
<Divider />
<Layout noPadding>
<div class="config-heading">
<Heading size="S">AI Configurations</Heading>
<div class="header">
<Heading size="M">AI</Heading>
{#if !isCloud && !customAIConfigsEnabled}
<Tags>
<Tag icon="LockClosed">Premium</Tag>
@ -144,24 +136,43 @@
<Tags>
<Tag icon="LockClosed">Enterprise</Tag>
</Tags>
{:else}
<Button size="S" cta on:click={newConfig}>Add configuration</Button>
{/if}
</div>
<Body size="S"
>Use the following interface to select your preferred AI configuration.</Body
>
<Body size="S">Select your AI Model:</Body>
{#if fullAIConfig?.config}
{#each Object.keys(fullAIConfig.config) as key}
<AIConfigTile
config={fullAIConfig.config[key]}
editHandler={() => editConfig(key)}
deleteHandler={() => deleteConfig(key)}
/>
{/each}
{/if}
<Body>Configure your AI settings within this section:</Body>
</Layout>
<Divider />
<div style={`opacity: ${customAIConfigsEnabled ? 1 : 0.5}`}>
<Layout noPadding>
<div class="config-heading">
<Heading size="S">AI Configurations</Heading>
<Button
size="S"
cta={customAIConfigsEnabled}
secondary={!customAIConfigsEnabled}
on:click={customAIConfigsEnabled ? newConfig : null}
>
Add configuration
</Button>
</div>
<Body size="S"
>Use the following interface to select your preferred AI configuration.</Body
>
{#if customAIConfigsEnabled}
<Body size="S">Select your AI Model:</Body>
{/if}
{#if fullAIConfig?.config}
{#each Object.keys(fullAIConfig.config) as key}
<AIConfigTile
config={fullAIConfig.config[key]}
editHandler={customAIConfigsEnabled ? () => editConfig(key) : null}
deleteHandler={customAIConfigsEnabled
? () => deleteConfig(key)
: null}
/>
{/each}
{/if}
</Layout>
</div>
</Layout>
<style>
@ -169,5 +180,12 @@
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: -18px;
}
.header {
display: flex;
align-items: center;
gap: 12px;
}
</style>

64
packages/builder/src/pages/builder/portal/settings/ai/logos/AzureOpenAI.svelte Normal file
View File

@ -0,0 +1,64 @@
<script>
export let width
export let height
</script>
<svg xmlns="http://www.w3.org/2000/svg" {width} {height} viewBox="0 0 96 96">
<defs>
<linearGradient
id="e399c19f-b68f-429d-b176-18c2117ff73c"
x1="-1032.172"
x2="-1059.213"
y1="145.312"
y2="65.426"
gradientTransform="matrix(1 0 0 -1 1075 158)"
gradientUnits="userSpaceOnUse"
>
<stop offset="0" stop-color="#114a8b" />
<stop offset="1" stop-color="#0669bc" />
</linearGradient>
<linearGradient
id="ac2a6fc2-ca48-4327-9a3c-d4dcc3256e15"
x1="-1023.725"
x2="-1029.98"
y1="108.083"
y2="105.968"
gradientTransform="matrix(1 0 0 -1 1075 158)"
gradientUnits="userSpaceOnUse"
>
<stop offset="0" stop-opacity=".3" />
<stop offset=".071" stop-opacity=".2" />
<stop offset=".321" stop-opacity=".1" />
<stop offset=".623" stop-opacity=".05" />
<stop offset="1" stop-opacity="0" />
</linearGradient>
<linearGradient
id="a7fee970-a784-4bb1-af8d-63d18e5f7db9"
x1="-1027.165"
x2="-997.482"
y1="147.642"
y2="68.561"
gradientTransform="matrix(1 0 0 -1 1075 158)"
gradientUnits="userSpaceOnUse"
>
<stop offset="0" stop-color="#3ccbf4" />
<stop offset="1" stop-color="#2892df" />
</linearGradient>
</defs>
<path
fill="url(#e399c19f-b68f-429d-b176-18c2117ff73c)"
d="M33.338 6.544h26.038l-27.03 80.087a4.152 4.152 0 0 1-3.933 2.824H8.149a4.145 4.145 0 0 1-3.928-5.47L29.404 9.368a4.152 4.152 0 0 1 3.934-2.825z"
/>
<path
fill="#0078d4"
d="M71.175 60.261h-41.29a1.911 1.911 0 0 0-1.305 3.309l26.532 24.764a4.171 4.171 0 0 0 2.846 1.121h23.38z"
/>
<path
fill="url(#ac2a6fc2-ca48-4327-9a3c-d4dcc3256e15)"
d="M33.338 6.544a4.118 4.118 0 0 0-3.943 2.879L4.252 83.917a4.14 4.14 0 0 0 3.908 5.538h20.787a4.443 4.443 0 0 0 3.41-2.9l5.014-14.777 17.91 16.705a4.237 4.237 0 0 0 2.666.972H81.24L71.024 60.261l-29.781.007L59.47 6.544z"
/>
<path
fill="url(#a7fee970-a784-4bb1-af8d-63d18e5f7db9)"
d="M66.595 9.364a4.145 4.145 0 0 0-3.928-2.82H33.648a4.146 4.146 0 0 1 3.928 2.82l25.184 74.62a4.146 4.146 0 0 1-3.928 5.472h29.02a4.146 4.146 0 0 0 3.927-5.472z"
/>
</svg>

7
packages/builder/src/pages/builder/portal/settings/index.svelte
View File

@ -1,5 +1,10 @@
<script>
import { redirect } from "@roxi/routify"
import { licensing } from "stores/portal"
$redirect("./auth")
if ($licensing.customAIConfigsEnabled) {
$redirect("./ai")
} else {
$redirect("./auth")
}
</script>

2
packages/cli/package.json
View File

@ -28,7 +28,7 @@
"inquirer": "8.0.0",
"lookpath": "1.1.0",
"node-fetch": "2.6.7",
"posthog-node": "1.3.0",
"posthog-node": "4.0.1",
"pouchdb": "7.3.0",
"@budibase/pouchdb-replication-stream": "1.2.11",
"randomstring": "1.1.5",

2
packages/cli/src/analytics/Client.ts
View File

@ -1,4 +1,4 @@
import PostHog from "posthog-node"
import { PostHog } from "posthog-node"
import { POSTHOG_TOKEN, AnalyticsEvent } from "../constants"
import { ConfigManager } from "../structures/ConfigManager"

4
packages/client/src/components/app/deprecated/table/Table.svelte
View File

@ -2,7 +2,7 @@
import { getContext, onDestroy } from "svelte"
import { Table } from "@budibase/bbui"
import SlotRenderer from "./SlotRenderer.svelte"
import { canBeSortColumn } from "@budibase/shared-core"
import { canBeSortColumn } from "@budibase/frontend-core"
import Provider from "components/context/Provider.svelte"
export let dataProvider
@ -146,7 +146,7 @@
return
}
newSchema[columnName] = schema[columnName]
if (!canBeSortColumn(schema[columnName].type)) {
if (!canBeSortColumn(schema[columnName])) {
newSchema[columnName].sortable = false
}

11
packages/frontend-core/src/api/ai.js Normal file
View File

@ -0,0 +1,11 @@
export const buildAIEndpoints = API => ({
/**
* Generates a cron expression from a prompt
*/
generateCronExpression: async ({ prompt }) => {
return await API.post({
url: "/api/ai/cron",
body: { prompt },
})
},
})

2
packages/frontend-core/src/api/index.js
View File

@ -2,6 +2,7 @@ import { Helpers } from "@budibase/bbui"
import { Header } from "@budibase/shared-core"
import { ApiVersion } from "../constants"
import { buildAnalyticsEndpoints } from "./analytics"
import { buildAIEndpoints } from "./ai"
import { buildAppEndpoints } from "./app"
import { buildAttachmentEndpoints } from "./attachments"
import { buildAuthEndpoints } from "./auth"
@ -269,6 +270,7 @@ export const createAPIClient = config => {
// Attach all endpoints
return {
...API,
...buildAIEndpoints(API),
...buildAnalyticsEndpoints(API),
...buildAppEndpoints(API),
...buildAttachmentEndpoints(API),

13
packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
View File

@ -2,6 +2,7 @@
import { onMount, getContext } from "svelte"
import { Dropzone } from "@budibase/bbui"
import GridPopover from "../overlays/GridPopover.svelte"
import { FieldType } from "@budibase/types"
export let value
export let focused = false
@ -81,7 +82,12 @@
>
{#each value || [] as attachment}
{#if isImage(attachment.extension)}
<img src={attachment.url} alt={attachment.extension} />
<img
class:light={!$props?.darkMode &&
schema.type === FieldType.SIGNATURE_SINGLE}
src={attachment.url}
alt={attachment.extension}
/>
{:else}
<div class="file" title={attachment.name}>
{attachment.extension}
@ -140,4 +146,9 @@
width: 320px;
padding: var(--cell-padding);
}
.attachment-cell img.light {
-webkit-filter: invert(100%);
filter: invert(100%);
}
</style>

21
packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
View File

@ -1,6 +1,6 @@
<script>
import { getContext, onMount, tick } from "svelte"
import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
import { canBeSortColumn, canBeDisplayColumn } from "@budibase/frontend-core"
import { Icon, Menu, MenuItem, Modal } from "@budibase/bbui"
import GridCell from "./GridCell.svelte"
import { getColumnIcon } from "../../../utils/schema"
@ -165,7 +165,17 @@
}
const hideColumn = () => {
datasource.actions.addSchemaMutation(column.name, { visible: false })
const { related } = column
const mutation = { visible: false }
if (!related) {
datasource.actions.addSchemaMutation(column.name, mutation)
} else {
datasource.actions.addSubSchemaMutation(
related.subField,
related.field,
mutation
)
}
datasource.actions.saveSchemaMutations()
open = false
}
@ -347,15 +357,14 @@
<MenuItem
icon="Label"
on:click={makeDisplayColumn}
disabled={column.primaryDisplay ||
!canBeDisplayColumn(column.schema.type)}
disabled={column.primaryDisplay || !canBeDisplayColumn(column.schema)}
>
Use as display column
</MenuItem>
<MenuItem
icon="SortOrderUp"
on:click={sortAscending}
disabled={!canBeSortColumn(column.schema.type) ||
disabled={!canBeSortColumn(column.schema) ||
(column.name === $sort.column && $sort.order === "ascending")}
>
Sort {sortingLabels.ascending}
@ -363,7 +372,7 @@
<MenuItem
icon="SortOrderDown"
on:click={sortDescending}
disabled={!canBeSortColumn(column.schema.type) ||
disabled={!canBeSortColumn(column.schema) ||
(column.name === $sort.column && $sort.order === "descending")}
>
Sort {sortingLabels.descending}

6
packages/frontend-core/src/components/grid/lib/renderers.js
View File

@ -35,5 +35,9 @@ const TypeComponentMap = {
[FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
}
export const getCellRenderer = column => {
return TypeComponentMap[column?.schema?.type] || TextCell
return (
TypeComponentMap[column?.schema?.cellRenderType] ||
TypeComponentMap[column?.schema?.type] ||
TextCell
)
}

29
packages/frontend-core/src/components/grid/stores/columns.js
View File

@ -42,6 +42,11 @@ export const deriveStores = context => {
return map
})
// Derived list of columns which are direct part of the table
const tableColumns = derived(columns, $columns => {
return $columns.filter(col => !col.related)
})
// Derived list of columns which have not been explicitly hidden
const visibleColumns = derived(columns, $columns => {
return $columns.filter(col => col.visible)
@ -64,6 +69,7 @@ export const deriveStores = context => {
})
return {
tableColumns,
displayColumn,
columnLookupMap,
visibleColumns,
@ -73,16 +79,24 @@ export const deriveStores = context => {
}
export const createActions = context => {
const { columns, datasource, schema } = context
const { columns, datasource } = context
// Updates the width of all columns
const changeAllColumnWidths = async width => {
const $schema = get(schema)
let mutations = {}
Object.keys($schema).forEach(field => {
mutations[field] = { width }
const $columns = get(columns)
$columns.forEach(column => {
const { related } = column
const mutation = { width }
if (!related) {
datasource.actions.addSchemaMutation(column.name, mutation)
} else {
datasource.actions.addSubSchemaMutation(
related.subField,
related.field,
mutation
)
}
})
datasource.actions.addSchemaMutations(mutations)
await datasource.actions.saveSchemaMutations()
}
@ -136,7 +150,7 @@ export const initialise = context => {
.map(field => {
const fieldSchema = $enrichedSchema[field]
const oldColumn = $columns?.find(col => col.name === field)
let column = {
const column = {
name: field,
label: fieldSchema.displayName || field,
schema: fieldSchema,
@ -145,6 +159,7 @@ export const initialise = context => {
readonly: fieldSchema.readonly,
order: fieldSchema.order ?? oldColumn?.order,
conditions: fieldSchema.conditions,
related: fieldSchema.related,
}
// Override a few properties for primary display
if (field === primaryDisplay) {

30
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -1,6 +1,6 @@
import { derived, get } from "svelte/store"
import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
import { memo } from "../../../utils"
import { enrichSchemaWithRelColumns, memo } from "../../../utils"
import { cloneDeep } from "lodash"
export const createStores = () => {
@ -54,10 +54,13 @@ export const deriveStores = context => {
if (!$schema) {
return null
}
let enrichedSchema = {}
Object.keys($schema).forEach(field => {
const schemaWithRelatedColumns = enrichSchemaWithRelColumns($schema)
const enrichedSchema = {}
Object.keys(schemaWithRelatedColumns).forEach(field => {
enrichedSchema[field] = {
...$schema[field],
...schemaWithRelatedColumns[field],
...$schemaOverrides?.[field],
...$schemaMutations[field],
}
@ -211,24 +214,6 @@ export const createActions = context => {
})
}
// Adds schema mutations for multiple fields at once
const addSchemaMutations = mutations => {
const fields = Object.keys(mutations || {})
if (!fields.length) {
return
}
schemaMutations.update($schemaMutations => {
let newSchemaMutations = { ...$schemaMutations }
fields.forEach(field => {
newSchemaMutations[field] = {
...newSchemaMutations[field],
...mutations[field],
}
})
return newSchemaMutations
})
}
// Saves schema changes to the server, if possible
const saveSchemaMutations = async () => {
// If we can't save schema changes then we just want to keep this in memory
@ -318,7 +303,6 @@ export const createActions = context => {
changePrimaryDisplay,
addSchemaMutation,
addSubSchemaMutation,
addSchemaMutations,
saveSchemaMutations,
resetSchemaMutations,
},

37
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -133,24 +133,29 @@ export const initialise = context => {
// When sorting changes, ensure view definition is kept up to date
unsubscribers.push(
sort.subscribe(async $sort => {
// Ensure we're updating the correct view
const $view = get(definition)
if ($view?.id !== $datasource.id) {
return
}
// Skip if nothing actually changed
if (
$sort?.column === $view.sort?.field &&
$sort?.order === $view.sort?.order
) {
return
}
// If we can mutate schema then update the view definition
if (get(config).canSaveSchema) {
const $view = get(definition)
if ($view?.id !== $datasource.id) {
return
}
if (
$sort?.column !== $view.sort?.field ||
$sort?.order !== $view.sort?.order
) {
await datasource.actions.saveDefinition({
...$view,
sort: {
field: $sort.column,
order: $sort.order || "ascending",
},
})
}
await datasource.actions.saveDefinition({
...$view,
sort: {
field: $sort.column,
order: $sort.order || "ascending",
},
})
}
// Also update the fetch to ensure the new sort is respected.

15
packages/frontend-core/src/components/grid/stores/reorder.js
View File

@ -214,11 +214,20 @@ export const createActions = context => {
})
// Extract new orders as schema mutations
let mutations = {}
get(columns).forEach((column, idx) => {
mutations[column.name] = { order: idx }
const { related } = column
const mutation = { order: idx }
if (!related) {
datasource.actions.addSchemaMutation(column.name, mutation)
} else {
datasource.actions.addSubSchemaMutation(
related.subField,
related.field,
mutation
)
}
})
datasource.actions.addSchemaMutations(mutations)
await datasource.actions.saveSchemaMutations()
}

11
packages/frontend-core/src/components/grid/stores/resize.js
View File

@ -38,6 +38,7 @@ export const createActions = context => {
initialWidth: column.width,
initialMouseX: x,
column: column.name,
related: column.related,
})
// Add mouse event listeners to handle resizing
@ -50,7 +51,7 @@ export const createActions = context => {
// Handler for moving the mouse to resize columns
const onResizeMouseMove = e => {
const { initialMouseX, initialWidth, width, column } = get(resize)
const { initialMouseX, initialWidth, width, column, related } = get(resize)
const { x } = parseEventLocation(e)
const dx = x - initialMouseX
const newWidth = Math.round(Math.max(MinColumnWidth, initialWidth + dx))
@ -61,7 +62,13 @@ export const createActions = context => {
}
// Update column state
datasource.actions.addSchemaMutation(column, { width })
if (!related) {
datasource.actions.addSchemaMutation(column, { width })
} else {
datasource.actions.addSubSchemaMutation(related.subField, related.field, {
width,
})
}
// Update state
resize.update(state => ({

26
packages/frontend-core/src/components/grid/stores/rows.js
View File

@ -6,6 +6,7 @@ import { tick } from "svelte"
import { Helpers } from "@budibase/bbui"
import { sleep } from "../../../utils/utils"
import { FieldType } from "@budibase/types"
import { getRelatedTableValues } from "../../../utils"
export const createStores = () => {
const rows = writable([])
@ -42,15 +43,26 @@ export const createStores = () => {
}
export const deriveStores = context => {
const { rows } = context
const { rows, enrichedSchema } = context
// Enrich rows with an index property and any pending changes
const enrichedRows = derived(rows, $rows => {
return $rows.map((row, idx) => ({
...row,
__idx: idx,
}))
})
const enrichedRows = derived(
[rows, enrichedSchema],
([$rows, $enrichedSchema]) => {
const customColumns = Object.values($enrichedSchema || {}).filter(
f => f.related
)
return $rows.map((row, idx) => ({
...row,
__idx: idx,
...customColumns.reduce((map, column) => {
const fromField = $enrichedSchema[column.related.field]
map[column.name] = getRelatedTableValues(row, column, fromField)
return map
}, {}),
}))
}
)
// Generate a lookup map to quick find a row by ID
const rowLookupMap = derived(enrichedRows, $enrichedRows => {

2
packages/frontend-core/src/utils/index.js
View File

@ -11,3 +11,5 @@ export { createWebsocket } from "./websocket"
export * from "./download"
export * from "./theme"
export * from "./settings"
export * from "./relatedColumns"
export * from "./table"

107
packages/frontend-core/src/utils/relatedColumns.js Normal file
View File

@ -0,0 +1,107 @@
import { FieldType, RelationshipType } from "@budibase/types"
import { Helpers } from "@budibase/bbui"
const columnTypeManyTypeOverrides = {
[FieldType.DATETIME]: FieldType.STRING,
[FieldType.BOOLEAN]: FieldType.STRING,
[FieldType.SIGNATURE_SINGLE]: FieldType.ATTACHMENTS,
}
const columnTypeManyParser = {
[FieldType.DATETIME]: (value, field) => {
function parseDate(value) {
const { timeOnly, dateOnly, ignoreTimezones } = field || {}
const enableTime = !dateOnly
const parsedValue = Helpers.parseDate(value, {
timeOnly,
enableTime,
ignoreTimezones,
})
const parsed = Helpers.getDateDisplayValue(parsedValue, {
enableTime,
timeOnly,
})
return parsed
}
return value?.map(v => parseDate(v))
},
[FieldType.BOOLEAN]: value => value?.map(v => !!v),
[FieldType.BB_REFERENCE_SINGLE]: value => [
...new Map(value.map(i => [i._id, i])).values(),
],
[FieldType.BB_REFERENCE]: value => [
...new Map(value.map(i => [i._id, i])).values(),
],
[FieldType.ARRAY]: value => Array.from(new Set(value)),
}
export function enrichSchemaWithRelColumns(schema) {
if (!schema) {
return
}
const result = Object.keys(schema).reduce((result, fieldName) => {
const field = schema[fieldName]
result[fieldName] = field
if (field.visible !== false && field.columns) {
const fromSingle =
field?.relationshipType === RelationshipType.ONE_TO_MANY
for (const relColumn of Object.keys(field.columns)) {
const relField = field.columns[relColumn]
if (!relField.visible) {
continue
}
const name = `${field.name}.${relColumn}`
result[name] = {
...relField,
name,
related: { field: fieldName, subField: relColumn },
cellRenderType:
(!fromSingle && columnTypeManyTypeOverrides[relField.type]) ||
relField.type,
}
}
}
return result
}, {})
return result
}
export function getRelatedTableValues(row, field, fromField) {
const fromSingle =
fromField?.relationshipType === RelationshipType.ONE_TO_MANY
let result = ""
if (fromSingle) {
result = row[field.related.field]?.[0]?.[field.related.subField]
} else {
const parser = columnTypeManyParser[field.type] || (value => value)
result = parser(
row[field.related.field]
?.flatMap(r => r[field.related.subField])
?.filter(i => i !== undefined && i !== null),
field
)
if (
[
FieldType.STRING,
FieldType.NUMBER,
FieldType.BIGINT,
FieldType.BOOLEAN,
FieldType.DATETIME,
FieldType.LONGFORM,
FieldType.BARCODEQR,
].includes(field.type)
) {
result = result?.join(", ")
}
}
return result
}

27
packages/frontend-core/src/utils/table.js Normal file
View File

@ -0,0 +1,27 @@
import * as sharedCore from "@budibase/shared-core"
export function canBeDisplayColumn(column) {
if (!sharedCore.canBeDisplayColumn(column.type)) {
return false
}
if (column.related) {
// If it's a related column (only available in the frontend), don't allow using it as display column
return false
}
return true
}
export function canBeSortColumn(column) {
if (!sharedCore.canBeSortColumn(column.type)) {
return false
}
if (column.related) {
// If it's a related column (only available in the frontend), don't allow using it as display column
return false
}
return true
}

2
packages/server/package.json
View File

@ -101,7 +101,7 @@
"mysql2": "3.9.8",
"node-fetch": "2.6.7",
"object-sizeof": "2.6.1",
"openai": "^4.52.1",
"openai": "4.59.0",
"openapi-types": "9.3.1",
"oracledb": "6.5.1",
"pg": "8.10.0",

108
packages/server/src/api/controllers/permission.ts
View File

@ -1,9 +1,7 @@
import { permissions, roles, context } from "@budibase/backend-core"
import {
UserCtx,
Database,
Role,
PermissionLevel,
GetResourcePermsResponse,
ResourcePermissionInfo,
GetDependantResourcesResponse,
@ -12,107 +10,15 @@ import {
RemovePermissionRequest,
RemovePermissionResponse,
} from "@budibase/types"
import { getRoleParams } from "../../db/utils"
import {
CURRENTLY_SUPPORTED_LEVELS,
getBasePermissions,
} from "../../utilities/security"
import { removeFromArray } from "../../utilities"
import sdk from "../../sdk"
const enum PermissionUpdateType {
REMOVE = "remove",
ADD = "add",
}
import { PermissionUpdateType } from "../../sdk/app/permissions"
const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
// utility function to stop this repetition - permissions always stored under roles
async function getAllDBRoles(db: Database) {
const body = await db.allDocs<Role>(
getRoleParams(null, {
include_docs: true,
})
)
return body.rows.map(row => row.doc!)
}
async function updatePermissionOnRole(
{
roleId,
resourceId,
level,
}: { roleId: string; resourceId: string; level: PermissionLevel },
updateType: PermissionUpdateType
) {
const db = context.getAppDB()
const remove = updateType === PermissionUpdateType.REMOVE
const isABuiltin = roles.isBuiltin(roleId)
const dbRoleId = roles.getDBRoleID(roleId)
const dbRoles = await getAllDBRoles(db)
const docUpdates: Role[] = []
// the permission is for a built in, make sure it exists
if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) {
const builtin = roles.getBuiltinRoles()[roleId]
builtin._id = roles.getDBRoleID(builtin._id!)
dbRoles.push(builtin)
}
// now try to find any roles which need updated, e.g. removing the
// resource from another role and then adding to the new role
for (let role of dbRoles) {
let updated = false
const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
? role.permissions
: {}
// make sure its an array, also handle migrating
if (
!rolePermissions[resourceId] ||
!Array.isArray(rolePermissions[resourceId])
) {
rolePermissions[resourceId] =
typeof rolePermissions[resourceId] === "string"
? [rolePermissions[resourceId] as unknown as PermissionLevel]
: []
}
// handle the removal/updating the role which has this permission first
// the updating (role._id !== dbRoleId) is required because a resource/level can
// only be permitted in a single role (this reduces hierarchy confusion and simplifies
// the general UI for this, rather than needing to show everywhere it is used)
if (
(role._id !== dbRoleId || remove) &&
rolePermissions[resourceId].indexOf(level) !== -1
) {
removeFromArray(rolePermissions[resourceId], level)
updated = true
}
// handle the adding, we're on the correct role, at it to this
if (!remove && role._id === dbRoleId) {
const set = new Set(rolePermissions[resourceId])
rolePermissions[resourceId] = [...set.add(level)]
updated = true
}
// handle the update, add it to bulk docs to perform at end
if (updated) {
role.permissions = rolePermissions
docUpdates.push(role)
}
}
const response = await db.bulkDocs(docUpdates)
return response.map(resp => {
const version = docUpdates.find(role => role._id === resp.id)?.version
const _id = roles.getExternalRoleID(resp.id, version)
return {
_id,
rev: resp.rev,
error: resp.error,
reason: resp.reason,
}
})
}
export function fetchBuiltin(ctx: UserCtx) {
ctx.body = Object.values(permissions.getBuiltinPermissions())
}
@ -124,7 +30,7 @@ export function fetchLevels(ctx: UserCtx) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const dbRoles: Role[] = await getAllDBRoles(db)
const dbRoles: Role[] = await sdk.permissions.getAllDBRoles(db)
let permissions: any = {}
// create an object with structure role ID -> resource ID -> level
for (let role of dbRoles) {
@ -186,12 +92,18 @@ export async function getDependantResources(
export async function addPermission(ctx: UserCtx<void, AddPermissionResponse>) {
const params: AddPermissionRequest = ctx.params
ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.ADD)
ctx.body = await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.ADD
)
}
export async function removePermission(
ctx: UserCtx<void, RemovePermissionResponse>
) {
const params: RemovePermissionRequest = ctx.params
ctx.body = await updatePermissionOnRole(params, PermissionUpdateType.REMOVE)
ctx.body = await sdk.permissions.updatePermissionOnRole(
params,
PermissionUpdateType.REMOVE
)
}

111
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -1,5 +1,6 @@
import dayjs from "dayjs"
import {
Aggregation,
AutoFieldSubType,
AutoReason,
Datasource,
@ -19,6 +20,7 @@ import {
SortJson,
SortType,
Table,
ViewV2,
} from "@budibase/types"
import {
breakExternalTableId,
@ -46,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import env from "../../../environment"
import { makeExternalQuery } from "../../../integrations/base/query"
import { dataFilters } from "@budibase/shared-core"
import { dataFilters, helpers } from "@budibase/shared-core"
export interface ManyRelationship {
tableId?: string
@ -159,17 +161,41 @@ function isEditableColumn(column: FieldSchema) {
export class ExternalRequest<T extends Operation> {
private readonly operation: T
private readonly tableId: string
private datasource?: Datasource
private tables: { [key: string]: Table } = {}
private readonly source: Table | ViewV2
private datasource: Datasource
constructor(operation: T, tableId: string, datasource?: Datasource) {
this.operation = operation
this.tableId = tableId
this.datasource = datasource
if (datasource && datasource.entities) {
this.tables = datasource.entities
public static async for<T extends Operation>(
operation: T,
source: Table | ViewV2,
opts: { datasource?: Datasource } = {}
) {
if (!opts.datasource) {
if (sdk.views.isView(source)) {
const table = await sdk.views.getTable(source.id)
opts.datasource = await sdk.datasources.get(table.sourceId!)
} else {
opts.datasource = await sdk.datasources.get(source.sourceId!)
}
}
return new ExternalRequest(operation, source, opts.datasource)
}
private get tables(): { [key: string]: Table } {
if (!this.datasource.entities) {
throw new Error("Datasource does not have entities")
}
return this.datasource.entities
}
private constructor(
operation: T,
source: Table | ViewV2,
datasource: Datasource
) {
this.operation = operation
this.source = source
this.datasource = datasource
}
private prepareFilters(
@ -290,20 +316,6 @@ export class ExternalRequest<T extends Operation> {
return this.tables[tableName]
}
// seeds the object with table and datasource information
async retrieveMetadata(
datasourceId: string
): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
if (!this.datasource) {
this.datasource = await sdk.datasources.get(datasourceId)
if (!this.datasource || !this.datasource.entities) {
throw "No tables found, fetch tables before query."
}
this.tables = this.datasource.entities
}
return { tables: this.tables, datasource: this.datasource }
}
async getRow(table: Table, rowId: string): Promise<Row> {
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(table._id!, Operation.READ),
@ -619,24 +631,16 @@ export class ExternalRequest<T extends Operation> {
}
async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
const { operation, tableId } = this
if (!tableId) {
throw new Error("Unable to run without a table ID")
}
let { datasourceId, tableName } = breakExternalTableId(tableId)
let datasource = this.datasource
if (!datasource) {
const { datasource: ds } = await this.retrieveMetadata(datasourceId)
datasource = ds
}
const tables = this.tables
const table = tables[tableName]
let isSql = isSQL(datasource)
if (!table) {
throw new Error(
`Unable to process query, table "${tableName}" not defined.`
)
const { operation } = this
let table: Table
if (sdk.views.isView(this.source)) {
table = await sdk.views.getTable(this.source.id)
} else {
table = this.source
}
let isSql = isSQL(this.datasource)
// look for specific components of config which may not be considered acceptable
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
config,
@ -679,25 +683,40 @@ export class ExternalRequest<T extends Operation> {
}
}
}
if (
operation === Operation.DELETE &&
(filters == null || Object.keys(filters).length === 0)
) {
throw "Deletion must be filtered"
}
let aggregations: Aggregation[] = []
if (sdk.views.isView(this.source)) {
const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({
name: key,
field: field.field,
calculationType: field.calculationType,
})
}
}
let json: QueryJson = {
endpoint: {
datasourceId: datasourceId!,
entityId: tableName,
datasourceId: this.datasource._id!,
entityId: table.name,
operation,
},
resource: {
// have to specify the fields to avoid column overlap (for SQL)
fields: isSql
? buildSqlFieldList(table, this.tables, {
? await buildSqlFieldList(this.source, this.tables, {
relationships: incRelationships,
})
: [],
aggregations,
},
filters,
sort,
@ -714,7 +733,7 @@ export class ExternalRequest<T extends Operation> {
},
meta: {
table,
tables: tables,
tables: this.tables,
},
}
@ -745,7 +764,7 @@ export class ExternalRequest<T extends Operation> {
}
const output = await sqlOutputProcessing(
response,
table,
this.source,
this.tables,
relationships
)

46
packages/server/src/api/controllers/row/external.ts
View File

@ -17,6 +17,7 @@ import {
Row,
Table,
UserCtx,
ViewV2,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
@ -29,39 +30,40 @@ import { generateIdForRow } from "./utils"
export async function handleRequest<T extends Operation>(
operation: T,
tableId: string,
source: Table | ViewV2,
opts?: RunConfig
): Promise<ExternalRequestReturnType<T>> {
return new ExternalRequest<T>(operation, tableId, opts?.datasource).run(
opts || {}
)
return (
await ExternalRequest.for<T>(operation, source, {
datasource: opts?.datasource,
})
).run(opts || {})
}
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const { tableId, viewId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
const table = await utils.getTableFromSource(source)
const { _id, ...rowData } = ctx.request.body
const table = await sdk.tables.getTable(tableId)
const { row: dataToUpdate } = await inputProcessing(
const dataToUpdate = await inputProcessing(
ctx.user?._id,
cloneDeep(table),
cloneDeep(source),
rowData
)
const validateResult = await sdk.rows.utils.validate({
row: dataToUpdate,
tableId,
source,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
const beforeRow = await sdk.rows.external.getRow(tableId, _id, {
const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
relationships: true,
})
const response = await handleRequest(Operation.UPDATE, tableId, {
const response = await handleRequest(Operation.UPDATE, source, {
id: breakRowIdField(_id),
row: dataToUpdate,
})
@ -69,17 +71,16 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
// The id might have been changed, so the refetching would fail. Recalculating the id just in case
const updatedId =
generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
const row = await sdk.rows.external.getRow(tableId, updatedId, {
const row = await sdk.rows.external.getRow(table._id!, updatedId, {
relationships: true,
})
const [enrichedRow, oldRow] = await Promise.all([
outputProcessing(table, row, {
outputProcessing(source, row, {
squash: true,
preserveLinks: true,
fromViewId: viewId,
}),
outputProcessing(table, beforeRow, {
outputProcessing(source, beforeRow, {
squash: true,
preserveLinks: true,
}),
@ -94,9 +95,9 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
}
export async function destroy(ctx: UserCtx) {
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
const _id = ctx.request.body._id
const { row } = await handleRequest(Operation.DELETE, tableId, {
const { row } = await handleRequest(Operation.DELETE, source, {
id: breakRowIdField(_id),
includeSqlRelationships: IncludeRelationship.EXCLUDE,
})
@ -105,11 +106,11 @@ export async function destroy(ctx: UserCtx) {
export async function bulkDestroy(ctx: UserCtx) {
const { rows } = ctx.request.body
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
let promises: Promise<{ row: Row; table: Table }>[] = []
for (let row of rows) {
promises.push(
handleRequest(Operation.DELETE, tableId, {
handleRequest(Operation.DELETE, source, {
id: breakRowIdField(row._id),
includeSqlRelationships: IncludeRelationship.EXCLUDE,
})
@ -124,6 +125,7 @@ export async function bulkDestroy(ctx: UserCtx) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const source = await utils.getSource(ctx)
const { tableId } = utils.getSourceId(ctx)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const datasource: Datasource = await sdk.datasources.get(datasourceId)
@ -131,7 +133,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
const tables = datasource.entities
const response = await handleRequest(Operation.READ, tableId, {
const response = await handleRequest(Operation.READ, source, {
id,
datasource,
includeSqlRelationships: IncludeRelationship.INCLUDE,
@ -155,7 +157,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
// don't support composite keys right now
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
const primaryLink = linkedTable.primary?.[0] as string
const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
const relatedRows = await handleRequest(Operation.READ, linkedTable, {
tables,
filters: {
oneOf: {

10
packages/server/src/api/controllers/row/index.ts
View File

@ -207,7 +207,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
}
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const { tableId } = utils.getSourceId(ctx)
const { tableId, viewId } = utils.getSourceId(ctx)
await context.ensureSnippetContext(true)
@ -222,6 +222,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
...ctx.request.body,
query: enrichedQuery,
tableId,
viewId,
}
ctx.status = 200
@ -229,14 +230,15 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
}
export async function validate(ctx: Ctx<Row, ValidateResponse>) {
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
const table = await utils.getTableFromSource(source)
// external tables are hard to validate currently
if (isExternalTableID(tableId)) {
if (isExternalTableID(table._id!)) {
ctx.body = { valid: true, errors: {} }
} else {
ctx.body = await sdk.rows.utils.validate({
row: ctx.request.body,
tableId,
source,
})
}
}

32
packages/server/src/api/controllers/row/internal.ts
View File

@ -21,18 +21,19 @@ import {
import sdk from "../../../sdk"
import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
import { flatten } from "lodash"
import { findRow } from "../../../sdk/app/rows/internal"
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const { tableId, viewId } = utils.getSourceId(ctx)
const { tableId } = utils.getSourceId(ctx)
const source = await utils.getSource(ctx)
const table = sdk.views.isView(source)
? await sdk.views.getTable(source.id)
: source
const inputs = ctx.request.body
const isUserTable = tableId === InternalTables.USER_METADATA
let oldRow
const dbTable = await sdk.tables.getTable(tableId)
try {
oldRow = await outputProcessing(
dbTable,
await utils.findRow(tableId, inputs._id!)
)
oldRow = await outputProcessing(source, await findRow(tableId, inputs._id!))
} catch (err) {
if (isUserTable) {
// don't include the rev, it'll be the global rev
@ -48,22 +49,15 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
// need to build up full patch fields before coerce
let combinedRow: any = cloneDeep(oldRow)
for (let key of Object.keys(inputs)) {
if (!dbTable.schema[key]) continue
if (!table.schema[key]) continue
combinedRow[key] = inputs[key]
}
// need to copy the table so it can be differenced on way out
const tableClone = cloneDeep(dbTable)
// this returns the table and row incase they have been updated
let { table, row } = await inputProcessing(
ctx.user?._id,
tableClone,
combinedRow
)
let row = await inputProcessing(ctx.user?._id, source, combinedRow)
const validateResult = await sdk.rows.utils.validate({
row,
table,
source,
})
if (!validateResult.valid) {
@ -87,10 +81,8 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
return { row: ctx.body as Row, table, oldRow }
}
const result = await finaliseRow(table, row, {
oldTable: dbTable,
const result = await finaliseRow(source, row, {
updateFormula: true,
fromViewId: viewId,
})
return { ...result, oldRow }
@ -186,7 +178,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
sdk.tables.getTable(tableId),
linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
])
let row = await utils.findRow(tableId, rowId)
let row = await findRow(tableId, rowId)
row = await outputProcessing(table, row)
const linkVals = links as LinkDocumentValue[]

32
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -4,10 +4,11 @@ import {
processFormulas,
} from "../../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
import { Table, Row, FormulaType, FieldType } from "@budibase/types"
import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
import * as linkRows from "../../../db/linkedRows"
import isEqual from "lodash/isEqual"
import { cloneDeep } from "lodash/fp"
import sdk from "../../../sdk"
/**
* This function runs through a list of enriched rows, looks at the rows which
@ -121,33 +122,26 @@ export async function updateAllFormulasInTable(table: Table) {
* expects the row to be totally enriched/contain all relationships.
*/
export async function finaliseRow(
table: Table,
source: Table | ViewV2,
row: Row,
{
oldTable,
updateFormula,
fromViewId,
}: { oldTable?: Table; updateFormula: boolean; fromViewId?: string } = {
updateFormula: true,
}
opts?: { updateFormula: boolean }
) {
const db = context.getAppDB()
const { updateFormula = true } = opts || {}
const table = sdk.views.isView(source)
? await sdk.views.getTable(source.id)
: source
row.type = "row"
// process the row before return, to include relationships
let enrichedRow = (await outputProcessing(table, cloneDeep(row), {
let enrichedRow = await outputProcessing(source, cloneDeep(row), {
squash: false,
})) as Row
})
// use enriched row to generate formulas for saving, specifically only use as context
row = await processFormulas(table, row, {
dynamic: false,
contextRows: [enrichedRow],
})
// don't worry about rev, tables handle rev/lastID updates
// if another row has been written since processing this will
// handle the auto ID clash
if (oldTable && !isEqual(oldTable, table)) {
await db.put(table)
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
@ -158,8 +152,6 @@ export async function finaliseRow(
if (updateFormula) {
await updateRelatedFormula(table, enrichedRow)
}
const squashed = await linkRows.squashLinks(table, enrichedRow, {
fromViewId,
})
const squashed = await linkRows.squashLinks(source, enrichedRow)
return { row: enrichedRow, squashed, table }
}

83
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -1,11 +1,19 @@
// need to handle table name + field or just field, depending on if relationships used
import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types"
import {
FieldSchema,
FieldType,
Row,
Table,
JsonTypes,
ViewV2,
} from "@budibase/types"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { generateRowIdField } from "../../../../integrations/utils"
import sdk from "../../../../sdk"
function extractFieldValue({
row,
@ -78,20 +86,30 @@ function fixJsonTypes(row: Row, table: Table) {
return row
}
export function basicProcessing({
export async function basicProcessing({
row,
table,
source,
tables,
isLinked,
sqs,
}: {
row: Row
table: Table
source: Table | ViewV2
tables: Table[]
isLinked: boolean
sqs?: boolean
}): Row {
}): Promise<Row> {
let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else {
table = source
}
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
for (let fieldName of Object.keys(table.schema)) {
let value = extractFieldValue({
@ -108,13 +126,20 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
if (sdk.views.isView(source)) {
for (const key of Object.keys(helpers.views.calculationFields(source))) {
thisRow[key] = row[key]
}
}
let columns: string[] = Object.keys(table.schema)
if (!sqs) {
if (!sqs && !isCalculationView) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
} else {
} else if (!isCalculationView) {
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
@ -149,28 +174,30 @@ export function basicProcessing({
thisRow[col] = array
// make sure all of them have an _id
const sortField = relatedTable.primaryDisplay || relatedTable.primary![0]!
thisRow[col] = (thisRow[col] as Row[])
.map(relatedRow =>
basicProcessing({
row: relatedRow,
table: relatedTable,
tables,
isLinked: false,
sqs,
})
thisRow[col] = (
await Promise.all(
(thisRow[col] as Row[]).map(relatedRow =>
basicProcessing({
row: relatedRow,
source: relatedTable,
tables,
isLinked: false,
sqs,
})
)
)
.sort((a, b) => {
const aField = a?.[sortField],
bField = b?.[sortField]
if (!aField) {
return 1
} else if (!bField) {
return -1
}
return aField.localeCompare
? aField.localeCompare(bField)
: aField - bField
})
).sort((a, b) => {
const aField = a?.[sortField],
bField = b?.[sortField]
if (!aField) {
return 1
} else if (!bField) {
return -1
}
return aField.localeCompare
? aField.localeCompare(bField)
: aField - bField
})
}
}
return fixJsonTypes(thisRow, table)

46
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -7,10 +7,14 @@ import {
ManyToManyRelationshipFieldMetadata,
RelationshipFieldMetadata,
RelationshipsJson,
Row,
Table,
ViewV2,
} from "@budibase/types"
import { breakExternalTableId } from "../../../../integrations/utils"
import { generateJunctionTableID } from "../../../../db/utils"
import sdk from "../../../../sdk"
import { helpers } from "@budibase/shared-core"
type TableMap = Record<string, Table>
@ -108,11 +112,12 @@ export function buildInternalRelationships(
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us
* is more performant and has the added benefit of protecting against this scenario.
*/
export function buildSqlFieldList(
table: Table,
export async function buildSqlFieldList(
source: Table | ViewV2,
tables: TableMap,
opts?: { relationships: boolean }
) {
const { relationships } = opts || {}
function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema)
.filter(
@ -123,22 +128,33 @@ export function buildSqlFieldList(
)
.map(column => `${table.name}.${column[0]}`)
}
let fields = extractRealFields(table)
let fields: string[] = []
if (sdk.views.isView(source)) {
fields = Object.keys(helpers.views.basicFields(source)).filter(
key => source.schema?.[key]?.visible !== false
)
} else {
fields = extractRealFields(source)
}
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
for (let field of Object.values(table.schema)) {
if (
field.type !== FieldType.LINK ||
!opts?.relationships ||
!field.tableId
) {
if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
const linkTable = tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
const { tableName } = breakExternalTableId(field.tableId)
if (tables[tableName]) {
fields = fields.concat(extractRealFields(tables[tableName], fields))
}
}
return fields
}
@ -149,3 +165,7 @@ export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {
(DSPlusOperation.READ in resp[0] && resp[0].read === true)
)
}
export function isKnexRows(resp: DatasourcePlusQueryResponse): resp is Row[] {
return !isKnexEmptyReadResponse(resp)
}

102
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -1,6 +1,6 @@
import * as utils from "../../../../db/utils"
import { context } from "@budibase/backend-core"
import { docIds } from "@budibase/backend-core"
import {
Ctx,
DatasourcePlusQueryResponse,
@ -8,17 +8,18 @@ import {
RelationshipsJson,
Row,
Table,
ViewV2,
} from "@budibase/types"
import {
processDates,
processFormulas,
} from "../../../../utilities/rowProcessor"
import { isKnexEmptyReadResponse } from "./sqlUtils"
import { isKnexRows } from "./sqlUtils"
import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
import sdk from "../../../../sdk"
import { processStringSync } from "@budibase/string-templates"
import validateJs from "validate.js"
import { getFullUser } from "../../../../utilities/users"
import { helpers } from "@budibase/shared-core"
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -58,26 +59,11 @@ export async function processRelationshipFields(
return row
}
export async function findRow(tableId: string, rowId: string) {
const db = context.getAppDB()
let row: Row
// TODO remove special user case in future
if (tableId === utils.InternalTables.USER_METADATA) {
row = await getFullUser(rowId)
} else {
row = await db.get(rowId)
}
if (row.tableId !== tableId) {
throw "Supplied tableId does not match the rows tableId"
}
return row
}
export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
// top priority, use the URL first
if (ctx.params?.sourceId) {
const { sourceId } = ctx.params
if (utils.isViewID(sourceId)) {
if (docIds.isViewId(sourceId)) {
return {
tableId: utils.extractViewInfoFromID(sourceId).tableId,
viewId: sourceId,
@ -96,22 +82,22 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
throw new Error("Unable to find table ID in request")
}
export async function validate(
opts: { row: Row } & ({ tableId: string } | { table: Table })
) {
let fetchedTable: Table
if ("tableId" in opts) {
fetchedTable = await sdk.tables.getTable(opts.tableId)
} else {
fetchedTable = opts.table
export async function getSource(ctx: Ctx): Promise<Table | ViewV2> {
const { tableId, viewId } = getSourceId(ctx)
if (viewId) {
return sdk.views.get(viewId)
}
return sdk.rows.utils.validate({
...opts,
table: fetchedTable,
})
return sdk.tables.getTable(tableId)
}
function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
export async function getTableFromSource(source: Table | ViewV2) {
if (sdk.views.isView(source)) {
return await sdk.views.getTable(source.id)
}
return source
}
function fixBooleanFields(row: Row, table: Table) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.BOOLEAN) {
if (row[col.name] === 1) {
@ -126,49 +112,45 @@ function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
export async function sqlOutputProcessing(
rows: DatasourcePlusQueryResponse,
table: Table,
source: Table | ViewV2,
tables: Record<string, Table>,
relationships: RelationshipsJson[],
opts?: { sqs?: boolean }
): Promise<Row[]> {
if (isKnexEmptyReadResponse(rows)) {
if (!isKnexRows(rows)) {
return []
}
let finalRows: { [key: string]: Row } = {}
for (let row of rows as Row[]) {
let rowId = row._id
let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else {
table = source
}
let processedRows: Row[] = []
for (let row of rows) {
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
row._id = rowId
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
row._id = getInternalRowId(row, table)
} else if (row._id == null && !isCalculationView) {
row._id = generateIdForRow(row, table)
}
const thisRow = basicProcessing({
row = await basicProcessing({
row,
table,
source,
tables: Object.values(tables),
isLinked: false,
sqs: opts?.sqs,
})
if (thisRow._id == null) {
throw new Error("Unable to generate row ID for SQL rows")
}
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
row = fixBooleanFields(row, table)
row = await processRelationshipFields(table, tables, row, relationships)
processedRows.push(row)
}
// make sure all related rows are correct
let finalRowArray = []
for (let row of Object.values(finalRows)) {
finalRowArray.push(
await processRelationshipFields(table, tables, row, relationships)
)
}
// process some additional types
finalRowArray = processDates(table, finalRowArray)
return finalRowArray
return processDates(table, processedRows)
}
export function isUserMetadataTable(tableId: string) {

95
packages/server/src/api/controllers/row/views.ts
View File

@ -3,17 +3,9 @@ import {
ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
RowSearchParams,
SearchFilterKey,
LogicalOperator,
SearchFilter,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
import { db, context, features } from "@budibase/backend-core"
import { enrichSearchContext } from "./utils"
import { isExternalTableID } from "../../../integrations/utils"
import { context } from "@budibase/backend-core"
export async function searchView(
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@ -28,81 +20,26 @@ export async function searchView(
ctx.throw(400, `This method only supports viewsV2`)
}
const viewFields = Object.entries(view.schema || {})
.filter(([_, value]) => value.visible)
.map(([key]) => key)
const { body } = ctx.request
const sqsEnabled = await features.flags.isEnabled("SQS")
const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled
// Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access.
let query = supportsLogicalOperators
? dataFilters.buildQuery(view.query)
: dataFilters.buildQueryLegacy(view.query)
delete query?.onEmptyFilter
if (body.query) {
// Delete extraneous search params that cannot be overridden
delete body.query.onEmptyFilter
if (!supportsLogicalOperators) {
// In the unlikely event that a Grouped Filter is in a non-SQS environment
// It needs to be ignored entirely
let queryFilters: SearchFilter[] = Array.isArray(view.query)
? view.query
: []
// Extract existing fields
const existingFields =
queryFilters
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Carry over filters for unused fields
Object.keys(body.query).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(body.query[operator] || {}).forEach(field => {
if (query && !existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = body.query[operator]![field]
}
})
})
} else {
const conditions = query ? [query] : []
query = {
$and: {
conditions: [...conditions, body.query],
},
}
}
}
await context.ensureSnippetContext(true)
const enrichedQuery = await enrichSearchContext(query || {}, {
user: sdk.users.getUserContextBindings(ctx.user),
})
const result = await sdk.rows.search(
{
viewId: view.id,
tableId: view.tableId,
query: body.query,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
},
{
user: sdk.users.getUserContextBindings(ctx.user),
}
)
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<
Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
> = {
tableId: view.tableId,
viewId: view.id,
query: enrichedQuery,
fields: viewFields,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
countRows: body.countRows,
}
const result = await sdk.rows.search(searchOptions)
result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result
}

5
packages/server/src/api/controllers/table/external.ts
View File

@ -113,11 +113,10 @@ export async function bulkImport(
const processed = await inputProcessing(ctx.user?._id, table, row, {
noAutoRelationships: true,
})
parsedRows.push(processed.row)
table = processed.table
parsedRows.push(processed)
}
await handleRequest(Operation.BULK_UPSERT, table._id!, {
await handleRequest(Operation.BULK_UPSERT, table, {
rows: parsedRows,
})
await events.rows.imported(table, parsedRows.length)

9
packages/server/src/api/controllers/table/index.ts
View File

@ -33,7 +33,7 @@ import {
import sdk from "../../../sdk"
import { jsonFromCsvString } from "../../../utilities/csv"
import { builderSocket } from "../../../websockets"
import { cloneDeep, isEqual } from "lodash"
import { cloneDeep } from "lodash"
import {
helpers,
PROTECTED_EXTERNAL_COLUMNS,
@ -149,12 +149,7 @@ export async function bulkImport(
ctx: UserCtx<BulkImportRequest, BulkImportResponse>
) {
const tableId = ctx.params.tableId
let tableBefore = await sdk.tables.getTable(tableId)
let tableAfter = await pickApi({ tableId }).bulkImport(ctx)
if (!isEqual(tableBefore, tableAfter)) {
await sdk.tables.saveTable(tableAfter)
}
await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it
// can only be done in the builder, but in the future we may need to

23
packages/server/src/api/controllers/table/internal.ts
View File

@ -3,7 +3,6 @@ import { handleDataImport } from "./utils"
import {
BulkImportRequest,
BulkImportResponse,
FieldType,
RenameColumn,
SaveTableRequest,
SaveTableResponse,
@ -70,22 +69,10 @@ export async function bulkImport(
) {
const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows, identifierFields } = ctx.request.body
await handleDataImport(
{
...table,
schema: {
_id: {
name: "_id",
type: FieldType.STRING,
},
...table.schema,
},
},
{
importRows: rows,
identifierFields,
user: ctx.user,
}
)
await handleDataImport(table, {
importRows: rows,
identifierFields,
user: ctx.user,
})
return table
}

3
packages/server/src/api/controllers/table/utils.ts
View File

@ -139,8 +139,7 @@ export async function importToRows(
const processed = await inputProcessing(user?._id, table, row, {
noAutoRelationships: true,
})
row = processed.row
table = processed.table
row = processed
// However here we must reference the original table, as we want to mutate
// the real schema of the table passed in, not the clone used for

63
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -7,10 +7,49 @@ import {
ViewResponse,
ViewResponseEnriched,
ViewV2,
ViewFieldMetadata,
BasicViewFieldMetadata,
ViewCalculationFieldMetadata,
RelationSchemaField,
ViewFieldMetadata,
} from "@budibase/types"
import { builderSocket, gridSocket } from "../../../websockets"
import { helpers } from "@budibase/shared-core"
function stripUnknownFields(
field: BasicViewFieldMetadata
): RequiredKeys<BasicViewFieldMetadata> {
if (helpers.views.isCalculationField(field)) {
const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
calculationType: field.calculationType,
field: field.field,
columns: field.columns,
}
return strippedField
} else {
const strippedField: RequiredKeys<BasicViewFieldMetadata> = {
order: field.order,
width: field.width,
visible: field.visible,
readonly: field.readonly,
icon: field.icon,
columns: field.columns,
}
return strippedField
}
}
function stripUndefinedFields(obj: Record<string, any>): void {
Object.keys(obj)
.filter(key => obj[key] === undefined)
.forEach(key => {
delete obj[key]
})
}
async function parseSchema(view: CreateViewRequest) {
if (!view.schema) {
@ -22,6 +61,7 @@ async function parseSchema(view: CreateViewRequest) {
let fieldRelatedSchema:
| Record<string, RequiredKeys<RelationSchemaField>>
| undefined
if (schemaValue.columns) {
fieldRelatedSchema = Object.entries(schemaValue.columns).reduce<
NonNullable<typeof fieldRelatedSchema>
@ -35,25 +75,12 @@ async function parseSchema(view: CreateViewRequest) {
}
return acc
}, {})
schemaValue.columns = fieldRelatedSchema
}
const fieldSchema: RequiredKeys<
ViewFieldMetadata & {
columns: typeof fieldRelatedSchema
}
> = {
order: schemaValue.order,
width: schemaValue.width,
visible: schemaValue.visible,
readonly: schemaValue.readonly,
icon: schemaValue.icon,
columns: fieldRelatedSchema,
}
Object.entries(fieldSchema)
.filter(([, val]) => val === undefined)
.forEach(([key]) => {
delete fieldSchema[key as keyof ViewFieldMetadata]
})
const fieldSchema = stripUnknownFields(schemaValue)
stripUndefinedFields(fieldSchema)
p[fieldName] = fieldSchema
return p
}, {} as Record<string, RequiredKeys<ViewFieldMetadata>>)

2
packages/server/src/api/routes/index.ts
View File

@ -33,6 +33,7 @@ import rowActionRoutes from "./rowAction"
export { default as staticRoutes } from "./static"
export { default as publicRoutes } from "./public"
const aiRoutes = pro.ai
const appBackupRoutes = pro.appBackups
const environmentVariableRoutes = pro.environmentVariables
@ -67,6 +68,7 @@ export const mainRoutes: Router[] = [
debugRoutes,
environmentVariableRoutes,
rowActionRoutes,
aiRoutes,
// these need to be handled last as they still use /api/:tableId
// this could be breaking as koa may recognise other routes as this
tableRoutes,

12
packages/server/src/api/routes/tests/permissions.spec.ts
View File

@ -125,6 +125,12 @@ describe("/permission", () => {
})
it("should be able to access the view data when the table is set to public and with no view permissions overrides", async () => {
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()
@ -138,6 +144,12 @@ describe("/permission", () => {
resourceId: table._id,
level: PermissionLevel.READ,
})
// Make view inherit table permissions. Needed for backwards compatibility with existing views.
await config.api.permission.revoke({
roleId: STD_ROLE_ID,
resourceId: view.id,
level: PermissionLevel.READ,
})
// replicate changes before checking permissions
await config.publish()

14
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -76,7 +76,7 @@ async function waitForEvent(
}
describe.each([
["internal", undefined],
["lucene", undefined],
["sqs", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
@ -2453,9 +2453,15 @@ describe.each([
let flagCleanup: (() => void) | undefined
beforeAll(async () => {
flagCleanup = setCoreEnv({
const env = {
TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
})
}
if (isSqs) {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
} else {
env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
}
flagCleanup = setCoreEnv(env)
const aux2Table = await config.api.table.save(saveTableRequest())
const aux2Data = await config.api.row.save(aux2Table._id!, {})
@ -2684,7 +2690,7 @@ describe.each([
async (__, retrieveDelegate) => {
await withCoreEnv(
{
TENANT_FEATURE_FLAGS: ``,
TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
},
async () => {
const otherRows = _.sampleSize(auxData, 5)

9
packages/server/src/api/routes/tests/rowAction.spec.ts
View File

@ -826,11 +826,20 @@ describe("/rowsActions", () => {
)
).id
// Allow row action on view
await config.api.rowAction.setViewPermission(
tableId,
viewId,
rowAction.id
)
// Delete explicit view permissions so they inherit table permissions
await config.api.permission.revoke({
roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission
level: PermissionLevel.READ,
resourceId: viewId,
})
return { permissionResource: tableId, triggerResouce: viewId }
},
],

125
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -39,9 +39,10 @@ import tk from "timekeeper"
import { encodeJSBinding } from "@budibase/string-templates"
import { dataFilters } from "@budibase/shared-core"
import { Knex } from "knex"
import { structures } from "@budibase/backend-core/tests"
import { generator, structures } from "@budibase/backend-core/tests"
import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
import { generateRowIdField } from "../../../integrations/utils"
import { cloneDeep } from "lodash/fp"
describe.each([
["in-memory", undefined],
@ -66,6 +67,36 @@ describe.each([
let table: Table
let rows: Row[]
async function basicRelationshipTables(type: RelationshipType) {
const relatedTable = await createTable(
{
name: { name: "name", type: FieldType.STRING },
},
generator.guid().substring(0, 10)
)
table = await createTable(
{
name: { name: "name", type: FieldType.STRING },
//@ts-ignore - API accepts this structure, will build out rest of definition
productCat: {
type: FieldType.LINK,
relationshipType: type,
name: "productCat",
fieldName: "product",
tableId: relatedTable._id!,
constraints: {
type: "array",
},
},
},
generator.guid().substring(0, 10)
)
return {
relatedTable: await config.api.table.get(relatedTable._id!),
table,
}
}
beforeAll(async () => {
await withCoreEnv({ TENANT_FEATURE_FLAGS: "*:SQS" }, () => config.init())
if (isLucene) {
@ -126,7 +157,11 @@ describe.each([
if (isInMemory) {
return dataFilters.search(_.cloneDeep(rows), this.query)
} else {
return config.api.row.search(this.query.tableId, this.query)
const sourceId = this.query.viewId || this.query.tableId
if (!sourceId) {
throw new Error("No source ID provided")
}
return config.api.row.search(sourceId, this.query)
}
}
@ -201,6 +236,7 @@ describe.each([
// rows returned by the query will also cause the assertion to fail.
async toMatchExactly(expectedRows: any[]) {
const response = await this.performSearch()
const cloned = cloneDeep(response)
const foundRows = response.rows
// eslint-disable-next-line jest/no-standalone-expect
@ -211,7 +247,7 @@ describe.each([
expect.objectContaining(this.popRow(expectedRow, foundRows))
)
)
return response
return cloned
}
// Asserts that the query returns rows matching exactly the set of rows
@ -219,6 +255,7 @@ describe.each([
// cause the assertion to fail.
async toContainExactly(expectedRows: any[]) {
const response = await this.performSearch()
const cloned = cloneDeep(response)
const foundRows = response.rows
// eslint-disable-next-line jest/no-standalone-expect
@ -231,7 +268,7 @@ describe.each([
)
)
)
return response
return cloned
}
// Asserts that the query returns some property values - this cannot be used
@ -239,6 +276,7 @@ describe.each([
// typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...)
async toMatch(properties: Record<string, any>) {
const response = await this.performSearch()
const cloned = cloneDeep(response)
const keys = Object.keys(properties) as Array<keyof SearchResponse<Row>>
for (let key of keys) {
// eslint-disable-next-line jest/no-standalone-expect
@ -248,17 +286,18 @@ describe.each([
expect(response[key]).toEqual(properties[key])
}
}
return response
return cloned
}
// Asserts that the query doesn't return a property, e.g. pagination parameters.
async toNotHaveProperty(properties: (keyof SearchResponse<Row>)[]) {
const response = await this.performSearch()
const cloned = cloneDeep(response)
for (let property of properties) {
// eslint-disable-next-line jest/no-standalone-expect
expect(response[property]).toBeUndefined()
}
return response
return cloned
}
// Asserts that the query returns rows matching the set of rows passed in.
@ -266,6 +305,7 @@ describe.each([
// assertion to fail.
async toContain(expectedRows: any[]) {
const response = await this.performSearch()
const cloned = cloneDeep(response)
const foundRows = response.rows
// eslint-disable-next-line jest/no-standalone-expect
@ -276,7 +316,7 @@ describe.each([
)
)
)
return response
return cloned
}
async toFindNothing() {
@ -368,7 +408,6 @@ describe.each([
})
})
// We've decided not to try and support binding for in-memory search just now.
!isInMemory &&
describe("bindings", () => {
let globalUsers: any = []
@ -488,6 +527,20 @@ describe.each([
])
})
!isLucene &&
it("should return all rows matching the session user firstname when logical operator used", async () => {
await expectQuery({
$and: {
conditions: [{ equal: { name: "{{ [user].firstName }}" } }],
},
}).toContainExactly([
{
name: config.getUser().firstName,
appointment: future.toISOString(),
},
])
})
it("should parse the date binding and return all rows after the resolved value", async () => {
await tk.withFreeze(serverTime, async () => {
await expectQuery({
@ -2196,28 +2249,10 @@ describe.each([
let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => {
productCategoryTable = await createTable(
{
name: { name: "name", type: FieldType.STRING },
},
"productCategory"
)
table = await createTable(
{
name: { name: "name", type: FieldType.STRING },
productCat: {
type: FieldType.LINK,
relationshipType: RelationshipType.ONE_TO_MANY,
name: "productCat",
fieldName: "product",
tableId: productCategoryTable._id!,
constraints: {
type: "array",
},
},
},
"product"
const { relatedTable } = await basicRelationshipTables(
RelationshipType.ONE_TO_MANY
)
productCategoryTable = relatedTable
productCatRows = await Promise.all([
config.api.row.save(productCategoryTable._id!, { name: "foo" }),
@ -2250,7 +2285,7 @@ describe.each([
it("should be able to filter by relationship using table name", async () => {
await expectQuery({
equal: { ["productCategory.name"]: "foo" },
equal: { [`${productCategoryTable.name}.name`]: "foo" },
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
@ -2262,6 +2297,36 @@ describe.each([
}).toContainExactly([{ name: "baz", productCat: undefined }])
})
})
isSql &&
describe("big relations", () => {
beforeAll(async () => {
const { relatedTable } = await basicRelationshipTables(
RelationshipType.MANY_TO_ONE
)
const mainRow = await config.api.row.save(table._id!, {
name: "foo",
})
for (let i = 0; i < 11; i++) {
await config.api.row.save(relatedTable._id!, {
name: i,
product: [mainRow._id!],
})
}
})
it("can only pull 10 related rows", async () => {
await withCoreEnv({ SQL_MAX_RELATED_ROWS: "10" }, async () => {
const response = await expectQuery({}).toContain([{ name: "foo" }])
expect(response.rows[0].productCat).toBeArrayOfSize(10)
})
})
it("can pull max rows when env not set (defaults to 500)", async () => {
const response = await expectQuery({}).toContain([{ name: "foo" }])
expect(response.rows[0].productCat).toBeArrayOfSize(11)
})
})
;(isSqs || isLucene) &&
describe("relations to same table", () => {
let relatedTable: Table, relatedRows: Row[]

345
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -18,10 +18,13 @@ import {
ViewV2,
SearchResponse,
BasicOperator,
CalculationType,
RelationshipType,
TableSchema,
ViewFieldMetadata,
RenameColumn,
ViewFieldMetadata,
FeatureFlag,
BBReferenceFieldSubType,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -32,8 +35,8 @@ import {
roles,
withEnv as withCoreEnv,
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import sdk from "../../../sdk"
describe.each([
["lucene", undefined],
@ -694,22 +697,23 @@ describe.each([
)
})
it("cannot update views v1", async () => {
const viewV1 = await config.api.legacyView.save({
tableId: table._id!,
name: generator.guid(),
filters: [],
schema: {},
})
isInternal &&
it("cannot update views v1", async () => {
const viewV1 = await config.api.legacyView.save({
tableId: table._id!,
name: generator.guid(),
filters: [],
schema: {},
})
await config.api.viewV2.update(viewV1 as unknown as ViewV2, {
status: 400,
body: {
message: "Only views V2 can be updated",
await config.api.viewV2.update(viewV1 as unknown as ViewV2, {
status: 400,
},
body: {
message: "Only views V2 can be updated",
status: 400,
},
})
})
})
it("cannot update the a view with unmatching ids between url and body", async () => {
const anotherView = await config.api.viewV2.create({
@ -1734,6 +1738,40 @@ describe.each([
})
})
it("views filters are respected even if the column is hidden", async () => {
await config.api.row.save(table._id!, {
one: "foo",
two: "bar",
})
const two = await config.api.row.save(table._id!, {
one: "foo2",
two: "bar2",
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
query: [
{
operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
],
schema: {
id: { visible: true },
one: { visible: false },
two: { visible: false },
},
})
const response = await config.api.viewV2.search(view.id)
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual([
expect.objectContaining({ _id: two._id }),
])
})
it("views without data can be returned", async () => {
const response = await config.api.viewV2.search(view.id)
expect(response.rows).toHaveLength(0)
@ -2192,27 +2230,267 @@ describe.each([
expect(response.rows).toHaveLength(0)
})
it("queries the row api passing the view fields only", async () => {
const searchSpy = jest.spyOn(sdk.rows, "search")
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
one: { visible: false },
},
describe("foreign relationship columns", () => {
let envCleanup: () => void
beforeAll(() => {
const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
if (env.TENANT_FEATURE_FLAGS) {
flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
}
envCleanup = setCoreEnv({
TENANT_FEATURE_FLAGS: flags.join(","),
})
})
await config.api.viewV2.search(view.id, { query: {} })
expect(searchSpy).toHaveBeenCalledTimes(1)
afterAll(() => {
envCleanup?.()
})
expect(searchSpy).toHaveBeenCalledWith(
expect.objectContaining({
fields: ["id"],
const createMainTable = async (
links: {
name: string
tableId: string
fk: string
}[]
) => {
const table = await config.api.table.save(
saveTableRequest({
schema: { title: { name: "title", type: FieldType.STRING } },
})
)
await config.api.table.save({
...table,
schema: {
...table.schema,
...links.reduce<TableSchema>((acc, c) => {
acc[c.name] = {
name: c.name,
relationshipType: RelationshipType.ONE_TO_MANY,
type: FieldType.LINK,
tableId: c.tableId,
fieldName: c.fk,
constraints: { type: "array" },
}
return acc
}, {}),
},
})
)
return table
}
const createAuxTable = (schema: TableSchema) =>
config.api.table.save(
saveTableRequest({
primaryDisplay: "name",
schema: {
...schema,
name: { name: "name", type: FieldType.STRING },
},
})
)
it("returns squashed fields respecting the view config", async () => {
const auxTable = await createAuxTable({
age: { name: "age", type: FieldType.NUMBER },
})
const auxRow = await config.api.row.save(auxTable._id!, {
name: generator.name(),
age: generator.age(),
})
const table = await createMainTable([
{ name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
])
await config.api.row.save(table._id!, {
title: generator.word(),
aux: [auxRow],
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
title: { visible: true },
aux: {
visible: true,
columns: {
name: { visible: false, readonly: false },
age: { visible: true, readonly: true },
},
},
},
})
const response = await config.api.viewV2.search(view.id)
expect(response.rows).toEqual([
expect.objectContaining({
aux: [
{
_id: auxRow._id,
primaryDisplay: auxRow.name,
age: auxRow.age,
},
],
}),
])
})
it("enriches squashed fields", async () => {
const auxTable = await createAuxTable({
user: {
name: "user",
type: FieldType.BB_REFERENCE_SINGLE,
subtype: BBReferenceFieldSubType.USER,
constraints: { presence: true },
},
})
const table = await createMainTable([
{ name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
])
const user = config.getUser()
const auxRow = await config.api.row.save(auxTable._id!, {
name: generator.name(),
user: user._id,
})
await config.api.row.save(table._id!, {
title: generator.word(),
aux: [auxRow],
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
title: { visible: true },
aux: {
visible: true,
columns: {
name: { visible: true, readonly: true },
user: { visible: true, readonly: true },
},
},
},
})
const response = await config.api.viewV2.search(view.id)
expect(response.rows).toEqual([
expect.objectContaining({
aux: [
{
_id: auxRow._id,
primaryDisplay: auxRow.name,
name: auxRow.name,
user: {
_id: user._id,
email: user.email,
firstName: user.firstName,
lastName: user.lastName,
primaryDisplay: user.email,
},
},
],
}),
])
})
})
!isLucene &&
describe("calculations", () => {
let table: Table
let rows: Row[]
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
quantity: {
type: FieldType.NUMBER,
name: "quantity",
},
price: {
type: FieldType.NUMBER,
name: "price",
},
},
})
)
rows = await Promise.all(
Array.from({ length: 10 }, () =>
config.api.row.save(table._id!, {
quantity: generator.natural({ min: 1, max: 10 }),
price: generator.natural({ min: 1, max: 10 }),
})
)
)
})
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
"Quantity Sum": {
visible: true,
calculationType: CalculationType.SUM,
field: "quantity",
},
},
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
}),
])
)
// Calculation views do not return rows that can be linked back to
// the source table, and so should not have an _id field.
for (const row of response.rows) {
expect("_id" in row).toBe(false)
}
})
it("should be able to group by a basic field", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
quantity: {
visible: true,
field: "quantity",
},
"Total Price": {
visible: true,
calculationType: CalculationType.SUM,
field: "price",
},
},
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
const priceByQuantity: Record<number, number> = {}
for (const row of rows) {
priceByQuantity[row.quantity] ??= 0
priceByQuantity[row.quantity] += row.price
}
for (const row of response.rows) {
expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity])
}
})
})
})
describe("permissions", () => {
@ -2248,6 +2526,11 @@ describe.each([
level: PermissionLevel.READ,
resourceId: table._id!,
})
await config.api.permission.revoke({
roleId: roles.BUILTIN_ROLE_IDS.PUBLIC, // Don't think this matters since we are revoking the permission
level: PermissionLevel.READ,
resourceId: view.id,
})
await config.publish()
const response = await config.api.viewV2.publicSearch(view.id)

61
packages/server/src/automations/steps/openai.ts
View File

@ -10,6 +10,7 @@ import {
} from "@budibase/types"
import { env } from "@budibase/backend-core"
import * as automationUtils from "../automationUtils"
import * as pro from "@budibase/pro"
enum Model {
GPT_35_TURBO = "gpt-3.5-turbo",
@ -62,19 +63,33 @@ export const definition: AutomationStepDefinition = {
},
}
/**
* Maintains backward compatibility with automation steps created before the introduction
* of custom configurations and Budibase AI
* @param inputs - automation inputs from the OpenAI automation step.
*/
async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
return completion?.choices[0]?.message?.content
}
export async function run({
inputs,
}: {
inputs: OpenAIStepInputs
}): Promise<OpenAIStepOutputs> {
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
if (inputs.prompt == null) {
return {
success: false,
@ -83,20 +98,24 @@ export async function run({
}
try {
const openai = new OpenAI({
apiKey: env.OPENAI_API_KEY,
})
let response
const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
const completion = await openai.chat.completions.create({
model: inputs.model,
messages: [
{
role: "user",
content: inputs.prompt,
},
],
})
const response = completion?.choices[0]?.message?.content
if (budibaseAIEnabled || customConfigsEnabled) {
const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model)
response = await llm.run(inputs.prompt)
} else {
// fallback to the default that uses the environment variable for backwards compat
if (!env.OPENAI_API_KEY) {
return {
success: false,
response:
"OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.",
}
}
response = await legacyOpenAIPrompt(inputs)
}
return {
response,

39
packages/server/src/automations/tests/openai.spec.ts
View File

@ -4,6 +4,7 @@ import {
withEnv as withCoreEnv,
setEnv as setCoreEnv,
} from "@budibase/backend-core"
import * as pro from "@budibase/pro"
jest.mock("openai", () => ({
OpenAI: jest.fn().mockImplementation(() => ({
@ -22,7 +23,23 @@ jest.mock("openai", () => ({
},
})),
}))
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: jest.fn().mockImplementation(() => ({
init: jest.fn(),
run: jest.fn(),
})),
},
},
features: {
isAICustomConfigsEnabled: jest.fn(),
isBudibaseAIEnabled: jest.fn(),
},
}))
const mockedPro = jest.mocked(pro)
const mockedOpenAI = OpenAI as jest.MockedClass<typeof OpenAI>
const OPENAI_PROMPT = "What is the meaning of life?"
@ -41,6 +58,7 @@ describe("test the openai action", () => {
afterEach(() => {
resetEnv()
jest.clearAllMocks()
})
afterAll(_afterAll)
@ -94,4 +112,25 @@ describe("test the openai action", () => {
)
expect(res.success).toBeFalsy()
})
it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => {
jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true)
jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true)
const prompt = "What is the meaning of life?"
await runStep("OPENAI", {
model: "gpt-4o-mini",
prompt,
})
expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith(
"gpt-4o-mini"
)
const llmInstance =
mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value
// init does not appear to be called currently
// expect(llmInstance.init).toHaveBeenCalled()
expect(llmInstance.run).toHaveBeenCalledWith(prompt)
})
})

83
packages/server/src/automations/tests/scenarios/branching.spec.ts
View File

@ -17,44 +17,65 @@ describe("Branching automations", () => {
afterAll(setup.afterAll)
it("should run a multiple nested branching automation", async () => {
const firstLogId = "11111111-1111-1111-1111-111111111111"
const branch1LogId = "22222222-2222-2222-2222-222222222222"
const branch2LogId = "33333333-3333-3333-3333-333333333333"
const branch2Id = "44444444-4444-4444-4444-444444444444"
const builder = createAutomationBuilder({
name: "Test Trigger with Loop and Create Row",
})
const results = await builder
.appAction({ fields: {} })
.serverLog({ text: "Starting automation" })
.serverLog(
{ text: "Starting automation" },
{ stepName: "FirstLog", stepId: firstLogId }
)
.branch({
topLevelBranch1: {
steps: stepBuilder =>
stepBuilder.serverLog({ text: "Branch 1" }).branch({
branch1: {
steps: stepBuilder =>
stepBuilder.serverLog({ text: "Branch 1.1" }),
condition: {
equal: { "steps.1.success": true },
stepBuilder
.serverLog(
{ text: "Branch 1" },
{ stepId: "66666666-6666-6666-6666-666666666666" }
)
.branch({
branch1: {
steps: stepBuilder =>
stepBuilder.serverLog(
{ text: "Branch 1.1" },
{ stepId: branch1LogId }
),
condition: {
equal: { [`{{ steps.${firstLogId}.success }}`]: true },
},
},
},
branch2: {
steps: stepBuilder =>
stepBuilder.serverLog({ text: "Branch 1.2" }),
condition: {
equal: { "steps.1.success": false },
branch2: {
steps: stepBuilder =>
stepBuilder.serverLog(
{ text: "Branch 1.2" },
{ stepId: branch2LogId }
),
condition: {
equal: { [`{{ steps.${firstLogId}.success }}`]: false },
},
},
},
}),
}),
condition: {
equal: { "steps.1.success": true },
equal: { [`{{ steps.${firstLogId}.success }}`]: true },
},
},
topLevelBranch2: {
steps: stepBuilder => stepBuilder.serverLog({ text: "Branch 2" }),
steps: stepBuilder =>
stepBuilder.serverLog({ text: "Branch 2" }, { stepId: branch2Id }),
condition: {
equal: { "steps.1.success": false },
equal: { [`{{ steps.${firstLogId}.success }}`]: false },
},
},
})
.run()
expect(results.steps[3].outputs.status).toContain("branch1 branch taken")
expect(results.steps[4].outputs.message).toContain("Branch 1.1")
})
@ -70,14 +91,14 @@ describe("Branching automations", () => {
activeBranch: {
steps: stepBuilder => stepBuilder.serverLog({ text: "Active user" }),
condition: {
equal: { "trigger.fields.status": "active" },
equal: { "{{trigger.fields.status}}": "active" },
},
},
inactiveBranch: {
steps: stepBuilder =>
stepBuilder.serverLog({ text: "Inactive user" }),
condition: {
equal: { "trigger.fields.status": "inactive" },
equal: { "{{trigger.fields.status}}": "inactive" },
},
},
})
@ -102,8 +123,8 @@ describe("Branching automations", () => {
condition: {
$and: {
conditions: [
{ equal: { "trigger.fields.status": "active" } },
{ equal: { "trigger.fields.role": "admin" } },
{ equal: { "{{trigger.fields.status}}": "active" } },
{ equal: { "{{trigger.fields.role}}": "admin" } },
],
},
},
@ -111,7 +132,7 @@ describe("Branching automations", () => {
otherBranch: {
steps: stepBuilder => stepBuilder.serverLog({ text: "Other user" }),
condition: {
notEqual: { "trigger.fields.status": "active" },
notEqual: { "{{trigger.fields.status}}": "active" },
},
},
})
@ -133,8 +154,8 @@ describe("Branching automations", () => {
condition: {
$or: {
conditions: [
{ equal: { "trigger.fields.status": "test" } },
{ equal: { "trigger.fields.role": "admin" } },
{ equal: { "{{trigger.fields.status}}": "test" } },
{ equal: { "{{trigger.fields.role}}": "admin" } },
],
},
},
@ -144,8 +165,8 @@ describe("Branching automations", () => {
condition: {
$and: {
conditions: [
{ notEqual: { "trigger.fields.status": "active" } },
{ notEqual: { "trigger.fields.role": "admin" } },
{ notEqual: { "{{trigger.fields.status}}": "active" } },
{ notEqual: { "{{trigger.fields.role}}": "admin" } },
],
},
},
@ -170,8 +191,8 @@ describe("Branching automations", () => {
condition: {
$or: {
conditions: [
{ equal: { "trigger.fields.status": "new" } },
{ equal: { "trigger.fields.role": "admin" } },
{ equal: { "{{trigger.fields.status}}": "new" } },
{ equal: { "{{trigger.fields.role}}": "admin" } },
],
},
},
@ -181,8 +202,8 @@ describe("Branching automations", () => {
condition: {
$and: {
conditions: [
{ equal: { "trigger.fields.status": "active" } },
{ equal: { "trigger.fields.role": "admin" } },
{ equal: { "{{trigger.fields.status}}": "active" } },
{ equal: { "{{trigger.fields.role}}": "admin" } },
],
},
},

142
packages/server/src/automations/tests/scenarios/looping.spec.ts
View File

@ -5,6 +5,7 @@ import {
LoopStepType,
CreateRowStepOutputs,
ServerLogStepOutputs,
FieldType,
} from "@budibase/types"
import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
@ -269,4 +270,145 @@ describe("Loop automations", () => {
expect(results.steps[1].outputs.message).toContain("- 3")
})
it("should run an automation with a loop and update row step", async () => {
const table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
value: {
name: "value",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const rows = [
{ name: "Row 1", value: 1, tableId: table._id },
{ name: "Row 2", value: 2, tableId: table._id },
{ name: "Row 3", value: 3, tableId: table._id },
]
await config.api.row.bulkImport(table._id!, { rows })
const builder = createAutomationBuilder({
name: "Test Loop and Update Row",
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.rows }}",
})
.updateRow({
rowId: "{{ loop.currentItem._id }}",
row: {
name: "Updated {{ loop.currentItem.name }}",
value: "{{ loop.currentItem.value }}",
tableId: table._id,
},
meta: {},
})
.queryRows({
tableId: table._id!,
})
.run()
const expectedRows = [
{ name: "Updated Row 1", value: 1 },
{ name: "Updated Row 2", value: 2 },
{ name: "Updated Row 3", value: 3 },
]
expect(results.steps[1].outputs.items).toEqual(
expect.arrayContaining(
expectedRows.map(row =>
expect.objectContaining({
success: true,
row: expect.objectContaining(row),
})
)
)
)
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining(
expectedRows.map(row => expect.objectContaining(row))
)
)
expect(results.steps[1].outputs.items).toHaveLength(expectedRows.length)
expect(results.steps[2].outputs.rows).toHaveLength(expectedRows.length)
})
it("should run an automation with a loop and delete row step", async () => {
const table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
value: {
name: "value",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const rows = [
{ name: "Row 1", value: 1, tableId: table._id },
{ name: "Row 2", value: 2, tableId: table._id },
{ name: "Row 3", value: 3, tableId: table._id },
]
await config.api.row.bulkImport(table._id!, { rows })
const builder = createAutomationBuilder({
name: "Test Loop and Delete Row",
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.rows }}",
})
.deleteRow({
tableId: table._id!,
id: "{{ loop.currentItem._id }}",
})
.queryRows({
tableId: table._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(0)
})
})

251
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View File

@ -1,8 +1,9 @@
import * as automation from "../../index"
import * as setup from "../utilities"
import { LoopStepType, FieldType } from "@budibase/types"
import { LoopStepType, FieldType, Table } from "@budibase/types"
import { createAutomationBuilder } from "../utilities/AutomationTestBuilder"
import { DatabaseName } from "../../../integrations/tests/utils"
import { FilterConditions } from "../../../automations/steps/filter"
describe("Automation Scenarios", () => {
let config = setup.getConfig()
@ -195,6 +196,91 @@ describe("Automation Scenarios", () => {
)
})
})
it("should trigger an automation which creates and then updates a row", async () => {
const table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
value: {
name: "value",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const builder = createAutomationBuilder({
name: "Test Create and Update Row",
})
const results = await builder
.appAction({ fields: {} })
.createRow(
{
row: {
name: "Initial Row",
value: 1,
tableId: table._id,
},
},
{ stepName: "CreateRowStep" }
)
.updateRow(
{
rowId: "{{ steps.CreateRowStep.row._id }}",
row: {
name: "Updated Row",
value: 2,
tableId: table._id,
},
meta: {},
},
{ stepName: "UpdateRowStep" }
)
.queryRows(
{
tableId: table._id!,
},
{ stepName: "QueryRowsStep" }
)
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[0].outputs).toMatchObject({
success: true,
row: {
name: "Initial Row",
value: 1,
},
})
expect(results.steps[1].outputs).toMatchObject({
success: true,
row: {
name: "Updated Row",
value: 2,
},
})
const expectedRows = [{ name: "Updated Row", value: 2 }]
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining(
expectedRows.map(row => expect.objectContaining(row))
)
)
})
})
describe("Name Based Automations", () => {
@ -233,4 +319,167 @@ describe("Automation Scenarios", () => {
expect(results.steps[2].outputs.rows).toHaveLength(1)
})
})
describe("Automations with filter", () => {
let table: Table
beforeEach(async () => {
table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
value: {
name: "value",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
})
it("should stop an automation if the condition is not met", async () => {
const builder = createAutomationBuilder({
name: "Test Equal",
})
const results = await builder
.appAction({ fields: {} })
.createRow({
row: {
name: "Equal Test",
value: 10,
tableId: table._id,
},
})
.queryRows({
tableId: table._id!,
})
.filter({
field: "{{ steps.2.rows.0.value }}",
condition: FilterConditions.EQUAL,
value: 20,
})
.serverLog({ text: "Equal condition met" })
.run()
expect(results.steps[2].outputs.success).toBeTrue()
expect(results.steps[2].outputs.result).toBeFalse()
expect(results.steps[3]).toBeUndefined()
})
it("should continue the automation if the condition is met", async () => {
const builder = createAutomationBuilder({
name: "Test Not Equal",
})
const results = await builder
.appAction({ fields: {} })
.createRow({
row: {
name: "Not Equal Test",
value: 10,
tableId: table._id,
},
})
.queryRows({
tableId: table._id!,
})
.filter({
field: "{{ steps.2.rows.0.value }}",
condition: FilterConditions.NOT_EQUAL,
value: 20,
})
.serverLog({ text: "Not Equal condition met" })
.run()
expect(results.steps[2].outputs.success).toBeTrue()
expect(results.steps[2].outputs.result).toBeTrue()
expect(results.steps[3].outputs.success).toBeTrue()
})
const testCases = [
{
condition: FilterConditions.EQUAL,
value: 10,
rowValue: 10,
expectPass: true,
},
{
condition: FilterConditions.NOT_EQUAL,
value: 10,
rowValue: 20,
expectPass: true,
},
{
condition: FilterConditions.GREATER_THAN,
value: 10,
rowValue: 15,
expectPass: true,
},
{
condition: FilterConditions.LESS_THAN,
value: 10,
rowValue: 5,
expectPass: true,
},
{
condition: FilterConditions.GREATER_THAN,
value: 10,
rowValue: 5,
expectPass: false,
},
{
condition: FilterConditions.LESS_THAN,
value: 10,
rowValue: 15,
expectPass: false,
},
]
it.each(testCases)(
"should pass the filter when condition is $condition",
async ({ condition, value, rowValue, expectPass }) => {
const builder = createAutomationBuilder({
name: `Test ${condition}`,
})
const results = await builder
.appAction({ fields: {} })
.createRow({
row: {
name: `${condition} Test`,
value: rowValue,
tableId: table._id,
},
})
.queryRows({
tableId: table._id!,
})
.filter({
field: "{{ steps.2.rows.0.value }}",
condition,
value,
})
.serverLog({
text: `${condition} condition ${expectPass ? "passed" : "failed"}`,
})
.run()
expect(results.steps[2].outputs.result).toBe(expectPass)
if (expectPass) {
expect(results.steps[3].outputs.success).toBeTrue()
} else {
expect(results.steps[3]).toBeUndefined()
}
}
)
})
})

70
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View File

@ -33,6 +33,7 @@ import {
BranchStepInputs,
SearchFilters,
Branch,
FilterStepInputs,
} from "@budibase/types"
import TestConfiguration from "../../../tests/utilities/TestConfiguration"
import * as setup from "../utilities"
@ -63,18 +64,18 @@ class BaseStepBuilder {
stepId: TStep,
stepSchema: Omit<AutomationStep, "id" | "stepId" | "inputs">,
inputs: AutomationStepInputs<TStep>,
stepName?: string
opts?: { stepName?: string; stepId?: string }
): this {
const id = uuidv4()
const id = opts?.stepId || uuidv4()
this.steps.push({
...stepSchema,
inputs: inputs as any,
id,
stepId,
name: stepName || stepSchema.name,
name: opts?.stepName || stepSchema.name,
})
if (stepName) {
this.stepNames[id] = stepName
if (opts?.stepName) {
this.stepNames[id] = opts.stepName
}
return this
}
@ -94,7 +95,6 @@ class BaseStepBuilder {
})
branchStepInputs.children![key] = stepBuilder.build()
})
const branchStep: AutomationStep = {
...definition,
id: uuidv4(),
@ -105,80 +105,106 @@ class BaseStepBuilder {
}
// STEPS
createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this {
createRow(
inputs: CreateRowStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.CREATE_ROW,
BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
inputs,
opts?.stepName
opts
)
}
updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this {
updateRow(
inputs: UpdateRowStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.UPDATE_ROW,
BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
inputs,
opts?.stepName
opts
)
}
deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this {
deleteRow(
inputs: DeleteRowStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.DELETE_ROW,
BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
inputs,
opts?.stepName
opts
)
}
sendSmtpEmail(
inputs: SmtpEmailStepInputs,
opts?: { stepName?: string }
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.SEND_EMAIL_SMTP,
BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
inputs,
opts?.stepName
opts
)
}
executeQuery(
inputs: ExecuteQueryStepInputs,
opts?: { stepName?: string }
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.EXECUTE_QUERY,
BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
inputs,
opts?.stepName
opts
)
}
queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this {
queryRows(
inputs: QueryRowsStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.QUERY_ROWS,
BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
inputs,
opts?.stepName
opts
)
}
loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this {
loop(
inputs: LoopStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.LOOP,
BUILTIN_ACTION_DEFINITIONS.LOOP,
inputs,
opts?.stepName
opts
)
}
serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this {
serverLog(
input: ServerLogStepInputs,
opts?: { stepName?: string; stepId?: string }
): this {
return this.step(
AutomationActionStepId.SERVER_LOG,
BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
input,
opts?.stepName
opts
)
}
filter(input: FilterStepInputs): this {
return this.step(
AutomationActionStepId.FILTER,
BUILTIN_ACTION_DEFINITIONS.FILTER,
input
)
}
}

75
packages/server/src/db/linkedRows/index.ts
View File

@ -10,7 +10,10 @@ import flatten from "lodash/flatten"
import { USER_METDATA_PREFIX } from "../utils"
import partition from "lodash/partition"
import { getGlobalUsersFromMetadata } from "../../utilities/global"
import { processFormulas } from "../../utilities/rowProcessor"
import {
coreOutputProcessing,
processFormulas,
} from "../../utilities/rowProcessor"
import { context, features } from "@budibase/backend-core"
import {
ContextUser,
@ -24,6 +27,7 @@ import {
ViewV2,
} from "@budibase/types"
import sdk from "../../sdk"
import { helpers } from "@budibase/shared-core"
export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@ -156,9 +160,6 @@ export async function updateLinks(args: {
/**
* Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
* This is required for formula fields, this may only be utilised internally (for now).
* @param table The table from which the rows originated.
* @param rows The rows which are to be enriched.
* @param opts optional - options like passing in a base row to use for enrichment.
* @return returns the rows with all of the enriched relationships on it.
*/
export async function attachFullLinkedDocs(
@ -247,35 +248,42 @@ function getPrimaryDisplayValue(row: Row, table?: Table) {
export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
/**
* This function will take the given enriched rows and squash the links to only contain the primary display field.
* @param table The table from which the rows originated.
* @param enriched The pre-enriched rows (full docs) which are to be squashed.
* @param squashFields Per link column (key) define which columns are allowed while squashing.
* @returns The rows after having their links squashed to only contain the ID and primary display.
* This function will take the given enriched rows and squash the links to only
* contain the primary display field.
*
* @returns The rows after having their links squashed to only contain the ID
* and primary display.
*/
export async function squashLinks<T = Row[] | Row>(
table: Table,
enriched: T,
options?: {
fromViewId?: string
}
source: Table | ViewV2,
enriched: T
): Promise<T> {
const allowRelationshipSchemas = await features.flags.isEnabled(
FeatureFlag.ENRICHED_RELATIONSHIPS
)
let viewSchema: Record<string, ViewFieldMetadata> = {}
if (options?.fromViewId && allowRelationshipSchemas) {
const view = Object.values(table.views || {}).find(
(v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId
)
viewSchema = view?.schema || {}
if (sdk.views.isView(source)) {
if (helpers.views.isCalculationView(source)) {
return enriched
}
if (allowRelationshipSchemas) {
viewSchema = source.schema || {}
}
}
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
// will populate this as we find them
const linkedTables = [table]
const isArray = Array.isArray(enriched)
const enrichedArray = !isArray ? [enriched] : enriched
const enrichedArray = !isArray ? [enriched as Row] : (enriched as Row[])
for (const row of enrichedArray) {
// this only fetches the table if its not already in array
const rowTable = await getLinkedTable(row.tableId!, linkedTables)
@ -283,18 +291,18 @@ export async function squashLinks<T = Row[] | Row>(
if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
continue
}
const newLinks = []
for (const link of row[column]) {
const linkTblId =
link.tableId || getRelatedTableForField(table.schema, column)
const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
if (viewSchema[column]?.columns) {
row[column] = await coreOutputProcessing(relatedTable, row[column])
}
row[column] = row[column].map((link: Row) => {
const obj: any = { _id: link._id }
obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
if (viewSchema[column]?.columns) {
const squashFields = Object.entries(viewSchema[column].columns)
const squashFields = Object.entries(viewSchema[column].columns || {})
.filter(([columnName, viewColumnConfig]) => {
const tableColumn = linkedTable.schema[columnName]
const tableColumn = relatedTable.schema[columnName]
if (!tableColumn) {
return false
}
@ -312,14 +320,15 @@ export async function squashLinks<T = Row[] | Row>(
.map(([columnName]) => columnName)
for (const relField of squashFields) {
obj[relField] = link[relField]
if (link[relField] != null) {
obj[relField] = link[relField]
}
}
}
newLinks.push(obj)
}
row[column] = newLinks
return obj
})
}
}
return isArray ? enrichedArray : enrichedArray[0]
return (isArray ? enrichedArray : enrichedArray[0]) as T
}

8
packages/server/src/db/utils.ts
View File

@ -1,4 +1,4 @@
import { context, db as dbCore, utils } from "@budibase/backend-core"
import { context, db as dbCore, docIds, utils } from "@budibase/backend-core"
import {
DatabaseQueryOpts,
Datasource,
@ -318,12 +318,8 @@ export function generateViewID(tableId: string) {
}${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
}
export function isViewID(viewId: string) {
return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW
}
export function extractViewInfoFromID(viewId: string) {
if (!isViewID(viewId)) {
if (!docIds.isViewId(viewId)) {
throw new Error("Unable to extract table ID, is not a view ID")
}
const split = viewId.split(SEPARATOR)

3
packages/server/src/definitions/automations.ts
View File

@ -15,7 +15,8 @@ export interface TriggerOutput {
export interface AutomationContext extends AutomationResults {
steps: any[]
stepsByName?: Record<string, any>
stepsById: Record<string, any>
stepsByName: Record<string, any>
env?: Record<string, string>
trigger: any
}

11
packages/server/src/integrations/googlesheets.ts
View File

@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
rows = await sheet.getRows()
}
if (hasFilters && query.paginate) {
rows = rows.slice(offset, offset + limit)
}
const headerValues = sheet.headerValues
let response = rows.map(row =>
this.buildRowObject(headerValues, row.toObject(), row.rowNumber)
this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber)
)
response = dataFilters.runQuery(response, query.filters || {})
if (hasFilters && query.paginate) {
response = response.slice(offset, offset + limit)
}
if (query.sort) {
if (Object.keys(query.sort).length !== 1) {
console.warn("Googlesheets does not support multiple sorting", {

10
packages/server/src/integrations/mysql.ts
View File

@ -241,6 +241,16 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
async connect() {
this.client = await mysql.createConnection(this.config)
const res = await this.internalQuery(
{
sql: "SELECT VERSION();",
},
{ connect: false }
)
const version = res?.[0]?.["VERSION()"]
if (version?.toLowerCase().includes("mariadb")) {
this.setExtendedSqlClient(SqlClient.MARIADB)
}
}
async disconnect() {

190
packages/server/src/integrations/tests/googlesheets.spec.ts
View File

@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration"
import {
Datasource,
FieldType,
Row,
SourceName,
Table,
TableSourceType,
@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => {
)
})
})
describe("search", () => {
let table: Table
beforeEach(async () => {
table = await config.api.table.save({
name: "Test Table",
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
type: "string",
},
},
},
})
await config.api.row.bulkImport(table._id!, {
rows: [
{
name: "Foo",
},
{
name: "Bar",
},
{
name: "Baz",
},
],
})
})
it("should be able to find rows with equals filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
equal: {
name: "Foo",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].name).toEqual("Foo")
})
it("should be able to find rows with not equals filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
notEqual: {
name: "Foo",
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Bar")
expect(response.rows[1].name).toEqual("Baz")
})
it("should be able to find rows with empty filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
empty: {
name: null,
},
},
})
expect(response.rows).toHaveLength(0)
})
it("should be able to find rows with not empty filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
notEmpty: {
name: null,
},
},
})
expect(response.rows).toHaveLength(3)
})
it("should be able to find rows with one of filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
oneOf: {
name: ["Foo", "Bar"],
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Foo")
expect(response.rows[1].name).toEqual("Bar")
})
it("should be able to find rows with fuzzy filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
fuzzy: {
name: "oo",
},
},
})
expect(response.rows).toHaveLength(1)
expect(response.rows[0].name).toEqual("Foo")
})
it("should be able to find rows with range filter", async () => {
const response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {
range: {
name: {
low: "A",
high: "C",
},
},
},
})
expect(response.rows).toHaveLength(2)
expect(response.rows[0].name).toEqual("Bar")
expect(response.rows[1].name).toEqual("Baz")
})
it("should paginate correctly", async () => {
await config.api.row.bulkImport(table._id!, {
rows: Array.from({ length: 50 }, () => ({
name: `Unique value!`,
})),
})
await config.api.row.bulkImport(table._id!, {
rows: Array.from({ length: 50 }, () => ({
name: `Non-unique value!`,
})),
})
let response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: { equal: { name: "Unique value!" } },
paginate: true,
limit: 10,
})
let rows: Row[] = response.rows
while (response.hasNextPage) {
response = await config.api.row.search(table._id!, {
tableId: table._id!,
query: { equal: { name: "Unique value!" } },
paginate: true,
limit: 10,
bookmark: response.bookmark,
})
expect(response.rows.length).toBeLessThanOrEqual(10)
rows = rows.concat(response.rows)
}
// Make sure we only get rows matching the query.
expect(rows.length).toEqual(50)
expect(rows.map(row => row.name)).toEqual(
expect.arrayContaining(
Array.from({ length: 50 }, () => "Unique value!")
)
)
// Make sure all of the rows have a unique ID.
const ids = Object.keys(
rows.reduce((acc, row) => {
acc[row._id!] = true
return acc
}, {})
)
expect(ids.length).toEqual(50)
})
})
})

2
packages/server/src/integrations/tests/utils/googlesheets.ts
View File

@ -440,6 +440,8 @@ export class GoogleSheetsMock {
endColumnIndex: 0,
})
sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length
return {
spreadsheetId: this.spreadsheet.spreadsheetId,
tableRange: range,

2
packages/server/src/middleware/triggerRowActionAuthorised.ts
View File

@ -15,7 +15,7 @@ export function triggerRowActionAuthorised(
const rowActionId: string = ctx.params[actionPath]
const isTableId = docIds.isTableId(sourceId)
const isViewId = utils.isViewID(sourceId)
const isViewId = docIds.isViewId(sourceId)
if (!isTableId && !isViewId) {
ctx.throw(400, `'${sourceId}' is not a valid source id`)
}

100
packages/server/src/sdk/app/permissions/index.ts
View File

@ -1,26 +1,34 @@
import { db, roles } from "@budibase/backend-core"
import { db, roles, context, docIds } from "@budibase/backend-core"
import {
PermissionLevel,
PermissionSource,
VirtualDocumentType,
Role,
Database,
} from "@budibase/types"
import { extractViewInfoFromID, isViewID } from "../../../db/utils"
import { extractViewInfoFromID, getRoleParams } from "../../../db/utils"
import {
CURRENTLY_SUPPORTED_LEVELS,
getBasePermissions,
} from "../../../utilities/security"
import sdk from "../../../sdk"
import { isV2 } from "../views"
import { removeFromArray } from "../../../utilities"
type ResourcePermissions = Record<
string,
{ role: string; type: PermissionSource }
>
export const enum PermissionUpdateType {
REMOVE = "remove",
ADD = "add",
}
export async function getInheritablePermissions(
resourceId: string
): Promise<ResourcePermissions | undefined> {
if (isViewID(resourceId)) {
if (docIds.isViewId(resourceId)) {
return await getResourcePerms(extractViewInfoFromID(resourceId).tableId)
}
}
@ -100,3 +108,89 @@ export async function getDependantResources(
return
}
export async function updatePermissionOnRole(
{
roleId,
resourceId,
level,
}: { roleId: string; resourceId: string; level: PermissionLevel },
updateType: PermissionUpdateType
) {
const db = context.getAppDB()
const remove = updateType === PermissionUpdateType.REMOVE
const isABuiltin = roles.isBuiltin(roleId)
const dbRoleId = roles.getDBRoleID(roleId)
const dbRoles = await getAllDBRoles(db)
const docUpdates: Role[] = []
// the permission is for a built in, make sure it exists
if (isABuiltin && !dbRoles.some(role => role._id === dbRoleId)) {
const builtin = roles.getBuiltinRoles()[roleId]
builtin._id = roles.getDBRoleID(builtin._id!)
dbRoles.push(builtin)
}
// now try to find any roles which need updated, e.g. removing the
// resource from another role and then adding to the new role
for (let role of dbRoles) {
let updated = false
const rolePermissions: Record<string, PermissionLevel[]> = role.permissions
? role.permissions
: {}
// make sure its an array, also handle migrating
if (
!rolePermissions[resourceId] ||
!Array.isArray(rolePermissions[resourceId])
) {
rolePermissions[resourceId] =
typeof rolePermissions[resourceId] === "string"
? [rolePermissions[resourceId] as unknown as PermissionLevel]
: []
}
// handle the removal/updating the role which has this permission first
// the updating (role._id !== dbRoleId) is required because a resource/level can
// only be permitted in a single role (this reduces hierarchy confusion and simplifies
// the general UI for this, rather than needing to show everywhere it is used)
if (
(role._id !== dbRoleId || remove) &&
rolePermissions[resourceId].indexOf(level) !== -1
) {
removeFromArray(rolePermissions[resourceId], level)
updated = true
}
// handle the adding, we're on the correct role, at it to this
if (!remove && role._id === dbRoleId) {
const set = new Set(rolePermissions[resourceId])
rolePermissions[resourceId] = [...set.add(level)]
updated = true
}
// handle the update, add it to bulk docs to perform at end
if (updated) {
role.permissions = rolePermissions
docUpdates.push(role)
}
}
const response = await db.bulkDocs(docUpdates)
return response.map(resp => {
const version = docUpdates.find(role => role._id === resp.id)?.version
const _id = roles.getExternalRoleID(resp.id, version)
return {
_id,
rev: resp.rev,
error: resp.error,
reason: resp.reason,
}
})
}
// utility function to stop this repetition - permissions always stored under roles
export async function getAllDBRoles(db: Database) {
const body = await db.allDocs<Role>(
getRoleParams(null, {
include_docs: true,
})
)
return body.rows.map(row => row.doc!)
}

6
packages/server/src/sdk/app/rowActions.ts
View File

@ -1,11 +1,11 @@
import { context, HTTPError, utils } from "@budibase/backend-core"
import { context, docIds, HTTPError, utils } from "@budibase/backend-core"
import {
AutomationTriggerStepId,
SEPARATOR,
TableRowActions,
VirtualDocumentType,
} from "@budibase/types"
import { generateRowActionsID, isViewID } from "../../db/utils"
import { generateRowActionsID } from "../../db/utils"
import automations from "./automations"
import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo"
import * as triggers from "../../automations/triggers"
@ -155,7 +155,7 @@ export async function update(
async function guardView(tableId: string, viewId: string) {
let view
if (isViewID(viewId)) {
if (docIds.isViewId(viewId)) {
view = await sdk.views.get(viewId)
}
if (!view || view.tableId !== tableId) {

72
packages/server/src/sdk/app/rows/external.ts
View File

@ -1,5 +1,11 @@
import { IncludeRelationship, Operation, Row } from "@budibase/types"
import { HTTPError } from "@budibase/backend-core"
import {
IncludeRelationship,
Operation,
Row,
Table,
ViewV2,
} from "@budibase/types"
import { docIds, HTTPError } from "@budibase/backend-core"
import { handleRequest } from "../../../api/controllers/row/external"
import { breakRowIdField } from "../../../integrations/utils"
import sdk from "../../../sdk"
@ -8,15 +14,24 @@ import {
outputProcessing,
} from "../../../utilities/rowProcessor"
import cloneDeep from "lodash/fp/cloneDeep"
import isEqual from "lodash/fp/isEqual"
import { tryExtractingTableAndViewId } from "./utils"
export async function getRow(
tableId: string,
sourceId: string | Table | ViewV2,
rowId: string,
opts?: { relationships?: boolean }
) {
const response = await handleRequest(Operation.READ, tableId, {
let source: Table | ViewV2
if (typeof sourceId === "string") {
if (docIds.isViewId(sourceId)) {
source = await sdk.views.get(sourceId)
} else {
source = await sdk.tables.getTable(sourceId)
}
} else {
source = sourceId
}
const response = await handleRequest(Operation.READ, source, {
id: breakRowIdField(rowId),
includeSqlRelationships: opts?.relationships
? IncludeRelationship.INCLUDE
@ -27,45 +42,42 @@ export async function getRow(
}
export async function save(
tableOrViewId: string,
sourceId: string,
inputs: Row,
userId: string | undefined
) {
const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
const table = await sdk.tables.getTable(tableId)
const { table: updatedTable, row } = await inputProcessing(
userId,
cloneDeep(table),
inputs
)
const { tableId, viewId } = tryExtractingTableAndViewId(sourceId)
let source: Table | ViewV2
if (viewId) {
source = await sdk.views.get(viewId)
} else {
source = await sdk.tables.getTable(tableId)
}
const row = await inputProcessing(userId, cloneDeep(source), inputs)
const validateResult = await sdk.rows.utils.validate({
row,
tableId,
source,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
const response = await handleRequest(Operation.CREATE, tableId, {
const response = await handleRequest(Operation.CREATE, source, {
row,
})
if (!isEqual(table, updatedTable)) {
await sdk.tables.saveTable(updatedTable)
}
const rowId = response.row._id
if (rowId) {
const row = await getRow(tableId, rowId, {
const row = await getRow(source, rowId, {
relationships: true,
})
return {
...response,
row: await outputProcessing(table, row, {
row: await outputProcessing(source, row, {
preserveLinks: true,
squash: true,
fromViewId: viewId,
}),
}
} else {
@ -76,7 +88,14 @@ export async function save(
export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
const row = await getRow(tableId, rowId, {
let source: Table | ViewV2
if (viewId) {
source = await sdk.views.get(viewId)
} else {
source = await sdk.tables.getTable(tableId)
}
const row = await getRow(source, rowId, {
relationships: true,
})
@ -84,11 +103,10 @@ export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
throw new HTTPError("Row not found", 404)
}
const table = await sdk.tables.getTable(tableId)
// Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
return await outputProcessing(table, row, {
// Preserving links, as the outputProcessing does not support external rows
// yet and we don't need it in this use case
return await outputProcessing(source, row, {
squash: true,
preserveLinks: true,
fromViewId: viewId,
})
}

49
packages/server/src/sdk/app/rows/internal.ts
View File

@ -1,7 +1,6 @@
import { context, db } from "@budibase/backend-core"
import { Row } from "@budibase/types"
import { Row, Table, ViewV2 } from "@budibase/types"
import sdk from "../../../sdk"
import cloneDeep from "lodash/fp/cloneDeep"
import { finaliseRow } from "../../../api/controllers/row/staticFormula"
import {
inputProcessing,
@ -10,7 +9,7 @@ import {
import * as linkRows from "../../../db/linkedRows"
import { InternalTables } from "../../../db/utils"
import { getFullUser } from "../../../utilities/users"
import { tryExtractingTableAndViewId } from "./utils"
import { getSource, tryExtractingTableAndViewId } from "./utils"
export async function save(
tableOrViewId: string,
@ -20,21 +19,25 @@ export async function save(
const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
inputs.tableId = tableId
let source: Table | ViewV2
let table: Table
if (viewId) {
source = await sdk.views.get(viewId)
table = await sdk.views.getTable(viewId)
} else {
source = await sdk.tables.getTable(tableId)
table = source
}
if (!inputs._rev && !inputs._id) {
inputs._id = db.generateRowID(inputs.tableId)
}
// this returns the table and row incase they have been updated
const dbTable = await sdk.tables.getTable(inputs.tableId)
// need to copy the table so it can be differenced on way out
const tableClone = cloneDeep(dbTable)
let { table, row } = await inputProcessing(userId, tableClone, inputs)
let row = await inputProcessing(userId, source, inputs)
const validateResult = await sdk.rows.utils.validate({
row,
table,
source,
})
if (!validateResult.valid) {
@ -49,24 +52,18 @@ export async function save(
table,
})) as Row
return finaliseRow(table, row, {
oldTable: dbTable,
updateFormula: true,
fromViewId: viewId,
return finaliseRow(source, row, { updateFormula: true })
}
export async function find(sourceId: string, rowId: string): Promise<Row> {
const source = await getSource(sourceId)
return await outputProcessing(source, await findRow(sourceId, rowId), {
squash: true,
})
}
export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
const table = await sdk.tables.getTable(tableId)
let row = await findRow(tableId, rowId)
row = await outputProcessing(table, row, { squash: true, fromViewId: viewId })
return row
}
async function findRow(tableId: string, rowId: string) {
export async function findRow(sourceId: string, rowId: string) {
const { tableId } = tryExtractingTableAndViewId(sourceId)
const db = context.getAppDB()
let row: Row
// TODO remove special user case in future

20
packages/server/src/sdk/app/rows/queryUtils.ts
View File

@ -18,11 +18,11 @@ export const removeInvalidFilters = (
for (const filterKey of Object.keys(
result || {}
) as (keyof SearchFilters)[]) {
const filter = result[filterKey]
if (!filter || typeof filter !== "object") {
continue
}
if (isLogicalSearchOperator(filterKey)) {
const filter = result[filterKey]
if (!filter || typeof filter !== "object") {
continue
}
const resultingConditions: SearchFilters[] = []
for (const condition of filter.conditions) {
const resultingCondition = removeInvalidFilters(condition, validFields)
@ -38,6 +38,11 @@ export const removeInvalidFilters = (
continue
}
const filter = result[filterKey]
if (!filter || typeof filter !== "object") {
continue
}
for (const columnKey of Object.keys(filter)) {
const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map(
c => c.toLowerCase()
@ -55,8 +60,8 @@ export const removeInvalidFilters = (
}
export const getQueryableFields = async (
fields: string[],
table: Table
table: Table,
fields?: string[]
): Promise<string[]> => {
const extractTableFields = async (
table: Table,
@ -112,6 +117,9 @@ export const getQueryableFields = async (
"_id", // Querying by _id is always allowed, even if it's never part of the schema
]
if (fields == null) {
fields = Object.keys(table.schema)
}
result.push(...(await extractTableFields(table, fields, [table._id!])))
return result

18
packages/server/src/sdk/app/rows/rows.ts
View File

@ -1,10 +1,6 @@
import { db as dbCore, context } from "@budibase/backend-core"
import { db as dbCore, context, docIds } from "@budibase/backend-core"
import { Database, Row } from "@budibase/types"
import {
extractViewInfoFromID,
getRowParams,
isViewID,
} from "../../../db/utils"
import { extractViewInfoFromID, getRowParams } from "../../../db/utils"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./internal"
import * as external from "./external"
@ -26,7 +22,7 @@ export async function getAllInternalRows(appId?: string) {
function pickApi(tableOrViewId: string) {
let tableId = tableOrViewId
if (isViewID(tableOrViewId)) {
if (docIds.isViewId(tableOrViewId)) {
tableId = extractViewInfoFromID(tableOrViewId).tableId
}
@ -37,13 +33,13 @@ function pickApi(tableOrViewId: string) {
}
export async function save(
tableOrViewId: string,
sourceId: string,
row: Row,
userId: string | undefined
) {
return pickApi(tableOrViewId).save(tableOrViewId, row, userId)
return pickApi(sourceId).save(sourceId, row, userId)
}
export async function find(tableOrViewId: string, rowId: string) {
return pickApi(tableOrViewId).find(tableOrViewId, rowId)
export async function find(sourceId: string, rowId: string) {
return pickApi(sourceId).find(sourceId, rowId)
}

110
packages/server/src/sdk/app/rows/search.ts
View File

@ -1,9 +1,16 @@
import {
EmptyFilterOption,
LogicalOperator,
Row,
RowSearchParams,
SearchFilter,
SearchFilterGroup,
SearchFilterKey,
SearchFilters,
SearchResponse,
SortOrder,
Table,
ViewV2,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./search/internal"
@ -12,9 +19,10 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
import { features } from "@budibase/backend-core"
import { db, features } from "@budibase/backend-core"
import tracer from "dd-trace"
import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
import { enrichSearchContext } from "../../../api/controllers/row/utils"
export { isValidFilter } from "../../../integrations/utils"
@ -32,11 +40,13 @@ function pickApi(tableId: any) {
}
export async function search(
options: RowSearchParams
options: RowSearchParams,
context?: Record<string, any>
): Promise<SearchResponse<Row>> {
return await tracer.trace("search", async span => {
span?.addTags({
tableId: options.tableId,
viewId: options.viewId,
query: options.query,
sort: options.sort,
sortOrder: options.sortOrder,
@ -48,20 +58,85 @@ export async function search(
countRows: options.countRows,
})
const isExternalTable = isExternalTableID(options.tableId)
options.query = dataFilters.cleanupQuery(options.query || {})
let source: Table | ViewV2
let table: Table
if (options.viewId) {
source = await sdk.views.get(options.viewId)
table = await sdk.views.getTable(source)
options = searchInputMapping(table, options)
} else if (options.tableId) {
source = await sdk.tables.getTable(options.tableId)
table = source
} else {
throw new Error(`Must supply either a view ID or a table ID`)
}
const isExternalTable = isExternalTableID(table._id!)
if (options.query) {
const visibleFields = (
options.fields || Object.keys(table.schema)
).filter(field => table.schema[field].visible !== false)
const queryableFields = await getQueryableFields(table, visibleFields)
options.query = removeInvalidFilters(options.query, queryableFields)
} else {
options.query = {}
}
if (options.viewId) {
const view = await sdk.views.get(options.viewId)
// Enrich saved query with ephemeral query params.
// We prevent searching on any fields that are saved as part of the query, as
// that could let users find rows they should not be allowed to access.
let viewQuery = dataFilters.buildQuery(view.query || [])
if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
// Lucene does not accept conditional filters, so we need to keep the old logic
const query: SearchFilters = viewQuery || {}
const viewFilters = view.query as SearchFilter[]
// Extract existing fields
const existingFields =
viewFilters
?.filter(filter => filter.field)
.map(filter => db.removeKeyNumbering(filter.field)) || []
// Carry over filters for unused fields
Object.keys(options.query || {}).forEach(key => {
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
Object.keys(options.query[operator] || {}).forEach(field => {
if (!existingFields.includes(db.removeKeyNumbering(field))) {
query[operator]![field] = options.query[operator]![field]
}
})
})
options.query = query
} else {
options.query = {
$and: {
conditions: [viewQuery as SearchFilterGroup, options.query],
},
}
}
}
if (context) {
options.query = await enrichSearchContext(options.query, context)
}
options.query = dataFilters.cleanupQuery(options.query)
options.query = dataFilters.fixupFilterArrays(options.query)
span?.addTags({
span.addTags({
cleanedQuery: options.query,
isExternalTable,
})
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
) {
span?.addTags({ emptyQuery: true })
span.addTags({ emptyQuery: true })
return {
rows: [],
}
@ -71,34 +146,21 @@ export async function search(
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (options.query) {
const tableFields = Object.keys(table.schema).filter(
f => table.schema[f].visible !== false
)
const queriableFields = await getQueryableFields(
options.fields?.filter(f => tableFields.includes(f)) ?? tableFields,
table
)
options.query = removeInvalidFilters(options.query, queriableFields)
}
let result: SearchResponse<Row>
if (isExternalTable) {
span?.addTags({ searchType: "external" })
result = await external.search(options, table)
result = await external.search(options, source)
} else if (await features.flags.isEnabled("SQS")) {
span?.addTags({ searchType: "sqs" })
result = await internal.sqs.search(options, table)
result = await internal.sqs.search(options, source)
} else {
span?.addTags({ searchType: "lucene" })
result = await internal.lucene.search(options, table)
result = await internal.lucene.search(options, source)
}
span?.addTags({
span.addTags({
foundRows: result.rows.length,
totalRows: result.totalRows,
})

31
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -9,6 +9,7 @@ import {
SortJson,
SortOrder,
Table,
ViewV2,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import { handleRequest } from "../../../../api/controllers/row/external"
@ -60,9 +61,8 @@ function getPaginationAndLimitParameters(
export async function search(
options: RowSearchParams,
table: Table
source: Table | ViewV2
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { countRows, paginate, query, ...params } = options
const { limit } = params
let bookmark =
@ -106,16 +106,15 @@ export async function search(
includeSqlRelationships: IncludeRelationship.INCLUDE,
}
const [{ rows, rawResponseSize }, totalRows] = await Promise.all([
handleRequest(Operation.READ, tableId, parameters),
handleRequest(Operation.READ, source, parameters),
countRows
? handleRequest(Operation.COUNT, tableId, parameters)
? handleRequest(Operation.COUNT, source, parameters)
: Promise.resolve(undefined),
])
let processed = await outputProcessing(table, rows, {
let processed = await outputProcessing(source, rows, {
preserveLinks: true,
squash: true,
fromViewId: options.viewId,
})
let hasNextPage = false
@ -128,10 +127,13 @@ export async function search(
}
}
if (options.fields) {
const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS]
processed = processed.map((r: any) => pick(r, fields))
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS]
processed = processed.map((r: any) => pick(r, allowedFields))
// need wrapper object for bookmarks etc when paginating
const response: SearchResponse<Row> = { rows: processed, hasNextPage }
@ -201,7 +203,7 @@ export async function exportRows(
}
let result = await search(
{ tableId, query: requestQuery, sort, sortOrder },
{ tableId: table._id!, query: requestQuery, sort, sortOrder },
table
)
let rows: Row[] = []
@ -257,10 +259,10 @@ export async function exportRows(
}
export async function fetch(tableId: string): Promise<Row[]> {
const response = await handleRequest(Operation.READ, tableId, {
const table = await sdk.tables.getTable(tableId)
const response = await handleRequest(Operation.READ, table, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
const table = await sdk.tables.getTable(tableId)
return await outputProcessing(table, response.rows, {
preserveLinks: true,
squash: true,
@ -268,7 +270,8 @@ export async function fetch(tableId: string): Promise<Row[]> {
}
export async function fetchRaw(tableId: string): Promise<Row[]> {
const response = await handleRequest(Operation.READ, tableId, {
const table = await sdk.tables.getTable(tableId)
const response = await handleRequest(Operation.READ, table, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
return response.rows

28
packages/server/src/sdk/app/rows/search/internal/lucene.ts
View File

@ -8,21 +8,29 @@ import {
SortType,
Table,
User,
ViewV2,
} from "@budibase/types"
import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
import { outputProcessing } from "../../../../../utilities/rowProcessor"
import pick from "lodash/pick"
import sdk from "../../../../"
export async function search(
options: RowSearchParams,
table: Table
source: Table | ViewV2
): Promise<SearchResponse<Row>> {
const { tableId } = options
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
const { paginate, query } = options
const params: RowSearchParams = {
tableId: options.tableId,
viewId: options.viewId,
sort: options.sort,
sortOrder: options.sortOrder,
sortType: options.sortType,
@ -50,18 +58,20 @@ export async function search(
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
if (table._id === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
if (options.fields) {
const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, fields))
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, allowedFields))
response.rows = await outputProcessing(table, response.rows, {
response.rows = await outputProcessing(source, response.rows, {
squash: true,
fromViewId: options.viewId,
})
}

Some files were not shown because too many files have changed in this diff Show More