MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v3-ui' of github.com:Budibase/budibase into view-calculation-ui

This commit is contained in:
Andrew Kingston 2024-10-18 10:00:49 +01:00
parent 21383cf1a0 02a60ba55d
commit 18a207f87a
No known key found for this signature in database
83 changed files with 1604 additions and 392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.github/workflows/deploy-featurebranch.yml vendored
View File

@ -2,13 +2,11 @@ name: deploy-featurebranch
on:
pull_request:
types: [
labeled,
# default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
opened,
synchronize,
reopened,
]
types:
- labeled
- opened
- synchronize
- reopened
jobs:
release:
@ -22,31 +20,21 @@ jobs:
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
)
runs-on: ubuntu-latest
env:
PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: |
${{
contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
}}
steps:
- uses: actions/checkout@v4
- name: Set PAYLOAD_LICENSE_TYPE
id: set_license_type
run: |
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
else
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
fi
- uses: passeidireto/trigger-external-workflow-action@main
env:
PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
with:
repository: budibase/budibase-deploys
event: featurebranch-qa-deploy

10
charts/budibase/templates/app-service-deployment.yaml
View File

@ -42,14 +42,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
{{ else }}
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

6
charts/budibase/templates/automation-worker-service-deployment.yaml
View File

@ -43,6 +43,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

10
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -56,14 +56,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
{{ else }}
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
{{ end }}
- name: API_ENCRYPTION_KEY
valueFrom:
secretKeyRef:

3
charts/budibase/values.yaml
View File

@ -139,9 +139,6 @@ globals:
password: ""
sqs:
# -- Whether to use the CouchDB "structured query service" or not. This is disabled by
# default for now, but will become the default in a future release.
enabled: false
# @ignore
url: ""
# @ignore

6
hosting/docker-compose.yaml
View File

@ -5,7 +5,7 @@ version: "3"
services:
app-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/apps
image: budibase/apps
container_name: bbapps
environment:
SELF_HOSTED: 1
@ -35,7 +35,7 @@ services:
worker-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/worker
image: budibase/worker
container_name: bbworker
environment:
SELF_HOSTED: 1
@ -97,7 +97,7 @@ services:
couchdb-service:
restart: unless-stopped
image: budibase/couchdb
image: budibase/couchdb:v3.3.3-sqs-v2.1.1
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}

3
hosting/single/Dockerfile
View File

@ -69,6 +69,9 @@ WORKDIR /minio
COPY scripts/install-minio.sh ./install.sh
RUN chmod +x install.sh && ./install.sh
# setup redis
COPY hosting/single/redis.conf /etc/redis/redis.conf
# setup runner file
WORKDIR /
COPY hosting/single/runner.sh .

7
hosting/single/redis.conf Normal file
View File

@ -0,0 +1,7 @@
dir "DATA_DIR/redis"
appendonly yes
appendfsync everysec
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb

8
hosting/single/runner.sh
View File

@ -75,13 +75,17 @@ fi
for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
ln -s ${DATA_DIR}/.env /app/.env
ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
mkdir -p ${DATA_DIR}/minio
mkdir -p ${DATA_DIR}/redis
chown -R couchdb:couchdb ${DATA_DIR}/couch
sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
if [[ -n "${REDIS_PASSWORD}" ]]; then
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
else
redis-server > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
fi
/bbcouch-runner.sh &

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.17",
"version": "2.33.2",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/backend-core/src/constants/misc.ts
View File

@ -28,6 +28,7 @@ export enum Config {
OIDC = "oidc",
OIDC_LOGOS = "logos_oidc",
SCIM = "scim",
AI = "AI",
}
export const MIN_VALID_DATE = new Date(-2147483647000)

50
packages/backend-core/src/environment.ts
View File

@ -54,30 +54,46 @@ function getPackageJsonFields(): {
VERSION: string
SERVICE_NAME: string
} {
function findFileInAncestors(
fileName: string,
currentDir: string
): string | null {
const filePath = `${currentDir}/${fileName}`
if (existsSync(filePath)) {
return filePath
function getParentFile(file: string) {
function findFileInAncestors(
fileName: string,
currentDir: string
): string | null {
const filePath = `${currentDir}/${fileName}`
if (existsSync(filePath)) {
return filePath
}
const parentDir = `${currentDir}/..`
if (parentDir === currentDir) {
// reached root directory
return null
}
return findFileInAncestors(fileName, parentDir)
}
const parentDir = `${currentDir}/..`
if (parentDir === currentDir) {
// reached root directory
return null
}
const packageJsonFile = findFileInAncestors(file, process.cwd())
const content = readFileSync(packageJsonFile!, "utf-8")
const parsedContent = JSON.parse(content)
return parsedContent
}
return findFileInAncestors(fileName, parentDir)
let localVersion: string | undefined
if (isDev() && !isTest()) {
try {
const lerna = getParentFile("lerna.json")
localVersion = lerna.version
} catch {
//
}
}
try {
const packageJsonFile = findFileInAncestors("package.json", process.cwd())
const content = readFileSync(packageJsonFile!, "utf-8")
const parsedContent = JSON.parse(content)
const parsedContent = getParentFile("package.json")
return {
VERSION: process.env.BUDIBASE_VERSION || parsedContent.version,
VERSION:
localVersion || process.env.BUDIBASE_VERSION || parsedContent.version,
SERVICE_NAME: parsedContent.name,
}
} catch {

2
packages/backend-core/src/features/features.ts
View File

@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
SQS: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
[FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

6
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -10,6 +10,7 @@ const schema = {
TEST_BOOLEAN: Flag.boolean(false),
TEST_STRING: Flag.string("default value"),
TEST_NUMBER: Flag.number(0),
TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
}
const flags = new FlagSet(schema)
@ -123,6 +124,11 @@ describe("feature flags", () => {
},
expected: flags.defaults(),
},
{
it: "should be possible to override a default true flag to false",
environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
},
])(
"$it",
async ({

51
packages/backend-core/src/security/roles.ts
View File

@ -1,3 +1,4 @@
import semver from "semver"
import { BuiltinPermissionID, PermissionLevel } from "./permissions"
import {
prefixRoleID,
@ -7,7 +8,13 @@ import {
doWithDB,
} from "../db"
import { getAppDB } from "../context"
import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types"
import {
Screen,
Role as RoleDoc,
RoleUIMetadata,
Database,
App,
} from "@budibase/types"
import cloneDeep from "lodash/fp/cloneDeep"
import { RoleColor } from "@budibase/shared-core"
@ -23,14 +30,6 @@ const BUILTIN_IDS = {
BUILDER: "BUILDER",
}
// exclude internal roles like builder
const EXTERNAL_BUILTIN_ROLE_IDS = [
BUILTIN_IDS.ADMIN,
BUILTIN_IDS.POWER,
BUILTIN_IDS.BASIC,
BUILTIN_IDS.PUBLIC,
]
export const RoleIDVersion = {
// original version, with a UUID based ID
UUID: undefined,
@ -319,7 +318,7 @@ export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
}
return internal(appDB)
}
async function internal(db: any) {
async function internal(db: Database | undefined) {
let roles: RoleDoc[] = []
if (db) {
const body = await db.allDocs(
@ -334,8 +333,26 @@ export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
}
const builtinRoles = getBuiltinRoles()
// exclude internal roles like builder
let externalBuiltinRoles = []
if (!db || (await shouldIncludePowerRole(db))) {
externalBuiltinRoles = [
BUILTIN_IDS.ADMIN,
BUILTIN_IDS.POWER,
BUILTIN_IDS.BASIC,
BUILTIN_IDS.PUBLIC,
]
} else {
externalBuiltinRoles = [
BUILTIN_IDS.ADMIN,
BUILTIN_IDS.BASIC,
BUILTIN_IDS.PUBLIC,
]
}
// need to combine builtin with any DB record of them (for sake of permissions)
for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) {
for (let builtinRoleId of externalBuiltinRoles) {
const builtinRole = builtinRoles[builtinRoleId]
const dbBuiltin = roles.filter(
dbRole =>
@ -366,6 +383,18 @@ export async function getAllRoles(appId?: string): Promise<RoleDoc[]> {
}
}
async function shouldIncludePowerRole(db: Database) {
const app = await db.tryGet<App>(DocumentType.APP_METADATA)
const creationVersion = app?.creationVersion
if (!creationVersion || !semver.valid(creationVersion)) {
// Old apps don't have creationVersion, so we should include it for backward compatibility
return true
}
const isGreaterThan3x = semver.gte(creationVersion, "3.0.0")
return !isGreaterThan3x
}
export class AccessController {
userHierarchies: { [key: string]: string[] }
constructor() {

126
packages/backend-core/src/sql/sql.ts
View File

@ -23,12 +23,14 @@ import {
InternalSearchFilterOperator,
JsonFieldMetadata,
JsonTypes,
LogicalOperator,
Operation,
prefixed,
QueryJson,
QueryOptions,
RangeOperator,
RelationshipsJson,
SearchFilterKey,
SearchFilters,
SortOrder,
SqlClient,
@ -96,6 +98,22 @@ function isSqs(table: Table): boolean {
)
}
const allowEmptyRelationships: Record<SearchFilterKey, boolean> = {
[BasicOperator.EQUAL]: false,
[BasicOperator.NOT_EQUAL]: true,
[BasicOperator.EMPTY]: false,
[BasicOperator.NOT_EMPTY]: true,
[BasicOperator.FUZZY]: false,
[BasicOperator.STRING]: false,
[RangeOperator.RANGE]: false,
[ArrayOperator.CONTAINS]: false,
[ArrayOperator.NOT_CONTAINS]: true,
[ArrayOperator.CONTAINS_ANY]: false,
[ArrayOperator.ONE_OF]: false,
[LogicalOperator.AND]: false,
[LogicalOperator.OR]: false,
}
class InternalBuilder {
private readonly client: SqlClient
private readonly query: QueryJson
@ -405,31 +423,48 @@ class InternalBuilder {
addRelationshipForFilter(
query: Knex.QueryBuilder,
allowEmptyRelationships: boolean,
filterKey: string,
whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder
): Knex.QueryBuilder {
const mainKnex = this.knex
const { relationships, endpoint, tableAliases: aliases } = this.query
const tableName = endpoint.entityId
const fromAlias = aliases?.[tableName] || tableName
const matches = (possibleTable: string) =>
filterKey.startsWith(`${possibleTable}`)
const matches = (value: string) =>
filterKey.match(new RegExp(`^${value}\\.`))
if (!relationships) {
return query
}
for (const relationship of relationships) {
const relatedTableName = relationship.tableName
const toAlias = aliases?.[relatedTableName] || relatedTableName
const matchesTableName = matches(relatedTableName) || matches(toAlias)
const matchesRelationName = matches(relationship.column)
// this is the relationship which is being filtered
if (
(matches(relatedTableName) || matches(toAlias)) &&
(matchesTableName || matchesRelationName) &&
relationship.to &&
relationship.tableName
) {
let subQuery = mainKnex
const joinTable = mainKnex
.select(mainKnex.raw(1))
.from({ [toAlias]: relatedTableName })
let subQuery = joinTable.clone()
const manyToMany = validateManyToMany(relationship)
let updatedKey
if (!matchesTableName) {
updatedKey = filterKey.replace(
new RegExp(`^${relationship.column}.`),
`${aliases![relationship.tableName]}.`
)
} else {
updatedKey = filterKey
}
if (manyToMany) {
const throughAlias =
aliases?.[manyToMany.through] || relationship.through
@ -440,7 +475,6 @@ class InternalBuilder {
subQuery = subQuery
// add a join through the junction table
.innerJoin(throughTable, function () {
// @ts-ignore
this.on(
`${toAlias}.${manyToMany.toPrimary}`,
"=",
@ -460,18 +494,38 @@ class InternalBuilder {
if (this.client === SqlClient.SQL_LITE) {
subQuery = this.addJoinFieldCheck(subQuery, manyToMany)
}
query = query.where(q => {
q.whereExists(whereCb(updatedKey, subQuery))
if (allowEmptyRelationships) {
q.orWhereNotExists(
joinTable.clone().innerJoin(throughTable, function () {
this.on(
`${fromAlias}.${manyToMany.fromPrimary}`,
"=",
`${throughAlias}.${manyToMany.from}`
)
})
)
}
})
} else {
const toKey = `${toAlias}.${relationship.to}`
const foreignKey = `${fromAlias}.${relationship.from}`
// "join" to the main table, making sure the ID matches that of the main
subQuery = subQuery.where(
`${toAlias}.${relationship.to}`,
toKey,
"=",
mainKnex.raw(
this.quotedIdentifier(`${fromAlias}.${relationship.from}`)
)
mainKnex.raw(this.quotedIdentifier(foreignKey))
)
query = query.where(q => {
q.whereExists(whereCb(updatedKey, subQuery.clone()))
if (allowEmptyRelationships) {
q.orWhereNotExists(subQuery)
}
})
}
query = query.whereExists(whereCb(subQuery))
break
}
}
return query
@ -502,6 +556,7 @@ class InternalBuilder {
}
function iterate(
structure: AnySearchFilter,
operation: SearchFilterKey,
fn: (
query: Knex.QueryBuilder,
key: string,
@ -521,8 +576,11 @@ class InternalBuilder {
const [filterTableName, ...otherProperties] = key.split(".")
const property = otherProperties.join(".")
const alias = getTableAlias(filterTableName)
return fn(q, alias ? `${alias}.${property}` : property, value)
return q.andWhere(subquery =>
fn(subquery, alias ? `${alias}.${property}` : property, value)
)
}
for (const key in structure) {
const value = structure[key]
const updatedKey = dbCore.removeKeyNumbering(key)
@ -552,9 +610,17 @@ class InternalBuilder {
value
)
} else if (shouldProcessRelationship) {
query = builder.addRelationshipForFilter(query, updatedKey, q => {
return handleRelationship(q, updatedKey, value)
})
if (allOr) {
query = query.or
}
query = builder.addRelationshipForFilter(
query,
allowEmptyRelationships[operation],
updatedKey,
(updatedKey, q) => {
return handleRelationship(q, updatedKey, value)
}
)
}
}
}
@ -586,7 +652,7 @@ class InternalBuilder {
return `[${value.join(",")}]`
}
if (this.client === SqlClient.POSTGRES) {
iterate(mode, (q, key, value) => {
iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
const wrap = any ? "" : "'"
const op = any ? "\\?| array" : "@>"
const fieldNames = key.split(/\./g)
@ -604,7 +670,7 @@ class InternalBuilder {
this.client === SqlClient.MARIADB
) {
const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
iterate(mode, (q, key, value) => {
iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
return q[rawFnc](
`${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
value
@ -613,7 +679,7 @@ class InternalBuilder {
})
} else {
const andOr = mode === filters?.containsAny ? " OR " : " AND "
iterate(mode, (q, key, value) => {
iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => {
let statement = ""
const identifier = this.quotedIdentifier(key)
for (let i in value) {
@ -667,6 +733,7 @@ class InternalBuilder {
const fnc = allOr ? "orWhereIn" : "whereIn"
iterate(
filters.oneOf,
ArrayOperator.ONE_OF,
(q, key: string, array) => {
if (this.client === SqlClient.ORACLE) {
key = this.convertClobs(key)
@ -691,7 +758,7 @@ class InternalBuilder {
)
}
if (filters.string) {
iterate(filters.string, (q, key, value) => {
iterate(filters.string, BasicOperator.STRING, (q, key, value) => {
const fnc = allOr ? "orWhere" : "where"
// postgres supports ilike, nothing else does
if (this.client === SqlClient.POSTGRES) {
@ -706,10 +773,10 @@ class InternalBuilder {
})
}
if (filters.fuzzy) {
iterate(filters.fuzzy, like)
iterate(filters.fuzzy, BasicOperator.FUZZY, like)
}
if (filters.range) {
iterate(filters.range, (q, key, value) => {
iterate(filters.range, RangeOperator.RANGE, (q, key, value) => {
const isEmptyObject = (val: any) => {
return (
val &&
@ -775,7 +842,7 @@ class InternalBuilder {
})
}
if (filters.equal) {
iterate(filters.equal, (q, key, value) => {
iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => {
const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) {
return q[fnc](
@ -795,7 +862,7 @@ class InternalBuilder {
})
}
if (filters.notEqual) {
iterate(filters.notEqual, (q, key, value) => {
iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => {
const fnc = allOr ? "orWhereRaw" : "whereRaw"
if (this.client === SqlClient.MS_SQL) {
return q[fnc](
@ -816,13 +883,13 @@ class InternalBuilder {
})
}
if (filters.empty) {
iterate(filters.empty, (q, key) => {
iterate(filters.empty, BasicOperator.EMPTY, (q, key) => {
const fnc = allOr ? "orWhereNull" : "whereNull"
return q[fnc](key)
})
}
if (filters.notEmpty) {
iterate(filters.notEmpty, (q, key) => {
iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => {
const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
return q[fnc](key)
})
@ -1218,12 +1285,10 @@ class InternalBuilder {
})
: undefined
if (!throughTable) {
// @ts-ignore
query = query.leftJoin(toTableWithSchema, function () {
for (let relationship of columns) {
const from = relationship.from,
to = relationship.to
// @ts-ignore
this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
}
})
@ -1234,7 +1299,6 @@ class InternalBuilder {
for (let relationship of columns) {
const fromPrimary = relationship.fromPrimary
const from = relationship.from
// @ts-ignore
this.orOn(
`${fromAlias}.${fromPrimary}`,
"=",
@ -1246,7 +1310,6 @@ class InternalBuilder {
for (let relationship of columns) {
const toPrimary = relationship.toPrimary
const to = relationship.to
// @ts-ignore
this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
}
})
@ -1288,7 +1351,8 @@ class InternalBuilder {
schema.constraints?.presence === true ||
schema.type === FieldType.FORMULA ||
schema.type === FieldType.AUTO ||
schema.type === FieldType.LINK
schema.type === FieldType.LINK ||
schema.type === FieldType.AI
) {
continue
}

5
packages/backend-core/src/sql/sqlTable.ts
View File

@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
return ignored.indexOf(type) !== -1
}
@ -144,6 +144,9 @@ function generateSchema(
case FieldType.FORMULA:
// This is allowed, but nothing to do on the external datasource
break
case FieldType.AI:
// This is allowed, but nothing to do on the external datasource
break
case FieldType.ATTACHMENTS:
case FieldType.ATTACHMENT_SINGLE:
case FieldType.SIGNATURE_SINGLE:

8
packages/backend-core/tests/core/utilities/mocks/licenses.ts
View File

@ -102,6 +102,14 @@ export const useAppBuilders = () => {
return useFeature(Feature.APP_BUILDERS)
}
export const useBudibaseAI = () => {
return useFeature(Feature.BUDIBASE_AI)
}
export const useAICustomConfigs = () => {
return useFeature(Feature.AI_CUSTOM_CONFIGS)
}
// QUOTAS
export const setAutomationLogsQuota = (value: number) => {

83
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -26,6 +26,7 @@
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
import { licensing } from "stores/portal"
import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
import {
FIELDS,
@ -35,6 +36,7 @@
} from "constants/backend"
import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
import ConfirmDialog from "components/common/ConfirmDialog.svelte"
import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte"
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
@ -50,18 +52,13 @@
import { isEnabled } from "helpers/featureFlags"
import { getUserBindings } from "dataBinding"
const AUTO_TYPE = FieldType.AUTO
const FORMULA_TYPE = FieldType.FORMULA
const LINK_TYPE = FieldType.LINK
const STRING_TYPE = FieldType.STRING
const NUMBER_TYPE = FieldType.NUMBER
const JSON_TYPE = FieldType.JSON
const DATE_TYPE = FieldType.DATETIME
export let field
const dispatch = createEventDispatcher()
const { dispatch: gridDispatch, rows } = getContext("grid")
export let field
const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}`
const SingleUserDefault = `{{ ${SafeID} }}`
const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}`
let mounted = false
let originalName
@ -104,13 +101,15 @@
let optionsValid = true
$: rowGoldenSample = RowUtils.generateGoldenSample($rows)
$: aiEnabled =
$licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
$: if (primaryDisplay) {
editableColumn.constraints.presence = { allowEmpty: false }
}
$: {
// this parses any changes the user has made when creating a new internal relationship
// into what we expect the schema to look like
if (editableColumn.type === LINK_TYPE) {
if (editableColumn.type === FieldType.LINK) {
relationshipTableIdPrimary = table._id
if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
relationshipOpts2 = relationshipOpts2.filter(
@ -147,7 +146,7 @@
UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
$: invalid =
!editableColumn?.name ||
(editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) ||
(editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) ||
Object.keys(errors).length !== 0 ||
!optionsValid
$: errors = checkErrors(editableColumn)
@ -173,9 +172,9 @@
$: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
$: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
$: canBeRequired =
editableColumn?.type !== LINK_TYPE &&
editableColumn?.type !== FieldType.LINK &&
!uneditable &&
editableColumn?.type !== AUTO_TYPE &&
editableColumn?.type !== FieldType.AUTO &&
!editableColumn.autocolumn
$: hasDefault =
editableColumn?.default != null && editableColumn?.default !== ""
@ -224,7 +223,7 @@
function makeFieldId(type, subtype, autocolumn) {
// don't make field IDs for auto types
if (type === AUTO_TYPE || autocolumn) {
if (type === FieldType.AUTO || autocolumn) {
return type.toUpperCase()
} else if (
type === FieldType.BB_REFERENCE ||
@ -249,7 +248,7 @@
// Here we are setting the relationship values based on the editableColumn
// This part of the code is used when viewing an existing field hence the check
// for the tableId
if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
if (editableColumn.type === FieldType.LINK && editableColumn.tableId) {
relationshipTableIdPrimary = table._id
relationshipTableIdSecondary = editableColumn.tableId
if (editableColumn.relationshipType in relationshipMap) {
@ -290,14 +289,14 @@
delete saveColumn.fieldId
if (saveColumn.type === AUTO_TYPE) {
if (saveColumn.type === FieldType.AUTO) {
saveColumn = buildAutoColumn(
$tables.selected.name,
saveColumn.name,
saveColumn.subtype
)
}
if (saveColumn.type !== LINK_TYPE) {
if (saveColumn.type !== FieldType.LINK) {
delete saveColumn.fieldName
}
@ -384,9 +383,9 @@
editableColumn.subtype = definition.subtype
// Default relationships many to many
if (editableColumn.type === LINK_TYPE) {
if (editableColumn.type === FieldType.LINK) {
editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
} else if (editableColumn.type === FORMULA_TYPE) {
} else if (editableColumn.type === FieldType.FORMULA) {
editableColumn.formulaType = "dynamic"
}
}
@ -452,6 +451,7 @@
FIELDS.BOOLEAN,
FIELDS.DATETIME,
FIELDS.LINK,
...(aiEnabled ? [FIELDS.AI] : []),
FIELDS.LONGFORM,
FIELDS.USER,
FIELDS.USERS,
@ -505,17 +505,23 @@
fieldToCheck.constraints = {}
}
// some string types may have been built by server, may not always have constraints
if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) {
if (
fieldToCheck.type === FieldType.STRING &&
!fieldToCheck.constraints.length
) {
fieldToCheck.constraints.length = {}
}
// some number types made server-side will be missing constraints
if (
fieldToCheck.type === NUMBER_TYPE &&
fieldToCheck.type === FieldType.NUMBER &&
!fieldToCheck.constraints.numericality
) {
fieldToCheck.constraints.numericality = {}
}
if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
if (
fieldToCheck.type === FieldType.DATETIME &&
!fieldToCheck.constraints.datetime
) {
fieldToCheck.constraints.datetime = {}
}
}
@ -590,13 +596,13 @@
on:input={e => {
if (
!uneditable &&
!(linkEditDisabled && editableColumn.type === LINK_TYPE)
!(linkEditDisabled && editableColumn.type === FieldType.LINK)
) {
editableColumn.name = e.target.value
}
}}
disabled={uneditable ||
(linkEditDisabled && editableColumn.type === LINK_TYPE)}
(linkEditDisabled && editableColumn.type === FieldType.LINK)}
error={errors?.name}
/>
{/if}
@ -610,7 +616,7 @@
getOptionValue={field => field.fieldId}
getOptionIcon={field => field.icon}
isOptionEnabled={option => {
if (option.type === AUTO_TYPE) {
if (option.type === FieldType.AUTO) {
return availableAutoColumnKeys?.length > 0
}
return true
@ -653,7 +659,7 @@
bind:optionColors={editableColumn.optionColors}
bind:valid={optionsValid}
/>
{:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
{:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
<div class="split-label">
<div class="label-length">
<Label size="M">Earliest</Label>
@ -740,7 +746,7 @@
{tableOptions}
{errors}
/>
{:else if editableColumn.type === FORMULA_TYPE}
{:else if editableColumn.type === FieldType.FORMULA}
{#if !externalTable}
<div class="split-label">
<div class="label-length">
@ -783,12 +789,19 @@
/>
</div>
</div>
{:else if editableColumn.type === JSON_TYPE}
{:else if editableColumn.type === FieldType.AI}
<AIFieldConfiguration
aiField={editableColumn}
context={rowGoldenSample}
bindings={getBindings({ table })}
schema={table.schema}
/>
{:else if editableColumn.type === FieldType.JSON}
<Button primary text on:click={openJsonSchemaEditor}>
Open schema editor
</Button>
{/if}
{#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
{#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn}
<Select
label="Auto column type"
value={editableColumn.subtype}
@ -835,6 +848,18 @@
(editableColumn.default = e.detail?.length ? e.detail : undefined)}
placeholder="None"
/>
{:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
{@const defaultValue =
editableColumn.type === FieldType.BB_REFERENCE_SINGLE
? SingleUserDefault
: MultiUserDefault}
<Toggle
disabled={!canHaveDefault}
text="Default to current user"
value={editableColumn.default === defaultValue}
on:change={e =>
(editableColumn.default = e.detail ? defaultValue : undefined)}
/>
{:else}
<ModalBindableInput
disabled={!canHaveDefault}

3
packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte
View File

@ -7,6 +7,7 @@
import { FIELDS } from "constants/backend"
const FORMULA_TYPE = FIELDS.FORMULA.type
const AI_TYPE = FIELDS.AI.type
export let row = {}
@ -60,7 +61,7 @@
}}
>
{#each tableSchema as [key, meta]}
{#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
{#if !meta.autocolumn && meta.type !== FORMULA_TYPE && meta.type !== AI_TYPE}
<div>
<RowFieldControl error={errors[key]} {meta} bind:value={row[key]} />
</div>

59
packages/builder/src/components/common/AIFieldConfiguration.svelte Normal file
View File

@ -0,0 +1,59 @@
<script>
import { Helpers, Multiselect, Select } from "@budibase/bbui"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import {
AIOperations,
OperationFields,
OperationFieldTypes,
} from "@budibase/shared-core"
const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
label: AIOperations[key].label,
value: AIOperations[key].value,
}))
export let bindings
export let context
export let schema
export let aiField = {}
$: OperationField = OperationFields[aiField.operation]
$: schemaWithoutRelations = Object.keys(schema).filter(
key => schema[key].type !== "link"
)
</script>
<Select
label={"Operation"}
options={AIFieldConfigOptions}
bind:value={aiField.operation}
/>
{#if aiField.operation}
{#each Object.keys(OperationField) as key}
{#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
<ModalBindableInput
label={Helpers.capitalise(key)}
panel={ServerBindingPanel}
title="Prompt"
on:change={e => (aiField[key] = e.detail)}
value={aiField[key]}
{bindings}
allowJS
{context}
/>
{:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
<Multiselect
bind:value={aiField[key]}
label={Helpers.capitalise(key)}
options={schemaWithoutRelations}
/>
{:else if OperationField[key] === OperationFieldTypes.COLUMN}
<Select
bind:value={aiField[key]}
label={Helpers.capitalise(key)}
options={schemaWithoutRelations}
/>
{/if}
{/each}
{/if}

1
packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
View File

@ -59,6 +59,7 @@
bind:this={drawer}
title="Filtering"
on:drawerHide
on:drawerShow
on:drawerShow={() => {
// Reset to the currently available value.
localFilters = Helpers.cloneDeep(value)

6
packages/builder/src/constants/backend/index.js
View File

@ -159,6 +159,12 @@ export const FIELDS = {
icon: TypeIconMap[FieldType.FORMULA],
constraints: {},
},
AI: {
name: "AI",
type: FieldType.AI,
icon: TypeIconMap[FieldType.AI],
constraints: {},
},
JSON: {
name: "JSON",
type: FieldType.JSON,

1
packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte
View File

@ -23,6 +23,7 @@
sourceType: DB_TYPE_EXTERNAL,
schema: {
id: {
name: "id",
autocolumn: true,
type: "number",
},

3
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
View File

@ -1,6 +1,6 @@
<script>
import { viewsV2, rowActions } from "stores/builder"
import { admin, themeStore } from "stores/portal"
import { admin, themeStore, licensing } from "stores/portal"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import { notifications } from "@budibase/bbui"
@ -53,6 +53,7 @@
{buttons}
allowAddRows
allowDeleteRows
aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
showAvatars={false}
on:updatedatasource={handleGridViewUpdate}
isCloud={$admin.cloud}

4
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
View File

@ -7,7 +7,7 @@
appStore,
rowActions,
} from "stores/builder"
import { themeStore, admin } from "stores/portal"
import { themeStore, admin, licensing } from "stores/portal"
import { TableNames } from "constants"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
@ -125,6 +125,8 @@
schemaOverrides={isUsersTable ? userSchemaOverrides : null}
showAvatars={false}
isCloud={$admin.cloud}
aiEnabled={$licensing.budibaseAIEnabled ||
$licensing.customAIConfigsEnabled}
{buttons}
buttonsCollapsed
on:updatedatasource={handleGridTableUpdate}

14
packages/builder/src/stores/builder/viewsV2.js
View File

@ -6,16 +6,22 @@ import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view?.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
return {
...view,
queryUI: view.query,
query: dataFilters.buildQuery(view.query),
}
}
return view
}
function convertToSearchFilterGroup(view) {
if (view?.queryUI) {
view.query = view.queryUI
delete view.queryUI
return {
...view,
query: view.queryUI,
queryUI: undefined,
}
}
return view
}

3
packages/client/src/components/app/DataProvider.svelte
View File

@ -126,6 +126,9 @@
}
const extendQuery = (defaultQuery, extensions) => {
if (!Object.keys(extensions).length) {
return defaultQuery
}
const extended = {
[LogicalOperator.AND]: {
conditions: [

2
packages/client/src/components/app/GridBlock.svelte
View File

@ -3,6 +3,7 @@
// because it functions similarly to one
import { getContext, onMount } from "svelte"
import { get, derived, readable } from "svelte/store"
import { featuresStore } from "stores"
import { Grid } from "@budibase/frontend-core"
// table is actually any datasource, but called table for legacy compatibility
@ -186,6 +187,7 @@
{buttonsCollapsed}
{buttonsCollapsedText}
isCloud={$environmentStore.cloud}
aiEnabled={$featuresStore.aiEnabled}
on:rowclick={e => onRowClick?.({ row: e.detail })}
/>
</div>

9
packages/client/src/components/app/container/GridContainer.svelte
View File

@ -4,6 +4,8 @@
import { GridRowHeight, GridColumns } from "constants"
import { memo } from "@budibase/frontend-core"
export let onClick
const component = getContext("component")
const { styleable, builderStore } = getContext("sdk")
const context = getContext("context")
@ -121,15 +123,19 @@
})
</script>
<!-- svelte-ignore a11y-no-static-element-interactions -->
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div
bind:this={ref}
class="grid"
class:mobile
class:clickable={!!onClick}
bind:clientWidth={width}
bind:clientHeight={height}
use:styleable={$styles}
data-cols={GridColumns}
data-col-size={colSize}
on:click={onClick}
>
{#if inBuilder}
<div class="underlay">
@ -176,6 +182,9 @@
.placeholder.first-col {
border-left: 1px solid var(--spectrum-global-color-gray-900);
}
.clickable {
cursor: pointer;
}
/* Highlight grid lines when resizing children */
:global(.grid.highlight > .underlay) {

6
packages/client/src/stores/features.js
View File

@ -2,6 +2,7 @@ import { derived } from "svelte/store"
import { appStore } from "./app"
import { authStore } from "./auth"
import { Constants } from "@budibase/frontend-core"
import { Feature } from "@budibase/types"
const createFeaturesStore = () => {
return derived([authStore, appStore], ([$authStore, $appStore]) => {
@ -33,8 +34,13 @@ const createFeaturesStore = () => {
}
}
const license = getUserLicense()
return {
logoEnabled: isFreePlan(),
aiEnabled:
license?.features?.includes(Feature.AI_CUSTOM_CONFIGS) ||
license?.features?.includes(Feature.BUDIBASE_AI),
}
})
}

2
packages/frontend-core/src/components/FilterField.svelte
View File

@ -174,7 +174,7 @@
/>
{:else}
<div>
{#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
{#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
<Input
disabled={filter.noValue}
value={readableValue}

35
packages/frontend-core/src/components/Updating.svelte
View File

@ -2,36 +2,31 @@
export let isMigrationDone
export let onMigrationDone
export let timeoutSeconds = 60 // 1 minute
export let minTimeSeconds = 3
const loadTime = Date.now()
const intervalMs = 1000
let timedOut = false
let secondsWaited = 0
async function checkMigrationsFinished() {
setTimeout(async () => {
let totalWaitMs = 0
// eslint-disable-next-line no-constant-condition
while (true) {
const waitForMs = 5000 + Math.random() * 5000
await new Promise(resolve => setTimeout(resolve, waitForMs))
totalWaitMs += waitForMs
const isMigrated = await isMigrationDone()
const timeoutMs = timeoutSeconds * 1000
if (!isMigrated || secondsWaited <= minTimeSeconds) {
if (loadTime + timeoutMs > Date.now()) {
secondsWaited += 1
return checkMigrationsFinished()
}
return migrationTimeout()
if (isMigrated) {
onMigrationDone()
return
}
onMigrationDone()
}, intervalMs)
if (totalWaitMs > timeoutSeconds * 1000) {
timedOut = true
return
}
}
}
checkMigrationsFinished()
function migrationTimeout() {
timedOut = true
}
</script>
<div class="loading" class:timeout={timedOut}>

99
packages/frontend-core/src/components/grid/cells/AICell.svelte Normal file
View File

@ -0,0 +1,99 @@
<script>
import { onMount } from "svelte"
import { clickOutside } from "@budibase/bbui"
import GridPopover from "../overlays/GridPopover.svelte"
export let value
export let focused = false
export let api
let textarea
let isOpen = false
let anchor
$: {
if (!focused) {
isOpen = false
}
}
const onKeyDown = () => {
return isOpen
}
const open = async () => {
isOpen = true
}
const close = () => {
textarea?.blur()
isOpen = false
}
onMount(() => {
api = {
focus: () => open(),
blur: () => close(),
isActive: () => isOpen,
onKeyDown,
}
})
</script>
<!-- svelte-ignore a11y-no-static-element-interactions -->
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="long-form-cell" on:click={open} bind:this={anchor}>
<div class="value">
{value || ""}
</div>
</div>
{#if isOpen}
<GridPopover {anchor} on:close={close}>
<textarea
disabled
bind:this={textarea}
value={value || ""}
on:wheel|stopPropagation
spellcheck="false"
use:clickOutside={close}
/>
</GridPopover>
{/if}
<style>
.long-form-cell {
flex: 1 1 auto;
padding: var(--cell-padding);
align-self: stretch;
display: flex;
align-items: flex-start;
overflow: hidden;
}
.value {
display: -webkit-box;
-webkit-line-clamp: var(--content-lines);
-webkit-box-orient: vertical;
overflow: hidden;
line-height: 20px;
}
textarea {
border: none;
width: 320px;
flex: 1 1 auto;
height: var(--max-cell-render-overflow);
padding: var(--cell-padding);
margin: 0;
background: var(--cell-background);
font-size: var(--cell-font-size);
font-family: var(--font-sans);
color: inherit;
z-index: 1;
resize: none;
line-height: 20px;
overflow: auto;
}
textarea:focus {
outline: none;
}
</style>

3
packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
View File

@ -95,7 +95,8 @@
const { type, formulaType } = col.schema
return (
searchableTypes.includes(type) ||
(type === FieldType.FORMULA && formulaType === FormulaType.STATIC)
(type === FieldType.FORMULA && formulaType === FormulaType.STATIC) ||
type === FieldType.AI
)
}

1
packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
View File

@ -198,7 +198,6 @@
// Toggles whether a row is included in the relationship or not
const toggleRow = async row => {
hideRelationshipFields()
if (fieldValue?.some(x => x._id === row._id)) {
// If the row is already included, remove it and update the candidate
// row to be the same position if possible

2
packages/frontend-core/src/components/grid/layout/Grid.svelte
View File

@ -48,6 +48,7 @@
export let darkMode = false
export let isCloud = null
export let rowConditions = null
export let aiEnabled = false
// Unique identifier for DOM nodes inside this instance
const gridID = `grid-${Math.random().toString().slice(2)}`
@ -104,6 +105,7 @@
buttonsCollapsedText,
darkMode,
isCloud,
aiEnabled,
rowConditions,
})

2
packages/frontend-core/src/components/grid/lib/renderers.js
View File

@ -9,6 +9,7 @@ import TextCell from "../cells/TextCell.svelte"
import LongFormCell from "../cells/LongFormCell.svelte"
import BooleanCell from "../cells/BooleanCell.svelte"
import FormulaCell from "../cells/FormulaCell.svelte"
import AICell from "../cells/AICell.svelte"
import JSONCell from "../cells/JSONCell.svelte"
import AttachmentCell from "../cells/AttachmentCell.svelte"
import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
@ -30,6 +31,7 @@ const TypeComponentMap = {
[FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
[FieldType.LINK]: RelationshipCell,
[FieldType.FORMULA]: FormulaCell,
[FieldType.AI]: AICell,
[FieldType.JSON]: JSONCell,
[FieldType.BB_REFERENCE]: BBReferenceCell,
[FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,

19
packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
View File

@ -1,5 +1,6 @@
<script>
import { Menu, MenuItem, Helpers } from "@budibase/bbui"
import { FieldType } from "@budibase/types"
import { getContext } from "svelte"
import { NewRowID } from "../lib/constants"
import GridPopover from "./GridPopover.svelte"
@ -26,6 +27,9 @@
$: style = makeStyle($menu)
$: isNewRow = $focusedRowId === NewRowID
$: hasAIColumns = $visibleColumns.some(
col => col.schema.type === FieldType.AI
)
const makeStyle = menu => {
return `left:${menu.left}px; top:${menu.top}px;`
@ -53,6 +57,12 @@
await Helpers.copyToClipboard(value)
$notifications.success("Copied to clipboard")
}
const generateAIColumns = async () => {
menu.actions.close()
await rows.actions.applyRowChanges({ rowId: $focusedRowId })
$notifications.success("Generated AI columns")
}
</script>
<div bind:this={anchor} {style} class="menu-anchor" />
@ -161,6 +171,15 @@
>
Delete row
</MenuItem>
{#if $config.aiEnabled}
<MenuItem
icon="MagicWand"
disabled={isNewRow || !hasAIColumns}
on:click={generateAIColumns}
>
Generate AI Columns
</MenuItem>
{/if}
{/if}
</Menu>
</GridPopover>

1
packages/frontend-core/src/components/grid/stores/columns.js
View File

@ -109,6 +109,7 @@ export const createActions = context => {
column.schema.autocolumn ||
column.schema.disabled ||
column.schema.type === "formula" ||
column.schema.type === "ai" ||
column.schema.readonly
)
}

9
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -3,9 +3,12 @@ import { dataFilters } from "@budibase/shared-core"
function convertToSearchFilters(view) {
// convert from SearchFilterGroup type
if (view.query) {
view.queryUI = view.query
view.query = dataFilters.buildQuery(view.query)
if (view?.query) {
return {
...view,
queryUI: view.query,
query: dataFilters.buildQuery(view.query),
}
}
return view
}

3
packages/frontend-core/src/constants.js
View File

@ -160,9 +160,10 @@ export const TypeIconMap = {
[FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
[FieldType.LINK]: "DataCorrelated",
[FieldType.FORMULA]: "Calculator",
[FieldType.AI]: "MagicWand",
[FieldType.JSON]: "Brackets",
[FieldType.BIGINT]: "TagBold",
[FieldType.AUTO]: "MagicWand",
[FieldType.AUTO]: "Shapes",
[FieldType.BB_REFERENCE]: {
[BBReferenceFieldSubType.USER]: "UserGroup",
[BBReferenceFieldSubType.USERS]: "UserGroup",

16
packages/frontend-core/src/utils/relatedColumns.js
View File

@ -24,9 +24,9 @@ const columnTypeManyParser = {
return parsed
}
return value?.map(v => parseDate(v))
return value.map(v => parseDate(v))
},
[FieldType.BOOLEAN]: value => value?.map(v => !!v),
[FieldType.BOOLEAN]: value => value.map(v => !!v),
[FieldType.BB_REFERENCE_SINGLE]: value => [
...new Map(value.map(i => [i._id, i])).values(),
],
@ -80,14 +80,10 @@ export function getRelatedTableValues(row, field, fromField) {
result = row[field.related.field]?.[0]?.[field.related.subField]
} else {
const parser = columnTypeManyParser[field.type] || (value => value)
result = parser(
row[field.related.field]
?.flatMap(r => r[field.related.subField])
?.filter(i => i !== undefined && i !== null),
field
)
const value = row[field.related.field]
?.flatMap(r => r[field.related.subField])
?.filter(i => i !== undefined && i !== null)
result = parser(value || [], field)
if (
[
FieldType.STRING,

2
packages/pro

@ -1 +1 @@
Subproject commit fc4c7f4925139af078480217965c3d6338dc0a7f
Subproject commit 1a749caba9c85aab2645e5d00db479eb53d3f80f

12
packages/server/specs/openapi.json
View File

@ -833,7 +833,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -857,6 +858,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",
@ -1042,7 +1044,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -1066,6 +1069,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",
@ -1262,7 +1266,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -1286,6 +1291,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",

6
packages/server/specs/openapi.yaml
View File

@ -761,6 +761,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -779,6 +780,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr
@ -929,6 +931,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -947,6 +950,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr
@ -1104,6 +1108,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -1122,6 +1127,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr

41
packages/server/src/api/controllers/application.ts
View File

@ -208,9 +208,8 @@ export async function fetchAppDefinition(
export async function fetchAppPackage(
ctx: UserCtx<void, FetchAppPackageResponse>
) {
const db = context.getAppDB()
const appId = context.getAppId()
let application = await db.get<App>(DocumentType.APP_METADATA)
const application = await sdk.applications.metadata.get()
const layouts = await getLayouts()
let screens = await getScreens()
const license = await licensing.cache.getCachedLicense()
@ -272,6 +271,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
path: ctx.request.body.file?.path,
}
}
const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
const appId = generateDevAppID(generateAppID(tenantId))
@ -279,7 +279,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
const instance = await createInstance(appId, instanceConfig)
const db = context.getAppDB()
let newApplication: App = {
const newApplication: App = {
_id: DocumentType.APP_METADATA,
_rev: undefined,
appId,
@ -310,12 +310,18 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
disableUserMetadata: true,
skeletonLoader: true,
},
creationVersion: undefined,
}
const isImport = !!instanceConfig.file
if (!isImport) {
newApplication.creationVersion = envCore.VERSION
}
const existing = await sdk.applications.metadata.tryGet()
// If we used a template or imported an app there will be an existing doc.
// Fetch and migrate some metadata from the existing app.
try {
const existing: App = await db.get(DocumentType.APP_METADATA)
if (existing) {
const keys: (keyof App)[] = [
"_rev",
"navigation",
@ -323,6 +329,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
"customTheme",
"icon",
"snippets",
"creationVersion",
]
keys.forEach(key => {
if (existing[key]) {
@ -340,14 +347,10 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
}
// Migrate navigation settings and screens if required
if (existing) {
const navigation = await migrateAppNavigation()
if (navigation) {
newApplication.navigation = navigation
}
const navigation = await migrateAppNavigation()
if (navigation) {
newApplication.navigation = navigation
}
} catch (err) {
// Nothing to do
}
const response = await db.put(newApplication, { force: true })
@ -489,8 +492,7 @@ export async function update(
export async function updateClient(ctx: UserCtx) {
// Get current app version
const db = context.getAppDB()
const application = await db.get<App>(DocumentType.APP_METADATA)
const application = await sdk.applications.metadata.get()
const currentVersion = application.version
let manifest
@ -518,8 +520,7 @@ export async function updateClient(ctx: UserCtx) {
export async function revertClient(ctx: UserCtx) {
// Check app can be reverted
const db = context.getAppDB()
const application = await db.get<App>(DocumentType.APP_METADATA)
const application = await sdk.applications.metadata.get()
if (!application.revertableVersion) {
ctx.throw(400, "There is no version to revert to")
}
@ -577,7 +578,7 @@ async function destroyApp(ctx: UserCtx) {
const db = dbCore.getDB(devAppId)
// standard app deletion flow
const app = await db.get<App>(DocumentType.APP_METADATA)
const app = await sdk.applications.metadata.get()
const result = await db.destroy()
await quotas.removeApp()
await events.app.deleted(app)
@ -728,7 +729,7 @@ export async function updateAppPackage(
) {
return context.doInAppContext(appId, async () => {
const db = context.getAppDB()
const application = await db.get<App>(DocumentType.APP_METADATA)
const application = await sdk.applications.metadata.get()
const newAppPackage: App = { ...application, ...appPackage }
if (appPackage._rev !== application._rev) {
@ -754,7 +755,7 @@ export async function setRevertableVersion(
return
}
const db = context.getAppDB()
const app = await db.get<App>(DocumentType.APP_METADATA)
const app = await sdk.applications.metadata.get()
app.revertableVersion = ctx.request.body.revertableVersion
await db.put(app)
@ -763,7 +764,7 @@ export async function setRevertableVersion(
async function migrateAppNavigation() {
const db = context.getAppDB()
const existing: App = await db.get(DocumentType.APP_METADATA)
const existing = await sdk.applications.metadata.get()
const layouts: Layout[] = await getLayouts()
const screens: Screen[] = await getScreens()

19
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -157,7 +157,8 @@ function isEditableColumn(column: FieldSchema) {
column.autoReason !== AutoReason.FOREIGN_KEY &&
column.subtype !== AutoFieldSubType.AUTO_ID
const isFormula = column.type === FieldType.FORMULA
return !(isExternalAutoColumn || isFormula)
const isAIColumn = column.type === FieldType.AI
return !(isExternalAutoColumn || isFormula || isAIColumn)
}
export class ExternalRequest<T extends Operation> {
@ -173,9 +174,9 @@ export class ExternalRequest<T extends Operation> {
if (!opts.datasource) {
if (sdk.views.isView(source)) {
const table = await sdk.views.getTable(source.id)
opts.datasource = await sdk.datasources.get(table.sourceId!)
opts.datasource = await sdk.datasources.get(table.sourceId)
} else {
opts.datasource = await sdk.datasources.get(source.sourceId!)
opts.datasource = await sdk.datasources.get(source.sourceId)
}
}
@ -204,18 +205,6 @@ export class ExternalRequest<T extends Operation> {
filters: SearchFilters,
table: Table
): SearchFilters {
// replace any relationship columns initially, table names and relationship column names are acceptable
const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
filters = sdk.rows.filters.updateFilterKeys(
filters,
relationshipColumns.map(({ name, definition }) => {
const { tableName } = breakExternalTableId(definition.tableId)
return {
original: name,
updated: tableName,
}
})
)
const primary = table.primary
// if passed in array need to copy for shifting etc
let idCopy: undefined | string | any[] = cloneDeep(id)

60
packages/server/src/api/controllers/row/index.ts
View File

@ -15,13 +15,16 @@ import {
ExportRowsResponse,
FieldType,
GetRowResponse,
isRelationshipField,
PatchRowRequest,
PatchRowResponse,
Row,
RowAttachment,
RowSearchParams,
SearchFilters,
SearchRowRequest,
SearchRowResponse,
Table,
UserCtx,
ValidateResponse,
} from "@budibase/types"
@ -33,6 +36,7 @@ import sdk from "../../../sdk"
import * as exporters from "../view/exporters"
import { Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import { dataFilters } from "@budibase/shared-core"
export * as views from "./views"
@ -211,12 +215,15 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
await context.ensureSnippetContext(true)
const enrichedQuery = await utils.enrichSearchContext(
{ ...ctx.request.body.query },
{
user: sdk.users.getUserContextBindings(ctx.user),
}
)
let { query } = ctx.request.body
if (query) {
const allTables = await sdk.tables.getAllTables()
query = replaceTableNamesInFilters(tableId, query, allTables)
}
let enrichedQuery: SearchFilters = await utils.enrichSearchContext(query, {
user: sdk.users.getUserContextBindings(ctx.user),
})
const searchParams: RowSearchParams = {
...ctx.request.body,
@ -229,6 +236,47 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
ctx.body = await sdk.rows.search(searchParams)
}
function replaceTableNamesInFilters(
tableId: string,
filters: SearchFilters,
allTables: Table[]
): SearchFilters {
for (const filter of Object.values(filters)) {
for (const key of Object.keys(filter)) {
const matches = key.match(`^(?<relation>.+)\\.(?<field>.+)`)
const relation = matches?.groups?.["relation"]
const field = matches?.groups?.["field"]
if (!relation || !field) {
continue
}
const table = allTables.find(r => r._id === tableId)!
if (Object.values(table.schema).some(f => f.name === relation)) {
continue
}
const matchedTable = allTables.find(t => t.name === relation)
const relationship = Object.values(table.schema).find(
f => isRelationshipField(f) && f.tableId === matchedTable?._id
)
if (!relationship) {
continue
}
const updatedField = `${relationship.name}.${field}`
if (updatedField && updatedField !== key) {
filter[updatedField] = filter[key]
delete filter[key]
}
}
}
return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => {
return replaceTableNamesInFilters(tableId, f, allTables)
})
}
export async function validate(ctx: Ctx<Row, ValidateResponse>) {
const source = await utils.getSource(ctx)
const table = await utils.getTableFromSource(source)

19
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -1,6 +1,7 @@
import { getRowParams } from "../../../db/utils"
import {
outputProcessing,
processAIColumns,
processFormulas,
} from "../../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
@ -9,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
import isEqual from "lodash/isEqual"
import { cloneDeep } from "lodash/fp"
import sdk from "../../../sdk"
import * as pro from "@budibase/pro"
/**
* This function runs through a list of enriched rows, looks at the rows which
@ -102,7 +104,7 @@ export async function updateAllFormulasInTable(table: Table) {
(enriched: Row) => enriched._id === row._id
)
if (enrichedRow) {
const processed = await processFormulas(table, cloneDeep(row), {
let processed = await processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: [enrichedRow],
})
@ -142,12 +144,27 @@ export async function finaliseRow(
dynamic: false,
contextRows: [enrichedRow],
})
const aiEnabled =
(await pro.features.isBudibaseAIEnabled()) ||
(await pro.features.isAICustomConfigsEnabled())
if (aiEnabled) {
row = await processAIColumns(table, row, {
contextRows: [enrichedRow],
})
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, {
dynamic: false,
})
if (aiEnabled) {
enrichedRow = await processAIColumns(table, row, {
contextRows: [enrichedRow],
})
}
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await updateRelatedFormula(table, enrichedRow)

1
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -124,6 +124,7 @@ export async function buildSqlFieldList(
([columnName, column]) =>
column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI &&
!existing.find(
(field: string) => field === `${table.name}.${columnName}`
)

2
packages/server/src/api/controllers/table/bulkFormula.ts
View File

@ -6,10 +6,10 @@ import uniq from "lodash/uniq"
import { updateAllFormulasInTable } from "../row/staticFormula"
import { context } from "@budibase/backend-core"
import {
FormulaType,
FieldSchema,
FieldType,
FormulaFieldMetadata,
FormulaType,
Table,
} from "@budibase/types"
import sdk from "../../../sdk"

21
packages/server/src/api/controllers/table/index.ts
View File

@ -71,19 +71,20 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
const datasources = await sdk.datasources.getExternalDatasources()
const external = datasources.flatMap(datasource => {
const external: Table[] = []
for (const datasource of datasources) {
let entities = datasource.entities
if (entities) {
return Object.values(entities).map<Table>((entity: Table) => ({
...entity,
sourceType: TableSourceType.EXTERNAL,
sourceId: datasource._id!,
sql: isSQL(datasource),
}))
} else {
return []
for (const entity of Object.values(entities)) {
external.push({
...(await processTable(entity)),
sourceType: TableSourceType.EXTERNAL,
sourceId: datasource._id!,
sql: isSQL(datasource),
})
}
}
})
}
const result: FetchTablesResponse = []
for (const table of [...internal, ...external]) {

99
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -18,6 +18,7 @@ import {
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
AIOperationEnum,
AttachmentFieldMetadata,
AutoFieldSubType,
Datasource,
@ -50,6 +51,18 @@ import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => `Mock LLM Response`),
buildPromptFromAIOperation: jest.fn(),
}),
},
},
}))
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
interface WaitOptions {
@ -789,6 +802,39 @@ describe.each([
})
})
describe("multi-user column", () => {
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
users: {
name: "users",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
default: ["{{ [Current User]._id }}"],
},
},
})
)
})
it("creates a new row with a default value successfully", async () => {
const row = await config.api.row.save(table._id!, {})
expect(row.users).toHaveLength(1)
expect(row.users[0]._id).toEqual(config.getUser()._id)
})
it("does not use default value if value specified", async () => {
const id = `us_${utils.newid()}`
await config.createUser({ _id: id })
const row = await config.api.row.save(table._id!, {
users: [id],
})
expect(row.users).toHaveLength(1)
expect(row.users[0]._id).toEqual(id)
})
})
describe("bindings", () => {
describe("string column", () => {
beforeAll(async () => {
@ -2053,6 +2099,7 @@ describe.each([
[FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
[FieldType.FORMULA]: undefined, // generated field
[FieldType.AUTO]: undefined, // generated field
[FieldType.AI]: undefined, // generated field
[FieldType.JSON]: { name: generator.guid() },
[FieldType.INTERNAL]: generator.guid(),
[FieldType.BARCODEQR]: generator.guid(),
@ -2156,6 +2203,7 @@ describe.each([
expectedRowData["bb_reference_single"].sample,
false
),
ai: null,
},
])
})
@ -2942,6 +2990,57 @@ describe.each([
)
})
isSqs &&
describe("AI fields", () => {
let table: Table
beforeAll(async () => {
mocks.licenses.useBudibaseAI()
mocks.licenses.useAICustomConfigs()
table = await config.api.table.save(
saveTableRequest({
schema: {
ai: {
name: "ai",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Convert the following to German: '{{ product }}'",
},
product: {
name: "product",
type: FieldType.STRING,
},
},
})
)
await config.api.row.save(table._id!, {
product: generator.word(),
})
})
afterAll(() => {
jest.unmock("@budibase/pro")
})
it("should be able to save a row with an AI column", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
expect(rows[0].ai).toEqual("Mock LLM Response")
})
it("should be able to update a row with an AI column", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
await config.api.row.save(table._id!, {
product: generator.word(),
...rows[0],
})
expect(rows.length).toBe(1)
expect(rows[0].ai).toEqual("Mock LLM Response")
})
})
describe("Formula fields", () => {
let table: Table
let otherTable: Table

404
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -17,12 +17,14 @@ import {
import * as setup from "./utilities"
import {
AIOperationEnum,
AutoFieldSubType,
BBReferenceFieldSubType,
Datasource,
EmptyFilterOption,
FieldType,
JsonFieldSubType,
LogicalOperator,
RelationshipType,
Row,
RowSearchParams,
@ -41,11 +43,23 @@ import tk from "timekeeper"
import { encodeJSBinding } from "@budibase/string-templates"
import { dataFilters } from "@budibase/shared-core"
import { Knex } from "knex"
import { generator, structures } from "@budibase/backend-core/tests"
import { generator, structures, mocks } from "@budibase/backend-core/tests"
import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
import { generateRowIdField } from "../../../integrations/utils"
import { cloneDeep } from "lodash/fp"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => `Mock LLM Response`),
buildPromptFromAIOperation: jest.fn(),
}),
},
},
}))
describe.each([
["in-memory", undefined],
["lucene", undefined],
@ -1601,6 +1615,79 @@ describe.each([
})
})
isSqs &&
describe("AI Column", () => {
const UNEXISTING_AI_COLUMN = "Real LLM Response"
beforeAll(async () => {
mocks.licenses.useBudibaseAI()
mocks.licenses.useAICustomConfigs()
tableOrViewId = await createTableOrView({
product: { name: "product", type: FieldType.STRING },
ai: {
name: "AI",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
})
await createRows([{ product: "Big Mac" }, { product: "McCrispy" }])
})
describe("equal", () => {
it("successfully finds rows based on AI column", async () => {
await expectQuery({
equal: { ai: "Mock LLM Response" },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
it("fails to find nonexistent row", async () => {
await expectQuery({
equal: { ai: UNEXISTING_AI_COLUMN },
}).toFindNothing()
})
})
describe("notEqual", () => {
it("Returns nothing when searching notEqual on the mock AI response", async () => {
await expectQuery({
notEqual: { ai: "Mock LLM Response" },
}).toContainExactly([])
})
it("return all when requesting non-existing response", async () => {
await expectQuery({
notEqual: { ai: "Real LLM Response" },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
})
describe("oneOf", () => {
it("successfully finds a row", async () => {
await expectQuery({
oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
it("fails to find nonexistent row", async () => {
await expectQuery({
oneOf: { ai: ["Whopper"] },
}).toFindNothing()
})
})
})
describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
beforeAll(async () => {
tableOrViewId = await createTableOrView({
@ -2277,12 +2364,16 @@ describe.each([
// It also can't work for in-memory searching because the related table name
// isn't available.
!isInMemory &&
describe("relations", () => {
describe.each([
RelationshipType.ONE_TO_MANY,
RelationshipType.MANY_TO_ONE,
RelationshipType.MANY_TO_MANY,
])("relations (%s)", relationshipType => {
let productCategoryTable: Table, productCatRows: Row[]
beforeAll(async () => {
const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.ONE_TO_MANY
relationshipType
)
tableOrViewId = tableId
productCategoryTable = relatedTable
@ -2329,13 +2420,233 @@ describe.each([
equal: { ["name"]: "baz" },
}).toContainExactly([{ name: "baz", productCat: undefined }])
})
describe("logical filters", () => {
const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
describe("$and", () => {
it("should allow single conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
})
it.each([logicalOperators])(
"should allow nested ands with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
])
}
)
it.each([logicalOperators])(
"should allow nested ands with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([])
}
)
it.each([logicalOperators])(
"should allow nested ands with multiple conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
}
)
})
describe("$ors", () => {
it("should allow single conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
{ name: "baz", productCat: undefined },
])
})
it.each([logicalOperators])(
"should allow nested ors with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
])
}
)
it.each([logicalOperators])(
"should allow nested ors with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{
name: "foo",
productCat: [{ _id: productCatRows[0]._id }],
},
{
name: "bar",
productCat: [{ _id: productCatRows[1]._id }],
},
{ name: "baz", productCat: undefined },
])
}
)
it("should allow nested ors with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
{ name: "baz", productCat: undefined },
])
})
})
})
})
isSql &&
describe("big relations", () => {
describe.each([
RelationshipType.MANY_TO_ONE,
RelationshipType.MANY_TO_MANY,
])("big relations (%s)", relationshipType => {
beforeAll(async () => {
const { relatedTable, tableId } = await basicRelationshipTables(
RelationshipType.MANY_TO_ONE
relationshipType
)
tableOrViewId = tableId
const mainRow = await config.api.row.save(tableOrViewId, {
@ -2361,7 +2672,8 @@ describe.each([
expect(response.rows[0].productCat).toBeArrayOfSize(11)
})
})
;(isSqs || isLucene) &&
isSql &&
describe("relations to same table", () => {
let relatedTable: string, relatedRows: Row[]
@ -2403,6 +2715,11 @@ describe.each([
related1: [relatedRows[2]._id!],
related2: [relatedRows[3]._id!],
}),
config.api.row.save(tableOrViewId, {
name: "test3",
related1: [relatedRows[1]._id],
related2: [relatedRows[2]._id!],
}),
])
})
@ -2420,42 +2737,59 @@ describe.each([
related1: [{ _id: relatedRows[2]._id }],
related2: [{ _id: relatedRows[3]._id }],
},
{
name: "test3",
related1: [{ _id: relatedRows[1]._id }],
related2: [{ _id: relatedRows[2]._id }],
},
])
})
isSqs &&
it("should be able to filter down to second row with equal", async () => {
await expectSearch({
query: {
equal: {
["related1.name"]: "baz",
},
it("should be able to filter via the first relation field with equal", async () => {
await expectSearch({
query: {
equal: {
["related1.name"]: "baz",
},
}).toContainExactly([
{
name: "test2",
related1: [{ _id: relatedRows[2]._id }],
},
])
})
},
}).toContainExactly([
{
name: "test2",
related1: [{ _id: relatedRows[2]._id }],
},
])
})
isSqs &&
it("should be able to filter down to first row with not equal", async () => {
await expectSearch({
query: {
notEqual: {
["1:related2.name"]: "bar",
["2:related2.name"]: "baz",
["3:related2.name"]: "boo",
},
it("should be able to filter via the second relation field with not equal", async () => {
await expectSearch({
query: {
notEqual: {
["1:related2.name"]: "foo",
["2:related2.name"]: "baz",
["3:related2.name"]: "boo",
},
}).toContainExactly([
{
name: "test",
related1: [{ _id: relatedRows[0]._id }],
},
}).toContainExactly([
{
name: "test",
},
])
})
it("should be able to filter on both fields", async () => {
await expectSearch({
query: {
notEqual: {
["related1.name"]: "foo",
["related2.name"]: "baz",
},
])
})
},
}).toContainExactly([
{
name: "test2",
},
])
})
})
isInternal &&

8
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -206,7 +206,7 @@ describe.each([
visible: false,
icon: "ic",
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
}
const createdView = await config.api.viewV2.create(newView)
@ -250,7 +250,7 @@ describe.each([
name: "Category",
type: FieldType.STRING,
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
}
await config.api.viewV2.create(newView, {
@ -1044,7 +1044,7 @@ describe.each([
visible: false,
icon: "ic",
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
})
expect(updatedView).toEqual({
@ -1078,7 +1078,7 @@ describe.each([
name: "Category",
type: FieldType.STRING,
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
},
{
status: 200,

4
packages/server/src/db/linkedRows/index.ts
View File

@ -307,7 +307,9 @@ export async function squashLinks<T = Row[] | Row>(
return false
}
if (
[FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type)
[FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
tableColumn.type
)
) {
return false
}

4
packages/server/src/integrations/googlesheets.ts
View File

@ -56,6 +56,7 @@ interface AuthTokenResponse {
const isTypeAllowed: Record<FieldType, boolean> = {
[FieldType.STRING]: true,
[FieldType.FORMULA]: true,
[FieldType.AI]: true,
[FieldType.NUMBER]: true,
[FieldType.LONGFORM]: true,
[FieldType.DATETIME]: true,
@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
}
if (
!sheet.headerValues.includes(key) &&
column.type !== FieldType.FORMULA
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI
) {
updatedHeaderValues.push(key)
}

7
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -78,8 +78,7 @@ describe("Captures of real examples", () => {
bindings: ["assembling", primaryLimit, relationshipLimit],
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid"
and COALESCE("b"."taskname" = $1, FALSE)`
`where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))`
)
),
})
@ -133,6 +132,8 @@ describe("Captures of real examples", () => {
expect(query).toEqual({
bindings: [
rangeValue.low,
rangeValue.high,
rangeValue.low,
rangeValue.high,
equalValue,
@ -144,7 +145,7 @@ describe("Captures of real examples", () => {
],
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
`where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))`
)
),
})

1
packages/server/src/integrations/utils/utils.ts
View File

@ -242,6 +242,7 @@ function copyExistingPropsOver(
let shouldKeepSchema = false
switch (existingColumnType) {
case FieldType.FORMULA:
case FieldType.AI:
case FieldType.AUTO:
case FieldType.INTERNAL:
shouldKeepSchema = true

2
packages/server/src/sdk/app/applications/index.ts
View File

@ -2,10 +2,12 @@ import * as sync from "./sync"
import * as utils from "./utils"
import * as applications from "./applications"
import * as imports from "./import"
import * as metadata from "./metadata"
export default {
...sync,
...utils,
...applications,
...imports,
metadata,
}

18
packages/server/src/sdk/app/applications/metadata.ts Normal file
View File

@ -0,0 +1,18 @@
import { context, DocumentType } from "@budibase/backend-core"
import { App } from "@budibase/types"
/**
* @deprecated the plan is to get everything using `tryGet` instead, then rename
* `tryGet` to `get`.
*/
export async function get() {
const db = context.getAppDB()
const application = await db.get<App>(DocumentType.APP_METADATA)
return application
}
export async function tryGet() {
const db = context.getAppDB()
const application = await db.tryGet<App>(DocumentType.APP_METADATA)
return application
}

2
packages/server/src/sdk/app/rows/index.ts
View File

@ -3,14 +3,12 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
import * as filters from "./search/filters"
import AliasTables from "./sqlAlias"
export default {
...attachments,
...rows,
...search,
filters,
utils,
external,
AliasTables,

65
packages/server/src/sdk/app/rows/search/filters.ts
View File

@ -1,65 +0,0 @@
import {
FieldType,
RelationshipFieldMetadata,
SearchFilters,
Table,
} from "@budibase/types"
import { isPlainObject } from "lodash"
import { dataFilters } from "@budibase/shared-core"
export function getRelationshipColumns(table: Table): {
name: string
definition: RelationshipFieldMetadata
}[] {
// performing this with a for loop rather than an array filter improves
// type guarding, as no casts are required
const linkEntries: [string, RelationshipFieldMetadata][] = []
for (let entry of Object.entries(table.schema)) {
if (entry[1].type === FieldType.LINK) {
const linkColumn: RelationshipFieldMetadata = entry[1]
linkEntries.push([entry[0], linkColumn])
}
}
return linkEntries.map(entry => ({
name: entry[0],
definition: entry[1],
}))
}
export function getTableIDList(
tables: Table[]
): { name: string; id: string }[] {
return tables
.filter(table => table.originalName && table._id)
.map(table => ({ id: table._id!, name: table.originalName! }))
}
export function updateFilterKeys(
filters: SearchFilters,
updates: { original: string; updated: string }[]
): SearchFilters {
const makeFilterKeyRegex = (str: string) =>
new RegExp(`^${str}\\.|:${str}\\.`)
for (let filter of Object.values(filters)) {
if (!isPlainObject(filter)) {
continue
}
for (let [key, keyFilter] of Object.entries(filter)) {
if (keyFilter === "") {
delete filter[key]
}
const possibleKey = updates.find(({ original }) =>
key.match(makeFilterKeyRegex(original))
)
if (possibleKey && possibleKey.original !== possibleKey.updated) {
// only replace the first, not replaceAll
filter[key.replace(possibleKey.original, possibleKey.updated)] =
filter[key]
delete filter[key]
}
}
}
return dataFilters.recurseLogicalOperators(filters, (f: SearchFilters) => {
return updateFilterKeys(f, updates)
})
}

33
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -39,11 +39,6 @@ import AliasTables from "../../sqlAlias"
import { outputProcessing } from "../../../../../utilities/rowProcessor"
import pick from "lodash/pick"
import { processRowCountResponse } from "../../utils"
import {
getRelationshipColumns,
getTableIDList,
updateFilterKeys,
} from "../filters"
import {
dataFilters,
helpers,
@ -133,31 +128,7 @@ async function buildInternalFieldList(
return [...new Set(fieldList)]
}
function cleanupFilters(
filters: SearchFilters,
table: Table,
allTables: Table[]
) {
// get a list of all relationship columns in the table for updating
const relationshipColumns = getRelationshipColumns(table)
// get table names to ID map for relationships
const tableNameToID = getTableIDList(allTables)
// all should be applied at once
filters = updateFilterKeys(
filters,
relationshipColumns
.map(({ name, definition }) => ({
original: name,
updated: definition.tableId,
}))
.concat(
tableNameToID.map(({ name, id }) => ({
original: name,
updated: id,
}))
)
)
function cleanupFilters(filters: SearchFilters, allTables: Table[]) {
// generate a map of all possible column names (these can be duplicated across tables
// the map of them will always be the same
const userColumnMap: Record<string, string> = {}
@ -356,7 +327,7 @@ export async function search(
const relationships = buildInternalRelationships(table, allTables)
const searchFilters: SearchFilters = {
...cleanupFilters(query, table, allTables),
...cleanupFilters(query, allTables),
documentType: DocumentType.ROW,
}

12
packages/server/src/sdk/app/tables/getters.ts
View File

@ -20,7 +20,13 @@ export async function processTable(table: Table): Promise<Table> {
if (!table) {
return table
}
table = { ...table }
if (table._id && isExternalTableID(table._id)) {
// Old created external tables via Budibase might have a missing field name breaking some UI such as filters
if (table.schema["id"] && !table.schema["id"].name) {
table.schema["id"].name = "id"
}
return {
...table,
type: "table",
@ -84,7 +90,11 @@ export async function getExternalTable(
if (!entities[tableName]) {
throw new Error(`Unable to find table named "${tableName}"`)
}
return processTable(entities[tableName])
const table = await processTable(entities[tableName])
if (!table.sourceId) {
table.sourceId = datasourceId
}
return table
}
export async function getTable(tableId: string): Promise<Table> {

1
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -19,6 +19,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
[FieldType.DATETIME]: SQLiteType.TEXT,
[FieldType.FORMULA]: SQLiteType.TEXT,
[FieldType.AI]: SQLiteType.TEXT,
[FieldType.LONGFORM]: SQLiteType.TEXT,
[FieldType.NUMBER]: SQLiteType.REAL,
[FieldType.STRING]: SQLiteType.TEXT,

6
packages/server/src/sdk/app/views/index.ts
View File

@ -313,7 +313,11 @@ export async function enrichSchema(
const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName]
if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
if (
[FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
relTableField.type
)
) {
continue
}

7
packages/server/src/tests/utilities/structures.ts
View File

@ -7,6 +7,7 @@ import {
TRIGGER_DEFINITIONS,
} from "../../automations"
import {
AIOperationEnum,
Automation,
AutomationActionStepId,
AutomationResults,
@ -666,6 +667,12 @@ export function fullSchemaWithoutLinks({
presence: allRequired,
},
},
[FieldType.AI]: {
name: "ai",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Translate this into German :'{{ product }}'",
},
[FieldType.BARCODEQR]: {
name: "barcodeqr",
type: FieldType.BARCODEQR,

19
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -33,7 +33,7 @@ import {
PROTECTED_EXTERNAL_COLUMNS,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { processString } from "@budibase/string-templates"
import { processStringSync } from "@budibase/string-templates"
import {
getTableFromSource,
isUserMetadataTable,
@ -134,10 +134,15 @@ async function processDefaultValues(table: Table, row: Row) {
for (const [key, schema] of Object.entries(table.schema)) {
if ("default" in schema && schema.default != null && row[key] == null) {
const processed =
typeof schema.default === "string"
? await processString(schema.default, ctx)
: schema.default
let processed: string | string[]
if (Array.isArray(schema.default)) {
processed = schema.default.map(val => processStringSync(val, ctx))
} else if (typeof schema.default === "string") {
processed = processStringSync(schema.default, ctx)
} else {
processed = schema.default
}
try {
row[key] = coerce(processed, schema.type)
} catch (err: any) {
@ -205,6 +210,10 @@ export async function inputProcessing(
if (field.type === FieldType.FORMULA) {
delete clonedRow[key]
}
// remove any AI values, they are to be generated
if (field.type === FieldType.AI) {
delete clonedRow[key]
}
// otherwise coerce what is there to correct types
else {
clonedRow[key] = coerce(value, field.type)

75
packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
View File

@ -1,11 +1,29 @@
import { fixAutoColumnSubType } from "../utils"
import { fixAutoColumnSubType, processAIColumns } from "../utils"
import { AutoFieldDefaultNames } from "../../../constants"
import {
AIOperationEnum,
AutoFieldSubType,
FieldSchema,
FieldType,
INTERNAL_TABLE_SOURCE_ID,
RelationshipType,
Table,
TableSourceType,
} from "@budibase/types"
import { generator } from "@budibase/backend-core/tests"
const buildPromptMock = jest.fn()
jest.mock("@budibase/pro", () => ({
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => "response from LLM"),
buildPromptFromAIOperation: buildPromptMock,
}),
},
},
}))
describe("rowProcessor utility", () => {
describe("fixAutoColumnSubType", () => {
@ -60,4 +78,59 @@ describe("rowProcessor utility", () => {
expect(fixAutoColumnSubType(schema)).toEqual(schema)
})
})
describe("processAIColumns", () => {
it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
const table: Table = {
_id: generator.guid(),
name: "AITestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
product: {
type: FieldType.STRING,
name: "product",
constraints: {
presence: true,
type: "string",
},
},
aicol: {
type: FieldType.AI,
name: "aicol",
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
},
}
const inputRows = [
{
product: "Car Battery",
},
]
const result = await processAIColumns(table, inputRows, {
contextRows: inputRows,
})
expect(buildPromptMock).toHaveBeenCalledWith({
row: {
product: "Car Battery",
},
schema: {
name: "aicol",
operation: "PROMPT",
prompt: "Translate 'Car Battery' into German",
type: "ai",
},
})
expect(result).toEqual([
{
aicol: "response from LLM",
product: "Car Battery",
},
])
})
})
})

64
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -8,9 +8,13 @@ import {
FormulaType,
AutoFieldSubType,
FieldType,
OperationFieldTypeEnum,
AIOperationEnum,
} from "@budibase/types"
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
interface FormulaOpts {
dynamic?: boolean
@ -91,6 +95,66 @@ export async function processFormulas<T extends Row | Row[]>(
})
}
/**
* Looks through the rows provided and finds AI columns - which it then processes.
*/
export async function processAIColumns<T extends Row | Row[]>(
table: Table,
inputRows: T,
{ contextRows }: FormulaOpts
): Promise<T> {
return tracer.trace("processAIColumns", {}, async span => {
const numRows = Array.isArray(inputRows) ? inputRows.length : 1
span?.addTags({ table_id: table._id, numRows })
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini")
if (rows) {
// Ensure we have snippet context
await context.ensureSnippetContext()
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldType.AI) {
continue
}
const rowUpdates = rows.map((row, i) => {
const contextRow = contextRows ? contextRows[i] : row
// Check if the type is bindable and pass through HBS if so
const operationField =
OperationFields[schema.operation as AIOperationEnum]
for (const key in schema) {
const fieldType = operationField[key as keyof typeof operationField]
if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
// @ts-ignore
schema[key] = processStringSync(schema[key], contextRow)
}
}
const prompt = llm.buildPromptFromAIOperation({ schema, row })
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })
const llmResponse = await llm.run(prompt!)
return {
...row,
[column]: llmResponse,
}
})
})
const processedRows = await Promise.all(rowUpdates)
// Promise.all is deterministic so can rely on the indexing here
processedRows.forEach(
(processedRow, index) => (rows[index] = processedRow)
)
}
}
return Array.isArray(inputRows) ? rows : rows[0]
})
}
/**
* Processes any date columns and ensures that those without the ignoreTimezones
* flag set are parsed as UTC rather than local time.

68
packages/shared-core/src/constants/ai.ts Normal file
View File

@ -0,0 +1,68 @@
import {
AIOperationEnum,
OperationFieldsType,
OperationFieldTypeEnum,
} from "@budibase/types"
export const AIOperations = {
SUMMARISE_TEXT: {
label: "Summarise Text",
value: "SUMMARISE_TEXT",
},
CLEAN_DATA: {
label: "Clean Data",
value: "CLEAN_DATA",
},
TRANSLATE: {
label: "Translate",
value: "TRANSLATE",
},
CATEGORISE_TEXT: {
label: "Categorise Text",
value: "CATEGORISE_TEXT",
},
SENTIMENT_ANALYSIS: {
label: "Sentiment Analysis",
value: "SENTIMENT_ANALYSIS",
},
PROMPT: {
label: "Prompt",
value: "PROMPT",
},
SEARCH_WEB: {
label: "Search Web",
value: "SEARCH_WEB",
},
}
export const OperationFieldTypes = {
MULTI_COLUMN: "columns",
COLUMN: "column",
BINDABLE_TEXT: "prompt",
}
export const OperationFields: OperationFieldsType = {
[AIOperationEnum.SUMMARISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
},
[AIOperationEnum.CLEAN_DATA]: {
column: OperationFieldTypeEnum.COLUMN,
},
[AIOperationEnum.TRANSLATE]: {
column: OperationFieldTypeEnum.COLUMN,
language: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.CATEGORISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
categories: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.SENTIMENT_ANALYSIS]: {
column: OperationFieldTypeEnum.COLUMN,
},
[AIOperationEnum.PROMPT]: {
prompt: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.SEARCH_WEB]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
},
}

1
packages/shared-core/src/constants/index.ts
View File

@ -1,3 +1,4 @@
export * from "./ai"
export * from "./api"
export * from "./fields"
export * from "./rows"

2
packages/shared-core/src/filters.ts
View File

@ -90,6 +90,8 @@ export const getValidOperatorsForType = (
ops = numOps
} else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
} else if (type === FieldType.AI) {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
} else if (
type === FieldType.BB_REFERENCE_SINGLE ||
schema.isDeprecatedSingleUserColumn(fieldType)

5
packages/shared-core/src/table.ts
View File

@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
[FieldType.NUMBER]: true,
[FieldType.DATETIME]: true,
[FieldType.FORMULA]: true,
[FieldType.AI]: true,
[FieldType.AUTO]: true,
[FieldType.INTERNAL]: true,
[FieldType.BARCODEQR]: true,
@ -38,6 +39,7 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
[FieldType.JSON]: true,
[FieldType.FORMULA]: false,
[FieldType.AI]: false,
[FieldType.ATTACHMENTS]: false,
[FieldType.ATTACHMENT_SINGLE]: false,
[FieldType.SIGNATURE_SINGLE]: false,
@ -62,11 +64,12 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
[FieldType.BIGINT]: false,
[FieldType.BOOLEAN]: false,
[FieldType.FORMULA]: false,
[FieldType.AI]: false,
[FieldType.ATTACHMENTS]: false,
[FieldType.ATTACHMENT_SINGLE]: false,
[FieldType.SIGNATURE_SINGLE]: false,
[FieldType.LINK]: false,
[FieldType.BB_REFERENCE]: false,
[FieldType.BB_REFERENCE]: true,
[FieldType.BB_REFERENCE_SINGLE]: true,
}

1
packages/types/src/documents/app/app.ts
View File

@ -27,6 +27,7 @@ export interface App extends Document {
usedPlugins?: Plugin[]
upgradableVersion?: string
snippets?: Snippet[]
creationVersion?: string
}
export interface AppInstance {

7
packages/types/src/documents/app/row.ts
View File

@ -76,6 +76,13 @@ export enum FieldType {
* that is part of the initial formula definition, the formula will be live evaluated in the browser.
*/
AUTO = "auto",
/**
* A complex type, called an AI column within Budibase. This type is only supported against internal tables
* and calculates the output based on a chosen operation (summarise text, translation etc) which passes to
* the configured Budibase Large Language Model to retrieve the output and write it back into the row.
* AI fields function in a similar fashion to static formulas, and possess many of the same characteristics.
*/
AI = "ai",
/**
* a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
* type, which will be represented as a JSON object in the row. This type depends on a schema being

1
packages/types/src/documents/app/table/constants.ts
View File

@ -30,6 +30,7 @@ export enum JsonFieldSubType {
export enum FormulaType {
STATIC = "static",
DYNAMIC = "dynamic",
AI = "ai",
}
export enum BBReferenceFieldSubType {

14
packages/types/src/documents/app/table/schema.ts
View File

@ -9,6 +9,7 @@ import {
JsonFieldSubType,
RelationshipType,
} from "./constants"
import { AIOperationEnum } from "../../../sdk/ai"
export interface UIFieldMetadata {
order?: number
@ -116,11 +117,22 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
formulaType?: FormulaType
}
export interface AIFieldMetadata extends BaseFieldSchema {
type: FieldType.AI
operation: AIOperationEnum
columns?: string[]
column?: string
categories?: string[]
prompt?: string
language?: string
}
export interface BBReferenceFieldMetadata
extends Omit<BaseFieldSchema, "subtype"> {
type: FieldType.BB_REFERENCE
subtype: BBReferenceFieldSubType
relationshipType?: RelationshipType
default?: string[]
}
export interface BBReferenceSingleFieldMetadata
extends Omit<BaseFieldSchema, "subtype"> {
@ -193,6 +205,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
| FieldType.LINK
| FieldType.AUTO
| FieldType.FORMULA
| FieldType.AI
| FieldType.NUMBER
| FieldType.LONGFORM
| FieldType.BB_REFERENCE
@ -210,6 +223,7 @@ export type FieldSchema =
| RelationshipFieldMetadata
| AutoColumnFieldMetadata
| FormulaFieldMetadata
| AIFieldMetadata
| NumberFieldMetadata
| LongFormFieldMetadata
| StringFieldMetadata

91
packages/types/src/sdk/ai.ts Normal file
View File

@ -0,0 +1,91 @@
export enum AIOperationEnum {
SUMMARISE_TEXT = "SUMMARISE_TEXT",
CLEAN_DATA = "CLEAN_DATA",
TRANSLATE = "TRANSLATE",
CATEGORISE_TEXT = "CATEGORISE_TEXT",
SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS",
PROMPT = "PROMPT",
SEARCH_WEB = "SEARCH_WEB",
}
export enum OperationFieldTypeEnum {
MULTI_COLUMN = "columns",
COLUMN = "column",
BINDABLE_TEXT = "prompt",
}
export type OperationFieldsType = {
[AIOperationEnum.SUMMARISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
}
[AIOperationEnum.CLEAN_DATA]: {
column: OperationFieldTypeEnum.COLUMN
}
[AIOperationEnum.TRANSLATE]: {
column: OperationFieldTypeEnum.COLUMN
language: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.CATEGORISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
categories: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.SENTIMENT_ANALYSIS]: {
column: OperationFieldTypeEnum.COLUMN
}
[AIOperationEnum.PROMPT]: {
prompt: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.SEARCH_WEB]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
}
}
type BaseSchema = {
operation: AIOperationEnum
}
type SummariseTextSchema = BaseSchema & {
operation: AIOperationEnum.SUMMARISE_TEXT
columns: string[]
}
type CleanDataSchema = BaseSchema & {
operation: AIOperationEnum.CLEAN_DATA
column: string
}
type TranslateSchema = BaseSchema & {
operation: AIOperationEnum.TRANSLATE
column: string
language: string
}
type CategoriseTextSchema = BaseSchema & {
operation: AIOperationEnum.CATEGORISE_TEXT
columns: string[]
categories: string[]
}
type SentimentAnalysisSchema = BaseSchema & {
operation: AIOperationEnum.SENTIMENT_ANALYSIS
column: string
}
type PromptSchema = BaseSchema & {
operation: AIOperationEnum.PROMPT
prompt: string
}
type SearchWebSchema = BaseSchema & {
operation: AIOperationEnum.SEARCH_WEB
columns: string[]
}
export type AIColumnSchema =
| SummariseTextSchema
| CleanDataSchema
| TranslateSchema
| CategoriseTextSchema
| SentimentAnalysisSchema
| PromptSchema
| SearchWebSchema

1
packages/types/src/sdk/index.ts
View File

@ -1,3 +1,4 @@
export * from "./ai"
export * from "./automations"
export * from "./hosting"
export * from "./context"

57
packages/worker/src/api/routes/global/tests/roles.spec.ts
View File

@ -1,6 +1,6 @@
import { structures, TestConfiguration } from "../../../../tests"
import { context, db, permissions, roles } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import { App, Database } from "@budibase/types"
jest.mock("@budibase/backend-core", () => {
const core = jest.requireActual("@budibase/backend-core")
@ -30,6 +30,14 @@ async function addAppMetadata() {
})
}
async function updateAppMetadata(update: Partial<Omit<App, "_id" | "_rev">>) {
const app = await appDb.get("app_metadata")
await appDb.put({
...app,
...update,
})
}
describe("/api/global/roles", () => {
const config = new TestConfiguration()
@ -69,6 +77,53 @@ describe("/api/global/roles", () => {
expect(res.body[appId].roles.length).toEqual(5)
expect(res.body[appId].roles.map((r: any) => r._id)).toContain(ROLE_NAME)
})
it.each(["3.0.0", "3.0.1", "3.1.0", "3.0.0+2146.b125a7c"])(
"exclude POWER roles after v3 (%s)",
async creationVersion => {
await updateAppMetadata({ creationVersion })
const res = await config.api.roles.get()
expect(res.body).toBeDefined()
expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([
ROLE_NAME,
roles.BUILTIN_ROLE_IDS.ADMIN,
roles.BUILTIN_ROLE_IDS.BASIC,
roles.BUILTIN_ROLE_IDS.PUBLIC,
])
}
)
it.each(["2.9.0", "1.0.0", "0.0.0", "2.32.17+2146.b125a7c"])(
"include POWER roles before v3 (%s)",
async creationVersion => {
await updateAppMetadata({ creationVersion })
const res = await config.api.roles.get()
expect(res.body).toBeDefined()
expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([
ROLE_NAME,
roles.BUILTIN_ROLE_IDS.ADMIN,
roles.BUILTIN_ROLE_IDS.POWER,
roles.BUILTIN_ROLE_IDS.BASIC,
roles.BUILTIN_ROLE_IDS.PUBLIC,
])
}
)
it.each(["invalid", ""])(
"include POWER roles when the version is corrupted (%s)",
async creationVersion => {
await updateAppMetadata({ creationVersion })
const res = await config.api.roles.get()
expect(res.body[appId].roles.map((r: any) => r._id)).toEqual([
ROLE_NAME,
roles.BUILTIN_ROLE_IDS.ADMIN,
roles.BUILTIN_ROLE_IDS.POWER,
roles.BUILTIN_ROLE_IDS.BASIC,
roles.BUILTIN_ROLE_IDS.PUBLIC,
])
}
)
})
describe("GET api/global/roles/:appId", () => {