MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v3-ui' into feature/automation-branching-ux

This commit is contained in:
Dean 2024-10-16 15:51:24 +01:00
parent 092febb7d9 38a57faee4
commit d9deba6337
63 changed files with 1073 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/deploy-featurebranch.yml vendored
View File

@ -2,13 +2,11 @@ name: deploy-featurebranch
on:
pull_request:
types: [
labeled,
# default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
opened,
synchronize,
reopened,
]
types:
- labeled
- opened
- synchronize
- reopened
jobs:
release:
@ -22,31 +20,21 @@ jobs:
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set PAYLOAD_LICENSE_TYPE
id: set_license_type
run: |
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
else
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
fi
- uses: passeidireto/trigger-external-workflow-action@main
env:
PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
PAYLOAD_LICENSE_TYPE: |
${{
contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
}}
steps:
- uses: actions/checkout@v4
- uses: passeidireto/trigger-external-workflow-action@main
with:
repository: budibase/budibase-deploys
event: featurebranch-qa-deploy

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -42,14 +42,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

6
charts/budibase/templates/automation-worker-service-deployment.yaml
View File

@ -43,6 +43,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

4
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -56,14 +56,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
- name: API_ENCRYPTION_KEY
valueFrom:
secretKeyRef:

3
charts/budibase/values.yaml
View File

@ -139,9 +139,6 @@ globals:
password: ""
sqs:
# -- Whether to use the CouchDB "structured query service" or not. This is disabled by
# default for now, but will become the default in a future release.
enabled: false
# @ignore
url: ""
# @ignore

6
hosting/docker-compose.yaml
View File

@ -5,7 +5,7 @@ version: "3"
services:
app-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/apps
image: budibase/apps
container_name: bbapps
environment:
SELF_HOSTED: 1
@ -35,7 +35,7 @@ services:
worker-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/worker
image: budibase/worker
container_name: bbworker
environment:
SELF_HOSTED: 1
@ -97,7 +97,7 @@ services:
couchdb-service:
restart: unless-stopped
image: budibase/couchdb
image: budibase/couchdb:v3.3.3-sqs-v2.1.1
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}

3
hosting/single/Dockerfile
View File

@ -69,6 +69,9 @@ WORKDIR /minio
COPY scripts/install-minio.sh ./install.sh
RUN chmod +x install.sh && ./install.sh
# setup redis
COPY hosting/single/redis.conf /etc/redis/redis.conf
# setup runner file
WORKDIR /
COPY hosting/single/runner.sh .

7
hosting/single/redis.conf Normal file
View File

@ -0,0 +1,7 @@
dir "DATA_DIR/redis"
appendonly yes
appendfsync everysec
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb

8
hosting/single/runner.sh
View File

@ -75,13 +75,17 @@ fi
for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
ln -s ${DATA_DIR}/.env /app/.env
ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
mkdir -p ${DATA_DIR}/minio
mkdir -p ${DATA_DIR}/redis
chown -R couchdb:couchdb ${DATA_DIR}/couch
sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
if [[ -n "${REDIS_PASSWORD}" ]]; then
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
else
redis-server > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
fi
/bbcouch-runner.sh &

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.17",
"version": "2.33.1",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/backend-core/src/constants/misc.ts
View File

@ -28,6 +28,7 @@ export enum Config {
OIDC = "oidc",
OIDC_LOGOS = "logos_oidc",
SCIM = "scim",
AI = "AI",
}
export const MIN_VALID_DATE = new Date(-2147483647000)

2
packages/backend-core/src/features/features.ts
View File

@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
SQS: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
[FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

6
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -10,6 +10,7 @@ const schema = {
TEST_BOOLEAN: Flag.boolean(false),
TEST_STRING: Flag.string("default value"),
TEST_NUMBER: Flag.number(0),
TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
}
const flags = new FlagSet(schema)
@ -123,6 +124,11 @@ describe("feature flags", () => {
},
expected: flags.defaults(),
},
{
it: "should be possible to override a default true flag to false",
environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
},
])(
"$it",
async ({

11
packages/backend-core/src/sql/sql.ts
View File

@ -521,8 +521,11 @@ class InternalBuilder {
const [filterTableName, ...otherProperties] = key.split(".")
const property = otherProperties.join(".")
const alias = getTableAlias(filterTableName)
return fn(q, alias ? `${alias}.${property}` : property, value)
return q.andWhere(subquery =>
fn(subquery, alias ? `${alias}.${property}` : property, value)
)
}
for (const key in structure) {
const value = structure[key]
const updatedKey = dbCore.removeKeyNumbering(key)
@ -552,6 +555,9 @@ class InternalBuilder {
value
)
} else if (shouldProcessRelationship) {
if (allOr) {
query = query.or
}
query = builder.addRelationshipForFilter(query, updatedKey, q => {
return handleRelationship(q, updatedKey, value)
})
@ -1288,7 +1294,8 @@ class InternalBuilder {
schema.constraints?.presence === true ||
schema.type === FieldType.FORMULA ||
schema.type === FieldType.AUTO ||
schema.type === FieldType.LINK
schema.type === FieldType.LINK ||
schema.type === FieldType.AI
) {
continue
}

5
packages/backend-core/src/sql/sqlTable.ts
View File

@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
return ignored.indexOf(type) !== -1
}
@ -144,6 +144,9 @@ function generateSchema(
case FieldType.FORMULA:
// This is allowed, but nothing to do on the external datasource
break
case FieldType.AI:
// This is allowed, but nothing to do on the external datasource
break
case FieldType.ATTACHMENTS:
case FieldType.ATTACHMENT_SINGLE:
case FieldType.SIGNATURE_SINGLE:

8
packages/backend-core/tests/core/utilities/mocks/licenses.ts
View File

@ -102,6 +102,14 @@ export const useAppBuilders = () => {
return useFeature(Feature.APP_BUILDERS)
}
export const useBudibaseAI = () => {
return useFeature(Feature.BUDIBASE_AI)
}
export const useAICustomConfigs = () => {
return useFeature(Feature.AI_CUSTOM_CONFIGS)
}
// QUOTAS
export const setAutomationLogsQuota = (value: number) => {

12
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -26,6 +26,7 @@
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
import { licensing } from "stores/portal"
import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
import {
FIELDS,
@ -35,6 +36,7 @@
} from "constants/backend"
import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
import ConfirmDialog from "components/common/ConfirmDialog.svelte"
import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte"
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
@ -99,6 +101,8 @@
let optionsValid = true
$: rowGoldenSample = RowUtils.generateGoldenSample($rows)
$: aiEnabled =
$licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
$: if (primaryDisplay) {
editableColumn.constraints.presence = { allowEmpty: false }
}
@ -447,6 +451,7 @@
FIELDS.BOOLEAN,
FIELDS.DATETIME,
FIELDS.LINK,
...(aiEnabled ? [FIELDS.AI] : []),
FIELDS.LONGFORM,
FIELDS.USER,
FIELDS.USERS,
@ -784,6 +789,13 @@
/>
</div>
</div>
{:else if editableColumn.type === FieldType.AI}
<AIFieldConfiguration
aiField={editableColumn}
context={rowGoldenSample}
bindings={getBindings({ table })}
schema={table.schema}
/>
{:else if editableColumn.type === FieldType.JSON}
<Button primary text on:click={openJsonSchemaEditor}>
Open schema editor

3
packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte
View File

@ -7,6 +7,7 @@
import { FIELDS } from "constants/backend"
const FORMULA_TYPE = FIELDS.FORMULA.type
const AI_TYPE = FIELDS.AI.type
export let row = {}
@ -60,7 +61,7 @@
}}
>
{#each tableSchema as [key, meta]}
{#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
{#if !meta.autocolumn && meta.type !== FORMULA_TYPE && meta.type !== AI_TYPE}
<div>
<RowFieldControl error={errors[key]} {meta} bind:value={row[key]} />
</div>

59
packages/builder/src/components/common/AIFieldConfiguration.svelte Normal file
View File

@ -0,0 +1,59 @@
<script>
import { Helpers, Multiselect, Select } from "@budibase/bbui"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import {
AIOperations,
OperationFields,
OperationFieldTypes,
} from "@budibase/shared-core"
const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
label: AIOperations[key].label,
value: AIOperations[key].value,
}))
export let bindings
export let context
export let schema
export let aiField = {}
$: OperationField = OperationFields[aiField.operation]
$: schemaWithoutRelations = Object.keys(schema).filter(
key => schema[key].type !== "link"
)
</script>
<Select
label={"Operation"}
options={AIFieldConfigOptions}
bind:value={aiField.operation}
/>
{#if aiField.operation}
{#each Object.keys(OperationField) as key}
{#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
<ModalBindableInput
label={Helpers.capitalise(key)}
panel={ServerBindingPanel}
title="Prompt"
on:change={e => (aiField[key] = e.detail)}
value={aiField[key]}
{bindings}
allowJS
{context}
/>
{:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
<Multiselect
bind:value={aiField[key]}
label={Helpers.capitalise(key)}
options={schemaWithoutRelations}
/>
{:else if OperationField[key] === OperationFieldTypes.COLUMN}
<Select
bind:value={aiField[key]}
label={Helpers.capitalise(key)}
options={schemaWithoutRelations}
/>
{/if}
{/each}
{/if}

1
packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
View File

@ -59,6 +59,7 @@
bind:this={drawer}
title="Filtering"
on:drawerHide
on:drawerShow
on:drawerShow={() => {
// Reset to the currently available value.
localFilters = Helpers.cloneDeep(value)

6
packages/builder/src/constants/backend/index.js
View File

@ -159,6 +159,12 @@ export const FIELDS = {
icon: TypeIconMap[FieldType.FORMULA],
constraints: {},
},
AI: {
name: "AI",
type: FieldType.AI,
icon: TypeIconMap[FieldType.AI],
constraints: {},
},
JSON: {
name: "JSON",
type: FieldType.JSON,

3
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte
View File

@ -1,6 +1,6 @@
<script>
import { viewsV2, rowActions } from "stores/builder"
import { admin, themeStore } from "stores/portal"
import { admin, themeStore, licensing } from "stores/portal"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import { notifications } from "@budibase/bbui"
@ -49,6 +49,7 @@
{buttons}
allowAddRows
allowDeleteRows
aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
showAvatars={false}
on:updatedatasource={handleGridViewUpdate}
isCloud={$admin.cloud}

4
packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte
View File

@ -7,7 +7,7 @@
appStore,
rowActions,
} from "stores/builder"
import { themeStore, admin } from "stores/portal"
import { themeStore, admin, licensing } from "stores/portal"
import { TableNames } from "constants"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
@ -125,6 +125,8 @@
schemaOverrides={isUsersTable ? userSchemaOverrides : null}
showAvatars={false}
isCloud={$admin.cloud}
aiEnabled={$licensing.budibaseAIEnabled ||
$licensing.customAIConfigsEnabled}
{buttons}
buttonsCollapsed
on:updatedatasource={handleGridTableUpdate}

2
packages/client/src/components/app/GridBlock.svelte
View File

@ -3,6 +3,7 @@
// because it functions similarly to one
import { getContext, onMount } from "svelte"
import { get, derived, readable } from "svelte/store"
import { featuresStore } from "stores"
import { Grid } from "@budibase/frontend-core"
// table is actually any datasource, but called table for legacy compatibility
@ -186,6 +187,7 @@
{buttonsCollapsed}
{buttonsCollapsedText}
isCloud={$environmentStore.cloud}
aiEnabled={$featuresStore.aiEnabled}
on:rowclick={e => onRowClick?.({ row: e.detail })}
/>
</div>

6
packages/client/src/stores/features.js
View File

@ -2,6 +2,7 @@ import { derived } from "svelte/store"
import { appStore } from "./app"
import { authStore } from "./auth"
import { Constants } from "@budibase/frontend-core"
import { Feature } from "@budibase/types"
const createFeaturesStore = () => {
return derived([authStore, appStore], ([$authStore, $appStore]) => {
@ -33,8 +34,13 @@ const createFeaturesStore = () => {
}
}
const license = getUserLicense()
return {
logoEnabled: isFreePlan(),
aiEnabled:
license?.features?.includes(Feature.AI_CUSTOM_CONFIGS) ||
license?.features?.includes(Feature.BUDIBASE_AI),
}
})
}

2
packages/frontend-core/src/components/FilterField.svelte
View File

@ -175,7 +175,7 @@
/>
{:else}
<div>
{#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
{#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
<Input
disabled={filter.noValue}
value={readableValue}

37
packages/frontend-core/src/components/Updating.svelte
View File

@ -2,36 +2,31 @@
export let isMigrationDone
export let onMigrationDone
export let timeoutSeconds = 60 // 1 minute
export let minTimeSeconds = 3
const loadTime = Date.now()
const intervalMs = 1000
let timedOut = false
let secondsWaited = 0
async function checkMigrationsFinished() {
setTimeout(async () => {
let totalWaitMs = 0
// eslint-disable-next-line no-constant-condition
while (true) {
const waitForMs = 5000 + Math.random() * 5000
await new Promise(resolve => setTimeout(resolve, waitForMs))
totalWaitMs += waitForMs
const isMigrated = await isMigrationDone()
const timeoutMs = timeoutSeconds * 1000
if (!isMigrated || secondsWaited <= minTimeSeconds) {
if (loadTime + timeoutMs > Date.now()) {
secondsWaited += 1
return checkMigrationsFinished()
}
return migrationTimeout()
}
if (isMigrated) {
onMigrationDone()
}, intervalMs)
return
}
if (totalWaitMs > timeoutSeconds * 1000) {
timedOut = true
return
}
}
}
checkMigrationsFinished()
function migrationTimeout() {
timedOut = true
}
</script>
<div class="loading" class:timeout={timedOut}>

99
packages/frontend-core/src/components/grid/cells/AICell.svelte Normal file
View File

@ -0,0 +1,99 @@
<script>
import { onMount } from "svelte"
import { clickOutside } from "@budibase/bbui"
import GridPopover from "../overlays/GridPopover.svelte"
export let value
export let focused = false
export let api
let textarea
let isOpen = false
let anchor
$: {
if (!focused) {
isOpen = false
}
}
const onKeyDown = () => {
return isOpen
}
const open = async () => {
isOpen = true
}
const close = () => {
textarea?.blur()
isOpen = false
}
onMount(() => {
api = {
focus: () => open(),
blur: () => close(),
isActive: () => isOpen,
onKeyDown,
}
})
</script>
<!-- svelte-ignore a11y-no-static-element-interactions -->
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div class="long-form-cell" on:click={open} bind:this={anchor}>
<div class="value">
{value || ""}
</div>
</div>
{#if isOpen}
<GridPopover {anchor} on:close={close}>
<textarea
disabled
bind:this={textarea}
value={value || ""}
on:wheel|stopPropagation
spellcheck="false"
use:clickOutside={close}
/>
</GridPopover>
{/if}
<style>
.long-form-cell {
flex: 1 1 auto;
padding: var(--cell-padding);
align-self: stretch;
display: flex;
align-items: flex-start;
overflow: hidden;
}
.value {
display: -webkit-box;
-webkit-line-clamp: var(--content-lines);
-webkit-box-orient: vertical;
overflow: hidden;
line-height: 20px;
}
textarea {
border: none;
width: 320px;
flex: 1 1 auto;
height: var(--max-cell-render-overflow);
padding: var(--cell-padding);
margin: 0;
background: var(--cell-background);
font-size: var(--cell-font-size);
font-family: var(--font-sans);
color: inherit;
z-index: 1;
resize: none;
line-height: 20px;
overflow: auto;
}
textarea:focus {
outline: none;
}
</style>

3
packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
View File

@ -95,7 +95,8 @@
const { type, formulaType } = col.schema
return (
searchableTypes.includes(type) ||
(type === FieldType.FORMULA && formulaType === FormulaType.STATIC)
(type === FieldType.FORMULA && formulaType === FormulaType.STATIC) ||
type === FieldType.AI
)
}

2
packages/frontend-core/src/components/grid/layout/Grid.svelte
View File

@ -48,6 +48,7 @@
export let darkMode = false
export let isCloud = null
export let rowConditions = null
export let aiEnabled = false
// Unique identifier for DOM nodes inside this instance
const gridID = `grid-${Math.random().toString().slice(2)}`
@ -104,6 +105,7 @@
buttonsCollapsedText,
darkMode,
isCloud,
aiEnabled,
rowConditions,
})

2
packages/frontend-core/src/components/grid/lib/renderers.js
View File

@ -9,6 +9,7 @@ import TextCell from "../cells/TextCell.svelte"
import LongFormCell from "../cells/LongFormCell.svelte"
import BooleanCell from "../cells/BooleanCell.svelte"
import FormulaCell from "../cells/FormulaCell.svelte"
import AICell from "../cells/AICell.svelte"
import JSONCell from "../cells/JSONCell.svelte"
import AttachmentCell from "../cells/AttachmentCell.svelte"
import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
@ -30,6 +31,7 @@ const TypeComponentMap = {
[FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
[FieldType.LINK]: RelationshipCell,
[FieldType.FORMULA]: FormulaCell,
[FieldType.AI]: AICell,
[FieldType.JSON]: JSONCell,
[FieldType.BB_REFERENCE]: BBReferenceCell,
[FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,

19
packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte
View File

@ -1,5 +1,6 @@
<script>
import { Menu, MenuItem, Helpers } from "@budibase/bbui"
import { FieldType } from "@budibase/types"
import { getContext } from "svelte"
import { NewRowID } from "../lib/constants"
import GridPopover from "./GridPopover.svelte"
@ -26,6 +27,9 @@
$: style = makeStyle($menu)
$: isNewRow = $focusedRowId === NewRowID
$: hasAIColumns = $visibleColumns.some(
col => col.schema.type === FieldType.AI
)
const makeStyle = menu => {
return `left:${menu.left}px; top:${menu.top}px;`
@ -53,6 +57,12 @@
await Helpers.copyToClipboard(value)
$notifications.success("Copied to clipboard")
}
const generateAIColumns = async () => {
menu.actions.close()
await rows.actions.applyRowChanges({ rowId: $focusedRowId })
$notifications.success("Generated AI columns")
}
</script>
<div bind:this={anchor} {style} class="menu-anchor" />
@ -161,6 +171,15 @@
>
Delete row
</MenuItem>
{#if $config.aiEnabled}
<MenuItem
icon="MagicWand"
disabled={isNewRow || !hasAIColumns}
on:click={generateAIColumns}
>
Generate AI Columns
</MenuItem>
{/if}
{/if}
</Menu>
</GridPopover>

1
packages/frontend-core/src/components/grid/stores/columns.js
View File

@ -109,6 +109,7 @@ export const createActions = context => {
column.schema.autocolumn ||
column.schema.disabled ||
column.schema.type === "formula" ||
column.schema.type === "ai" ||
column.schema.readonly
)
}

3
packages/frontend-core/src/constants.js
View File

@ -160,9 +160,10 @@ export const TypeIconMap = {
[FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
[FieldType.LINK]: "DataCorrelated",
[FieldType.FORMULA]: "Calculator",
[FieldType.AI]: "MagicWand",
[FieldType.JSON]: "Brackets",
[FieldType.BIGINT]: "TagBold",
[FieldType.AUTO]: "MagicWand",
[FieldType.AUTO]: "Shapes",
[FieldType.BB_REFERENCE]: {
[BBReferenceFieldSubType.USER]: "UserGroup",
[BBReferenceFieldSubType.USERS]: "UserGroup",

12
packages/server/specs/openapi.json
View File

@ -833,7 +833,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -857,6 +858,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",
@ -1042,7 +1044,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -1066,6 +1069,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",
@ -1262,7 +1266,8 @@
"type": "string",
"enum": [
"static",
"dynamic"
"dynamic",
"ai"
],
"description": "Defines whether this is a static or dynamic formula."
}
@ -1286,6 +1291,7 @@
"link",
"formula",
"auto",
"ai",
"json",
"internal",
"barcodeqr",

6
packages/server/specs/openapi.yaml
View File

@ -761,6 +761,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -779,6 +780,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr
@ -929,6 +931,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -947,6 +950,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr
@ -1104,6 +1108,7 @@ components:
enum:
- static
- dynamic
- ai
description: Defines whether this is a static or dynamic formula.
- type: object
properties:
@ -1122,6 +1127,7 @@ components:
- link
- formula
- auto
- ai
- json
- internal
- barcodeqr

7
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -157,7 +157,8 @@ function isEditableColumn(column: FieldSchema) {
column.autoReason !== AutoReason.FOREIGN_KEY &&
column.subtype !== AutoFieldSubType.AUTO_ID
const isFormula = column.type === FieldType.FORMULA
return !(isExternalAutoColumn || isFormula)
const isAIColumn = column.type === FieldType.AI
return !(isExternalAutoColumn || isFormula || isAIColumn)
}
export class ExternalRequest<T extends Operation> {
@ -173,9 +174,9 @@ export class ExternalRequest<T extends Operation> {
if (!opts.datasource) {
if (sdk.views.isView(source)) {
const table = await sdk.views.getTable(source.id)
opts.datasource = await sdk.datasources.get(table.sourceId!)
opts.datasource = await sdk.datasources.get(table.sourceId)
} else {
opts.datasource = await sdk.datasources.get(source.sourceId!)
opts.datasource = await sdk.datasources.get(source.sourceId)
}
}

19
packages/server/src/api/controllers/row/staticFormula.ts
View File

@ -1,6 +1,7 @@
import { getRowParams } from "../../../db/utils"
import {
outputProcessing,
processAIColumns,
processFormulas,
} from "../../../utilities/rowProcessor"
import { context } from "@budibase/backend-core"
@ -9,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
import isEqual from "lodash/isEqual"
import { cloneDeep } from "lodash/fp"
import sdk from "../../../sdk"
import * as pro from "@budibase/pro"
/**
* This function runs through a list of enriched rows, looks at the rows which
@ -102,7 +104,7 @@ export async function updateAllFormulasInTable(table: Table) {
(enriched: Row) => enriched._id === row._id
)
if (enrichedRow) {
const processed = await processFormulas(table, cloneDeep(row), {
let processed = await processFormulas(table, cloneDeep(row), {
dynamic: false,
contextRows: [enrichedRow],
})
@ -142,12 +144,27 @@ export async function finaliseRow(
dynamic: false,
contextRows: [enrichedRow],
})
const aiEnabled =
(await pro.features.isBudibaseAIEnabled()) ||
(await pro.features.isAICustomConfigsEnabled())
if (aiEnabled) {
row = await processAIColumns(table, row, {
contextRows: [enrichedRow],
})
}
const response = await db.put(row)
// for response, calculate the formulas for the enriched row
enrichedRow._rev = response.rev
enrichedRow = await processFormulas(table, enrichedRow, {
dynamic: false,
})
if (aiEnabled) {
enrichedRow = await processAIColumns(table, row, {
contextRows: [enrichedRow],
})
}
// this updates the related formulas in other rows based on the relations to this row
if (updateFormula) {
await updateRelatedFormula(table, enrichedRow)

1
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -124,6 +124,7 @@ export async function buildSqlFieldList(
([columnName, column]) =>
column.type !== FieldType.LINK &&
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI &&
!existing.find(
(field: string) => field === `${table.name}.${columnName}`
)

2
packages/server/src/api/controllers/table/bulkFormula.ts
View File

@ -6,10 +6,10 @@ import uniq from "lodash/uniq"
import { updateAllFormulasInTable } from "../row/staticFormula"
import { context } from "@budibase/backend-core"
import {
FormulaType,
FieldSchema,
FieldType,
FormulaFieldMetadata,
FormulaType,
Table,
} from "@budibase/types"
import sdk from "../../../sdk"

66
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -18,6 +18,7 @@ import {
} from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import {
AIOperationEnum,
AttachmentFieldMetadata,
AutoFieldSubType,
Datasource,
@ -50,6 +51,18 @@ import { InternalTables } from "../../../db/utils"
import { withEnv } from "../../../environment"
import { JsTimeoutError } from "@budibase/string-templates"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => `Mock LLM Response`),
buildPromptFromAIOperation: jest.fn(),
}),
},
},
}))
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
tk.freeze(timestamp)
interface WaitOptions {
@ -2086,6 +2099,7 @@ describe.each([
[FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
[FieldType.FORMULA]: undefined, // generated field
[FieldType.AUTO]: undefined, // generated field
[FieldType.AI]: undefined, // generated field
[FieldType.JSON]: { name: generator.guid() },
[FieldType.INTERNAL]: generator.guid(),
[FieldType.BARCODEQR]: generator.guid(),
@ -2189,6 +2203,7 @@ describe.each([
expectedRowData["bb_reference_single"].sample,
false
),
ai: null,
},
])
})
@ -2975,6 +2990,57 @@ describe.each([
)
})
isSqs &&
describe("AI fields", () => {
let table: Table
beforeAll(async () => {
mocks.licenses.useBudibaseAI()
mocks.licenses.useAICustomConfigs()
table = await config.api.table.save(
saveTableRequest({
schema: {
ai: {
name: "ai",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Convert the following to German: '{{ product }}'",
},
product: {
name: "product",
type: FieldType.STRING,
},
},
})
)
await config.api.row.save(table._id!, {
product: generator.word(),
})
})
afterAll(() => {
jest.unmock("@budibase/pro")
})
it("should be able to save a row with an AI column", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
expect(rows[0].ai).toEqual("Mock LLM Response")
})
it("should be able to update a row with an AI column", async () => {
const { rows } = await config.api.row.search(table._id!)
expect(rows.length).toBe(1)
await config.api.row.save(table._id!, {
product: generator.word(),
...rows[0],
})
expect(rows.length).toBe(1)
expect(rows[0].ai).toEqual("Mock LLM Response")
})
})
describe("Formula fields", () => {
let table: Table
let otherTable: Table

294
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -17,12 +17,14 @@ import {
import * as setup from "./utilities"
import {
AIOperationEnum,
AutoFieldSubType,
BBReferenceFieldSubType,
Datasource,
EmptyFilterOption,
FieldType,
JsonFieldSubType,
LogicalOperator,
RelationshipType,
Row,
RowSearchParams,
@ -41,11 +43,23 @@ import tk from "timekeeper"
import { encodeJSBinding } from "@budibase/string-templates"
import { dataFilters } from "@budibase/shared-core"
import { Knex } from "knex"
import { generator, structures } from "@budibase/backend-core/tests"
import { generator, structures, mocks } from "@budibase/backend-core/tests"
import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
import { generateRowIdField } from "../../../integrations/utils"
import { cloneDeep } from "lodash/fp"
jest.mock("@budibase/pro", () => ({
...jest.requireActual("@budibase/pro"),
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => `Mock LLM Response`),
buildPromptFromAIOperation: jest.fn(),
}),
},
},
}))
describe.each([
["in-memory", undefined],
["lucene", undefined],
@ -1601,6 +1615,79 @@ describe.each([
})
})
isSqs &&
describe("AI Column", () => {
const UNEXISTING_AI_COLUMN = "Real LLM Response"
beforeAll(async () => {
mocks.licenses.useBudibaseAI()
mocks.licenses.useAICustomConfigs()
tableOrViewId = await createTableOrView({
product: { name: "product", type: FieldType.STRING },
ai: {
name: "AI",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
})
await createRows([{ product: "Big Mac" }, { product: "McCrispy" }])
})
describe("equal", () => {
it("successfully finds rows based on AI column", async () => {
await expectQuery({
equal: { ai: "Mock LLM Response" },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
it("fails to find nonexistent row", async () => {
await expectQuery({
equal: { ai: UNEXISTING_AI_COLUMN },
}).toFindNothing()
})
})
describe("notEqual", () => {
it("Returns nothing when searching notEqual on the mock AI response", async () => {
await expectQuery({
notEqual: { ai: "Mock LLM Response" },
}).toContainExactly([])
})
it("return all when requesting non-existing response", async () => {
await expectQuery({
notEqual: { ai: "Real LLM Response" },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
})
describe("oneOf", () => {
it("successfully finds a row", async () => {
await expectQuery({
oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] },
}).toContainExactly([
{ product: "Big Mac" },
{ product: "McCrispy" },
])
})
it("fails to find nonexistent row", async () => {
await expectQuery({
oneOf: { ai: ["Whopper"] },
}).toFindNothing()
})
})
})
describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
beforeAll(async () => {
tableOrViewId = await createTableOrView({
@ -2329,6 +2416,211 @@ describe.each([
equal: { ["name"]: "baz" },
}).toContainExactly([{ name: "baz", productCat: undefined }])
})
describe("logical filters", () => {
const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
describe("$and", () => {
it("should allow single conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
})
it.each([logicalOperators])(
"should allow nested ands with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
}
)
it.each([logicalOperators])(
"should allow nested ands with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([])
}
)
it.each([logicalOperators])(
"should allow nested ands with multiple conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
}
)
})
describe("$ors", () => {
it("should allow single conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
})
it.each([logicalOperators])(
"should allow nested ors with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
}
)
it.each([logicalOperators])(
"should allow nested ors with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
}
)
it("should allow nested ors with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
})
})
})
})
isSql &&

8
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -206,7 +206,7 @@ describe.each([
visible: false,
icon: "ic",
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
}
const createdView = await config.api.viewV2.create(newView)
@ -250,7 +250,7 @@ describe.each([
name: "Category",
type: FieldType.STRING,
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
}
await config.api.viewV2.create(newView, {
@ -1044,7 +1044,7 @@ describe.each([
visible: false,
icon: "ic",
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
})
expect(updatedView).toEqual({
@ -1078,7 +1078,7 @@ describe.each([
name: "Category",
type: FieldType.STRING,
},
} as Record<string, FieldSchema>,
} as ViewV2Schema,
},
{
status: 200,

4
packages/server/src/db/linkedRows/index.ts
View File

@ -307,7 +307,9 @@ export async function squashLinks<T = Row[] | Row>(
return false
}
if (
[FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type)
[FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
tableColumn.type
)
) {
return false
}

4
packages/server/src/integrations/googlesheets.ts
View File

@ -56,6 +56,7 @@ interface AuthTokenResponse {
const isTypeAllowed: Record<FieldType, boolean> = {
[FieldType.STRING]: true,
[FieldType.FORMULA]: true,
[FieldType.AI]: true,
[FieldType.NUMBER]: true,
[FieldType.LONGFORM]: true,
[FieldType.DATETIME]: true,
@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
}
if (
!sheet.headerValues.includes(key) &&
column.type !== FieldType.FORMULA
column.type !== FieldType.FORMULA &&
column.type !== FieldType.AI
) {
updatedHeaderValues.push(key)
}

4
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid"
and COALESCE("b"."taskname" = $1, FALSE)`
and (COALESCE("b"."taskname" = $1, FALSE))`
)
),
})
@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
],
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
)
),
})

1
packages/server/src/integrations/utils/utils.ts
View File

@ -242,6 +242,7 @@ function copyExistingPropsOver(
let shouldKeepSchema = false
switch (existingColumnType) {
case FieldType.FORMULA:
case FieldType.AI:
case FieldType.AUTO:
case FieldType.INTERNAL:
shouldKeepSchema = true

6
packages/server/src/sdk/app/tables/getters.ts
View File

@ -90,7 +90,11 @@ export async function getExternalTable(
if (!entities[tableName]) {
throw new Error(`Unable to find table named "${tableName}"`)
}
return processTable(entities[tableName])
const table = await processTable(entities[tableName])
if (!table.sourceId) {
table.sourceId = datasourceId
}
return table
}
export async function getTable(tableId: string): Promise<Table> {

1
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -19,6 +19,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
[FieldType.DATETIME]: SQLiteType.TEXT,
[FieldType.FORMULA]: SQLiteType.TEXT,
[FieldType.AI]: SQLiteType.TEXT,
[FieldType.LONGFORM]: SQLiteType.TEXT,
[FieldType.NUMBER]: SQLiteType.REAL,
[FieldType.STRING]: SQLiteType.TEXT,

6
packages/server/src/sdk/app/views/index.ts
View File

@ -313,7 +313,11 @@ export async function enrichSchema(
const result: Record<string, ViewV2ColumnEnriched> = {}
for (const relTableFieldName of Object.keys(relTable.schema)) {
const relTableField = relTable.schema[relTableFieldName]
if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
if (
[FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
relTableField.type
)
) {
continue
}

7
packages/server/src/tests/utilities/structures.ts
View File

@ -7,6 +7,7 @@ import {
TRIGGER_DEFINITIONS,
} from "../../automations"
import {
AIOperationEnum,
Automation,
AutomationActionStepId,
AutomationResults,
@ -666,6 +667,12 @@ export function fullSchemaWithoutLinks({
presence: allRequired,
},
},
[FieldType.AI]: {
name: "ai",
type: FieldType.AI,
operation: AIOperationEnum.PROMPT,
prompt: "Translate this into German :'{{ product }}'",
},
[FieldType.BARCODEQR]: {
name: "barcodeqr",
type: FieldType.BARCODEQR,

4
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -210,6 +210,10 @@ export async function inputProcessing(
if (field.type === FieldType.FORMULA) {
delete clonedRow[key]
}
// remove any AI values, they are to be generated
if (field.type === FieldType.AI) {
delete clonedRow[key]
}
// otherwise coerce what is there to correct types
else {
clonedRow[key] = coerce(value, field.type)

75
packages/server/src/utilities/rowProcessor/tests/utils.spec.ts
View File

@ -1,11 +1,29 @@
import { fixAutoColumnSubType } from "../utils"
import { fixAutoColumnSubType, processAIColumns } from "../utils"
import { AutoFieldDefaultNames } from "../../../constants"
import {
AIOperationEnum,
AutoFieldSubType,
FieldSchema,
FieldType,
INTERNAL_TABLE_SOURCE_ID,
RelationshipType,
Table,
TableSourceType,
} from "@budibase/types"
import { generator } from "@budibase/backend-core/tests"
const buildPromptMock = jest.fn()
jest.mock("@budibase/pro", () => ({
ai: {
LargeLanguageModel: {
forCurrentTenant: async () => ({
run: jest.fn(() => "response from LLM"),
buildPromptFromAIOperation: buildPromptMock,
}),
},
},
}))
describe("rowProcessor utility", () => {
describe("fixAutoColumnSubType", () => {
@ -60,4 +78,59 @@ describe("rowProcessor utility", () => {
expect(fixAutoColumnSubType(schema)).toEqual(schema)
})
})
describe("processAIColumns", () => {
it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
const table: Table = {
_id: generator.guid(),
name: "AITestTable",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
product: {
type: FieldType.STRING,
name: "product",
constraints: {
presence: true,
type: "string",
},
},
aicol: {
type: FieldType.AI,
name: "aicol",
operation: AIOperationEnum.PROMPT,
prompt: "Translate '{{ product }}' into German",
},
},
}
const inputRows = [
{
product: "Car Battery",
},
]
const result = await processAIColumns(table, inputRows, {
contextRows: inputRows,
})
expect(buildPromptMock).toHaveBeenCalledWith({
row: {
product: "Car Battery",
},
schema: {
name: "aicol",
operation: "PROMPT",
prompt: "Translate 'Car Battery' into German",
type: "ai",
},
})
expect(result).toEqual([
{
aicol: "response from LLM",
product: "Car Battery",
},
])
})
})
})

64
packages/server/src/utilities/rowProcessor/utils.ts
View File

@ -8,9 +8,13 @@ import {
FormulaType,
AutoFieldSubType,
FieldType,
OperationFieldTypeEnum,
AIOperationEnum,
} from "@budibase/types"
import { OperationFields } from "@budibase/shared-core"
import tracer from "dd-trace"
import { context } from "@budibase/backend-core"
import * as pro from "@budibase/pro"
interface FormulaOpts {
dynamic?: boolean
@ -91,6 +95,66 @@ export async function processFormulas<T extends Row | Row[]>(
})
}
/**
* Looks through the rows provided and finds AI columns - which it then processes.
*/
export async function processAIColumns<T extends Row | Row[]>(
table: Table,
inputRows: T,
{ contextRows }: FormulaOpts
): Promise<T> {
return tracer.trace("processAIColumns", {}, async span => {
const numRows = Array.isArray(inputRows) ? inputRows.length : 1
span?.addTags({ table_id: table._id, numRows })
const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini")
if (rows) {
// Ensure we have snippet context
await context.ensureSnippetContext()
for (let [column, schema] of Object.entries(table.schema)) {
if (schema.type !== FieldType.AI) {
continue
}
const rowUpdates = rows.map((row, i) => {
const contextRow = contextRows ? contextRows[i] : row
// Check if the type is bindable and pass through HBS if so
const operationField =
OperationFields[schema.operation as AIOperationEnum]
for (const key in schema) {
const fieldType = operationField[key as keyof typeof operationField]
if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
// @ts-ignore
schema[key] = processStringSync(schema[key], contextRow)
}
}
const prompt = llm.buildPromptFromAIOperation({ schema, row })
return tracer.trace("processAIColumn", {}, async span => {
span?.addTags({ table_id: table._id, column })
const llmResponse = await llm.run(prompt!)
return {
...row,
[column]: llmResponse,
}
})
})
const processedRows = await Promise.all(rowUpdates)
// Promise.all is deterministic so can rely on the indexing here
processedRows.forEach(
(processedRow, index) => (rows[index] = processedRow)
)
}
}
return Array.isArray(inputRows) ? rows : rows[0]
})
}
/**
* Processes any date columns and ensures that those without the ignoreTimezones
* flag set are parsed as UTC rather than local time.

68
packages/shared-core/src/constants/ai.ts Normal file
View File

@ -0,0 +1,68 @@
import {
AIOperationEnum,
OperationFieldsType,
OperationFieldTypeEnum,
} from "@budibase/types"
export const AIOperations = {
SUMMARISE_TEXT: {
label: "Summarise Text",
value: "SUMMARISE_TEXT",
},
CLEAN_DATA: {
label: "Clean Data",
value: "CLEAN_DATA",
},
TRANSLATE: {
label: "Translate",
value: "TRANSLATE",
},
CATEGORISE_TEXT: {
label: "Categorise Text",
value: "CATEGORISE_TEXT",
},
SENTIMENT_ANALYSIS: {
label: "Sentiment Analysis",
value: "SENTIMENT_ANALYSIS",
},
PROMPT: {
label: "Prompt",
value: "PROMPT",
},
SEARCH_WEB: {
label: "Search Web",
value: "SEARCH_WEB",
},
}
export const OperationFieldTypes = {
MULTI_COLUMN: "columns",
COLUMN: "column",
BINDABLE_TEXT: "prompt",
}
export const OperationFields: OperationFieldsType = {
[AIOperationEnum.SUMMARISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
},
[AIOperationEnum.CLEAN_DATA]: {
column: OperationFieldTypeEnum.COLUMN,
},
[AIOperationEnum.TRANSLATE]: {
column: OperationFieldTypeEnum.COLUMN,
language: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.CATEGORISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
categories: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.SENTIMENT_ANALYSIS]: {
column: OperationFieldTypeEnum.COLUMN,
},
[AIOperationEnum.PROMPT]: {
prompt: OperationFieldTypeEnum.BINDABLE_TEXT,
},
[AIOperationEnum.SEARCH_WEB]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN,
},
}

1
packages/shared-core/src/constants/index.ts
View File

@ -1,3 +1,4 @@
export * from "./ai"
export * from "./api"
export * from "./fields"
export * from "./rows"

2
packages/shared-core/src/filters.ts
View File

@ -90,6 +90,8 @@ export const getValidOperatorsForType = (
ops = numOps
} else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
} else if (type === FieldType.AI) {
ops = stringOps.concat([Op.MoreThan, Op.LessThan])
} else if (
type === FieldType.BB_REFERENCE_SINGLE ||
schema.isDeprecatedSingleUserColumn(fieldType)

3
packages/shared-core/src/table.ts
View File

@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
[FieldType.NUMBER]: true,
[FieldType.DATETIME]: true,
[FieldType.FORMULA]: true,
[FieldType.AI]: true,
[FieldType.AUTO]: true,
[FieldType.INTERNAL]: true,
[FieldType.BARCODEQR]: true,
@ -38,6 +39,7 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
[FieldType.JSON]: true,
[FieldType.FORMULA]: false,
[FieldType.AI]: false,
[FieldType.ATTACHMENTS]: false,
[FieldType.ATTACHMENT_SINGLE]: false,
[FieldType.SIGNATURE_SINGLE]: false,
@ -62,6 +64,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
[FieldType.BIGINT]: false,
[FieldType.BOOLEAN]: false,
[FieldType.FORMULA]: false,
[FieldType.AI]: false,
[FieldType.ATTACHMENTS]: false,
[FieldType.ATTACHMENT_SINGLE]: false,
[FieldType.SIGNATURE_SINGLE]: false,

7
packages/types/src/documents/app/row.ts
View File

@ -76,6 +76,13 @@ export enum FieldType {
* that is part of the initial formula definition, the formula will be live evaluated in the browser.
*/
AUTO = "auto",
/**
* A complex type, called an AI column within Budibase. This type is only supported against internal tables
* and calculates the output based on a chosen operation (summarise text, translation etc) which passes to
* the configured Budibase Large Language Model to retrieve the output and write it back into the row.
* AI fields function in a similar fashion to static formulas, and possess many of the same characteristics.
*/
AI = "ai",
/**
* a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
* type, which will be represented as a JSON object in the row. This type depends on a schema being

1
packages/types/src/documents/app/table/constants.ts
View File

@ -30,6 +30,7 @@ export enum JsonFieldSubType {
export enum FormulaType {
STATIC = "static",
DYNAMIC = "dynamic",
AI = "ai",
}
export enum BBReferenceFieldSubType {

13
packages/types/src/documents/app/table/schema.ts
View File

@ -9,6 +9,7 @@ import {
JsonFieldSubType,
RelationshipType,
} from "./constants"
import { AIOperationEnum } from "../../../sdk/ai"
export interface UIFieldMetadata {
order?: number
@ -116,6 +117,16 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
formulaType?: FormulaType
}
export interface AIFieldMetadata extends BaseFieldSchema {
type: FieldType.AI
operation: AIOperationEnum
columns?: string[]
column?: string
categories?: string[]
prompt?: string
language?: string
}
export interface BBReferenceFieldMetadata
extends Omit<BaseFieldSchema, "subtype"> {
type: FieldType.BB_REFERENCE
@ -194,6 +205,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
| FieldType.LINK
| FieldType.AUTO
| FieldType.FORMULA
| FieldType.AI
| FieldType.NUMBER
| FieldType.LONGFORM
| FieldType.BB_REFERENCE
@ -211,6 +223,7 @@ export type FieldSchema =
| RelationshipFieldMetadata
| AutoColumnFieldMetadata
| FormulaFieldMetadata
| AIFieldMetadata
| NumberFieldMetadata
| LongFormFieldMetadata
| StringFieldMetadata

91
packages/types/src/sdk/ai.ts Normal file
View File

@ -0,0 +1,91 @@
export enum AIOperationEnum {
SUMMARISE_TEXT = "SUMMARISE_TEXT",
CLEAN_DATA = "CLEAN_DATA",
TRANSLATE = "TRANSLATE",
CATEGORISE_TEXT = "CATEGORISE_TEXT",
SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS",
PROMPT = "PROMPT",
SEARCH_WEB = "SEARCH_WEB",
}
export enum OperationFieldTypeEnum {
MULTI_COLUMN = "columns",
COLUMN = "column",
BINDABLE_TEXT = "prompt",
}
export type OperationFieldsType = {
[AIOperationEnum.SUMMARISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
}
[AIOperationEnum.CLEAN_DATA]: {
column: OperationFieldTypeEnum.COLUMN
}
[AIOperationEnum.TRANSLATE]: {
column: OperationFieldTypeEnum.COLUMN
language: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.CATEGORISE_TEXT]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
categories: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.SENTIMENT_ANALYSIS]: {
column: OperationFieldTypeEnum.COLUMN
}
[AIOperationEnum.PROMPT]: {
prompt: OperationFieldTypeEnum.BINDABLE_TEXT
}
[AIOperationEnum.SEARCH_WEB]: {
columns: OperationFieldTypeEnum.MULTI_COLUMN
}
}
type BaseSchema = {
operation: AIOperationEnum
}
type SummariseTextSchema = BaseSchema & {
operation: AIOperationEnum.SUMMARISE_TEXT
columns: string[]
}
type CleanDataSchema = BaseSchema & {
operation: AIOperationEnum.CLEAN_DATA
column: string
}
type TranslateSchema = BaseSchema & {
operation: AIOperationEnum.TRANSLATE
column: string
language: string
}
type CategoriseTextSchema = BaseSchema & {
operation: AIOperationEnum.CATEGORISE_TEXT
columns: string[]
categories: string[]
}
type SentimentAnalysisSchema = BaseSchema & {
operation: AIOperationEnum.SENTIMENT_ANALYSIS
column: string
}
type PromptSchema = BaseSchema & {
operation: AIOperationEnum.PROMPT
prompt: string
}
type SearchWebSchema = BaseSchema & {
operation: AIOperationEnum.SEARCH_WEB
columns: string[]
}
export type AIColumnSchema =
| SummariseTextSchema
| CleanDataSchema
| TranslateSchema
| CategoriseTextSchema
| SentimentAnalysisSchema
| PromptSchema
| SearchWebSchema

1
packages/types/src/sdk/index.ts
View File

@ -1,3 +1,4 @@
export * from "./ai"
export * from "./automations"
export * from "./hosting"
export * from "./context"