Merge remote-tracking branch 'origin/v3-ui' into feature/automation-branching-ux

.github/workflows/deploy-featurebranch.yml

@@ -2,13 +2,11 @@ name: deploy-featurebranch
 
 on:
   pull_request:
-    types: [
+    types: 
-      labeled,
+      - labeled
-      # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
+      - opened
-      opened,
+      - synchronize
-      synchronize,
+      - reopened
-      reopened,
-    ]
 
 jobs:
   release:
@@ -22,31 +20,21 @@ jobs:
         contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
       )
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set PAYLOAD_LICENSE_TYPE
-        id: set_license_type
-        run: |
-          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
-          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
-            echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
-          else
-            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
-          fi
-
-      - uses: passeidireto/trigger-external-workflow-action@main
     env:
       PAYLOAD_BRANCH: ${{ github.head_ref }}
       PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
-          PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
+      PAYLOAD_LICENSE_TYPE: |
+        ${{ 
+          contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
+          contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
+        }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: passeidireto/trigger-external-workflow-action@main
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

charts/budibase/templates/app-service-deployment.yaml

@@ -42,14 +42,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
           {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
           {{ end }}
-        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/automation-worker-service-deployment.yaml

@@ -43,6 +43,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        - name: COUCH_DB_SQL_URL
+          {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
+          {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+          {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:

charts/budibase/templates/worker-service-deployment.yaml

@@ -56,14 +56,12 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
-        {{ if .Values.globals.sqs.enabled }}
         - name: COUCH_DB_SQL_URL
           {{ if .Values.globals.sqs.url }}
-          value: {{ .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url | quote }}
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
           {{ end }}
-        {{ end }}
         - name: API_ENCRYPTION_KEY
           valueFrom:
             secretKeyRef:

charts/budibase/values.yaml

@@ -139,9 +139,6 @@ globals:
     password: ""
 
   sqs:
-    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
-    # default for now, but will become the default in a future release.
-    enabled: false
     # @ignore
     url: ""
     # @ignore

hosting/docker-compose.yaml

@@ -5,7 +5,7 @@ version: "3"
 services:
   app-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/apps
+    image: budibase/apps
     container_name: bbapps
     environment:
       SELF_HOSTED: 1
@@ -35,7 +35,7 @@ services:
 
   worker-service:
     restart: unless-stopped
-    image: budibase.docker.scarf.sh/budibase/worker
+    image: budibase/worker
     container_name: bbworker
     environment:
       SELF_HOSTED: 1
@@ -97,7 +97,7 @@ services:
 
   couchdb-service:
     restart: unless-stopped
-    image: budibase/couchdb
+    image: budibase/couchdb:v3.3.3-sqs-v2.1.1
     environment:
       - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
       - COUCHDB_USER=${COUCH_DB_USER}

hosting/single/Dockerfile

@@ -69,6 +69,9 @@ WORKDIR /minio
 COPY scripts/install-minio.sh ./install.sh
 RUN chmod +x install.sh && ./install.sh
 
+# setup redis
+COPY hosting/single/redis.conf /etc/redis/redis.conf
+
 # setup runner file
 WORKDIR /
 COPY hosting/single/runner.sh .

hosting/single/redis.conf

@@ -0,0 +1,7 @@
+dir "DATA_DIR/redis"
+
+appendonly yes
+appendfsync everysec
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb

hosting/single/runner.sh

@@ -75,13 +75,17 @@ fi
 for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
 ln -s ${DATA_DIR}/.env /app/.env
 ln -s ${DATA_DIR}/.env /worker/.env
+
 # make these directories in runner, incase of mount
 mkdir -p ${DATA_DIR}/minio
+mkdir -p ${DATA_DIR}/redis
 chown -R couchdb:couchdb ${DATA_DIR}/couch
+
+sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
 if [[ -n "${REDIS_PASSWORD}" ]]; then
-  redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
 else
-  redis-server > /dev/stdout 2>&1 &
+  redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
 fi
 /bbcouch-runner.sh &
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.17",
+  "version": "2.33.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/constants/misc.ts

@@ -28,6 +28,7 @@ export enum Config {
   OIDC = "oidc",
   OIDC_LOGOS = "logos_oidc",
   SCIM = "scim",
+  AI = "AI",
 }
 
 export const MIN_VALID_DATE = new Date(-2147483647000)

packages/backend-core/src/features/features.ts

@@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
-  SQS: Flag.boolean(env.isDev()),
+  SQS: Flag.boolean(true),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
   [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

packages/backend-core/src/features/tests/features.spec.ts

@@ -10,6 +10,7 @@ const schema = {
   TEST_BOOLEAN: Flag.boolean(false),
   TEST_STRING: Flag.string("default value"),
   TEST_NUMBER: Flag.number(0),
+  TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
 }
 const flags = new FlagSet(schema)
 
@@ -123,6 +124,11 @@ describe("feature flags", () => {
       },
       expected: flags.defaults(),
     },
+    {
+      it: "should be possible to override a default true flag to false",
+      environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
+      expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
+    },
   ])(
     "$it",
     async ({

packages/backend-core/src/sql/sql.ts

@@ -521,8 +521,11 @@ class InternalBuilder {
         const [filterTableName, ...otherProperties] = key.split(".")
         const property = otherProperties.join(".")
         const alias = getTableAlias(filterTableName)
-        return fn(q, alias ? `${alias}.${property}` : property, value)
+        return q.andWhere(subquery =>
+          fn(subquery, alias ? `${alias}.${property}` : property, value)
+        )
       }
+
       for (const key in structure) {
         const value = structure[key]
         const updatedKey = dbCore.removeKeyNumbering(key)
@@ -552,6 +555,9 @@ class InternalBuilder {
             value
           )
         } else if (shouldProcessRelationship) {
+          if (allOr) {
+            query = query.or
+          }
           query = builder.addRelationshipForFilter(query, updatedKey, q => {
             return handleRelationship(q, updatedKey, value)
           })
@@ -1288,7 +1294,8 @@ class InternalBuilder {
           schema.constraints?.presence === true ||
           schema.type === FieldType.FORMULA ||
           schema.type === FieldType.AUTO ||
-          schema.type === FieldType.LINK
+          schema.type === FieldType.LINK ||
+          schema.type === FieldType.AI
         ) {
           continue
         }

packages/backend-core/src/sql/sqlTable.ts

@@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 
 function isIgnoredType(type: FieldType) {
-  const ignored = [FieldType.LINK, FieldType.FORMULA]
+  const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
   return ignored.indexOf(type) !== -1
 }
 
@@ -144,6 +144,9 @@ function generateSchema(
       case FieldType.FORMULA:
         // This is allowed, but nothing to do on the external datasource
         break
+      case FieldType.AI:
+        // This is allowed, but nothing to do on the external datasource
+        break
       case FieldType.ATTACHMENTS:
       case FieldType.ATTACHMENT_SINGLE:
       case FieldType.SIGNATURE_SINGLE:

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -102,6 +102,14 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
+export const useBudibaseAI = () => {
+  return useFeature(Feature.BUDIBASE_AI)
+}
+
+export const useAICustomConfigs = () => {
+  return useFeature(Feature.AI_CUSTOM_CONFIGS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -26,6 +26,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
+  import { licensing } from "stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -35,6 +36,7 @@
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
@@ -99,6 +101,8 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
+  $: aiEnabled =
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
@@ -447,6 +451,7 @@
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
         FIELDS.LINK,
+        ...(aiEnabled ? [FIELDS.AI] : []),
         FIELDS.LONGFORM,
         FIELDS.USER,
         FIELDS.USERS,
@@ -784,6 +789,13 @@
         />
       </div>
     </div>
+  {:else if editableColumn.type === FieldType.AI}
+    <AIFieldConfiguration
+      aiField={editableColumn}
+      context={rowGoldenSample}
+      bindings={getBindings({ table })}
+      schema={table.schema}
+    />
   {:else if editableColumn.type === FieldType.JSON}
     <Button primary text on:click={openJsonSchemaEditor}>
       Open schema editor

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -7,6 +7,7 @@
   import { FIELDS } from "constants/backend"
 
   const FORMULA_TYPE = FIELDS.FORMULA.type
+  const AI_TYPE = FIELDS.AI.type
 
   export let row = {}
 
@@ -60,7 +61,7 @@
     }}
   >
     {#each tableSchema as [key, meta]}
-      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
+      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE && meta.type !== AI_TYPE}
         <div>
           <RowFieldControl error={errors[key]} {meta} bind:value={row[key]} />
         </div>

packages/builder/src/components/common/AIFieldConfiguration.svelte

@@ -0,0 +1,59 @@
+<script>
+  import { Helpers, Multiselect, Select } from "@budibase/bbui"
+  import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
+  import {
+    AIOperations,
+    OperationFields,
+    OperationFieldTypes,
+  } from "@budibase/shared-core"
+
+  const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
+    label: AIOperations[key].label,
+    value: AIOperations[key].value,
+  }))
+
+  export let bindings
+  export let context
+  export let schema
+  export let aiField = {}
+
+  $: OperationField = OperationFields[aiField.operation]
+  $: schemaWithoutRelations = Object.keys(schema).filter(
+    key => schema[key].type !== "link"
+  )
+</script>
+
+<Select
+  label={"Operation"}
+  options={AIFieldConfigOptions}
+  bind:value={aiField.operation}
+/>
+{#if aiField.operation}
+  {#each Object.keys(OperationField) as key}
+    {#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
+      <ModalBindableInput
+        label={Helpers.capitalise(key)}
+        panel={ServerBindingPanel}
+        title="Prompt"
+        on:change={e => (aiField[key] = e.detail)}
+        value={aiField[key]}
+        {bindings}
+        allowJS
+        {context}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
+      <Multiselect
+        bind:value={aiField[key]}
+        label={Helpers.capitalise(key)}
+        options={schemaWithoutRelations}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.COLUMN}
+      <Select
+        bind:value={aiField[key]}
+        label={Helpers.capitalise(key)}
+        options={schemaWithoutRelations}
+      />
+    {/if}
+  {/each}
+{/if}

packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte

@@ -59,6 +59,7 @@
   bind:this={drawer}
   title="Filtering"
   on:drawerHide
+  on:drawerShow
   on:drawerShow={() => {
     // Reset to the currently available value.
     localFilters = Helpers.cloneDeep(value)

packages/builder/src/constants/backend/index.js

@@ -159,6 +159,12 @@ export const FIELDS = {
     icon: TypeIconMap[FieldType.FORMULA],
     constraints: {},
   },
+  AI: {
+    name: "AI",
+    type: FieldType.AI,
+    icon: TypeIconMap[FieldType.AI],
+    constraints: {},
+  },
   JSON: {
     name: "JSON",
     type: FieldType.JSON,

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin, themeStore } from "stores/portal"
+  import { admin, themeStore, licensing } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import { notifications } from "@budibase/bbui"
@@ -49,6 +49,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
+  aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -7,7 +7,7 @@
     appStore,
     rowActions,
   } from "stores/builder"
-  import { themeStore, admin } from "stores/portal"
+  import { themeStore, admin, licensing } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -125,6 +125,8 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
+    aiEnabled={$licensing.budibaseAIEnabled ||
+      $licensing.customAIConfigsEnabled}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}

packages/client/src/components/app/GridBlock.svelte

@@ -3,6 +3,7 @@
   // because it functions similarly to one
   import { getContext, onMount } from "svelte"
   import { get, derived, readable } from "svelte/store"
+  import { featuresStore } from "stores"
   import { Grid } from "@budibase/frontend-core"
 
   // table is actually any datasource, but called table for legacy compatibility
@@ -186,6 +187,7 @@
     {buttonsCollapsed}
     {buttonsCollapsedText}
     isCloud={$environmentStore.cloud}
+    aiEnabled={$featuresStore.aiEnabled}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>

packages/client/src/stores/features.js

@@ -2,6 +2,7 @@ import { derived } from "svelte/store"
 import { appStore } from "./app"
 import { authStore } from "./auth"
 import { Constants } from "@budibase/frontend-core"
+import { Feature } from "@budibase/types"
 
 const createFeaturesStore = () => {
   return derived([authStore, appStore], ([$authStore, $appStore]) => {
@@ -33,8 +34,13 @@ const createFeaturesStore = () => {
       }
     }
 
+    const license = getUserLicense()
+
     return {
       logoEnabled: isFreePlan(),
+      aiEnabled:
+        license?.features?.includes(Feature.AI_CUSTOM_CONFIGS) ||
+        license?.features?.includes(Feature.BUDIBASE_AI),
     }
   })
 }

packages/frontend-core/src/components/FilterField.svelte

@@ -175,7 +175,7 @@
         />
       {:else}
         <div>
-          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
+          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
             <Input
               disabled={filter.noValue}
               value={readableValue}

packages/frontend-core/src/components/Updating.svelte

@@ -2,36 +2,31 @@
   export let isMigrationDone
   export let onMigrationDone
   export let timeoutSeconds = 60 // 1 minute
-  export let minTimeSeconds = 3
 
-  const loadTime = Date.now()
-  const intervalMs = 1000
   let timedOut = false
-  let secondsWaited = 0
 
   async function checkMigrationsFinished() {
-    setTimeout(async () => {
+    let totalWaitMs = 0
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const waitForMs = 5000 + Math.random() * 5000
+      await new Promise(resolve => setTimeout(resolve, waitForMs))
+      totalWaitMs += waitForMs
+
       const isMigrated = await isMigrationDone()
-
+      if (isMigrated) {
-      const timeoutMs = timeoutSeconds * 1000
-      if (!isMigrated || secondsWaited <= minTimeSeconds) {
-        if (loadTime + timeoutMs > Date.now()) {
-          secondsWaited += 1
-          return checkMigrationsFinished()
-        }
-
-        return migrationTimeout()
-      }
-
         onMigrationDone()
-    }, intervalMs)
+        return
+      }
+
+      if (totalWaitMs > timeoutSeconds * 1000) {
+        timedOut = true
+        return
+      }
+    }
   }
 
   checkMigrationsFinished()
-
-  function migrationTimeout() {
-    timedOut = true
-  }
 </script>
 
 <div class="loading" class:timeout={timedOut}>

packages/frontend-core/src/components/grid/cells/AICell.svelte

@@ -0,0 +1,99 @@
+<script>
+  import { onMount } from "svelte"
+  import { clickOutside } from "@budibase/bbui"
+  import GridPopover from "../overlays/GridPopover.svelte"
+
+  export let value
+  export let focused = false
+  export let api
+
+  let textarea
+  let isOpen = false
+  let anchor
+
+  $: {
+    if (!focused) {
+      isOpen = false
+    }
+  }
+
+  const onKeyDown = () => {
+    return isOpen
+  }
+
+  const open = async () => {
+    isOpen = true
+  }
+
+  const close = () => {
+    textarea?.blur()
+    isOpen = false
+  }
+
+  onMount(() => {
+    api = {
+      focus: () => open(),
+      blur: () => close(),
+      isActive: () => isOpen,
+      onKeyDown,
+    }
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<div class="long-form-cell" on:click={open} bind:this={anchor}>
+  <div class="value">
+    {value || ""}
+  </div>
+</div>
+
+{#if isOpen}
+  <GridPopover {anchor} on:close={close}>
+    <textarea
+      disabled
+      bind:this={textarea}
+      value={value || ""}
+      on:wheel|stopPropagation
+      spellcheck="false"
+      use:clickOutside={close}
+    />
+  </GridPopover>
+{/if}
+
+<style>
+  .long-form-cell {
+    flex: 1 1 auto;
+    padding: var(--cell-padding);
+    align-self: stretch;
+    display: flex;
+    align-items: flex-start;
+    overflow: hidden;
+  }
+  .value {
+    display: -webkit-box;
+    -webkit-line-clamp: var(--content-lines);
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+    line-height: 20px;
+  }
+  textarea {
+    border: none;
+    width: 320px;
+    flex: 1 1 auto;
+    height: var(--max-cell-render-overflow);
+    padding: var(--cell-padding);
+    margin: 0;
+    background: var(--cell-background);
+    font-size: var(--cell-font-size);
+    font-family: var(--font-sans);
+    color: inherit;
+    z-index: 1;
+    resize: none;
+    line-height: 20px;
+    overflow: auto;
+  }
+  textarea:focus {
+    outline: none;
+  }
+</style>

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -95,7 +95,8 @@
     const { type, formulaType } = col.schema
     return (
       searchableTypes.includes(type) ||
-      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC)
+      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) ||
+      type === FieldType.AI
     )
   }
 

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -48,6 +48,7 @@
   export let darkMode = false
   export let isCloud = null
   export let rowConditions = null
+  export let aiEnabled = false
 
   // Unique identifier for DOM nodes inside this instance
   const gridID = `grid-${Math.random().toString().slice(2)}`
@@ -104,6 +105,7 @@
     buttonsCollapsedText,
     darkMode,
     isCloud,
+    aiEnabled,
     rowConditions,
   })
 

packages/frontend-core/src/components/grid/lib/renderers.js

@@ -9,6 +9,7 @@ import TextCell from "../cells/TextCell.svelte"
 import LongFormCell from "../cells/LongFormCell.svelte"
 import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
+import AICell from "../cells/AICell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
 import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
@@ -30,6 +31,7 @@ const TypeComponentMap = {
   [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
+  [FieldType.AI]: AICell,
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -1,5 +1,6 @@
 <script>
   import { Menu, MenuItem, Helpers } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import { getContext } from "svelte"
   import { NewRowID } from "../lib/constants"
   import GridPopover from "./GridPopover.svelte"
@@ -26,6 +27,9 @@
 
   $: style = makeStyle($menu)
   $: isNewRow = $focusedRowId === NewRowID
+  $: hasAIColumns = $visibleColumns.some(
+    col => col.schema.type === FieldType.AI
+  )
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
@@ -53,6 +57,12 @@
     await Helpers.copyToClipboard(value)
     $notifications.success("Copied to clipboard")
   }
+
+  const generateAIColumns = async () => {
+    menu.actions.close()
+    await rows.actions.applyRowChanges({ rowId: $focusedRowId })
+    $notifications.success("Generated AI columns")
+  }
 </script>
 
 <div bind:this={anchor} {style} class="menu-anchor" />
@@ -161,6 +171,15 @@
           >
             Delete row
           </MenuItem>
+          {#if $config.aiEnabled}
+            <MenuItem
+              icon="MagicWand"
+              disabled={isNewRow || !hasAIColumns}
+              on:click={generateAIColumns}
+            >
+              Generate AI Columns
+            </MenuItem>
+          {/if}
         {/if}
       </Menu>
     </GridPopover>

packages/frontend-core/src/components/grid/stores/columns.js

@@ -109,6 +109,7 @@ export const createActions = context => {
       column.schema.autocolumn ||
       column.schema.disabled ||
       column.schema.type === "formula" ||
+      column.schema.type === "ai" ||
       column.schema.readonly
     )
   }

packages/frontend-core/src/constants.js

@@ -160,9 +160,10 @@ export const TypeIconMap = {
   [FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
   [FieldType.LINK]: "DataCorrelated",
   [FieldType.FORMULA]: "Calculator",
+  [FieldType.AI]: "MagicWand",
   [FieldType.JSON]: "Brackets",
   [FieldType.BIGINT]: "TagBold",
-  [FieldType.AUTO]: "MagicWand",
+  [FieldType.AUTO]: "Shapes",
   [FieldType.BB_REFERENCE]: {
     [BBReferenceFieldSubType.USER]: "UserGroup",
     [BBReferenceFieldSubType.USERS]: "UserGroup",

packages/server/specs/openapi.json

@@ -833,7 +833,8 @@
                       "type": "string",
                       "enum": [
                         "static",
-                        "dynamic"
+                        "dynamic",
+                        "ai"
                       ],
                       "description": "Defines whether this is a static or dynamic formula."
                     }
@@ -857,6 +858,7 @@
                         "link",
                         "formula",
                         "auto",
+                        "ai",
                         "json",
                         "internal",
                         "barcodeqr",
@@ -1042,7 +1044,8 @@
                           "type": "string",
                           "enum": [
                             "static",
-                            "dynamic"
+                            "dynamic",
+                            "ai"
                           ],
                           "description": "Defines whether this is a static or dynamic formula."
                         }
@@ -1066,6 +1069,7 @@
                             "link",
                             "formula",
                             "auto",
+                            "ai",
                             "json",
                             "internal",
                             "barcodeqr",
@@ -1262,7 +1266,8 @@
                             "type": "string",
                             "enum": [
                               "static",
-                              "dynamic"
+                              "dynamic",
+                              "ai"
                             ],
                             "description": "Defines whether this is a static or dynamic formula."
                           }
@@ -1286,6 +1291,7 @@
                               "link",
                               "formula",
                               "auto",
+                              "ai",
                               "json",
                               "internal",
                               "barcodeqr",

packages/server/specs/openapi.yaml

@@ -761,6 +761,7 @@ components:
                     enum:
                       - static
                       - dynamic
+                      - ai
                     description: Defines whether this is a static or dynamic formula.
               - type: object
                 properties:
@@ -779,6 +780,7 @@ components:
                       - link
                       - formula
                       - auto
+                      - ai
                       - json
                       - internal
                       - barcodeqr
@@ -929,6 +931,7 @@ components:
                         enum:
                           - static
                           - dynamic
+                          - ai
                         description: Defines whether this is a static or dynamic formula.
                   - type: object
                     properties:
@@ -947,6 +950,7 @@ components:
                           - link
                           - formula
                           - auto
+                          - ai
                           - json
                           - internal
                           - barcodeqr
@@ -1104,6 +1108,7 @@ components:
                           enum:
                             - static
                             - dynamic
+                            - ai
                           description: Defines whether this is a static or dynamic formula.
                     - type: object
                       properties:
@@ -1122,6 +1127,7 @@ components:
                             - link
                             - formula
                             - auto
+                            - ai
                             - json
                             - internal
                             - barcodeqr

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -157,7 +157,8 @@ function isEditableColumn(column: FieldSchema) {
     column.autoReason !== AutoReason.FOREIGN_KEY &&
     column.subtype !== AutoFieldSubType.AUTO_ID
   const isFormula = column.type === FieldType.FORMULA
-  return !(isExternalAutoColumn || isFormula)
+  const isAIColumn = column.type === FieldType.AI
+  return !(isExternalAutoColumn || isFormula || isAIColumn)
 }
 
 export class ExternalRequest<T extends Operation> {
@@ -173,9 +174,9 @@ export class ExternalRequest<T extends Operation> {
     if (!opts.datasource) {
       if (sdk.views.isView(source)) {
         const table = await sdk.views.getTable(source.id)
-        opts.datasource = await sdk.datasources.get(table.sourceId!)
+        opts.datasource = await sdk.datasources.get(table.sourceId)
       } else {
-        opts.datasource = await sdk.datasources.get(source.sourceId!)
+        opts.datasource = await sdk.datasources.get(source.sourceId)
       }
     }
 

packages/server/src/api/controllers/row/staticFormula.ts

@@ -1,6 +1,7 @@
 import { getRowParams } from "../../../db/utils"
 import {
   outputProcessing,
+  processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
@@ -9,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
+import * as pro from "@budibase/pro"
 
 /**
  * This function runs through a list of enriched rows, looks at the rows which
@@ -102,7 +104,7 @@ export async function updateAllFormulasInTable(table: Table) {
       (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
-      const processed = await processFormulas(table, cloneDeep(row), {
+      let processed = await processFormulas(table, cloneDeep(row), {
         dynamic: false,
         contextRows: [enrichedRow],
       })
@@ -142,12 +144,27 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+  const aiEnabled =
+    (await pro.features.isBudibaseAIEnabled()) ||
+    (await pro.features.isAICustomConfigsEnabled())
+  if (aiEnabled) {
+    row = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
+
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
+  if (aiEnabled) {
+    enrichedRow = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
+
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -124,6 +124,7 @@ export async function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI &&
           !existing.find(
             (field: string) => field === `${table.name}.${columnName}`
           )

packages/server/src/api/controllers/table/bulkFormula.ts

@@ -6,10 +6,10 @@ import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
 import {
-  FormulaType,
   FieldSchema,
   FieldType,
   FormulaFieldMetadata,
+  FormulaType,
   Table,
 } from "@budibase/types"
 import sdk from "../../../sdk"

packages/server/src/api/routes/tests/row.spec.ts

@@ -18,6 +18,7 @@ import {
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AIOperationEnum,
   AttachmentFieldMetadata,
   AutoFieldSubType,
   Datasource,
@@ -50,6 +51,18 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 interface WaitOptions {
@@ -2086,6 +2099,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
+          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2189,6 +2203,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
+            ai: null,
           },
         ])
       })
@@ -2975,6 +2990,57 @@ describe.each([
       )
     })
 
+  isSqs &&
+    describe("AI fields", () => {
+      let table: Table
+
+      beforeAll(async () => {
+        mocks.licenses.useBudibaseAI()
+        mocks.licenses.useAICustomConfigs()
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              ai: {
+                name: "ai",
+                type: FieldType.AI,
+                operation: AIOperationEnum.PROMPT,
+                prompt: "Convert the following to German: '{{ product }}'",
+              },
+              product: {
+                name: "product",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+        })
+      })
+
+      afterAll(() => {
+        jest.unmock("@budibase/pro")
+      })
+
+      it("should be able to save a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+
+      it("should be able to update a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+          ...rows[0],
+        })
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+    })
+
   describe("Formula fields", () => {
     let table: Table
     let otherTable: Table

packages/server/src/api/routes/tests/search.spec.ts

@@ -17,12 +17,14 @@ import {
 
 import * as setup from "./utilities"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   BBReferenceFieldSubType,
   Datasource,
   EmptyFilterOption,
   FieldType,
   JsonFieldSubType,
+  LogicalOperator,
   RelationshipType,
   Row,
   RowSearchParams,
@@ -41,11 +43,23 @@ import tk from "timekeeper"
 import { encodeJSBinding } from "@budibase/string-templates"
 import { dataFilters } from "@budibase/shared-core"
 import { Knex } from "knex"
-import { generator, structures } from "@budibase/backend-core/tests"
+import { generator, structures, mocks } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 import { generateRowIdField } from "../../../integrations/utils"
 import { cloneDeep } from "lodash/fp"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 describe.each([
   ["in-memory", undefined],
   ["lucene", undefined],
@@ -1601,6 +1615,79 @@ describe.each([
         })
       })
 
+    isSqs &&
+      describe("AI Column", () => {
+        const UNEXISTING_AI_COLUMN = "Real LLM Response"
+
+        beforeAll(async () => {
+          mocks.licenses.useBudibaseAI()
+          mocks.licenses.useAICustomConfigs()
+
+          tableOrViewId = await createTableOrView({
+            product: { name: "product", type: FieldType.STRING },
+            ai: {
+              name: "AI",
+              type: FieldType.AI,
+              operation: AIOperationEnum.PROMPT,
+              prompt: "Translate '{{ product }}' into German",
+            },
+          })
+
+          await createRows([{ product: "Big Mac" }, { product: "McCrispy" }])
+        })
+
+        describe("equal", () => {
+          it("successfully finds rows based on AI column", async () => {
+            await expectQuery({
+              equal: { ai: "Mock LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              equal: { ai: UNEXISTING_AI_COLUMN },
+            }).toFindNothing()
+          })
+        })
+
+        describe("notEqual", () => {
+          it("Returns nothing when searching notEqual on the mock AI response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Mock LLM Response" },
+            }).toContainExactly([])
+          })
+
+          it("return all when requesting non-existing response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Real LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+        })
+
+        describe("oneOf", () => {
+          it("successfully finds a row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Whopper"] },
+            }).toFindNothing()
+          })
+        })
+      })
+
     describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
       beforeAll(async () => {
         tableOrViewId = await createTableOrView({
@@ -2329,6 +2416,211 @@ describe.each([
             equal: { ["name"]: "baz" },
           }).toContainExactly([{ name: "baz", productCat: undefined }])
         })
+
+        describe("logical filters", () => {
+          const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
+
+          describe("$and", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $and: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ands with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ands with multiple conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $and: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                        notEqual: { ["productCat.name"]: "foo" },
+                      },
+                    ],
+                  },
+                }).toContainExactly([])
+              }
+            )
+          })
+
+          describe("$ors", () => {
+            it("should allow single conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+              ])
+            })
+
+            it("should allow exclusive conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      equal: { ["productCat.name"]: "foo" },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+
+            it.each([logicalOperators])(
+              "should allow nested ors with single conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                ])
+              }
+            )
+
+            it.each([logicalOperators])(
+              "should allow nested ors with exclusive conditions (with %s as root)",
+              async rootOperator => {
+                await expectQuery({
+                  [rootOperator]: {
+                    conditions: [
+                      {
+                        $or: {
+                          conditions: [
+                            {
+                              equal: { ["productCat.name"]: "foo" },
+                              notEqual: { ["productCat.name"]: "foo" },
+                            },
+                          ],
+                        },
+                      },
+                    ],
+                  },
+                }).toContainExactly([
+                  { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                  { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                  // { name: "baz", productCat: undefined }, // TODO
+                ])
+              }
+            )
+
+            it("should allow nested ors with multiple conditions", async () => {
+              await expectQuery({
+                $or: {
+                  conditions: [
+                    {
+                      $or: {
+                        conditions: [
+                          {
+                            equal: { ["productCat.name"]: "foo" },
+                          },
+                        ],
+                      },
+                      notEqual: { ["productCat.name"]: "foo" },
+                    },
+                  ],
+                },
+              }).toContainExactly([
+                { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+                { name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
+                // { name: "baz", productCat: undefined }, // TODO
+              ])
+            })
+          })
+        })
       })
 
     isSql &&

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -206,7 +206,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         const createdView = await config.api.viewV2.create(newView)
@@ -250,7 +250,7 @@ describe.each([
               name: "Category",
               type: FieldType.STRING,
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         await config.api.viewV2.create(newView, {
@@ -1044,7 +1044,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         })
 
         expect(updatedView).toEqual({
@@ -1078,7 +1078,7 @@ describe.each([
                 name: "Category",
                 type: FieldType.STRING,
               },
-            } as Record<string, FieldSchema>,
+            } as ViewV2Schema,
           },
           {
             status: 200,

packages/server/src/db/linkedRows/index.ts

@@ -307,7 +307,9 @@ export async function squashLinks<T = Row[] | Row>(
                 return false
               }
               if (
-                [FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type)
+                [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
+                  tableColumn.type
+                )
               ) {
                 return false
               }

packages/server/src/integrations/googlesheets.ts

@@ -56,6 +56,7 @@ interface AuthTokenResponse {
 const isTypeAllowed: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.NUMBER]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.DATETIME]: true,
@@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
         }
         if (
           !sheet.headerValues.includes(key) &&
-          column.type !== FieldType.FORMULA
+          column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI
         ) {
           updatedHeaderValues.push(key)
         }

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         sql: expect.stringContaining(
           multiline(
             `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" 
-                 and COALESCE("b"."taskname" = $1, FALSE)`
+                 and (COALESCE("b"."taskname" = $1, FALSE))`
           )
         ),
       })
@@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
         ],
         sql: expect.stringContaining(
           multiline(
-            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
+            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
           )
         ),
       })

packages/server/src/integrations/utils/utils.ts

@@ -242,6 +242,7 @@ function copyExistingPropsOver(
       let shouldKeepSchema = false
       switch (existingColumnType) {
         case FieldType.FORMULA:
+        case FieldType.AI:
         case FieldType.AUTO:
         case FieldType.INTERNAL:
           shouldKeepSchema = true

packages/server/src/sdk/app/tables/getters.ts

@@ -90,7 +90,11 @@ export async function getExternalTable(
   if (!entities[tableName]) {
     throw new Error(`Unable to find table named "${tableName}"`)
   }
-  return processTable(entities[tableName])
+  const table = await processTable(entities[tableName])
+  if (!table.sourceId) {
+    table.sourceId = datasourceId
+  }
+  return table
 }
 
 export async function getTable(tableId: string): Promise<Table> {

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -19,6 +19,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
   [FieldType.DATETIME]: SQLiteType.TEXT,
   [FieldType.FORMULA]: SQLiteType.TEXT,
+  [FieldType.AI]: SQLiteType.TEXT,
   [FieldType.LONGFORM]: SQLiteType.TEXT,
   [FieldType.NUMBER]: SQLiteType.REAL,
   [FieldType.STRING]: SQLiteType.TEXT,

packages/server/src/sdk/app/views/index.ts

@@ -313,7 +313,11 @@ export async function enrichSchema(
     const result: Record<string, ViewV2ColumnEnriched> = {}
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
-      if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
+      if (
+        [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
+          relTableField.type
+        )
+      ) {
         continue
       }
 

packages/server/src/tests/utilities/structures.ts

@@ -7,6 +7,7 @@ import {
   TRIGGER_DEFINITIONS,
 } from "../../automations"
 import {
+  AIOperationEnum,
   Automation,
   AutomationActionStepId,
   AutomationResults,
@@ -666,6 +667,12 @@ export function fullSchemaWithoutLinks({
         presence: allRequired,
       },
     },
+    [FieldType.AI]: {
+      name: "ai",
+      type: FieldType.AI,
+      operation: AIOperationEnum.PROMPT,
+      prompt: "Translate this into German :'{{ product }}'",
+    },
     [FieldType.BARCODEQR]: {
       name: "barcodeqr",
       type: FieldType.BARCODEQR,

packages/server/src/utilities/rowProcessor/index.ts

@@ -210,6 +210,10 @@ export async function inputProcessing(
     if (field.type === FieldType.FORMULA) {
       delete clonedRow[key]
     }
+    // remove any AI values, they are to be generated
+    if (field.type === FieldType.AI) {
+      delete clonedRow[key]
+    }
     // otherwise coerce what is there to correct types
     else {
       clonedRow[key] = coerce(value, field.type)

packages/server/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -1,11 +1,29 @@
-import { fixAutoColumnSubType } from "../utils"
+import { fixAutoColumnSubType, processAIColumns } from "../utils"
 import { AutoFieldDefaultNames } from "../../../constants"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   RelationshipType,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
+import { generator } from "@budibase/backend-core/tests"
+
+const buildPromptMock = jest.fn()
+
+jest.mock("@budibase/pro", () => ({
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => "response from LLM"),
+        buildPromptFromAIOperation: buildPromptMock,
+      }),
+    },
+  },
+}))
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
@@ -60,4 +78,59 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema)).toEqual(schema)
     })
   })
+
+  describe("processAIColumns", () => {
+    it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
+      const table: Table = {
+        _id: generator.guid(),
+        name: "AITestTable",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          product: {
+            type: FieldType.STRING,
+            name: "product",
+            constraints: {
+              presence: true,
+              type: "string",
+            },
+          },
+          aicol: {
+            type: FieldType.AI,
+            name: "aicol",
+            operation: AIOperationEnum.PROMPT,
+            prompt: "Translate '{{ product }}' into German",
+          },
+        },
+      }
+
+      const inputRows = [
+        {
+          product: "Car Battery",
+        },
+      ]
+
+      const result = await processAIColumns(table, inputRows, {
+        contextRows: inputRows,
+      })
+      expect(buildPromptMock).toHaveBeenCalledWith({
+        row: {
+          product: "Car Battery",
+        },
+        schema: {
+          name: "aicol",
+          operation: "PROMPT",
+          prompt: "Translate 'Car Battery' into German",
+          type: "ai",
+        },
+      })
+      expect(result).toEqual([
+        {
+          aicol: "response from LLM",
+          product: "Car Battery",
+        },
+      ])
+    })
+  })
 })

packages/server/src/utilities/rowProcessor/utils.ts

@@ -8,9 +8,13 @@ import {
   FormulaType,
   AutoFieldSubType,
   FieldType,
+  OperationFieldTypeEnum,
+  AIOperationEnum,
 } from "@budibase/types"
+import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
+import * as pro from "@budibase/pro"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -91,6 +95,66 @@ export async function processFormulas<T extends Row | Row[]>(
   })
 }
 
+/**
+ * Looks through the rows provided and finds AI columns - which it then processes.
+ */
+export async function processAIColumns<T extends Row | Row[]>(
+  table: Table,
+  inputRows: T,
+  { contextRows }: FormulaOpts
+): Promise<T> {
+  return tracer.trace("processAIColumns", {}, async span => {
+    const numRows = Array.isArray(inputRows) ? inputRows.length : 1
+    span?.addTags({ table_id: table._id, numRows })
+    const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+    const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini")
+    if (rows) {
+      // Ensure we have snippet context
+      await context.ensureSnippetContext()
+
+      for (let [column, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldType.AI) {
+          continue
+        }
+
+        const rowUpdates = rows.map((row, i) => {
+          const contextRow = contextRows ? contextRows[i] : row
+
+          // Check if the type is bindable and pass through HBS if so
+          const operationField =
+            OperationFields[schema.operation as AIOperationEnum]
+          for (const key in schema) {
+            const fieldType = operationField[key as keyof typeof operationField]
+            if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
+              // @ts-ignore
+              schema[key] = processStringSync(schema[key], contextRow)
+            }
+          }
+
+          const prompt = llm.buildPromptFromAIOperation({ schema, row })
+
+          return tracer.trace("processAIColumn", {}, async span => {
+            span?.addTags({ table_id: table._id, column })
+            const llmResponse = await llm.run(prompt!)
+            return {
+              ...row,
+              [column]: llmResponse,
+            }
+          })
+        })
+
+        const processedRows = await Promise.all(rowUpdates)
+
+        // Promise.all is deterministic so can rely on the indexing here
+        processedRows.forEach(
+          (processedRow, index) => (rows[index] = processedRow)
+        )
+      }
+    }
+    return Array.isArray(inputRows) ? rows : rows[0]
+  })
+}
+
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.

packages/shared-core/src/constants/ai.ts

@@ -0,0 +1,68 @@
+import {
+  AIOperationEnum,
+  OperationFieldsType,
+  OperationFieldTypeEnum,
+} from "@budibase/types"
+
+export const AIOperations = {
+  SUMMARISE_TEXT: {
+    label: "Summarise Text",
+    value: "SUMMARISE_TEXT",
+  },
+  CLEAN_DATA: {
+    label: "Clean Data",
+    value: "CLEAN_DATA",
+  },
+  TRANSLATE: {
+    label: "Translate",
+    value: "TRANSLATE",
+  },
+  CATEGORISE_TEXT: {
+    label: "Categorise Text",
+    value: "CATEGORISE_TEXT",
+  },
+  SENTIMENT_ANALYSIS: {
+    label: "Sentiment Analysis",
+    value: "SENTIMENT_ANALYSIS",
+  },
+  PROMPT: {
+    label: "Prompt",
+    value: "PROMPT",
+  },
+  SEARCH_WEB: {
+    label: "Search Web",
+    value: "SEARCH_WEB",
+  },
+}
+
+export const OperationFieldTypes = {
+  MULTI_COLUMN: "columns",
+  COLUMN: "column",
+  BINDABLE_TEXT: "prompt",
+}
+
+export const OperationFields: OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN,
+    language: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+}

packages/shared-core/src/constants/index.ts

@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./api"
 export * from "./fields"
 export * from "./rows"

packages/shared-core/src/filters.ts

@@ -90,6 +90,8 @@ export const getValidOperatorsForType = (
     ops = numOps
   } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  } else if (type === FieldType.AI) {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
   } else if (
     type === FieldType.BB_REFERENCE_SINGLE ||
     schema.isDeprecatedSingleUserColumn(fieldType)

packages/shared-core/src/table.ts

@@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.NUMBER]: true,
   [FieldType.DATETIME]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.AUTO]: true,
   [FieldType.INTERNAL]: true,
   [FieldType.BARCODEQR]: true,
@@ -38,6 +39,7 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
   [FieldType.JSON]: true,
 
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
@@ -62,6 +64,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.BIGINT]: false,
   [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,

packages/types/src/documents/app/row.ts

@@ -76,6 +76,13 @@ export enum FieldType {
    * that is part of the initial formula definition, the formula will be live evaluated in the browser.
    */
   AUTO = "auto",
+  /**
+   * A complex type, called an AI column within Budibase. This type is only supported against internal tables
+   * and calculates the output based on a chosen operation (summarise text, translation etc) which passes to
+   * the configured Budibase Large Language Model to retrieve the output and write it back into the row.
+   * AI fields function in a similar fashion to static formulas, and possess many of the same characteristics.
+   */
+  AI = "ai",
   /**
    * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
    * type, which will be represented as a JSON object in the row. This type depends on a schema being

packages/types/src/documents/app/table/constants.ts

@@ -30,6 +30,7 @@ export enum JsonFieldSubType {
 export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",
+  AI = "ai",
 }
 
 export enum BBReferenceFieldSubType {

packages/types/src/documents/app/table/schema.ts

@@ -9,6 +9,7 @@ import {
   JsonFieldSubType,
   RelationshipType,
 } from "./constants"
+import { AIOperationEnum } from "../../../sdk/ai"
 
 export interface UIFieldMetadata {
   order?: number
@@ -116,6 +117,16 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   formulaType?: FormulaType
 }
 
+export interface AIFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum
+  columns?: string[]
+  column?: string
+  categories?: string[]
+  prompt?: string
+  language?: string
+}
+
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
@@ -194,6 +205,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.LINK
     | FieldType.AUTO
     | FieldType.FORMULA
+    | FieldType.AI
     | FieldType.NUMBER
     | FieldType.LONGFORM
     | FieldType.BB_REFERENCE
@@ -211,6 +223,7 @@ export type FieldSchema =
   | RelationshipFieldMetadata
   | AutoColumnFieldMetadata
   | FormulaFieldMetadata
+  | AIFieldMetadata
   | NumberFieldMetadata
   | LongFormFieldMetadata
   | StringFieldMetadata

packages/types/src/sdk/ai.ts

@@ -0,0 +1,91 @@
+export enum AIOperationEnum {
+  SUMMARISE_TEXT = "SUMMARISE_TEXT",
+  CLEAN_DATA = "CLEAN_DATA",
+  TRANSLATE = "TRANSLATE",
+  CATEGORISE_TEXT = "CATEGORISE_TEXT",
+  SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS",
+  PROMPT = "PROMPT",
+  SEARCH_WEB = "SEARCH_WEB",
+}
+
+export enum OperationFieldTypeEnum {
+  MULTI_COLUMN = "columns",
+  COLUMN = "column",
+  BINDABLE_TEXT = "prompt",
+}
+
+export type OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN
+    language: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+}
+
+type BaseSchema = {
+  operation: AIOperationEnum
+}
+
+type SummariseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.SUMMARISE_TEXT
+  columns: string[]
+}
+
+type CleanDataSchema = BaseSchema & {
+  operation: AIOperationEnum.CLEAN_DATA
+  column: string
+}
+
+type TranslateSchema = BaseSchema & {
+  operation: AIOperationEnum.TRANSLATE
+  column: string
+  language: string
+}
+
+type CategoriseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.CATEGORISE_TEXT
+  columns: string[]
+  categories: string[]
+}
+
+type SentimentAnalysisSchema = BaseSchema & {
+  operation: AIOperationEnum.SENTIMENT_ANALYSIS
+  column: string
+}
+
+type PromptSchema = BaseSchema & {
+  operation: AIOperationEnum.PROMPT
+  prompt: string
+}
+
+type SearchWebSchema = BaseSchema & {
+  operation: AIOperationEnum.SEARCH_WEB
+  columns: string[]
+}
+
+export type AIColumnSchema =
+  | SummariseTextSchema
+  | CleanDataSchema
+  | TranslateSchema
+  | CategoriseTextSchema
+  | SentimentAnalysisSchema
+  | PromptSchema
+  | SearchWebSchema

packages/types/src/sdk/index.ts

@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./automations"
 export * from "./hosting"
 export * from "./context"