Merge branch 'v3-ui' of github.com:Budibase/budibase into new-rbac-ui

.github/workflows/deploy-featurebranch.yml

@@ -14,15 +14,39 @@ jobs:
   release:
     if: |
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase') &&
-      contains(github.event.pull_request.labels.*.name, 'feature-branch')
+      (
+        contains(github.event.pull_request.labels.*.name, 'feature-branch') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-team') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-business') ||
+        contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
+      )
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
+      - name: Set PAYLOAD_LICENSE_TYPE
+        id: set_license_type
+        run: |
+          if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
+          elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
+            echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
+          else
+            echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
+          fi
+
       - uses: passeidireto/trigger-external-workflow-action@main
         env:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
-          PAYLOAD_LICENSE_TYPE: "free"
+          PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.15",
+  "version": "2.32.17",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/constants/misc.ts

@@ -28,6 +28,7 @@ export enum Config {
   OIDC = "oidc",
   OIDC_LOGOS = "logos_oidc",
   SCIM = "scim",
+  AI = "AI",
 }
 
 export const MIN_VALID_DATE = new Date(-2147483647000)

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -211,6 +211,17 @@ export class DatabaseImpl implements Database {
     })
   }
 
+  async tryGet<T extends Document>(id?: string): Promise<T | undefined> {
+    try {
+      return await this.get<T>(id)
+    } catch (err: any) {
+      if (err.statusCode === 404) {
+        return undefined
+      }
+      throw err
+    }
+  }
+
   async getMultiple<T extends Document>(
     ids: string[],
     opts?: { allowMissing?: boolean; excludeDocs?: boolean }

packages/backend-core/src/db/instrumentation.ts

@@ -42,6 +42,13 @@ export class DDInstrumentedDatabase implements Database {
     })
   }
 
+  tryGet<T extends Document>(id?: string | undefined): Promise<T | undefined> {
+    return tracer.trace("db.tryGet", span => {
+      span?.addTags({ db_name: this.name, doc_id: id })
+      return this.db.tryGet(id)
+    })
+  }
+
   getMultiple<T extends Document>(
     ids: string[],
     opts?: { allowMissing?: boolean | undefined } | undefined

packages/backend-core/src/features/features.ts

@@ -272,6 +272,7 @@ export const flags = new FlagSet({
   SQS: Flag.boolean(env.isDev()),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),
 })
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

packages/backend-core/src/security/roles.ts

@@ -457,7 +457,9 @@ export function getExternalRoleID(roleId: string, version?: string) {
     roleId.startsWith(DocumentType.ROLE) &&
     (isBuiltin(roleId) || version === RoleIDVersion.NAME)
   ) {
-    return roleId.split(`${DocumentType.ROLE}${SEPARATOR}`)[1]
+    const parts = roleId.split(SEPARATOR)
+    parts.shift()
+    return parts.join(SEPARATOR)
   }
   return roleId
 }

packages/backend-core/src/sql/sql.ts

@@ -273,6 +273,7 @@ class InternalBuilder {
     const col = parts.pop()!
     const schema = this.table.schema[col]
     let identifier = this.quotedIdentifier(field)
+
     if (
       schema.type === FieldType.STRING ||
       schema.type === FieldType.LONGFORM ||
@@ -957,6 +958,13 @@ class InternalBuilder {
     return query
   }
 
+  isAggregateField(field: string): boolean {
+    const found = this.query.resource?.aggregations?.find(
+      aggregation => aggregation.name === field
+    )
+    return !!found
+  }
+
   addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
     let { sort, resource } = this.query
     const primaryKey = this.table.primary
@@ -979,6 +987,9 @@ class InternalBuilder {
           nulls = value.direction === SortOrder.ASCENDING ? "first" : "last"
         }
 
+        if (this.isAggregateField(key)) {
+          query = query.orderBy(key, direction, nulls)
+        } else {
           let composite = `${aliased}.${key}`
           if (this.client === SqlClient.ORACLE) {
             query = query.orderByRaw(
@@ -989,6 +1000,7 @@ class InternalBuilder {
           }
         }
       }
+    }
 
     // add sorting by the primary key if the result isn't already sorted by it,
     // to make sure result is deterministic
@@ -1276,7 +1288,8 @@ class InternalBuilder {
           schema.constraints?.presence === true ||
           schema.type === FieldType.FORMULA ||
           schema.type === FieldType.AUTO ||
-          schema.type === FieldType.LINK
+          schema.type === FieldType.LINK ||
+          schema.type === FieldType.AI
         ) {
           continue
         }

packages/backend-core/src/sql/sqlTable.ts

@@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 
 function isIgnoredType(type: FieldType) {
-  const ignored = [FieldType.LINK, FieldType.FORMULA]
+  const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
   return ignored.indexOf(type) !== -1
 }
 
@@ -144,6 +144,9 @@ function generateSchema(
       case FieldType.FORMULA:
         // This is allowed, but nothing to do on the external datasource
         break
+      case FieldType.AI:
+        // This is allowed, but nothing to do on the external datasource
+        break
       case FieldType.ATTACHMENTS:
       case FieldType.ATTACHMENT_SINGLE:
       case FieldType.SIGNATURE_SINGLE:

packages/backend-core/tests/core/utilities/mocks/licenses.ts

@@ -102,6 +102,14 @@ export const useAppBuilders = () => {
   return useFeature(Feature.APP_BUILDERS)
 }
 
+export const useBudibaseAI = () => {
+  return useFeature(Feature.BUDIBASE_AI)
+}
+
+export const useAICustomConfigs = () => {
+  return useFeature(Feature.AI_CUSTOM_CONFIGS)
+}
+
 // QUOTAS
 
 export const setAutomationLogsQuota = (value: number) => {

packages/builder/src/components/backend/DataTable/buttons/grid/ColumnsSettingContent.svelte

@@ -124,7 +124,7 @@
         subtype: column.subtype,
         visible: column.visible,
         readonly: column.readonly,
-        constraints: column.constraints, // This is needed to properly display "users" column
+        icon: column.icon,
       },
     }
   })

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -26,6 +26,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
+  import { licensing } from "stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -35,6 +36,7 @@
   } from "constants/backend"
   import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
+  import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte"
   import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
   import { getBindings } from "components/backend/DataTable/formula"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
@@ -50,18 +52,13 @@
   import { isEnabled } from "helpers/featureFlags"
   import { getUserBindings } from "dataBinding"
 
-  const AUTO_TYPE = FieldType.AUTO
-  const FORMULA_TYPE = FieldType.FORMULA
-  const LINK_TYPE = FieldType.LINK
-  const STRING_TYPE = FieldType.STRING
-  const NUMBER_TYPE = FieldType.NUMBER
-  const JSON_TYPE = FieldType.JSON
-  const DATE_TYPE = FieldType.DATETIME
+  export let field
 
   const dispatch = createEventDispatcher()
   const { dispatch: gridDispatch, rows } = getContext("grid")
-
-  export let field
+  const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}`
+  const SingleUserDefault = `{{ ${SafeID} }}`
+  const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}`
 
   let mounted = false
   let originalName
@@ -104,13 +101,15 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
+  $: aiEnabled =
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }
   $: {
     // this parses any changes the user has made when creating a new internal relationship
     // into what we expect the schema to look like
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       relationshipTableIdPrimary = table._id
       if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) {
         relationshipOpts2 = relationshipOpts2.filter(
@@ -147,7 +146,7 @@
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
   $: invalid =
     !editableColumn?.name ||
-    (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) ||
+    (editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) ||
     Object.keys(errors).length !== 0 ||
     !optionsValid
   $: errors = checkErrors(editableColumn)
@@ -173,9 +172,9 @@
   $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
   $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
-    editableColumn?.type !== LINK_TYPE &&
+    editableColumn?.type !== FieldType.LINK &&
     !uneditable &&
-    editableColumn?.type !== AUTO_TYPE &&
+    editableColumn?.type !== FieldType.AUTO &&
     !editableColumn.autocolumn
   $: hasDefault =
     editableColumn?.default != null && editableColumn?.default !== ""
@@ -224,7 +223,7 @@
 
   function makeFieldId(type, subtype, autocolumn) {
     // don't make field IDs for auto types
-    if (type === AUTO_TYPE || autocolumn) {
+    if (type === FieldType.AUTO || autocolumn) {
       return type.toUpperCase()
     } else if (
       type === FieldType.BB_REFERENCE ||
@@ -249,7 +248,7 @@
       // Here we are setting the relationship values based on the editableColumn
       // This part of the code is used when viewing an existing field hence the check
       // for the tableId
-      if (editableColumn.type === LINK_TYPE && editableColumn.tableId) {
+      if (editableColumn.type === FieldType.LINK && editableColumn.tableId) {
         relationshipTableIdPrimary = table._id
         relationshipTableIdSecondary = editableColumn.tableId
         if (editableColumn.relationshipType in relationshipMap) {
@@ -290,14 +289,14 @@
 
     delete saveColumn.fieldId
 
-    if (saveColumn.type === AUTO_TYPE) {
+    if (saveColumn.type === FieldType.AUTO) {
       saveColumn = buildAutoColumn(
         $tables.selected.name,
         saveColumn.name,
         saveColumn.subtype
       )
     }
-    if (saveColumn.type !== LINK_TYPE) {
+    if (saveColumn.type !== FieldType.LINK) {
       delete saveColumn.fieldName
     }
 
@@ -384,9 +383,9 @@
     editableColumn.subtype = definition.subtype
 
     // Default relationships many to many
-    if (editableColumn.type === LINK_TYPE) {
+    if (editableColumn.type === FieldType.LINK) {
       editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
-    } else if (editableColumn.type === FORMULA_TYPE) {
+    } else if (editableColumn.type === FieldType.FORMULA) {
       editableColumn.formulaType = "dynamic"
     }
   }
@@ -452,6 +451,7 @@
         FIELDS.BOOLEAN,
         FIELDS.DATETIME,
         FIELDS.LINK,
+        ...(aiEnabled ? [FIELDS.AI] : []),
         FIELDS.LONGFORM,
         FIELDS.USER,
         FIELDS.USERS,
@@ -505,17 +505,23 @@
       fieldToCheck.constraints = {}
     }
     // some string types may have been built by server, may not always have constraints
-    if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) {
+    if (
+      fieldToCheck.type === FieldType.STRING &&
+      !fieldToCheck.constraints.length
+    ) {
       fieldToCheck.constraints.length = {}
     }
     // some number types made server-side will be missing constraints
     if (
-      fieldToCheck.type === NUMBER_TYPE &&
+      fieldToCheck.type === FieldType.NUMBER &&
       !fieldToCheck.constraints.numericality
     ) {
       fieldToCheck.constraints.numericality = {}
     }
-    if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) {
+    if (
+      fieldToCheck.type === FieldType.DATETIME &&
+      !fieldToCheck.constraints.datetime
+    ) {
       fieldToCheck.constraints.datetime = {}
     }
   }
@@ -590,13 +596,13 @@
       on:input={e => {
         if (
           !uneditable &&
-          !(linkEditDisabled && editableColumn.type === LINK_TYPE)
+          !(linkEditDisabled && editableColumn.type === FieldType.LINK)
         ) {
           editableColumn.name = e.target.value
         }
       }}
       disabled={uneditable ||
-        (linkEditDisabled && editableColumn.type === LINK_TYPE)}
+        (linkEditDisabled && editableColumn.type === FieldType.LINK)}
       error={errors?.name}
     />
   {/if}
@@ -610,7 +616,7 @@
     getOptionValue={field => field.fieldId}
     getOptionIcon={field => field.icon}
     isOptionEnabled={option => {
-      if (option.type === AUTO_TYPE) {
+      if (option.type === FieldType.AUTO) {
         return availableAutoColumnKeys?.length > 0
       }
       return true
@@ -653,7 +659,7 @@
       bind:optionColors={editableColumn.optionColors}
       bind:valid={optionsValid}
     />
-  {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn}
+  {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
         <Label size="M">Earliest</Label>
@@ -740,7 +746,7 @@
       {tableOptions}
       {errors}
     />
-  {:else if editableColumn.type === FORMULA_TYPE}
+  {:else if editableColumn.type === FieldType.FORMULA}
     {#if !externalTable}
       <div class="split-label">
         <div class="label-length">
@@ -783,12 +789,19 @@
         />
       </div>
     </div>
-  {:else if editableColumn.type === JSON_TYPE}
+  {:else if editableColumn.type === FieldType.AI}
+    <AIFieldConfiguration
+      aiField={editableColumn}
+      context={rowGoldenSample}
+      bindings={getBindings({ table })}
+      schema={table.schema}
+    />
+  {:else if editableColumn.type === FieldType.JSON}
     <Button primary text on:click={openJsonSchemaEditor}>
       Open schema editor
     </Button>
   {/if}
-  {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
+  {#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn}
     <Select
       label="Auto column type"
       value={editableColumn.subtype}
@@ -835,6 +848,18 @@
           (editableColumn.default = e.detail?.length ? e.detail : undefined)}
         placeholder="None"
       />
+    {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
+      {@const defaultValue =
+        editableColumn.type === FieldType.BB_REFERENCE_SINGLE
+          ? SingleUserDefault
+          : MultiUserDefault}
+      <Toggle
+        disabled={!canHaveDefault}
+        text="Default to current user"
+        value={editableColumn.default === defaultValue}
+        on:change={e =>
+          (editableColumn.default = e.detail ? defaultValue : undefined)}
+      />
     {:else}
       <ModalBindableInput
         disabled={!canHaveDefault}

packages/builder/src/components/backend/DataTable/modals/CreateEditRow.svelte

@@ -7,6 +7,7 @@
   import { FIELDS } from "constants/backend"
 
   const FORMULA_TYPE = FIELDS.FORMULA.type
+  const AI_TYPE = FIELDS.AI.type
 
   export let row = {}
 
@@ -60,7 +61,7 @@
     }}
   >
     {#each tableSchema as [key, meta]}
-      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE}
+      {#if !meta.autocolumn && meta.type !== FORMULA_TYPE && meta.type !== AI_TYPE}
         <div>
           <RowFieldControl error={errors[key]} {meta} bind:value={row[key]} />
         </div>

packages/builder/src/components/common/AIFieldConfiguration.svelte

@@ -0,0 +1,59 @@
+<script>
+  import { Helpers, Multiselect, Select } from "@budibase/bbui"
+  import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
+  import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
+  import {
+    AIOperations,
+    OperationFields,
+    OperationFieldTypes,
+  } from "@budibase/shared-core"
+
+  const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({
+    label: AIOperations[key].label,
+    value: AIOperations[key].value,
+  }))
+
+  export let bindings
+  export let context
+  export let schema
+  export let aiField = {}
+
+  $: OperationField = OperationFields[aiField.operation]
+  $: schemaWithoutRelations = Object.keys(schema).filter(
+    key => schema[key].type !== "link"
+  )
+</script>
+
+<Select
+  label={"Operation"}
+  options={AIFieldConfigOptions}
+  bind:value={aiField.operation}
+/>
+{#if aiField.operation}
+  {#each Object.keys(OperationField) as key}
+    {#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT}
+      <ModalBindableInput
+        label={Helpers.capitalise(key)}
+        panel={ServerBindingPanel}
+        title="Prompt"
+        on:change={e => (aiField[key] = e.detail)}
+        value={aiField[key]}
+        {bindings}
+        allowJS
+        {context}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN}
+      <Multiselect
+        bind:value={aiField[key]}
+        label={Helpers.capitalise(key)}
+        options={schemaWithoutRelations}
+      />
+    {:else if OperationField[key] === OperationFieldTypes.COLUMN}
+      <Select
+        bind:value={aiField[key]}
+        label={Helpers.capitalise(key)}
+        options={schemaWithoutRelations}
+      />
+    {/if}
+  {/each}
+{/if}

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte

@@ -33,7 +33,7 @@
 
   const getSchemaFields = resourceId => {
     const { schema } = getSchemaForDatasourcePlus(resourceId)
-    return Object.values(schema || {})
+    return Object.values(schema || {}).filter(field => !field.readonly)
   }
 
   const onFieldsChanged = e => {

packages/builder/src/components/portal/licensing/EnterpriseBasicTrialBanner.svelte

@@ -14,7 +14,13 @@
   function daysUntilCancel() {
     const cancelAt = license?.billing?.subscription?.cancelAt
     const diffTime = Math.abs(cancelAt - new Date().getTime()) / 1000
-    return Math.floor(diffTime / oneDayInSeconds)
+    const days = Math.floor(diffTime / oneDayInSeconds)
+    if (days === 1) {
+      return "tomorrow."
+    } else if (days === 0) {
+      return "today."
+    }
+    return `in ${days} days.`
   }
 </script>
 
@@ -28,7 +34,7 @@
           extraLinkAction={$licensing.goToUpgradePage}
           showCloseButton={false}
         >
-          Your free trial will end in {daysUntilCancel()} days.
+          Your free trial will end {daysUntilCancel()}
         </Banner>
       </div>
     {/if}

packages/builder/src/constants/backend/index.js

@@ -159,6 +159,12 @@ export const FIELDS = {
     icon: TypeIconMap[FieldType.FORMULA],
     constraints: {},
   },
+  AI: {
+    name: "AI",
+    type: FieldType.AI,
+    icon: TypeIconMap[FieldType.AI],
+    constraints: {},
+  },
   JSON: {
     name: "JSON",
     type: FieldType.JSON,

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte

@@ -23,6 +23,7 @@
       sourceType: DB_TYPE_EXTERNAL,
       schema: {
         id: {
+          name: "id",
           autocolumn: true,
           type: "number",
         },

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin, themeStore } from "stores/portal"
+  import { admin, themeStore, licensing } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import { notifications } from "@budibase/bbui"
@@ -49,6 +49,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
+  aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/_components/ViewNavBar.svelte

@@ -196,7 +196,7 @@
     on:contextmenu={openTableContextMenu}
   >
     <div class="nav-item__title">
-      {table._id === TableNames.USERS ? "App users" : table.name}
+      {table?._id === TableNames.USERS ? "App users" : table?.name || ""}
     </div>
     {#if tableSelectedBy}
       <UserAvatars size="XS" users={tableSelectedBy} />

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -8,7 +8,7 @@
     rowActions,
     roles,
   } from "stores/builder"
-  import { themeStore, admin } from "stores/portal"
+  import { themeStore, admin, licensing } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -130,6 +130,8 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
+    aiEnabled={$licensing.budibaseAIEnabled ||
+      $licensing.customAIConfigsEnabled}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}

packages/builder/src/stores/builder/viewsV2.js

@@ -6,16 +6,22 @@ import { dataFilters } from "@budibase/shared-core"
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
   if (view?.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }
 
 function convertToSearchFilterGroup(view) {
   if (view?.queryUI) {
-    view.query = view.queryUI
-    delete view.queryUI
+    return {
+      ...view,
+      query: view.queryUI,
+      queryUI: undefined,
+    }
   }
   return view
 }

packages/client/src/components/app/GridBlock.svelte

@@ -3,6 +3,7 @@
   // because it functions similarly to one
   import { getContext, onMount } from "svelte"
   import { get, derived, readable } from "svelte/store"
+  import { featuresStore } from "stores"
   import { Grid } from "@budibase/frontend-core"
 
   // table is actually any datasource, but called table for legacy compatibility
@@ -186,6 +187,7 @@
     {buttonsCollapsed}
     {buttonsCollapsedText}
     isCloud={$environmentStore.cloud}
+    aiEnabled={$featuresStore.aiEnabled}
     on:rowclick={e => onRowClick?.({ row: e.detail })}
   />
 </div>

packages/client/src/components/app/container/GridContainer.svelte

@@ -4,6 +4,8 @@
   import { GridRowHeight, GridColumns } from "constants"
   import { memo } from "@budibase/frontend-core"
 
+  export let onClick
+
   const component = getContext("component")
   const { styleable, builderStore } = getContext("sdk")
   const context = getContext("context")
@@ -121,15 +123,19 @@
   })
 </script>
 
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
 <div
   bind:this={ref}
   class="grid"
   class:mobile
+  class:clickable={!!onClick}
   bind:clientWidth={width}
   bind:clientHeight={height}
   use:styleable={$styles}
   data-cols={GridColumns}
   data-col-size={colSize}
+  on:click={onClick}
 >
   {#if inBuilder}
     <div class="underlay">
@@ -176,6 +182,9 @@
   .placeholder.first-col {
     border-left: 1px solid var(--spectrum-global-color-gray-900);
   }
+  .clickable {
+    cursor: pointer;
+  }
 
   /* Highlight grid lines when resizing children */
   :global(.grid.highlight > .underlay) {

packages/client/src/stores/features.js

@@ -2,6 +2,7 @@ import { derived } from "svelte/store"
 import { appStore } from "./app"
 import { authStore } from "./auth"
 import { Constants } from "@budibase/frontend-core"
+import { Feature } from "@budibase/types"
 
 const createFeaturesStore = () => {
   return derived([authStore, appStore], ([$authStore, $appStore]) => {
@@ -33,8 +34,13 @@ const createFeaturesStore = () => {
       }
     }
 
+    const license = getUserLicense()
+
     return {
       logoEnabled: isFreePlan(),
+      aiEnabled:
+        license?.features?.includes(Feature.AI_CUSTOM_CONFIGS) ||
+        license?.features?.includes(Feature.BUDIBASE_AI),
     }
   })
 }

packages/frontend-core/src/components/FilterField.svelte

@@ -174,7 +174,7 @@
         />
       {:else}
         <div>
-          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
+          {#if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)}
             <Input
               disabled={filter.noValue}
               value={readableValue}

packages/frontend-core/src/components/grid/cells/AICell.svelte

@@ -0,0 +1,99 @@
+<script>
+  import { onMount } from "svelte"
+  import { clickOutside } from "@budibase/bbui"
+  import GridPopover from "../overlays/GridPopover.svelte"
+
+  export let value
+  export let focused = false
+  export let api
+
+  let textarea
+  let isOpen = false
+  let anchor
+
+  $: {
+    if (!focused) {
+      isOpen = false
+    }
+  }
+
+  const onKeyDown = () => {
+    return isOpen
+  }
+
+  const open = async () => {
+    isOpen = true
+  }
+
+  const close = () => {
+    textarea?.blur()
+    isOpen = false
+  }
+
+  onMount(() => {
+    api = {
+      focus: () => open(),
+      blur: () => close(),
+      isActive: () => isOpen,
+      onKeyDown,
+    }
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<div class="long-form-cell" on:click={open} bind:this={anchor}>
+  <div class="value">
+    {value || ""}
+  </div>
+</div>
+
+{#if isOpen}
+  <GridPopover {anchor} on:close={close}>
+    <textarea
+      disabled
+      bind:this={textarea}
+      value={value || ""}
+      on:wheel|stopPropagation
+      spellcheck="false"
+      use:clickOutside={close}
+    />
+  </GridPopover>
+{/if}
+
+<style>
+  .long-form-cell {
+    flex: 1 1 auto;
+    padding: var(--cell-padding);
+    align-self: stretch;
+    display: flex;
+    align-items: flex-start;
+    overflow: hidden;
+  }
+  .value {
+    display: -webkit-box;
+    -webkit-line-clamp: var(--content-lines);
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+    line-height: 20px;
+  }
+  textarea {
+    border: none;
+    width: 320px;
+    flex: 1 1 auto;
+    height: var(--max-cell-render-overflow);
+    padding: var(--cell-padding);
+    margin: 0;
+    background: var(--cell-background);
+    font-size: var(--cell-font-size);
+    font-family: var(--font-sans);
+    color: inherit;
+    z-index: 1;
+    resize: none;
+    line-height: 20px;
+    overflow: auto;
+  }
+  textarea:focus {
+    outline: none;
+  }
+</style>

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -95,7 +95,8 @@
     const { type, formulaType } = col.schema
     return (
       searchableTypes.includes(type) ||
-      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC)
+      (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) ||
+      type === FieldType.AI
     )
   }
 

packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte

@@ -27,10 +27,7 @@
   let candidateIndex
   let lastSearchId
   let searching = false
-  let container
   let anchor
-  let relationshipAnchor
-  let relationshipFields
 
   $: fieldValue = parseValue(value)
   $: oneRowOnly = schema?.relationshipType === "one-to-many"
@@ -57,13 +54,6 @@
     return acc
   }, {})
 
-  $: showRelationshipFields =
-    relationshipAnchor &&
-    relationshipFields &&
-    Object.keys(relationshipFields).length &&
-    focused &&
-    !isOpen
-
   const parseValue = value => {
     if (Array.isArray(value) && value.every(x => x?._id)) {
       return value
@@ -208,7 +198,6 @@
 
   // Toggles whether a row is included in the relationship or not
   const toggleRow = async row => {
-    hideRelationshipFields()
     if (fieldValue?.some(x => x._id === row._id)) {
       // If the row is already included, remove it and update the candidate
       // row to be the same position if possible
@@ -245,16 +234,6 @@
     return value
   }
 
-  const displayRelationshipFields = (e, relationship) => {
-    relationshipAnchor = e.target
-    relationshipFields = relationFields[relationship._id]
-  }
-
-  const hideRelationshipFields = () => {
-    relationshipAnchor = null
-    relationshipFields = undefined
-  }
-
   onMount(() => {
     api = {
       focus: open,
@@ -274,7 +253,7 @@
   style="--color:{color};"
   bind:this={anchor}
 >
-  <div class="container" bind:this={container}>
+  <div class="container">
     <div
       class="values"
       class:wrap={editable || contentLines > 1}
@@ -286,9 +265,7 @@
           <div
             class="badge"
             class:extra-info={!!relationFields[relationship._id]}
-            on:mouseenter={e => displayRelationshipFields(e, relationship)}
             on:focus={() => {}}
-            on:mouseleave={() => hideRelationshipFields()}
           >
             <span>
               {readable(
@@ -363,26 +340,6 @@
   </GridPopover>
 {/if}
 
-{#if showRelationshipFields}
-  <GridPopover
-    anchor={relationshipAnchor}
-    maxWidth={400}
-    offset={4}
-    clickOutsideOverride
-  >
-    <div class="relationship-fields">
-      {#each Object.entries(relationshipFields) as [fieldName, fieldValue]}
-        <div class="relationship-field-name">
-          {fieldName}
-        </div>
-        <div class="relationship-field-value">
-          {fieldValue}
-        </div>
-      {/each}
-    </div>
-  </GridPopover>
-{/if}
-
 <style>
   .wrapper {
     flex: 1 1 auto;
@@ -547,29 +504,4 @@
   .search :global(.spectrum-Form-item) {
     flex: 1 1 auto;
   }
-
-  .relationship-fields {
-    margin: var(--spacing-m) var(--spacing-l);
-    display: grid;
-    grid-template-columns: auto 1fr;
-    grid-row-gap: var(--spacing-m);
-    grid-column-gap: var(--spacing-xl);
-  }
-
-  .relationship-field-name {
-    text-transform: uppercase;
-    color: var(--spectrum-global-color-gray-600);
-    font-size: 12px;
-    overflow: hidden;
-    white-space: nowrap;
-    text-overflow: ellipsis;
-    max-width: 120px;
-  }
-  .relationship-field-value {
-    overflow: hidden;
-    display: -webkit-box;
-    -webkit-box-orient: vertical;
-    -webkit-line-clamp: 3;
-    line-clamp: 3;
-  }
 </style>

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -48,6 +48,7 @@
   export let darkMode = false
   export let isCloud = null
   export let rowConditions = null
+  export let aiEnabled = false
 
   // Unique identifier for DOM nodes inside this instance
   const gridID = `grid-${Math.random().toString().slice(2)}`
@@ -104,6 +105,7 @@
     buttonsCollapsedText,
     darkMode,
     isCloud,
+    aiEnabled,
     rowConditions,
   })
 

packages/frontend-core/src/components/grid/lib/renderers.js

@@ -9,6 +9,7 @@ import TextCell from "../cells/TextCell.svelte"
 import LongFormCell from "../cells/LongFormCell.svelte"
 import BooleanCell from "../cells/BooleanCell.svelte"
 import FormulaCell from "../cells/FormulaCell.svelte"
+import AICell from "../cells/AICell.svelte"
 import JSONCell from "../cells/JSONCell.svelte"
 import AttachmentCell from "../cells/AttachmentCell.svelte"
 import AttachmentSingleCell from "../cells/AttachmentSingleCell.svelte"
@@ -31,6 +32,7 @@ const TypeComponentMap = {
   [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell,
   [FieldType.LINK]: RelationshipCell,
   [FieldType.FORMULA]: FormulaCell,
+  [FieldType.AI]: AICell,
   [FieldType.JSON]: JSONCell,
   [FieldType.BB_REFERENCE]: BBReferenceCell,
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,

packages/frontend-core/src/components/grid/overlays/MenuOverlay.svelte

@@ -1,5 +1,6 @@
 <script>
   import { Menu, MenuItem, Helpers } from "@budibase/bbui"
+  import { FieldType } from "@budibase/types"
   import { getContext } from "svelte"
   import { NewRowID } from "../lib/constants"
   import GridPopover from "./GridPopover.svelte"
@@ -26,6 +27,9 @@
 
   $: style = makeStyle($menu)
   $: isNewRow = $focusedRowId === NewRowID
+  $: hasAIColumns = $visibleColumns.some(
+    col => col.schema.type === FieldType.AI
+  )
 
   const makeStyle = menu => {
     return `left:${menu.left}px; top:${menu.top}px;`
@@ -53,6 +57,12 @@
     await Helpers.copyToClipboard(value)
     $notifications.success("Copied to clipboard")
   }
+
+  const generateAIColumns = async () => {
+    menu.actions.close()
+    await rows.actions.applyRowChanges({ rowId: $focusedRowId })
+    $notifications.success("Generated AI columns")
+  }
 </script>
 
 <div bind:this={anchor} {style} class="menu-anchor" />
@@ -161,6 +171,15 @@
           >
             Delete row
           </MenuItem>
+          {#if $config.aiEnabled}
+            <MenuItem
+              icon="MagicWand"
+              disabled={isNewRow || !hasAIColumns}
+              on:click={generateAIColumns}
+            >
+              Generate AI Columns
+            </MenuItem>
+          {/if}
         {/if}
       </Menu>
     </GridPopover>

packages/frontend-core/src/components/grid/stores/columns.js

@@ -109,6 +109,7 @@ export const createActions = context => {
       column.schema.autocolumn ||
       column.schema.disabled ||
       column.schema.type === "formula" ||
+      column.schema.type === "ai" ||
       column.schema.readonly
     )
   }

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -3,9 +3,12 @@ import { dataFilters } from "@budibase/shared-core"
 
 function convertToSearchFilters(view) {
   // convert from SearchFilterGroup type
-  if (view.query) {
-    view.queryUI = view.query
-    view.query = dataFilters.buildQuery(view.query)
+  if (view?.query) {
+    return {
+      ...view,
+      queryUI: view.query,
+      query: dataFilters.buildQuery(view.query),
+    }
   }
   return view
 }

packages/frontend-core/src/constants.js

@@ -160,9 +160,10 @@ export const TypeIconMap = {
   [FieldType.ATTACHMENT_SINGLE]: "DocumentFragment",
   [FieldType.LINK]: "DataCorrelated",
   [FieldType.FORMULA]: "Calculator",
+  [FieldType.AI]: "MagicWand",
   [FieldType.JSON]: "Brackets",
   [FieldType.BIGINT]: "TagBold",
-  [FieldType.AUTO]: "MagicWand",
+  [FieldType.AUTO]: "Shapes",
   [FieldType.BB_REFERENCE]: {
     [BBReferenceFieldSubType.USER]: "UserGroup",
     [BBReferenceFieldSubType.USERS]: "UserGroup",

packages/frontend-core/src/utils/relatedColumns.js

@@ -24,9 +24,9 @@ const columnTypeManyParser = {
       return parsed
     }
 
-    return value?.map(v => parseDate(v))
+    return value.map(v => parseDate(v))
   },
-  [FieldType.BOOLEAN]: value => value?.map(v => !!v),
+  [FieldType.BOOLEAN]: value => value.map(v => !!v),
   [FieldType.BB_REFERENCE_SINGLE]: value => [
     ...new Map(value.map(i => [i._id, i])).values(),
   ],
@@ -80,14 +80,10 @@ export function getRelatedTableValues(row, field, fromField) {
     result = row[field.related.field]?.[0]?.[field.related.subField]
   } else {
     const parser = columnTypeManyParser[field.type] || (value => value)
-
-    result = parser(
-      row[field.related.field]
+    const value = row[field.related.field]
       ?.flatMap(r => r[field.related.subField])
-        ?.filter(i => i !== undefined && i !== null),
-      field
-    )
-
+      ?.filter(i => i !== undefined && i !== null)
+    result = parser(value || [], field)
     if (
       [
         FieldType.STRING,

packages/frontend-core/src/utils/schema.js

@@ -2,6 +2,10 @@ import { helpers } from "@budibase/shared-core"
 import { TypeIconMap } from "../constants"
 
 export const getColumnIcon = column => {
+  if (column.schema.icon) {
+    return column.schema.icon
+  }
+
   if (column.schema.autocolumn) {
     return "MagicWand"
   }

packages/pro

@@ -1 +1 @@
-Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7
+Subproject commit 61391549614b5ac153f267633d0aaea9b07f05c5

packages/server/specs/openapi.json

@@ -833,7 +833,8 @@
                       "type": "string",
                       "enum": [
                         "static",
-                        "dynamic"
+                        "dynamic",
+                        "ai"
                       ],
                       "description": "Defines whether this is a static or dynamic formula."
                     }
@@ -857,6 +858,7 @@
                         "link",
                         "formula",
                         "auto",
+                        "ai",
                         "json",
                         "internal",
                         "barcodeqr",
@@ -1042,7 +1044,8 @@
                           "type": "string",
                           "enum": [
                             "static",
-                            "dynamic"
+                            "dynamic",
+                            "ai"
                           ],
                           "description": "Defines whether this is a static or dynamic formula."
                         }
@@ -1066,6 +1069,7 @@
                             "link",
                             "formula",
                             "auto",
+                            "ai",
                             "json",
                             "internal",
                             "barcodeqr",
@@ -1262,7 +1266,8 @@
                             "type": "string",
                             "enum": [
                               "static",
-                              "dynamic"
+                              "dynamic",
+                              "ai"
                             ],
                             "description": "Defines whether this is a static or dynamic formula."
                           }
@@ -1286,6 +1291,7 @@
                               "link",
                               "formula",
                               "auto",
+                              "ai",
                               "json",
                               "internal",
                               "barcodeqr",

packages/server/specs/openapi.yaml

@@ -761,6 +761,7 @@ components:
                     enum:
                       - static
                       - dynamic
+                      - ai
                     description: Defines whether this is a static or dynamic formula.
               - type: object
                 properties:
@@ -779,6 +780,7 @@ components:
                       - link
                       - formula
                       - auto
+                      - ai
                       - json
                       - internal
                       - barcodeqr
@@ -929,6 +931,7 @@ components:
                         enum:
                           - static
                           - dynamic
+                          - ai
                         description: Defines whether this is a static or dynamic formula.
                   - type: object
                     properties:
@@ -947,6 +950,7 @@ components:
                           - link
                           - formula
                           - auto
+                          - ai
                           - json
                           - internal
                           - barcodeqr
@@ -1104,6 +1108,7 @@ components:
                           enum:
                             - static
                             - dynamic
+                            - ai
                           description: Defines whether this is a static or dynamic formula.
                     - type: object
                       properties:
@@ -1122,6 +1127,7 @@ components:
                             - link
                             - formula
                             - auto
+                            - ai
                             - json
                             - internal
                             - barcodeqr

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -157,7 +157,8 @@ function isEditableColumn(column: FieldSchema) {
     column.autoReason !== AutoReason.FOREIGN_KEY &&
     column.subtype !== AutoFieldSubType.AUTO_ID
   const isFormula = column.type === FieldType.FORMULA
-  return !(isExternalAutoColumn || isFormula)
+  const isAIColumn = column.type === FieldType.AI
+  return !(isExternalAutoColumn || isFormula || isAIColumn)
 }
 
 export class ExternalRequest<T extends Operation> {

packages/server/src/api/controllers/row/external.ts

@@ -28,6 +28,7 @@ import {
 import { cloneDeep } from "lodash"
 import { generateIdForRow } from "./utils"
 import { helpers } from "@budibase/shared-core"
+import { HTTPError } from "@budibase/backend-core"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
@@ -102,6 +103,11 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function destroy(ctx: UserCtx) {
   const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot delete rows through a calculation view", 400)
+  }
+
   const _id = ctx.request.body._id
   const { row } = await handleRequest(Operation.DELETE, source, {
     id: breakRowIdField(_id),

packages/server/src/api/controllers/row/internal.ts

@@ -8,7 +8,7 @@ import {
 } from "../../../utilities/rowProcessor"
 import * as utils from "./utils"
 import { cloneDeep } from "lodash/fp"
-import { context } from "@budibase/backend-core"
+import { context, HTTPError } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import {
   FieldType,
@@ -16,6 +16,7 @@ import {
   PatchRowRequest,
   PatchRowResponse,
   Row,
+  Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
@@ -97,15 +98,26 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
-  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
+
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
+    throw new HTTPError("Cannot delete rows through a calculation view", 400)
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   const { _id } = ctx.request.body
   let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
 
-  if (row.tableId !== tableId) {
+  if (row.tableId !== table._id) {
     throw "Supplied tableId doesn't match the row's tableId"
   }
-  const table = await sdk.tables.getTable(tableId)
   // update the row to include full relationships before deleting them
   row = await outputProcessing(table, row, {
     squash: false,
@@ -115,7 +127,7 @@ export async function destroy(ctx: UserCtx) {
   await linkRows.updateLinks({
     eventType: linkRows.EventType.ROW_DELETE,
     row,
-    tableId,
+    tableId: table._id!,
   })
   // remove any attachments that were on the row from object storage
   await AttachmentCleanup.rowDelete(table, [row])
@@ -123,7 +135,7 @@ export async function destroy(ctx: UserCtx) {
   await updateRelatedFormula(table, row)
 
   let response
-  if (tableId === InternalTables.USER_METADATA) {
+  if (table._id === InternalTables.USER_METADATA) {
     ctx.params = {
       id: _id,
     }

packages/server/src/api/controllers/row/staticFormula.ts

@@ -1,6 +1,7 @@
 import { getRowParams } from "../../../db/utils"
 import {
   outputProcessing,
+  processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
@@ -9,6 +10,7 @@ import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
 import sdk from "../../../sdk"
+import * as pro from "@budibase/pro"
 
 /**
  * This function runs through a list of enriched rows, looks at the rows which
@@ -102,7 +104,7 @@ export async function updateAllFormulasInTable(table: Table) {
       (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
-      const processed = await processFormulas(table, cloneDeep(row), {
+      let processed = await processFormulas(table, cloneDeep(row), {
         dynamic: false,
         contextRows: [enrichedRow],
       })
@@ -142,12 +144,27 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+  const aiEnabled =
+    (await pro.features.isBudibaseAIEnabled()) ||
+    (await pro.features.isAICustomConfigsEnabled())
+  if (aiEnabled) {
+    row = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
+
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
+  if (aiEnabled) {
+    enrichedRow = await processAIColumns(table, row, {
+      contextRows: [enrichedRow],
+    })
+  }
+
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -124,6 +124,7 @@ export async function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI &&
           !existing.find(
             (field: string) => field === `${table.name}.${columnName}`
           )
@@ -133,9 +134,7 @@ export async function buildSqlFieldList(
 
   let fields: string[] = []
   if (sdk.views.isView(source)) {
-    fields = Object.keys(helpers.views.basicFields(source)).filter(
-      key => source.schema?.[key]?.visible !== false
-    )
+    fields = Object.keys(helpers.views.basicFields(source))
   } else {
     fields = extractRealFields(source)
   }

packages/server/src/api/controllers/rowAction/crud.ts

@@ -21,14 +21,15 @@ export async function find(ctx: Ctx<void, RowActionsResponse>) {
   const table = await getTable(ctx)
   const tableId = table._id!
 
-  if (!(await sdk.rowActions.docExists(tableId))) {
+  const rowActions = await sdk.rowActions.getAll(tableId)
+  if (!rowActions) {
     ctx.body = {
       actions: {},
     }
     return
   }
 
-  const { actions } = await sdk.rowActions.getAll(tableId)
+  const { actions } = rowActions
   const result: RowActionsResponse = {
     actions: Object.entries(actions).reduce<Record<string, RowActionResponse>>(
       (acc, [key, action]) => ({

packages/server/src/api/controllers/table/bulkFormula.ts

@@ -6,10 +6,10 @@ import uniq from "lodash/uniq"
 import { updateAllFormulasInTable } from "../row/staticFormula"
 import { context } from "@budibase/backend-core"
 import {
-  FormulaType,
   FieldSchema,
   FieldType,
   FormulaFieldMetadata,
+  FormulaType,
   Table,
 } from "@budibase/types"
 import sdk from "../../../sdk"

packages/server/src/api/controllers/table/index.ts

@@ -71,19 +71,20 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
 
   const datasources = await sdk.datasources.getExternalDatasources()
 
-  const external = datasources.flatMap(datasource => {
+  const external: Table[] = []
+  for (const datasource of datasources) {
     let entities = datasource.entities
     if (entities) {
-      return Object.values(entities).map<Table>((entity: Table) => ({
-        ...entity,
+      for (const entity of Object.values(entities)) {
+        external.push({
+          ...(await processTable(entity)),
           sourceType: TableSourceType.EXTERNAL,
           sourceId: datasource._id!,
           sql: isSQL(datasource),
-      }))
-    } else {
-      return []
-    }
         })
+      }
+    }
+  }
 
   const result: FetchTablesResponse = []
   for (const table of [...internal, ...external]) {
@@ -139,6 +140,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
 export async function destroy(ctx: UserCtx) {
   const appId = ctx.appId
   const tableId = ctx.params.tableId
+  await sdk.rowActions.deleteAll(tableId)
   const deletedTable = await pickApi({ tableId }).destroy(ctx)
   await events.table.deleted(deletedTable)
   ctx.eventEmitter &&

packages/server/src/api/routes/tests/role.spec.js

@@ -0,0 +1,182 @@
+const { roles, events, permissions } = require("@budibase/backend-core")
+const setup = require("./utilities")
+const { PermissionLevel } = require("@budibase/types")
+const { basicRole } = setup.structures
+const { BUILTIN_ROLE_IDS } = roles
+const { BuiltinPermissionID } = permissions
+
+describe("/roles", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  const createRole = async role => {
+    if (!role) {
+      role = basicRole()
+    }
+
+    return request
+      .post(`/api/roles`)
+      .send(role)
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  }
+
+  describe("create", () => {
+    it("returns a success message when role is successfully created", async () => {
+      const role = basicRole()
+      const res = await createRole(role)
+
+      expect(res.body._id).toBeDefined()
+      expect(res.body._rev).toBeDefined()
+      expect(events.role.updated).not.toBeCalled()
+      expect(events.role.created).toBeCalledTimes(1)
+      expect(events.role.created).toBeCalledWith(res.body)
+    })
+  })
+
+  describe("update", () => {
+    it("updates a role", async () => {
+      const role = basicRole()
+      let res = await createRole(role)
+      jest.clearAllMocks()
+      res = await createRole(res.body)
+
+      expect(res.body._id).toBeDefined()
+      expect(res.body._rev).toBeDefined()
+      expect(events.role.created).not.toBeCalled()
+      expect(events.role.updated).toBeCalledTimes(1)
+      expect(events.role.updated).toBeCalledWith(res.body)
+    })
+  })
+
+  describe("fetch", () => {
+    beforeAll(async () => {
+      // Recreate the app
+      await config.init()
+    })
+
+    it("should list custom roles, plus 2 default roles", async () => {
+      const customRole = await config.createRole()
+
+      const res = await request
+        .get(`/api/roles`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+
+      expect(res.body.length).toBe(5)
+
+      const adminRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.ADMIN)
+      expect(adminRole).toBeDefined()
+      expect(adminRole.inherits).toEqual(BUILTIN_ROLE_IDS.POWER)
+      expect(adminRole.permissionId).toEqual(BuiltinPermissionID.ADMIN)
+
+      const powerUserRole = res.body.find(r => r._id === BUILTIN_ROLE_IDS.POWER)
+      expect(powerUserRole).toBeDefined()
+      expect(powerUserRole.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
+      expect(powerUserRole.permissionId).toEqual(BuiltinPermissionID.POWER)
+
+      const customRoleFetched = res.body.find(r => r._id === customRole.name)
+      expect(customRoleFetched).toBeDefined()
+      expect(customRoleFetched.inherits).toEqual(BUILTIN_ROLE_IDS.BASIC)
+      expect(customRoleFetched.permissionId).toEqual(
+        BuiltinPermissionID.READ_ONLY
+      )
+    })
+
+    it("should be able to get the role with a permission added", async () => {
+      const table = await config.createTable()
+      await config.api.permission.add({
+        roleId: BUILTIN_ROLE_IDS.POWER,
+        resourceId: table._id,
+        level: PermissionLevel.READ,
+      })
+      const res = await request
+        .get(`/api/roles`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.length).toBeGreaterThan(0)
+      const power = res.body.find(role => role._id === BUILTIN_ROLE_IDS.POWER)
+      expect(power.permissions[table._id]).toEqual(["read"])
+    })
+  })
+
+  describe("destroy", () => {
+    it("should delete custom roles", async () => {
+      const customRole = await config.createRole({
+        name: "user",
+        permissionId: BuiltinPermissionID.READ_ONLY,
+        inherits: BUILTIN_ROLE_IDS.BASIC,
+      })
+      delete customRole._rev_tree
+      await request
+        .delete(`/api/roles/${customRole._id}/${customRole._rev}`)
+        .set(config.defaultHeaders())
+        .expect(200)
+      await request
+        .get(`/api/roles/${customRole._id}`)
+        .set(config.defaultHeaders())
+        .expect(404)
+      expect(events.role.deleted).toBeCalledTimes(1)
+      expect(events.role.deleted).toBeCalledWith(customRole)
+    })
+  })
+
+  describe("accessible", () => {
+    it("should be able to fetch accessible roles (with builder)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.defaultHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(5)
+      expect(typeof res.body[0]).toBe("string")
+    })
+
+    it("should be able to fetch accessible roles (basic user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(await config.basicRoleHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(2)
+      expect(res.body[0]).toBe("BASIC")
+      expect(res.body[1]).toBe("PUBLIC")
+    })
+
+    it("should be able to fetch accessible roles (no user)", async () => {
+      const res = await request
+        .get("/api/roles/accessible")
+        .set(config.publicHeaders())
+        .expect(200)
+      expect(res.body.length).toBe(1)
+      expect(res.body[0]).toBe("PUBLIC")
+    })
+
+    it("should not fetch higher level accessible roles when a custom role header is provided", async () => {
+      await createRole({
+        name: `custom_role_1`,
+        inherits: roles.BUILTIN_ROLE_IDS.BASIC,
+        permissionId: permissions.BuiltinPermissionID.READ_ONLY,
+        version: "name",
+      })
+      const res = await request
+        .get("/api/roles/accessible")
+        .set({
+          ...config.defaultHeaders(),
+          "x-budibase-role": "custom_role_1",
+        })
+        .expect(200)
+      expect(res.body.length).toBe(3)
+      expect(res.body[0]).toBe("custom_role_1")
+      expect(res.body[1]).toBe("BASIC")
+      expect(res.body[2]).toBe("PUBLIC")
+    })
+  })
+})

packages/server/src/api/routes/tests/row.spec.ts

@@ -14,9 +14,11 @@ import {
   InternalTable,
   tenancy,
   features,
+  utils,
 } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  AIOperationEnum,
   AttachmentFieldMetadata,
   AutoFieldSubType,
   Datasource,
@@ -49,6 +51,18 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 interface WaitOptions {
@@ -757,6 +771,70 @@ describe.each([
         })
       })
 
+      describe("user column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                user: {
+                  name: "user",
+                  type: FieldType.BB_REFERENCE_SINGLE,
+                  subtype: BBReferenceFieldSubType.USER,
+                  default: "{{ [Current User]._id }}",
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.user._id).toEqual(config.getUser()._id)
+        })
+
+        it("does not use default value if value specified", async () => {
+          const id = `us_${utils.newid()}`
+          await config.createUser({ _id: id })
+          const row = await config.api.row.save(table._id!, {
+            user: id,
+          })
+          expect(row.user._id).toEqual(id)
+        })
+      })
+
+      describe("multi-user column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                users: {
+                  name: "users",
+                  type: FieldType.BB_REFERENCE,
+                  subtype: BBReferenceFieldSubType.USER,
+                  default: ["{{ [Current User]._id }}"],
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.users).toHaveLength(1)
+          expect(row.users[0]._id).toEqual(config.getUser()._id)
+        })
+
+        it("does not use default value if value specified", async () => {
+          const id = `us_${utils.newid()}`
+          await config.createUser({ _id: id })
+          const row = await config.api.row.save(table._id!, {
+            users: [id],
+          })
+          expect(row.users).toHaveLength(1)
+          expect(row.users[0]._id).toEqual(id)
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {
@@ -2021,6 +2099,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
+          [FieldType.AI]: undefined, // generated field
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2124,6 +2203,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
+            ai: null,
           },
         ])
       })
@@ -2910,6 +2990,57 @@ describe.each([
       )
     })
 
+  isSqs &&
+    describe("AI fields", () => {
+      let table: Table
+
+      beforeAll(async () => {
+        mocks.licenses.useBudibaseAI()
+        mocks.licenses.useAICustomConfigs()
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              ai: {
+                name: "ai",
+                type: FieldType.AI,
+                operation: AIOperationEnum.PROMPT,
+                prompt: "Convert the following to German: '{{ product }}'",
+              },
+              product: {
+                name: "product",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+        })
+      })
+
+      afterAll(() => {
+        jest.unmock("@budibase/pro")
+      })
+
+      it("should be able to save a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+
+      it("should be able to update a row with an AI column", async () => {
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows.length).toBe(1)
+        await config.api.row.save(table._id!, {
+          product: generator.word(),
+          ...rows[0],
+        })
+        expect(rows.length).toBe(1)
+        expect(rows[0].ai).toEqual("Mock LLM Response")
+      })
+    })
+
   describe("Formula fields", () => {
     let table: Table
     let otherTable: Table

packages/server/src/api/routes/tests/rowAction.spec.ts

@@ -1043,4 +1043,44 @@ describe("/rowsActions", () => {
       )
     })
   })
+
+  describe("scenarios", () => {
+    // https://linear.app/budibase/issue/BUDI-8717/
+    it("should not brick the app when deleting a table with row actions", async () => {
+      const view = await config.api.viewV2.create({
+        tableId,
+        name: generator.guid(),
+        schema: {
+          name: { visible: true },
+        },
+      })
+
+      const rowAction = await config.api.rowAction.save(tableId, {
+        name: generator.guid(),
+      })
+
+      await config.api.rowAction.setViewPermission(
+        tableId,
+        view.id,
+        rowAction.id
+      )
+
+      let actionsResp = await config.api.rowAction.find(tableId)
+      expect(actionsResp.actions[rowAction.id]).toEqual({
+        ...rowAction,
+        allowedSources: [tableId, view.id],
+      })
+
+      const table = await config.api.table.get(tableId)
+      await config.api.table.destroy(table._id!, table._rev!)
+
+      // In the bug reported by Conor, when a delete got deleted its row action
+      // document was not being cleaned up. This meant there existed code paths
+      // that would find it and try to reference the tables within it, resulting
+      // in errors.
+      await config.api.automation.fetchEnriched({
+        status: 200,
+      })
+    })
+  })
 })

packages/server/src/api/routes/tests/search.spec.ts

@@ -17,6 +17,7 @@ import {
 
 import * as setup from "./utilities"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   BBReferenceFieldSubType,
   Datasource,
@@ -41,11 +42,23 @@ import tk from "timekeeper"
 import { encodeJSBinding } from "@budibase/string-templates"
 import { dataFilters } from "@budibase/shared-core"
 import { Knex } from "knex"
-import { generator, structures } from "@budibase/backend-core/tests"
+import { generator, structures, mocks } from "@budibase/backend-core/tests"
 import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 import { generateRowIdField } from "../../../integrations/utils"
 import { cloneDeep } from "lodash/fp"
 
+jest.mock("@budibase/pro", () => ({
+  ...jest.requireActual("@budibase/pro"),
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => `Mock LLM Response`),
+        buildPromptFromAIOperation: jest.fn(),
+      }),
+    },
+  },
+}))
+
 describe.each([
   ["in-memory", undefined],
   ["lucene", undefined],
@@ -1601,6 +1614,79 @@ describe.each([
         })
       })
 
+    isSqs &&
+      describe("AI Column", () => {
+        const UNEXISTING_AI_COLUMN = "Real LLM Response"
+
+        beforeAll(async () => {
+          mocks.licenses.useBudibaseAI()
+          mocks.licenses.useAICustomConfigs()
+
+          tableOrViewId = await createTableOrView({
+            product: { name: "product", type: FieldType.STRING },
+            ai: {
+              name: "AI",
+              type: FieldType.AI,
+              operation: AIOperationEnum.PROMPT,
+              prompt: "Translate '{{ product }}' into German",
+            },
+          })
+
+          await createRows([{ product: "Big Mac" }, { product: "McCrispy" }])
+        })
+
+        describe("equal", () => {
+          it("successfully finds rows based on AI column", async () => {
+            await expectQuery({
+              equal: { ai: "Mock LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              equal: { ai: UNEXISTING_AI_COLUMN },
+            }).toFindNothing()
+          })
+        })
+
+        describe("notEqual", () => {
+          it("Returns nothing when searching notEqual on the mock AI response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Mock LLM Response" },
+            }).toContainExactly([])
+          })
+
+          it("return all when requesting non-existing response", async () => {
+            await expectQuery({
+              notEqual: { ai: "Real LLM Response" },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+        })
+
+        describe("oneOf", () => {
+          it("successfully finds a row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Mock LLM Response", "Other LLM Response"] },
+            }).toContainExactly([
+              { product: "Big Mac" },
+              { product: "McCrispy" },
+            ])
+          })
+
+          it("fails to find nonexistent row", async () => {
+            await expectQuery({
+              oneOf: { ai: ["Whopper"] },
+            }).toFindNothing()
+          })
+        })
+      })
+
     describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
       beforeAll(async () => {
         tableOrViewId = await createTableOrView({

packages/server/src/api/routes/tests/table.spec.ts

@@ -21,6 +21,7 @@ import {
   ViewCalculation,
   ViewV2Enriched,
   RowExportFormat,
+  PermissionLevel,
 } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
@@ -191,6 +192,55 @@ describe.each([
     )
   })
 
+  describe("permissions", () => {
+    it("get the base permissions for the table", async () => {
+      const table = await config.api.table.save(
+        tableForDatasource(datasource, {
+          schema: {
+            name: {
+              type: FieldType.STRING,
+              name: "name",
+            },
+          },
+        })
+      )
+
+      // get the explicit permissions
+      const { permissions } = await config.api.permission.get(table._id!, {
+        status: 200,
+      })
+      const explicitPermissions = {
+        role: "ADMIN",
+        permissionType: "EXPLICIT",
+      }
+      expect(permissions.write).toEqual(explicitPermissions)
+      expect(permissions.read).toEqual(explicitPermissions)
+
+      // revoke the explicit permissions
+      for (let level of [PermissionLevel.WRITE, PermissionLevel.READ]) {
+        await config.api.permission.revoke(
+          {
+            roleId: permissions[level].role,
+            resourceId: table._id!,
+            level,
+          },
+          { status: 200 }
+        )
+      }
+
+      // check base permissions
+      const { permissions: basePermissions } = await config.api.permission.get(
+        table._id!,
+        {
+          status: 200,
+        }
+      )
+      const basePerms = { role: "BASIC", permissionType: "BASE" }
+      expect(basePermissions.write).toEqual(basePerms)
+      expect(basePermissions.read).toEqual(basePerms)
+    })
+  })
+
   describe("update", () => {
     it("updates a table", async () => {
       const table = await config.api.table.save(

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -206,7 +206,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         const createdView = await config.api.viewV2.create(newView)
@@ -250,7 +250,7 @@ describe.each([
               name: "Category",
               type: FieldType.STRING,
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         }
 
         await config.api.viewV2.create(newView, {
@@ -800,6 +800,34 @@ describe.each([
             )
           }
         })
+
+      isInternal &&
+        it("shouldn't trigger a complex type check on a group by field if field is invisible", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                field: {
+                  name: "field",
+                  type: FieldType.JSON,
+                },
+              },
+            })
+          )
+
+          await config.api.viewV2.create(
+            {
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                field: { visible: false },
+              },
+            },
+            {
+              status: 201,
+            }
+          )
+        })
     })
 
     describe("update", () => {
@@ -1016,7 +1044,7 @@ describe.each([
               visible: false,
               icon: "ic",
             },
-          } as Record<string, FieldSchema>,
+          } as ViewV2Schema,
         })
 
         expect(updatedView).toEqual({
@@ -1050,7 +1078,7 @@ describe.each([
                 name: "Category",
                 type: FieldType.STRING,
               },
-            } as Record<string, FieldSchema>,
+            } as ViewV2Schema,
           },
           {
             status: 200,
@@ -1904,6 +1932,59 @@ describe.each([
         })
       })
     })
+
+    describe("calculation views", () => {
+      it("should not remove calculation columns when modifying table schema", async () => {
+        let table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+              age: {
+                name: "age",
+                type: FieldType.NUMBER,
+              },
+            },
+          })
+        )
+
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "age",
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save({
+          ...table,
+          schema: {
+            ...table.schema,
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+              constraints: { presence: true },
+            },
+          },
+        })
+
+        view = await config.api.viewV2.get(view.id)
+        expect(Object.keys(view.schema!).sort()).toEqual([
+          "age",
+          "id",
+          "name",
+          "sum",
+        ])
+      })
+    })
   })
 
   describe("row operations", () => {
@@ -2148,6 +2229,32 @@ describe.each([
         })
         await config.api.row.get(table._id!, rows[1]._id!, { status: 200 })
       })
+
+      it("should not be possible to delete a row in a calculation view", async () => {
+        const row = await config.api.row.save(table._id!, {})
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            id: { visible: true },
+            one: { visible: true },
+          },
+        })
+
+        await config.api.row.delete(
+          view.id,
+          { _id: row._id! },
+          {
+            status: 400,
+            body: {
+              message: "Cannot delete rows through a calculation view",
+              status: 400,
+            },
+          }
+        )
+      })
     })
 
     describe("read", () => {
@@ -3157,6 +3264,241 @@ describe.each([
               }
             )
           })
+
+          it("should be able to filter rows on the view itself", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+              schema: {
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
+
+          it("should be able to filter on group by fields", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+            })
+
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
+
+          it("should be able to sort by group by field", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+              sort: "quantity",
+              sortOrder: SortOrder.DESCENDING,
+            })
+
+            expect(rows).toEqual([
+              expect.objectContaining({ quantity: 2, sum: 10 }),
+              expect.objectContaining({ quantity: 1, sum: 3 }),
+            ])
+          })
+
+          it("should be able to sort by a calculation", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+              sort: "sum",
+              sortOrder: SortOrder.DESCENDING,
+            })
+
+            expect(rows).toEqual([
+              expect.objectContaining({ quantity: 2, sum: 10 }),
+              expect.objectContaining({ quantity: 1, sum: 3 }),
+            ])
+          })
         })
 
       !isLucene &&
@@ -3204,6 +3546,50 @@ describe.each([
           expect(response.rows).toHaveLength(1)
           expect(response.rows[0].sum).toEqual(61)
         })
+
+      it("should be able to filter on a single user field in both the view query and search query", async () => {
+        const table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              user: {
+                name: "user",
+                type: FieldType.BB_REFERENCE_SINGLE,
+                subtype: BBReferenceFieldSubType.USER,
+              },
+            },
+          })
+        )
+
+        await config.api.row.save(table._id!, {
+          user: config.getUser()._id,
+        })
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          query: {
+            equal: {
+              user: "{{ [user].[_id] }}",
+            },
+          },
+          schema: {
+            user: {
+              visible: true,
+            },
+          },
+        })
+
+        const { rows } = await config.api.viewV2.search(view.id, {
+          query: {
+            equal: {
+              user: "{{ [user].[_id] }}",
+            },
+          },
+        })
+
+        expect(rows).toHaveLength(1)
+        expect(rows[0].user._id).toEqual(config.getUser()._id)
+      })
     })
 
     describe("permissions", () => {

packages/server/src/api/routes/utils/validators.ts

@@ -20,31 +20,30 @@ const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
-const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
-  if (table.views && Object.entries(table.views).length) {
-    const requiredFields = Object.entries(table.schema)
-      .filter(([_, v]) => isRequired(v.constraints))
-      .map(([key]) => key)
-    if (requiredFields.length) {
+const validateViewSchemas: CustomValidator<Table> = (table, joiHelpers) => {
+  if (!table.views || Object.keys(table.views).length === 0) {
+    return table
+  }
+  const required = Object.keys(table.schema).filter(key =>
+    isRequired(table.schema[key].constraints)
+  )
+  if (required.length === 0) {
+    return table
+  }
   for (const view of Object.values(table.views)) {
-        if (!sdk.views.isV2(view)) {
+    if (!sdk.views.isV2(view) || helpers.views.isCalculationView(view)) {
       continue
     }
-
-        const editableViewFields = Object.entries(view.schema || {})
+    const editable = Object.entries(view.schema || {})
       .filter(([_, f]) => f.visible && !f.readonly)
       .map(([key]) => key)
-        const missingField = requiredFields.find(
-          f => !editableViewFields.includes(f)
-        )
+    const missingField = required.find(f => !editable.includes(f))
     if (missingField) {
-          return helpers.message({
+      return joiHelpers.message({
         custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
       })
     }
   }
-    }
-  }
   return table
 }
 

packages/server/src/db/linkedRows/index.ts

@@ -307,7 +307,9 @@ export async function squashLinks<T = Row[] | Row>(
                 return false
               }
               if (
-                [FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type)
+                [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
+                  tableColumn.type
+                )
               ) {
                 return false
               }

packages/server/src/integrations/googlesheets.ts

@@ -56,6 +56,7 @@ interface AuthTokenResponse {
 const isTypeAllowed: Record<FieldType, boolean> = {
   [FieldType.STRING]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.NUMBER]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.DATETIME]: true,
@@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus {
         }
         if (
           !sheet.headerValues.includes(key) &&
-          column.type !== FieldType.FORMULA
+          column.type !== FieldType.FORMULA &&
+          column.type !== FieldType.AI
         ) {
           updatedHeaderValues.push(key)
         }

packages/server/src/integrations/utils/utils.ts

@@ -242,6 +242,7 @@ function copyExistingPropsOver(
       let shouldKeepSchema = false
       switch (existingColumnType) {
         case FieldType.FORMULA:
+        case FieldType.AI:
         case FieldType.AUTO:
         case FieldType.INTERNAL:
           shouldKeepSchema = true

packages/server/src/sdk/app/automations/utils.ts

@@ -26,13 +26,13 @@ export async function getBuilderData(
     return tableNameCache[tableId]
   }
 
-  const rowActionNameCache: Record<string, TableRowActions> = {}
+  const rowActionNameCache: Record<string, TableRowActions | undefined> = {}
   async function getRowActionName(tableId: string, rowActionId: string) {
     if (!rowActionNameCache[tableId]) {
       rowActionNameCache[tableId] = await sdk.rowActions.getAll(tableId)
     }
 
-    return rowActionNameCache[tableId].actions[rowActionId]?.name
+    return rowActionNameCache[tableId]?.actions[rowActionId]?.name
   }
 
   const result: Record<string, AutomationBuilderData> = {}
@@ -51,6 +51,10 @@ export async function getBuilderData(
     const tableName = await getTableName(tableId)
     const rowActionName = await getRowActionName(tableId, rowActionId)
 
+    if (!rowActionName) {
+      throw new Error(`Row action not found: ${rowActionId}`)
+    }
+
     result[automation._id!] = {
       displayName: rowActionName,
       triggerInfo: {

packages/server/src/sdk/app/permissions/index.ts

@@ -73,8 +73,7 @@ export async function getResourcePerms(
     p[level] = { role, type: PermissionSource.BASE }
     return p
   }, {})
-  const result = Object.assign(basePermissions, permissions)
-  return result
+  return Object.assign(basePermissions, permissions)
 }
 
 export async function getDependantResources(

packages/server/src/sdk/app/rowActions.ts

@@ -1,5 +1,6 @@
 import { context, docIds, HTTPError, utils } from "@budibase/backend-core"
 import {
+  Automation,
   AutomationTriggerStepId,
   SEPARATOR,
   TableRowActions,
@@ -102,7 +103,25 @@ export async function get(tableId: string, rowActionId: string) {
 export async function getAll(tableId: string) {
   const db = context.getAppDB()
   const rowActionsId = generateRowActionsID(tableId)
-  return await db.get<TableRowActions>(rowActionsId)
+  return await db.tryGet<TableRowActions>(rowActionsId)
+}
+
+export async function deleteAll(tableId: string) {
+  const db = context.getAppDB()
+
+  const doc = await getAll(tableId)
+  if (!doc) {
+    return
+  }
+
+  const automationIds = Object.values(doc.actions).map(a => a.automationId)
+  const automations = await db.getMultiple<Automation>(automationIds)
+
+  for (const automation of automations) {
+    await sdk.automations.remove(automation._id!, automation._rev!)
+  }
+
+  await db.remove(doc)
 }
 
 export async function docExists(tableId: string) {
@@ -223,9 +242,8 @@ export async function run(tableId: any, rowActionId: any, rowId: string) {
     throw new HTTPError("Table not found", 404)
   }
 
-  const { actions } = await getAll(tableId)
-
-  const rowAction = actions[rowActionId]
+  const rowActions = await getAll(tableId)
+  const rowAction = rowActions?.actions[rowActionId]
   if (!rowAction) {
     throw new HTTPError("Row action not found", 404)
   }

packages/server/src/sdk/app/rows/search.ts

@@ -16,7 +16,7 @@ import * as external from "./search/external"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
-import { searchInputMapping } from "./search/utils"
+import { checkFilters, searchInputMapping } from "./search/utils"
 import { db, features } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
@@ -81,6 +81,10 @@ export async function search(
       options.query = {}
     }
 
+    if (context) {
+      options.query = await enrichSearchContext(options.query, context)
+    }
+
     // need to make sure filters in correct shape before checking for view
     options = searchInputMapping(table, options)
 
@@ -92,12 +96,15 @@ export async function search(
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
+      let viewQuery = await enrichSearchContext(view.query || {}, context)
+      viewQuery = dataFilters.buildQueryLegacy(viewQuery) || {}
+      viewQuery = checkFilters(table, viewQuery)
       delete viewQuery?.onEmptyFilter
 
       const sqsEnabled = await features.flags.isEnabled("SQS")
       const supportsLogicalOperators =
         isExternalTableID(view.tableId) || sqsEnabled
+
       if (!supportsLogicalOperators) {
         // In the unlikely event that a Grouped Filter is in a non-SQS environment
         // It needs to be ignored entirely
@@ -113,13 +120,12 @@ export async function search(
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
-        viewQuery ??= {}
         // Carry over filters for unused fields
         Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              viewQuery![operator]![field] = options.query[operator]![field]
+              viewQuery[operator]![field] = options.query[operator]![field]
             }
           })
         })
@@ -137,10 +143,6 @@ export async function search(
       }
     }
 
-    if (context) {
-      options.query = await enrichSearchContext(options.query, context)
-    }
-
     options.query = dataFilters.cleanupQuery(options.query)
     options.query = dataFilters.fixupFilterArrays(options.query)
 

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -68,9 +68,7 @@ async function buildInternalFieldList(
   const { relationships, allowedFields } = opts || {}
   let schemaFields: string[] = []
   if (sdk.views.isView(source)) {
-    schemaFields = Object.keys(helpers.views.basicFields(source)).filter(
-      key => source.schema?.[key]?.visible !== false
-    )
+    schemaFields = Object.keys(helpers.views.basicFields(source))
   } else {
     schemaFields = Object.keys(source.schema).filter(
       key => source.schema[key].visible !== false
@@ -420,6 +418,16 @@ export async function search(
 
   if (params.sort) {
     const sortField = table.schema[params.sort]
+    const isAggregateField = aggregations.some(agg => agg.name === params.sort)
+
+    if (isAggregateField) {
+      request.sort = {
+        [params.sort]: {
+          direction: params.sortOrder || SortOrder.ASCENDING,
+          type: SortType.NUMBER,
+        },
+      }
+    } else if (sortField) {
       const sortType =
         sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING
       request.sort = {
@@ -428,6 +436,9 @@ export async function search(
           type: sortType as SortType,
         },
       }
+    } else {
+      throw new Error(`Unable to sort by ${params.sort}`)
+    }
   }
 
   if (params.bookmark && typeof params.bookmark !== "number") {

packages/server/src/sdk/app/rows/search/utils.ts

@@ -80,11 +80,10 @@ function userColumnMapping(column: string, filters: SearchFilters) {
   })
 }
 
-// maps through the search parameters to check if any of the inputs are invalid
-// based on the table schema, converts them to something that is valid.
-export function searchInputMapping(table: Table, options: RowSearchParams) {
-  // need an internal function to loop over filters, because this takes the full options
-  function checkFilters(filters: SearchFilters) {
+export function checkFilters(
+  table: Table,
+  filters: SearchFilters
+): SearchFilters {
   for (let [key, column] of Object.entries(table.schema || {})) {
     switch (column.type) {
       case FieldType.BB_REFERENCE_SINGLE: {
@@ -105,10 +104,17 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
       }
     }
   }
-    return dataFilters.recurseLogicalOperators(filters, checkFilters)
+  return dataFilters.recurseLogicalOperators(filters, filters =>
+    checkFilters(table, filters)
+  )
 }
+
+// maps through the search parameters to check if any of the inputs are invalid
+// based on the table schema, converts them to something that is valid.
+export function searchInputMapping(table: Table, options: RowSearchParams) {
+  // need an internal function to loop over filters, because this takes the full options
   if (options.query) {
-    options.query = checkFilters(options.query)
+    options.query = checkFilters(table, options.query)
   }
   return options
 }

packages/server/src/sdk/app/tables/create.ts

@@ -1,10 +1,10 @@
-import { Row, Table } from "@budibase/types"
+import { FeatureFlag, Row, Table } from "@budibase/types"
 
 import * as external from "./external"
 import * as internal from "./internal"
 import { isExternal } from "./utils"
 import { setPermissions } from "../permissions"
-import { roles } from "@budibase/backend-core"
+import { features, roles } from "@budibase/backend-core"
 
 export async function create(
   table: Omit<Table, "_id" | "_rev">,
@@ -18,10 +18,16 @@ export async function create(
     createdTable = await internal.create(table, rows, userId)
   }
 
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+
+  if (setExplicitPermission) {
     await setPermissions(createdTable._id!, {
       writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
       readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
     })
+  }
 
   return createdTable
 }

packages/server/src/sdk/app/tables/getters.ts

@@ -20,7 +20,13 @@ export async function processTable(table: Table): Promise<Table> {
   if (!table) {
     return table
   }
+
+  table = { ...table }
   if (table._id && isExternalTableID(table._id)) {
+    // Old created external tables via Budibase might have a missing field name breaking some UI such as filters
+    if (table.schema["id"] && !table.schema["id"].name) {
+      table.schema["id"].name = "id"
+    }
     return {
       ...table,
       type: "table",

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -19,6 +19,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
   [FieldType.DATETIME]: SQLiteType.TEXT,
   [FieldType.FORMULA]: SQLiteType.TEXT,
+  [FieldType.AI]: SQLiteType.TEXT,
   [FieldType.LONGFORM]: SQLiteType.TEXT,
   [FieldType.NUMBER]: SQLiteType.REAL,
   [FieldType.STRING]: SQLiteType.TEXT,

packages/server/src/sdk/app/views/index.ts

@@ -1,11 +1,14 @@
 import {
+  BBReferenceFieldSubType,
   CalculationType,
   canGroupBy,
+  FeatureFlag,
   FieldType,
   isNumeric,
   PermissionLevel,
   RelationSchemaField,
   RenameColumn,
+  RequiredKeys,
   Table,
   TableSchema,
   View,
@@ -13,7 +16,7 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { context, docIds, HTTPError } from "@budibase/backend-core"
+import { context, docIds, features, HTTPError } from "@budibase/backend-core"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -94,6 +97,13 @@ async function guardCalculationViewSchema(
       continue
     }
 
+    if (!schema.field) {
+      throw new HTTPError(
+        `Calculation field "${name}" is missing a "field" property`,
+        400
+      )
+    }
+
     const targetSchema = table.schema[schema.field]
     if (!targetSchema) {
       throw new HTTPError(
@@ -241,12 +251,17 @@ export async function create(
   await guardViewSchema(tableId, viewRequest)
   const view = await pickApi(tableId).create(tableId, viewRequest)
 
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+  if (setExplicitPermission) {
     // Set permissions to be the same as the table
     const tablePerms = await sdk.permissions.getResourcePerms(tableId)
     await sdk.permissions.setPermissions(view.id, {
       writeRole: tablePerms[PermissionLevel.WRITE].role,
       readRole: tablePerms[PermissionLevel.READ].role,
     })
+  }
 
   return view
 }
@@ -298,7 +313,11 @@ export async function enrichSchema(
     const result: Record<string, ViewV2ColumnEnriched> = {}
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
-      if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
+      if (
+        [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(
+          relTableField.type
+        )
+      ) {
         continue
       }
 
@@ -309,13 +328,26 @@ export async function enrichSchema(
       const viewFieldSchema = viewFields[relTableFieldName]
       const isVisible = !!viewFieldSchema?.visible
       const isReadonly = !!viewFieldSchema?.readonly
-      result[relTableFieldName] = {
-        ...relTableField,
-        ...viewFieldSchema,
-        name: relTableField.name,
+      const enrichedFieldSchema: RequiredKeys<ViewV2ColumnEnriched> = {
         visible: isVisible,
         readonly: isReadonly,
+        order: viewFieldSchema?.order,
+        width: viewFieldSchema?.width,
+
+        icon: relTableField.icon,
+        type: relTableField.type,
+        subtype: relTableField.subtype,
       }
+      if (
+        !enrichedFieldSchema.icon &&
+        relTableField.type === FieldType.BB_REFERENCE &&
+        relTableField.subtype === BBReferenceFieldSubType.USER &&
+        !helpers.schema.isDeprecatedSingleUserColumn(relTableField)
+      ) {
+        // Forcing the icon, otherwise we would need to pass the constraints to show the proper icon
+        enrichedFieldSchema.icon = "UserGroup"
+      }
+      result[relTableFieldName] = enrichedFieldSchema
     }
     return result
   }
@@ -368,7 +400,8 @@ export function syncSchema(
 
   if (view.schema) {
     for (const fieldName of Object.keys(view.schema)) {
-      if (!schema[fieldName]) {
+      const viewSchema = view.schema[fieldName]
+      if (!helpers.views.isCalculationField(viewSchema) && !schema[fieldName]) {
         delete view.schema[fieldName]
       }
     }

packages/server/src/sdk/app/views/tests/views.spec.ts

@@ -355,13 +355,11 @@ describe("table sdk", () => {
               visible: true,
               columns: {
                 title: {
-                  name: "title",
                   type: "string",
                   visible: true,
                   readonly: true,
                 },
                 age: {
-                  name: "age",
                   type: "number",
                   visible: false,
                   readonly: false,

packages/server/src/tests/utilities/api/automation.ts

@@ -1,4 +1,4 @@
-import { Automation } from "@budibase/types"
+import { Automation, FetchAutomationResponse } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
 export class AutomationAPI extends TestAPI {
@@ -14,6 +14,26 @@ export class AutomationAPI extends TestAPI {
     )
     return result
   }
+
+  fetch = async (
+    expectations?: Expectations
+  ): Promise<FetchAutomationResponse> => {
+    return await this._get<FetchAutomationResponse>(`/api/automations`, {
+      expectations,
+    })
+  }
+
+  fetchEnriched = async (
+    expectations?: Expectations
+  ): Promise<FetchAutomationResponse> => {
+    return await this._get<FetchAutomationResponse>(
+      `/api/automations?enrich=true`,
+      {
+        expectations,
+      }
+    )
+  }
+
   post = async (
     body: Automation,
     expectations?: Expectations

packages/server/src/tests/utilities/structures.ts

@@ -7,6 +7,7 @@ import {
   TRIGGER_DEFINITIONS,
 } from "../../automations"
 import {
+  AIOperationEnum,
   Automation,
   AutomationActionStepId,
   AutomationResults,
@@ -668,6 +669,12 @@ export function fullSchemaWithoutLinks({
         presence: allRequired,
       },
     },
+    [FieldType.AI]: {
+      name: "ai",
+      type: FieldType.AI,
+      operation: AIOperationEnum.PROMPT,
+      prompt: "Translate this into German :'{{ product }}'",
+    },
     [FieldType.BARCODEQR]: {
       name: "barcodeqr",
       type: FieldType.BARCODEQR,

packages/server/src/utilities/rowProcessor/index.ts

@@ -33,7 +33,7 @@ import {
   PROTECTED_EXTERNAL_COLUMNS,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
-import { processString } from "@budibase/string-templates"
+import { processStringSync } from "@budibase/string-templates"
 import {
   getTableFromSource,
   isUserMetadataTable,
@@ -134,10 +134,15 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed =
-        typeof schema.default === "string"
-          ? await processString(schema.default, ctx)
-          : schema.default
+      let processed: string | string[]
+      if (Array.isArray(schema.default)) {
+        processed = schema.default.map(val => processStringSync(val, ctx))
+      } else if (typeof schema.default === "string") {
+        processed = processStringSync(schema.default, ctx)
+      } else {
+        processed = schema.default
+      }
+
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {
@@ -205,6 +210,10 @@ export async function inputProcessing(
     if (field.type === FieldType.FORMULA) {
       delete clonedRow[key]
     }
+    // remove any AI values, they are to be generated
+    if (field.type === FieldType.AI) {
+      delete clonedRow[key]
+    }
     // otherwise coerce what is there to correct types
     else {
       clonedRow[key] = coerce(value, field.type)

packages/server/src/utilities/rowProcessor/tests/utils.spec.ts

@@ -1,11 +1,29 @@
-import { fixAutoColumnSubType } from "../utils"
+import { fixAutoColumnSubType, processAIColumns } from "../utils"
 import { AutoFieldDefaultNames } from "../../../constants"
 import {
+  AIOperationEnum,
   AutoFieldSubType,
   FieldSchema,
   FieldType,
+  INTERNAL_TABLE_SOURCE_ID,
   RelationshipType,
+  Table,
+  TableSourceType,
 } from "@budibase/types"
+import { generator } from "@budibase/backend-core/tests"
+
+const buildPromptMock = jest.fn()
+
+jest.mock("@budibase/pro", () => ({
+  ai: {
+    LargeLanguageModel: {
+      forCurrentTenant: async () => ({
+        run: jest.fn(() => "response from LLM"),
+        buildPromptFromAIOperation: buildPromptMock,
+      }),
+    },
+  },
+}))
 
 describe("rowProcessor utility", () => {
   describe("fixAutoColumnSubType", () => {
@@ -60,4 +78,59 @@ describe("rowProcessor utility", () => {
       expect(fixAutoColumnSubType(schema)).toEqual(schema)
     })
   })
+
+  describe("processAIColumns", () => {
+    it("ensures that bindable inputs are mapped and passed to to LLM prompt generation", async () => {
+      const table: Table = {
+        _id: generator.guid(),
+        name: "AITestTable",
+        type: "table",
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        schema: {
+          product: {
+            type: FieldType.STRING,
+            name: "product",
+            constraints: {
+              presence: true,
+              type: "string",
+            },
+          },
+          aicol: {
+            type: FieldType.AI,
+            name: "aicol",
+            operation: AIOperationEnum.PROMPT,
+            prompt: "Translate '{{ product }}' into German",
+          },
+        },
+      }
+
+      const inputRows = [
+        {
+          product: "Car Battery",
+        },
+      ]
+
+      const result = await processAIColumns(table, inputRows, {
+        contextRows: inputRows,
+      })
+      expect(buildPromptMock).toHaveBeenCalledWith({
+        row: {
+          product: "Car Battery",
+        },
+        schema: {
+          name: "aicol",
+          operation: "PROMPT",
+          prompt: "Translate 'Car Battery' into German",
+          type: "ai",
+        },
+      })
+      expect(result).toEqual([
+        {
+          aicol: "response from LLM",
+          product: "Car Battery",
+        },
+      ])
+    })
+  })
 })

packages/server/src/utilities/rowProcessor/utils.ts

@@ -8,9 +8,13 @@ import {
   FormulaType,
   AutoFieldSubType,
   FieldType,
+  OperationFieldTypeEnum,
+  AIOperationEnum,
 } from "@budibase/types"
+import { OperationFields } from "@budibase/shared-core"
 import tracer from "dd-trace"
 import { context } from "@budibase/backend-core"
+import * as pro from "@budibase/pro"
 
 interface FormulaOpts {
   dynamic?: boolean
@@ -91,6 +95,66 @@ export async function processFormulas<T extends Row | Row[]>(
   })
 }
 
+/**
+ * Looks through the rows provided and finds AI columns - which it then processes.
+ */
+export async function processAIColumns<T extends Row | Row[]>(
+  table: Table,
+  inputRows: T,
+  { contextRows }: FormulaOpts
+): Promise<T> {
+  return tracer.trace("processAIColumns", {}, async span => {
+    const numRows = Array.isArray(inputRows) ? inputRows.length : 1
+    span?.addTags({ table_id: table._id, numRows })
+    const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+    const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini")
+    if (rows) {
+      // Ensure we have snippet context
+      await context.ensureSnippetContext()
+
+      for (let [column, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldType.AI) {
+          continue
+        }
+
+        const rowUpdates = rows.map((row, i) => {
+          const contextRow = contextRows ? contextRows[i] : row
+
+          // Check if the type is bindable and pass through HBS if so
+          const operationField =
+            OperationFields[schema.operation as AIOperationEnum]
+          for (const key in schema) {
+            const fieldType = operationField[key as keyof typeof operationField]
+            if (fieldType === OperationFieldTypeEnum.BINDABLE_TEXT) {
+              // @ts-ignore
+              schema[key] = processStringSync(schema[key], contextRow)
+            }
+          }
+
+          const prompt = llm.buildPromptFromAIOperation({ schema, row })
+
+          return tracer.trace("processAIColumn", {}, async span => {
+            span?.addTags({ table_id: table._id, column })
+            const llmResponse = await llm.run(prompt!)
+            return {
+              ...row,
+              [column]: llmResponse,
+            }
+          })
+        })
+
+        const processedRows = await Promise.all(rowUpdates)
+
+        // Promise.all is deterministic so can rely on the indexing here
+        processedRows.forEach(
+          (processedRow, index) => (rows[index] = processedRow)
+        )
+      }
+    }
+    return Array.isArray(inputRows) ? rows : rows[0]
+  })
+}
+
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.

packages/server/src/utilities/security.ts

@@ -1,5 +1,5 @@
-import { permissions, roles } from "@budibase/backend-core"
-import { DocumentType, VirtualDocumentType } from "../db/utils"
+import { DocumentType, permissions, roles } from "@budibase/backend-core"
+import { VirtualDocumentType } from "../db/utils"
 import { getDocumentType, getVirtualDocumentType } from "@budibase/types"
 
 export const CURRENTLY_SUPPORTED_LEVELS: string[] = [
@@ -19,6 +19,7 @@ export function getPermissionType(resourceId: string) {
   switch (docType) {
     case DocumentType.TABLE:
     case DocumentType.ROW:
+    case DocumentType.DATASOURCE_PLUS:
       return permissions.PermissionType.TABLE
     case DocumentType.AUTOMATION:
       return permissions.PermissionType.AUTOMATION

packages/shared-core/src/constants/ai.ts

@@ -0,0 +1,68 @@
+import {
+  AIOperationEnum,
+  OperationFieldsType,
+  OperationFieldTypeEnum,
+} from "@budibase/types"
+
+export const AIOperations = {
+  SUMMARISE_TEXT: {
+    label: "Summarise Text",
+    value: "SUMMARISE_TEXT",
+  },
+  CLEAN_DATA: {
+    label: "Clean Data",
+    value: "CLEAN_DATA",
+  },
+  TRANSLATE: {
+    label: "Translate",
+    value: "TRANSLATE",
+  },
+  CATEGORISE_TEXT: {
+    label: "Categorise Text",
+    value: "CATEGORISE_TEXT",
+  },
+  SENTIMENT_ANALYSIS: {
+    label: "Sentiment Analysis",
+    value: "SENTIMENT_ANALYSIS",
+  },
+  PROMPT: {
+    label: "Prompt",
+    value: "PROMPT",
+  },
+  SEARCH_WEB: {
+    label: "Search Web",
+    value: "SEARCH_WEB",
+  },
+}
+
+export const OperationFieldTypes = {
+  MULTI_COLUMN: "columns",
+  COLUMN: "column",
+  BINDABLE_TEXT: "prompt",
+}
+
+export const OperationFields: OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN,
+    language: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN,
+  },
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT,
+  },
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN,
+  },
+}

packages/shared-core/src/constants/index.ts

@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./api"
 export * from "./fields"
 export * from "./rows"

packages/shared-core/src/filters.ts

@@ -90,6 +90,8 @@ export const getValidOperatorsForType = (
     ops = numOps
   } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) {
     ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  } else if (type === FieldType.AI) {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
   } else if (
     type === FieldType.BB_REFERENCE_SINGLE ||
     schema.isDeprecatedSingleUserColumn(fieldType)

packages/shared-core/src/helpers/views.ts

@@ -33,6 +33,13 @@ export function calculationFields(view: UnsavedViewV2) {
   return pickBy(view.schema || {}, isCalculationField)
 }
 
-export function basicFields(view: UnsavedViewV2) {
-  return pickBy(view.schema || {}, field => !isCalculationField(field))
+export function isVisible(field: ViewFieldMetadata) {
+  return field.visible !== false
+}
+
+export function basicFields(view: UnsavedViewV2, opts?: { visible?: boolean }) {
+  const { visible = true } = opts || {}
+  return pickBy(view.schema || {}, field => {
+    return !isCalculationField(field) && (!visible || isVisible(field))
+  })
 }

packages/shared-core/src/table.ts

@@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record<FieldType, boolean> = {
   [FieldType.NUMBER]: true,
   [FieldType.DATETIME]: true,
   [FieldType.FORMULA]: true,
+  [FieldType.AI]: true,
   [FieldType.AUTO]: true,
   [FieldType.INTERNAL]: true,
   [FieldType.BARCODEQR]: true,
@@ -38,6 +39,7 @@ const allowSortColumnByType: Record<FieldType, boolean> = {
   [FieldType.JSON]: true,
 
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
@@ -62,12 +64,13 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.BIGINT]: false,
   [FieldType.BOOLEAN]: false,
   [FieldType.FORMULA]: false,
+  [FieldType.AI]: false,
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
   [FieldType.LINK]: false,
-  [FieldType.BB_REFERENCE]: false,
-  [FieldType.BB_REFERENCE_SINGLE]: false,
+  [FieldType.BB_REFERENCE]: true,
+  [FieldType.BB_REFERENCE_SINGLE]: true,
 }
 
 export function canBeDisplayColumn(type: FieldType): boolean {

packages/shared-core/src/utils.ts

@@ -172,7 +172,7 @@ export const processSearchFilters = (
         .sort((a, b) => {
           return a.localeCompare(b)
         })
-        .filter(key => key in filter)
+        .filter(key => filter[key])
 
       if (filterPropertyKeys.length == 1) {
         const key = filterPropertyKeys[0],

packages/types/src/documents/app/row.ts

@@ -76,6 +76,13 @@ export enum FieldType {
    * that is part of the initial formula definition, the formula will be live evaluated in the browser.
    */
   AUTO = "auto",
+  /**
+   * A complex type, called an AI column within Budibase. This type is only supported against internal tables
+   * and calculates the output based on a chosen operation (summarise text, translation etc) which passes to
+   * the configured Budibase Large Language Model to retrieve the output and write it back into the row.
+   * AI fields function in a similar fashion to static formulas, and possess many of the same characteristics.
+   */
+  AI = "ai",
   /**
    * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column
    * type, which will be represented as a JSON object in the row. This type depends on a schema being

packages/types/src/documents/app/table/constants.ts

@@ -30,6 +30,7 @@ export enum JsonFieldSubType {
 export enum FormulaType {
   STATIC = "static",
   DYNAMIC = "dynamic",
+  AI = "ai",
 }
 
 export enum BBReferenceFieldSubType {

packages/types/src/documents/app/table/schema.ts

@@ -9,6 +9,7 @@ import {
   JsonFieldSubType,
   RelationshipType,
 } from "./constants"
+import { AIOperationEnum } from "../../../sdk/ai"
 
 export interface UIFieldMetadata {
   order?: number
@@ -116,16 +117,28 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   formulaType?: FormulaType
 }
 
+export interface AIFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum
+  columns?: string[]
+  column?: string
+  categories?: string[]
+  prompt?: string
+  language?: string
+}
+
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE
   subtype: BBReferenceFieldSubType
   relationshipType?: RelationshipType
+  default?: string[]
 }
 export interface BBReferenceSingleFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE_SINGLE
   subtype: Exclude<BBReferenceFieldSubType, BBReferenceFieldSubType.USERS>
+  default?: string
 }
 
 export interface AttachmentFieldMetadata extends BaseFieldSchema {
@@ -192,6 +205,7 @@ interface OtherFieldMetadata extends BaseFieldSchema {
     | FieldType.LINK
     | FieldType.AUTO
     | FieldType.FORMULA
+    | FieldType.AI
     | FieldType.NUMBER
     | FieldType.LONGFORM
     | FieldType.BB_REFERENCE
@@ -209,6 +223,7 @@ export type FieldSchema =
   | RelationshipFieldMetadata
   | AutoColumnFieldMetadata
   | FormulaFieldMetadata
+  | AIFieldMetadata
   | NumberFieldMetadata
   | LongFormFieldMetadata
   | StringFieldMetadata

packages/types/src/sdk/ai.ts

@@ -0,0 +1,91 @@
+export enum AIOperationEnum {
+  SUMMARISE_TEXT = "SUMMARISE_TEXT",
+  CLEAN_DATA = "CLEAN_DATA",
+  TRANSLATE = "TRANSLATE",
+  CATEGORISE_TEXT = "CATEGORISE_TEXT",
+  SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS",
+  PROMPT = "PROMPT",
+  SEARCH_WEB = "SEARCH_WEB",
+}
+
+export enum OperationFieldTypeEnum {
+  MULTI_COLUMN = "columns",
+  COLUMN = "column",
+  BINDABLE_TEXT = "prompt",
+}
+
+export type OperationFieldsType = {
+  [AIOperationEnum.SUMMARISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+  [AIOperationEnum.CLEAN_DATA]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.TRANSLATE]: {
+    column: OperationFieldTypeEnum.COLUMN
+    language: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.CATEGORISE_TEXT]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+    categories: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SENTIMENT_ANALYSIS]: {
+    column: OperationFieldTypeEnum.COLUMN
+  }
+  [AIOperationEnum.PROMPT]: {
+    prompt: OperationFieldTypeEnum.BINDABLE_TEXT
+  }
+  [AIOperationEnum.SEARCH_WEB]: {
+    columns: OperationFieldTypeEnum.MULTI_COLUMN
+  }
+}
+
+type BaseSchema = {
+  operation: AIOperationEnum
+}
+
+type SummariseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.SUMMARISE_TEXT
+  columns: string[]
+}
+
+type CleanDataSchema = BaseSchema & {
+  operation: AIOperationEnum.CLEAN_DATA
+  column: string
+}
+
+type TranslateSchema = BaseSchema & {
+  operation: AIOperationEnum.TRANSLATE
+  column: string
+  language: string
+}
+
+type CategoriseTextSchema = BaseSchema & {
+  operation: AIOperationEnum.CATEGORISE_TEXT
+  columns: string[]
+  categories: string[]
+}
+
+type SentimentAnalysisSchema = BaseSchema & {
+  operation: AIOperationEnum.SENTIMENT_ANALYSIS
+  column: string
+}
+
+type PromptSchema = BaseSchema & {
+  operation: AIOperationEnum.PROMPT
+  prompt: string
+}
+
+type SearchWebSchema = BaseSchema & {
+  operation: AIOperationEnum.SEARCH_WEB
+  columns: string[]
+}
+
+export type AIColumnSchema =
+  | SummariseTextSchema
+  | CleanDataSchema
+  | TranslateSchema
+  | CategoriseTextSchema
+  | SentimentAnalysisSchema
+  | PromptSchema
+  | SearchWebSchema

packages/types/src/sdk/db.ts

@@ -129,7 +129,12 @@ export interface Database {
   name: string
 
   exists(): Promise<boolean>
+  /**
+   * @deprecated the plan is to get everything using `tryGet` instead, then rename
+   * `tryGet` to `get`.
+   */
   get<T extends Document>(id?: string): Promise<T>
+  tryGet<T extends Document>(id?: string): Promise<T | undefined>
   exists(docId: string): Promise<boolean>
   getMultiple<T extends Document>(
     ids: string[],

packages/types/src/sdk/featureFlag.ts

@@ -3,6 +3,7 @@ export enum FeatureFlag {
   PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
   AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
   ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",
+  TABLES_DEFAULT_ADMIN = "TABLES_DEFAULT_ADMIN",
 }
 
 export interface TenantFeatureFlags {

packages/types/src/sdk/index.ts

@@ -1,3 +1,4 @@
+export * from "./ai"
 export * from "./automations"
 export * from "./hosting"
 export * from "./context"

packages/types/src/sdk/view.ts

@@ -1,4 +1,10 @@
-import { FieldSchema, RelationSchemaField, ViewV2 } from "../documents"
+import {
+  FieldSchema,
+  FieldSubType,
+  FieldType,
+  RelationSchemaField,
+  ViewV2,
+} from "../documents"
 
 export interface ViewV2Enriched extends ViewV2 {
   schema?: {
@@ -8,4 +14,7 @@ export interface ViewV2Enriched extends ViewV2 {
   }
 }
 
-export type ViewV2ColumnEnriched = RelationSchemaField & FieldSchema
+export interface ViewV2ColumnEnriched extends RelationSchemaField {
+  type: FieldType
+  subtype?: FieldSubType
+}