From 2d21afbe101e646261652b5a6efbd27d504e4a41 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 12:52:11 +0100 Subject: [PATCH 01/69] rough v1 --- packages/backend-core/src/sql/sql.ts | 3 +- packages/backend-core/src/sql/sqlTable.ts | 5 +- .../DataTable/modals/CreateEditColumn.svelte | 10 ++ .../common/AIFieldConfiguration.svelte | 123 ++++++++++++++++++ .../builder/src/constants/backend/index.js | 6 + .../src/components/FilterBuilder.svelte | 2 +- .../components/grid/cells/HeaderCell.svelte | 3 +- .../src/components/grid/lib/renderers.js | 2 + packages/frontend-core/src/constants.js | 1 + .../api/controllers/row/ExternalRequest.ts | 3 +- .../src/api/controllers/row/aiColumn.ts | 43 ++++++ .../src/api/controllers/row/staticFormula.ts | 12 +- .../src/api/controllers/row/utils/sqlUtils.ts | 1 + .../src/api/controllers/table/bulkFormula.ts | 27 +++- .../server/src/api/controllers/table/utils.ts | 2 +- packages/server/src/db/linkedRows/index.ts | 2 +- .../server/src/integrations/googlesheets.ts | 4 +- .../src/sdk/app/tables/internal/index.ts | 3 +- .../server/src/sdk/app/tables/internal/sqs.ts | 1 + packages/server/src/sdk/app/views/index.ts | 2 +- .../src/utilities/rowProcessor/index.ts | 4 + .../src/utilities/rowProcessor/utils.ts | 51 ++++++++ packages/shared-core/src/filters.ts | 2 + packages/shared-core/src/table.ts | 3 + packages/types/src/documents/app/row.ts | 4 + .../src/documents/app/table/constants.ts | 1 + .../types/src/documents/app/table/schema.ts | 13 ++ 27 files changed, 314 insertions(+), 19 deletions(-) create mode 100644 packages/builder/src/components/common/AIFieldConfiguration.svelte create mode 100644 packages/server/src/api/controllers/row/aiColumn.ts diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b20938981..8c97b33418 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1141,7 +1141,8 @@ class InternalBuilder { schema.constraints?.presence === true || schema.type === FieldType.FORMULA || schema.type === FieldType.AUTO || - schema.type === FieldType.LINK + schema.type === FieldType.LINK || + schema.type === FieldType.AI ) { continue } diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index f5b02cc4e4..84f4e290aa 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder function isIgnoredType(type: FieldType) { - const ignored = [FieldType.LINK, FieldType.FORMULA] + const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI] return ignored.indexOf(type) !== -1 } @@ -144,6 +144,9 @@ function generateSchema( case FieldType.FORMULA: // This is allowed, but nothing to do on the external datasource break + case FieldType.AI: + // This is allowed, but nothing to do on the external datasource + break case FieldType.ATTACHMENTS: case FieldType.ATTACHMENT_SINGLE: case FieldType.SIGNATURE_SINGLE: diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 0130c39715..da68b4182b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -33,6 +33,7 @@ } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils" import ConfirmDialog from "components/common/ConfirmDialog.svelte" + import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" @@ -54,6 +55,7 @@ const NUMBER_TYPE = FieldType.NUMBER const JSON_TYPE = FieldType.JSON const DATE_TYPE = FieldType.DATETIME + const AI_TYPE = FieldType.AI const dispatch = createEventDispatcher() const { dispatch: gridDispatch, rows } = getContext("grid") @@ -421,6 +423,7 @@ FIELDS.ATTACHMENT_SINGLE, FIELDS.ATTACHMENTS, FIELDS.FORMULA, + FIELDS.AI, FIELDS.JSON, FIELDS.BARCODEQR, FIELDS.SIGNATURE_SINGLE, @@ -732,6 +735,13 @@ /> + {:else if editableColumn.type === AI_TYPE} + {:else if editableColumn.type === JSON_TYPE} + import { Input, Multiselect, Select, TextArea } from "@budibase/bbui" + import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" + import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" + + const AIOperations = { + SUMMARISE_TEXT: { + label: "Summarise Text", + value: "SUMMARISE_TEXT" + }, + CLEAN_DATA: { + label: "Clean Data", + value: "CLEAN_DATA" + }, + TRANSLATE: { + label: "Translate", + value: "TRANSLATE" + }, + CATEGORISE_TEXT: { + label: "Categorise Text", + value: "CATEGORISE_TEXT" + }, + SENTIMENT_ANALYSIS: { + label: "Sentiment Analysis", + value: "SENTIMENT_ANALYSIS" + }, + PROMPT: { + label: "Prompt", + value: "PROMPT" + }, + SEARCH_WEB: { + label: "Search Web", + value: "SEARCH_WEB" + } + } + + const OperationFieldTypes = { + MULTI_COLUMN: "columns", + COLUMN: "column", + BINDABLE_TEXT: "prompt", + // LANGUAGE: "language", + } + + const OperationFields = { + SUMMARISE_TEXT: { + columns: OperationFieldTypes.MULTI_COLUMN, + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + CLEAN_DATA: { + columns: OperationFieldTypes.MULTI_COLUMN, + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + TRANSLATE: { + columns: OperationFieldTypes.MULTI_COLUMN, + language: OperationFieldTypes.BINDABLE_TEXT, + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + CATEGORISE_TEXT: { + columns: OperationFieldTypes.MULTI_COLUMN, + categories: OperationFieldTypes.BINDABLE_TEXT, + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + SENTIMENT_ANALYSIS: { + column: OperationFieldTypes.COLUMN, + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + PROMPT: { + prompt: OperationFieldTypes.BINDABLE_TEXT, + }, + SEARCH_WEB: { + columns: OperationFieldTypes.MULTI_COLUMN, + prompt: OperationFieldTypes.BINDABLE_TEXT, + } + } + + const AIFieldConfigOptions = Object.keys(AIOperations).map(key => ({ + label: AIOperations[key].label, + value: AIOperations[key].value + })) + + export let bindings + export let context + export let schema + export let aiField = {} + + $: OperationField = OperationFields[aiField.operation] || null + $: console.log(aiField) + $: console.log(schema) + $: schemaWithoutRelations = Object.keys(schema).filter(key => schema[key].type !== "link") + + + + {/if} + {/each} +{/if} \ No newline at end of file diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 6fbc36afe2..6ddf4c2138 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -159,6 +159,12 @@ export const FIELDS = { icon: TypeIconMap[FieldType.FORMULA], constraints: {}, }, + AI: { + name: "AI", + type: FieldType.AI, + icon: TypeIconMap[FieldType.AI], + constraints: {}, + }, JSON: { name: "JSON", type: FieldType.JSON, diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 3a0c789b9e..d255d022bf 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -262,7 +262,7 @@ {/if} {#if allowBindings && filter.field && filter.valueType === "Binding"} - {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)} + {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA, FieldType.AI].includes(filter.type)} {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)} { diff --git a/packages/server/src/api/controllers/row/aiColumn.ts b/packages/server/src/api/controllers/row/aiColumn.ts new file mode 100644 index 0000000000..fc543c3487 --- /dev/null +++ b/packages/server/src/api/controllers/row/aiColumn.ts @@ -0,0 +1,43 @@ +import { getRowParams } from "../../../db/utils" +import { + outputProcessing, + processAIColumns, +} from "../../../utilities/rowProcessor" +import { context } from "@budibase/backend-core" +import { Table, Row } from "@budibase/types" +import isEqual from "lodash/isEqual" +import { cloneDeep } from "lodash/fp" + +export async function updateAllAIColumnsInTable(table: Table) { + const db = context.getAppDB() + // start by getting the raw rows (which will be written back to DB after update) + let rows = ( + await db.allDocs( + getRowParams(table._id, null, { + include_docs: true, + }) + ) + ).rows.map(row => row.doc!) + // now enrich the rows, note the clone so that we have the base state of the + // rows so that we don't write any of the enriched information back + let enrichedRows = await outputProcessing(table, cloneDeep(rows), { + squash: false, + }) + const updatedRows = [] + for (let row of rows) { + // find the enriched row, if found process the formulas + const enrichedRow = enrichedRows.find( + (enriched: Row) => enriched._id === row._id + ) + if (enrichedRow) { + let processed = await processAIColumns(table, cloneDeep(row), { + contextRows: [enrichedRow], + }) + // values have changed, need to add to bulk docs to update + if (!isEqual(processed, row)) { + updatedRows.push(processed) + } + } + } + await db.bulkDocs(updatedRows) +} diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 777379db14..54cb7d0728 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -1,6 +1,6 @@ import { getRowParams } from "../../../db/utils" import { - outputProcessing, + outputProcessing, processAIColumns, processFormulas, } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" @@ -101,7 +101,7 @@ export async function updateAllFormulasInTable(table: Table) { (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { - const processed = await processFormulas(table, cloneDeep(row), { + let processed = await processFormulas(table, cloneDeep(row), { dynamic: false, contextRows: [enrichedRow], }) @@ -142,6 +142,10 @@ export async function finaliseRow( dynamic: false, contextRows: [enrichedRow], }) + row = await processAIColumns(table, row, { + contextRows: [enrichedRow], + }) + // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will // handle the auto ID clash @@ -154,6 +158,10 @@ export async function finaliseRow( enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false, }) + enrichedRow = await processAIColumns(table, row, { + contextRows: [enrichedRow], + }) + // this updates the related formulas in other rows based on the relations to this row if (updateFormula) { await updateRelatedFormula(table, enrichedRow) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 249bb43bbc..22b34e1720 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -119,6 +119,7 @@ export function buildSqlFieldList( ([columnName, column]) => column.type !== FieldType.LINK && column.type !== FieldType.FORMULA && + column.type !== FieldType.AI && !existing.find((field: string) => field === columnName) ) .map(column => `${table.name}.${column[0]}`) diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts index 060b67e8ce..6e47f30233 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.ts +++ b/packages/server/src/api/controllers/table/bulkFormula.ts @@ -5,15 +5,10 @@ import isEqual from "lodash/isEqual" import uniq from "lodash/uniq" import { updateAllFormulasInTable } from "../row/staticFormula" import { context } from "@budibase/backend-core" -import { - FormulaType, - FieldSchema, - FieldType, - FormulaFieldMetadata, - Table, -} from "@budibase/types" +import { FieldSchema, FieldType, FormulaFieldMetadata, FormulaType, Table, } from "@budibase/types" import sdk from "../../../sdk" import { isRelationshipColumn } from "../../../db/utils" +import { updateAllAIColumnsInTable } from "../row/aiColumn" function isStaticFormula( column: FieldSchema @@ -198,3 +193,21 @@ export async function runStaticFormulaChecks( await checkIfFormulaUpdated(table, { oldTable }) } } + +export async function runAIColumnChecks( + table: Table, + { oldTable }: { oldTable?: Table } +) { + // look to see if any formula values have changed + const shouldUpdate = Object.values(table.schema).find( + column => + column.type === FieldType.AI && + (!oldTable || + !oldTable.schema[column.name] || + !isEqual(oldTable.schema[column.name], column)) + ) + // if a static formula column has updated, then need to run the update + if (shouldUpdate != null) { + await updateAllAIColumnsInTable(table) + } +} \ No newline at end of file diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 269f079ae8..b916cb9c04 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -33,7 +33,7 @@ import { } from "@budibase/types" import sdk from "../../../sdk" import env from "../../../environment" -import { runStaticFormulaChecks } from "./bulkFormula" +import { runAIColumnChecks, runStaticFormulaChecks } from "./bulkFormula" export async function clearColumns(table: Table, columnNames: string[]) { const db = context.getAppDB() diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2f7f7fd44c..f0e6c69563 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -296,7 +296,7 @@ export async function squashLinks( return false } if ( - [FieldType.LINK, FieldType.FORMULA].includes(tableColumn.type) + [FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(tableColumn.type) ) { return false } diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 831528f84d..5f61791683 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -56,6 +56,7 @@ interface AuthTokenResponse { const isTypeAllowed: Record = { [FieldType.STRING]: true, [FieldType.FORMULA]: true, + [FieldType.AI]: true, [FieldType.NUMBER]: true, [FieldType.LONGFORM]: true, [FieldType.DATETIME]: true, @@ -490,7 +491,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus { } if ( !sheet.headerValues.includes(key) && - column.type !== FieldType.FORMULA + column.type !== FieldType.FORMULA && + column.type !== FieldType.AI ) { updatedHeaderValues.push(key) } diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index c0beed0db8..c372b65bd0 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -15,7 +15,7 @@ import { import { EventType, updateLinks } from "../../../../db/linkedRows" import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" -import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" +import { runAIColumnChecks, runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" import { context } from "@budibase/backend-core" import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" @@ -133,6 +133,7 @@ export async function save( } // has to run after, make sure it has _id await runStaticFormulaChecks(table, { oldTable, deletion: false }) + await runAIColumnChecks(table, { oldTable, deletion: false }) return { table } } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index bd71644537..7533e2b22a 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -19,6 +19,7 @@ const FieldTypeMap: Record = { [FieldType.BOOLEAN]: SQLiteType.NUMERIC, [FieldType.DATETIME]: SQLiteType.TEXT, [FieldType.FORMULA]: SQLiteType.TEXT, + [FieldType.AI]: SQLiteType.TEXT, [FieldType.LONGFORM]: SQLiteType.TEXT, [FieldType.NUMBER]: SQLiteType.REAL, [FieldType.STRING]: SQLiteType.TEXT, diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 269158e61e..f2f6f764d1 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -172,7 +172,7 @@ export async function enrichSchema( for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] - if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) { + if ([FieldType.LINK, FieldType.FORMULA, FieldType.AI].includes(relTableField.type)) { continue } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 2f227dd646..c4b8130966 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -189,6 +189,10 @@ export async function inputProcessing( if (field.type === FieldType.FORMULA) { delete clonedRow[key] } + // remove any AI values, they are to be generated + if (field.type === FieldType.AI) { + delete clonedRow[key] + } // otherwise coerce what is there to correct types else { clonedRow[key] = coerce(value, field.type) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 0fa6f62807..30942edc93 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -11,6 +11,7 @@ import { } from "@budibase/types" import tracer from "dd-trace" import { context } from "@budibase/backend-core" +import * as pro from "@budibase/pro" interface FormulaOpts { dynamic?: boolean @@ -91,6 +92,56 @@ export async function processFormulas( }) } +/** + * Looks through the rows provided and finds AI columns - which it then processes. + */ +export async function processAIColumns( + table: Table, + inputRows: T, + { contextRows }: FormulaOpts +): Promise { + return tracer.trace("processAIColumns", {}, async span => { + const numRows = Array.isArray(inputRows) ? inputRows.length : 1 + span?.addTags({ table_id: table._id }) + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) { + // Ensure we have snippet context + await context.ensureSnippetContext() + + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldType.AI) { + continue + } + + // const llm = pro.ai.LargeLanguageModel() + // if ( + // schema.formula == null || + // (dynamic && isStatic) || + // (!dynamic && !isStatic) + // ) { + // continue + // } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + // let context = contextRows ? contextRows[i] : row + // let formula = schema.prompt + rows[i] = { + ...row, + [column]: tracer.trace("processAIColumn", {}, span => { + span?.addTags({ table_id: table._id, column }) + // return processStringSync(formula, context) + // TODO: Add the AI stuff in to this + return "YEET AI" + }), + } + } + } + } + return Array.isArray(inputRows) ? rows : rows[0] + }) +} + /** * Processes any date columns and ensures that those without the ignoreTimezones * flag set are parsed as UTC rather than local time. diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 45e9a7c6d0..6400b5852b 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -87,6 +87,8 @@ export const getValidOperatorsForType = ( ops = numOps } else if (type === FieldType.FORMULA && formulaType === FormulaType.STATIC) { ops = stringOps.concat([Op.MoreThan, Op.LessThan]) + } else if (type === FieldType.AI) { + ops = stringOps.concat([Op.MoreThan, Op.LessThan]) } else if ( type === FieldType.BB_REFERENCE_SINGLE || schema.isDeprecatedSingleUserColumn(fieldType) diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 8a8069ce4d..4067e251d1 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -8,6 +8,7 @@ const allowDisplayColumnByType: Record = { [FieldType.NUMBER]: true, [FieldType.DATETIME]: true, [FieldType.FORMULA]: true, + [FieldType.AI]: true, [FieldType.AUTO]: true, [FieldType.INTERNAL]: true, [FieldType.BARCODEQR]: true, @@ -38,6 +39,7 @@ const allowSortColumnByType: Record = { [FieldType.JSON]: true, [FieldType.FORMULA]: false, + [FieldType.AI]: false, [FieldType.ATTACHMENTS]: false, [FieldType.ATTACHMENT_SINGLE]: false, [FieldType.SIGNATURE_SINGLE]: false, @@ -61,6 +63,7 @@ const allowDefaultColumnByType: Record = { [FieldType.BIGINT]: false, [FieldType.BOOLEAN]: false, [FieldType.FORMULA]: false, + [FieldType.AI]: false, [FieldType.ATTACHMENTS]: false, [FieldType.ATTACHMENT_SINGLE]: false, [FieldType.SIGNATURE_SINGLE]: false, diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 42440b2988..cb4b1d560d 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -76,6 +76,10 @@ export enum FieldType { * that is part of the initial formula definition, the formula will be live evaluated in the browser. */ AUTO = "auto", + /** + * A complex type, called an AI column within Budibase. This type has a... TODO: fill out + */ + AI = "ai", /** * a JSON type, called JSON within Budibase. This type allows any arbitrary JSON to be input to this column * type, which will be represented as a JSON object in the row. This type depends on a schema being diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts index fffaddc5df..6ba23eba25 100644 --- a/packages/types/src/documents/app/table/constants.ts +++ b/packages/types/src/documents/app/table/constants.ts @@ -30,6 +30,7 @@ export enum JsonFieldSubType { export enum FormulaType { STATIC = "static", DYNAMIC = "dynamic", + AI = "ai" } export enum BBReferenceFieldSubType { diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index 6078f73d1d..b0c78f9dc5 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -116,6 +116,17 @@ export interface FormulaFieldMetadata extends BaseFieldSchema { formulaType?: FormulaType } +export interface AIFieldMetadata extends BaseFieldSchema { + type: FieldType.AI + formula: string + // TODO: needs better types + operation: string + columns?: string[] + column?: string + prompt?: string + language?: string +} + export interface BBReferenceFieldMetadata extends Omit { type: FieldType.BB_REFERENCE @@ -190,6 +201,7 @@ interface OtherFieldMetadata extends BaseFieldSchema { | FieldType.LINK | FieldType.AUTO | FieldType.FORMULA + | FieldType.AI | FieldType.NUMBER | FieldType.LONGFORM | FieldType.BB_REFERENCE @@ -207,6 +219,7 @@ export type FieldSchema = | RelationshipFieldMetadata | AutoColumnFieldMetadata | FormulaFieldMetadata + | AIFieldMetadata | NumberFieldMetadata | LongFormFieldMetadata | StringFieldMetadata From 0b9b9ab440cee0cd0e5413d28ea648cc1c7540cd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 14:56:16 +0100 Subject: [PATCH 02/69] AI functionaly working E2E --- .../common/AIFieldConfiguration.svelte | 4 +- .../src/utilities/rowProcessor/utils.ts | 52 +++++++++++-------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte index 66af1266be..73e0562dbe 100644 --- a/packages/builder/src/components/common/AIFieldConfiguration.svelte +++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte @@ -47,11 +47,11 @@ prompt: OperationFieldTypes.BINDABLE_TEXT, }, CLEAN_DATA: { - columns: OperationFieldTypes.MULTI_COLUMN, + column: OperationFieldTypes.COLUMN, prompt: OperationFieldTypes.BINDABLE_TEXT, }, TRANSLATE: { - columns: OperationFieldTypes.MULTI_COLUMN, + column: OperationFieldTypes.COLUMN, language: OperationFieldTypes.BINDABLE_TEXT, prompt: OperationFieldTypes.BINDABLE_TEXT, }, diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 30942edc93..6055a2511c 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -102,8 +102,9 @@ export async function processAIColumns( ): Promise { return tracer.trace("processAIColumns", {}, async span => { const numRows = Array.isArray(inputRows) ? inputRows.length : 1 - span?.addTags({ table_id: table._id }) + span?.addTags({ table_id: table._id, numRows }) const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + const llm = await pro.ai.LargeLanguageModel.forCurrentTenant("gpt-4o-mini") if (rows) { // Ensure we have snippet context await context.ensureSnippetContext() @@ -113,29 +114,34 @@ export async function processAIColumns( continue } - // const llm = pro.ai.LargeLanguageModel() - // if ( - // schema.formula == null || - // (dynamic && isStatic) || - // (!dynamic && !isStatic) - // ) { - // continue - // } - // iterate through rows and process formula - for (let i = 0; i < rows.length; i++) { - let row = rows[i] - // let context = contextRows ? contextRows[i] : row - // let formula = schema.prompt - rows[i] = { - ...row, - [column]: tracer.trace("processAIColumn", {}, span => { - span?.addTags({ table_id: table._id, column }) - // return processStringSync(formula, context) - // TODO: Add the AI stuff in to this - return "YEET AI" - }), + const rowUpdates = rows.map((row, i) => { + const contextRow = contextRows ? contextRows[i] : row + // TODO: Map the prompts with string-templates + // grab the operation based on the schema + // then check the types in the fields, and decide whether to pass them through string templates + // TODO: cleaner way to map to the schema, move things into BB types and check against the AI schema + for (const key in schema) { + if (["prompt", "categories"].includes(key)) { + schema[key] = processStringSync(schema[key], contextRow) + } } - } + + const prompt = llm.buildPromptFromAIOperation({ schema, row }) + + return tracer.trace("processAIColumn", {}, async span => { + span?.addTags({ table_id: table._id, column }) + const llmResponse = await llm.run(prompt) + return { + ...row, + [column]: llmResponse + } + }) + }) + + const processedRows = await Promise.all(rowUpdates) + + // Promise.all is deterministic so can rely on the indexing here + processedRows.forEach((processedRow, index) => rows[index] = processedRow) } } return Array.isArray(inputRows) ? rows : rows[0] From 2aaef12a1ae174a23eff4b5e92d272089d7664b0 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 16:36:37 +0100 Subject: [PATCH 03/69] better typing - discriminated unions --- .../DataTable/modals/CreateEditColumn.svelte | 2 +- .../common/AIFieldConfiguration.svelte | 15 +-- packages/pro | 2 +- .../src/api/controllers/table/bulkFormula.ts | 4 +- packages/types/src/documents/app/ai.ts | 105 ++++++++++++++++++ packages/types/src/documents/app/index.ts | 1 + .../types/src/documents/app/table/schema.ts | 5 +- 7 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 packages/types/src/documents/app/ai.ts diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index da68b4182b..30472c4939 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -417,13 +417,13 @@ FIELDS.BOOLEAN, FIELDS.DATETIME, FIELDS.LINK, + FIELDS.AI, FIELDS.LONGFORM, FIELDS.USER, FIELDS.USERS, FIELDS.ATTACHMENT_SINGLE, FIELDS.ATTACHMENTS, FIELDS.FORMULA, - FIELDS.AI, FIELDS.JSON, FIELDS.BARCODEQR, FIELDS.SIGNATURE_SINGLE, diff --git a/packages/builder/src/components/common/AIFieldConfiguration.svelte b/packages/builder/src/components/common/AIFieldConfiguration.svelte index 73e0562dbe..84a9deca95 100644 --- a/packages/builder/src/components/common/AIFieldConfiguration.svelte +++ b/packages/builder/src/components/common/AIFieldConfiguration.svelte @@ -1,5 +1,5 @@ {/if} diff --git a/packages/pro b/packages/pro index dcc9e50b80..d194be2339 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit dcc9e50b8064a2097d408771462ad80f48de7ff6 +Subproject commit d194be2339a1ee2e5b2ed1fe59d103249f2d8d2c diff --git a/packages/server/src/api/controllers/table/bulkFormula.ts b/packages/server/src/api/controllers/table/bulkFormula.ts index 6e47f30233..b2c9f6a27b 100644 --- a/packages/server/src/api/controllers/table/bulkFormula.ts +++ b/packages/server/src/api/controllers/table/bulkFormula.ts @@ -198,7 +198,7 @@ export async function runAIColumnChecks( table: Table, { oldTable }: { oldTable?: Table } ) { - // look to see if any formula values have changed + // look to see if any AI column values have changed const shouldUpdate = Object.values(table.schema).find( column => column.type === FieldType.AI && @@ -206,7 +206,7 @@ export async function runAIColumnChecks( !oldTable.schema[column.name] || !isEqual(oldTable.schema[column.name], column)) ) - // if a static formula column has updated, then need to run the update + // if an AI column has updated, then need to run the update if (shouldUpdate != null) { await updateAllAIColumnsInTable(table) } diff --git a/packages/types/src/documents/app/ai.ts b/packages/types/src/documents/app/ai.ts new file mode 100644 index 0000000000..6d3f164fa4 --- /dev/null +++ b/packages/types/src/documents/app/ai.ts @@ -0,0 +1,105 @@ + +export enum AIOperationEnum { + SUMMARISE_TEXT = "SUMMARISE_TEXT", + CLEAN_DATA = "CLEAN_DATA", + TRANSLATE = "TRANSLATE", + CATEGORISE_TEXT = "CATEGORISE_TEXT", + SENTIMENT_ANALYSIS = "SENTIMENT_ANALYSIS", + PROMPT = "PROMPT", + SEARCH_WEB = "SEARCH_WEB", +} + + +enum OperationFieldTypeEnum { + MULTI_COLUMN = "columns", + COLUMN = "column", + BINDABLE_TEXT = "prompt", +} + + +type OperationFieldsType = { + [AIOperationEnum.SUMMARISE_TEXT]: { + columns: OperationFieldTypeEnum.MULTI_COLUMN + } + [AIOperationEnum.CLEAN_DATA]: { + column: OperationFieldTypeEnum.COLUMN + } + [AIOperationEnum.TRANSLATE]: { + column: OperationFieldTypeEnum.COLUMN + language: OperationFieldTypeEnum.BINDABLE_TEXT + } + [AIOperationEnum.CATEGORISE_TEXT]: { + columns: OperationFieldTypeEnum.MULTI_COLUMN + categories: OperationFieldTypeEnum.BINDABLE_TEXT + } + [AIOperationEnum.SENTIMENT_ANALYSIS]: { + column: OperationFieldTypeEnum.COLUMN + } + [AIOperationEnum.PROMPT]: { + prompt: OperationFieldTypeEnum.BINDABLE_TEXT + } + [AIOperationEnum.SEARCH_WEB]: { + columns: OperationFieldTypeEnum.MULTI_COLUMN + } +} + +// Define the AI operations with just their labels (no need for separate 'value' property) +const AIOperations: { [key in AIOperationEnum]: string } = { + [AIOperationEnum.SUMMARISE_TEXT]: "Summarise Text", + [AIOperationEnum.CLEAN_DATA]: "Clean Data", + [AIOperationEnum.TRANSLATE]: "Translate", + [AIOperationEnum.CATEGORISE_TEXT]: "Categorise Text", + [AIOperationEnum.SENTIMENT_ANALYSIS]: "Sentiment Analysis", + [AIOperationEnum.PROMPT]: "Prompt", + [AIOperationEnum.SEARCH_WEB]: "Search Web", +} + +type BaseSchema = { + operation: AIOperationEnum +} + +type SummariseTextSchema = BaseSchema & { + operation: AIOperationEnum.SUMMARISE_TEXT + columns: string[] +} + +type CleanDataSchema = BaseSchema & { + operation: AIOperationEnum.CLEAN_DATA + column: string +} + +type TranslateSchema = BaseSchema & { + operation: AIOperationEnum.TRANSLATE + column: string + language: string +} + +type CategoriseTextSchema = BaseSchema & { + operation: AIOperationEnum.CATEGORISE_TEXT + columns: string[] + categories: string +} + +type SentimentAnalysisSchema = BaseSchema & { + operation: AIOperationEnum.SENTIMENT_ANALYSIS + column: string +} + +type PromptSchema = BaseSchema & { + operation: AIOperationEnum.PROMPT + prompt: string +} + +type SearchWebSchema = BaseSchema & { + operation: AIOperationEnum.SEARCH_WEB + columns: string[] +} + +export type AIColumnSchema = + | SummariseTextSchema + | CleanDataSchema + | TranslateSchema + | CategoriseTextSchema + | SentimentAnalysisSchema + | PromptSchema + | SearchWebSchema \ No newline at end of file diff --git a/packages/types/src/documents/app/index.ts b/packages/types/src/documents/app/index.ts index 2b13676ba1..0290e9ed2e 100644 --- a/packages/types/src/documents/app/index.ts +++ b/packages/types/src/documents/app/index.ts @@ -1,3 +1,4 @@ +export * from "./ai" export * from "./app" export * from "./automation" export * from "./datasource" diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts index b0c78f9dc5..de1395cf31 100644 --- a/packages/types/src/documents/app/table/schema.ts +++ b/packages/types/src/documents/app/table/schema.ts @@ -9,6 +9,7 @@ import { JsonFieldSubType, RelationshipType, } from "./constants" +import { AIOperationEnum } from "../ai" export interface UIFieldMetadata { order?: number @@ -119,10 +120,10 @@ export interface FormulaFieldMetadata extends BaseFieldSchema { export interface AIFieldMetadata extends BaseFieldSchema { type: FieldType.AI formula: string - // TODO: needs better types - operation: string + operation: AIOperationEnum columns?: string[] column?: string + categories?: string prompt?: string language?: string } From b3afbd57be0310d7dcfcbdeeeba0b957bd143c25 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 1 Oct 2024 16:55:20 +0100 Subject: [PATCH 04/69] longform text --- packages/frontend-core/src/components/grid/lib/renderers.js | 3 +-- packages/pro | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js index 3d0c0c7797..8939103fe0 100644 --- a/packages/frontend-core/src/components/grid/lib/renderers.js +++ b/packages/frontend-core/src/components/grid/lib/renderers.js @@ -30,8 +30,7 @@ const TypeComponentMap = { [FieldType.ATTACHMENT_SINGLE]: AttachmentSingleCell, [FieldType.LINK]: RelationshipCell, [FieldType.FORMULA]: FormulaCell, - // TODO: fix - [FieldType.AI]: FormulaCell, + [FieldType.AI]: LongFormCell, [FieldType.JSON]: JSONCell, [FieldType.BB_REFERENCE]: BBReferenceCell, [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell, diff --git a/packages/pro b/packages/pro index d194be2339..0c8f834932 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit d194be2339a1ee2e5b2ed1fe59d103249f2d8d2c +Subproject commit 0c8f834932670ce1d3b3f639e6e8d0d0086e508d From 2c5fe77740942093bb784d6441b18b6f52d70de1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 2 Oct 2024 16:11:52 +0100 Subject: [PATCH 05/69] don't run over all historical cells --- .../src/components/grid/cells/AICell.svelte | 106 ++++++++++++++++++ .../src/components/grid/lib/renderers.js | 3 +- .../src/sdk/app/tables/internal/index.ts | 3 +- 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 packages/frontend-core/src/components/grid/cells/AICell.svelte diff --git a/packages/frontend-core/src/components/grid/cells/AICell.svelte b/packages/frontend-core/src/components/grid/cells/AICell.svelte new file mode 100644 index 0000000000..4e11fb7ebd --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/AICell.svelte @@ -0,0 +1,106 @@ + + + + +
+
+ {value || ""} +
+
+ +{#if isOpen} + +