Merge branch 'master' into ts/notification-store

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.37",
+  "version": "3.2.38",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/builder/src/stores/portal/menu.ts

@@ -1,9 +1,7 @@
 import { derived, Readable } from "svelte/store"
 import { admin } from "./admin"
 import { auth } from "./auth"
-import { isEnabled } from "@/helpers/featureFlags"
 import { sdk } from "@budibase/shared-core"
-import { FeatureFlag } from "@budibase/types"
 
 interface MenuItem {
   title: string
@@ -73,13 +71,11 @@ export const menu: Readable<MenuItem[]> = derived(
           title: "Environment",
           href: "/builder/portal/settings/environment",
         },
-      ]
+        {
-      if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) {
-        settingsSubPages.push({
           title: "AI",
           href: "/builder/portal/settings/ai",
-        })
+        },
-      }
+      ]
 
       if (!cloud) {
         settingsSubPages.push({

packages/client/src/utils/schema.js

@@ -31,7 +31,7 @@ const getDatasourceFetchInstance = datasource => {
   if (!handler) {
     return null
   }
-  return new handler({ API })
+  return new handler({ API, datasource })
 }
 
 /**
@@ -52,7 +52,7 @@ export const fetchDatasourceSchema = async (
   // Get the normal schema as long as we aren't wanting a form schema
   let schema
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(datasource, definition)
+    schema = instance.getSchema(definition)
   } else if (definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {

packages/frontend-core/src/components/grid/lib/renderers.ts

@@ -1,4 +1,4 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, UIColumn } from "@budibase/types"
 
 import OptionsCell from "../cells/OptionsCell.svelte"
 import DateCell from "../cells/DateCell.svelte"
@@ -40,13 +40,23 @@ const TypeComponentMap = {
   // Custom types for UI only
   role: RoleCell,
 }
-export const getCellRenderer = column => {
+
+function getCellRendererByType(type: FieldType | "role" | undefined) {
+  if (!type) {
+    return
+  }
+
+  return TypeComponentMap[type as keyof typeof TypeComponentMap]
+}
+
+export const getCellRenderer = (column: UIColumn) => {
   if (column.calculationType) {
     return NumberCell
   }
+
   return (
-    TypeComponentMap[column?.schema?.cellRenderType] ||
+    getCellRendererByType(column.schema?.cellRenderType) ||
-    TypeComponentMap[column?.schema?.type] ||
+    getCellRendererByType(column.schema?.type) ||
     TextCell
   )
 }

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,32 +0,0 @@
-// TODO: remove when all stores are typed
-
-import { GeneratedIDPrefix, CellIDSeparator } from "./constants"
-import { Helpers } from "@budibase/bbui"
-
-export const parseCellID = cellId => {
-  if (!cellId) {
-    return { rowId: undefined, field: undefined }
-  }
-  const parts = cellId.split(CellIDSeparator)
-  const field = parts.pop()
-  return { rowId: parts.join(CellIDSeparator), field }
-}
-
-export const getCellID = (rowId, fieldName) => {
-  return `${rowId}${CellIDSeparator}${fieldName}`
-}
-
-export const parseEventLocation = e => {
-  return {
-    x: e.clientX ?? e.touches?.[0]?.clientX,
-    y: e.clientY ?? e.touches?.[0]?.clientY,
-  }
-}
-
-export const generateRowID = () => {
-  return `${GeneratedIDPrefix}${Helpers.uuid()}`
-}
-
-export const isGeneratedRowID = id => {
-  return id?.startsWith(GeneratedIDPrefix)
-}

packages/frontend-core/src/components/grid/lib/websocket.ts

@@ -1,12 +1,14 @@
 import { get } from "svelte/store"
 import { createWebsocket } from "../../../utils"
 import { SocketEvent, GridSocketEvent } from "@budibase/shared-core"
+import { Store } from "../stores"
+import { UIDatasource, UIUser } from "@budibase/types"
 
-export const createGridWebsocket = context => {
+export const createGridWebsocket = (context: Store) => {
   const { rows, datasource, users, focusedCellId, definition, API } = context
   const socket = createWebsocket("/socket/grid")
 
-  const connectToDatasource = datasource => {
+  const connectToDatasource = (datasource: UIDatasource) => {
     if (!socket.connected) {
       return
     }
@@ -18,7 +20,7 @@ export const createGridWebsocket = context => {
         datasource,
         appId,
       },
-      ({ users: gridUsers }) => {
+      ({ users: gridUsers }: { users: UIUser[] }) => {
         users.set(gridUsers)
       }
     )
@@ -65,7 +67,7 @@ export const createGridWebsocket = context => {
     GridSocketEvent.DatasourceChange,
     ({ datasource: newDatasource }) => {
       // Listen builder renames, as these aren't handled otherwise
-      if (newDatasource?.name !== get(definition).name) {
+      if (newDatasource?.name !== get(definition)?.name) {
         definition.set(newDatasource)
       }
     }

packages/frontend-core/src/fetch/DataFetch.ts

@@ -179,9 +179,6 @@ export default abstract class DataFetch<
       this.store.update($store => ({ ...$store, loaded: true }))
       return
     }
-
-    // Initially fetch data but don't bother waiting for the result
-    this.getInitialData()
   }
 
   /**

packages/frontend-core/src/fetch/index.ts

@@ -33,7 +33,12 @@ const DataFetchMap = {
 export const fetchData = ({ API, datasource, options }: any) => {
   const Fetch =
     DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch
-  return new Fetch({ API, datasource, ...options })
+  const fetch = new Fetch({ API, datasource, ...options })
+
+  // Initially fetch data but don't bother waiting for the result
+  fetch.getInitialData()
+
+  return fetch
 }
 
 // Creates an empty fetch instance with no datasource configured, so no data

packages/frontend-core/src/utils/utils.js

@@ -209,6 +209,9 @@ export const buildFormBlockButtonConfig = props => {
     {
       "##eventHandlerType": "Close Side Panel",
     },
+    {
+      "##eventHandlerType": "Close Modal",
+    },
 
     ...(actionUrl
       ? [

packages/pro

@@ -1 +1 @@
-Subproject commit 32d84f109d4edc526145472a7446327312151442
+Subproject commit 193476cdfade6d3c613e6972f16ee0c527e01ff6

packages/server/src/api/controllers/row/staticFormula.ts

@@ -4,15 +4,8 @@ import {
   processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context, features } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
-import {
+import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
-  Table,
-  Row,
-  FeatureFlag,
-  FormulaType,
-  FieldType,
-  ViewV2,
-} from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep, merge } from "lodash/fp"
@@ -162,11 +155,10 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+
   const aiEnabled =
-    ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
+    (await pro.features.isBudibaseAIEnabled()) ||
-      (await pro.features.isBudibaseAIEnabled())) ||
+    (await pro.features.isAICustomConfigsEnabled())
-    ((await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
-      (await pro.features.isAICustomConfigsEnabled()))
   if (aiEnabled) {
     row = await processAIColumns(table, row, {
       contextRows: [enrichedRow],
@@ -184,11 +176,6 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
-  if (aiEnabled) {
-    enrichedRow = await processAIColumns(table, enrichedRow, {
-      contextRows: [enrichedRow],
-    })
-  }
 
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {

packages/server/src/api/routes/tests/row.spec.ts

@@ -8,7 +8,13 @@ import {
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
 import { outputProcessing } from "../../../utilities/rowProcessor"
-import { context, InternalTable, tenancy, utils } from "@budibase/backend-core"
+import {
+  context,
+  setEnv,
+  InternalTable,
+  tenancy,
+  utils,
+} from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   AIOperationEnum,
@@ -42,19 +48,8 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 import { isDate } from "../../../utilities"
-
+import nock from "nock"
-jest.mock("@budibase/pro", () => ({
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
-  ...jest.requireActual("@budibase/pro"),
-  ai: {
-    LargeLanguageModel: {
-      forCurrentTenant: async () => ({
-        llm: {},
-        run: jest.fn(() => `Mock LLM Response`),
-        buildPromptFromAIOperation: jest.fn(),
-      }),
-    },
-  },
-}))
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -99,6 +94,8 @@ if (descriptions.length) {
         const ds = await dsProvider()
         datasource = ds.datasource
         client = ds.client
+
+        mocks.licenses.useCloudFree()
       })
 
       afterAll(async () => {
@@ -172,10 +169,6 @@ if (descriptions.length) {
         )
       }
 
-      beforeEach(async () => {
-        mocks.licenses.useCloudFree()
-      })
-
       const getRowUsage = async () => {
         const { total } = await config.doInContext(undefined, () =>
           quotas.getCurrentUsageValues(
@@ -3224,10 +3217,17 @@ if (descriptions.length) {
       isInternal &&
         describe("AI fields", () => {
           let table: Table
+          let envCleanup: () => void
 
           beforeAll(async () => {
             mocks.licenses.useBudibaseAI()
             mocks.licenses.useAICustomConfigs()
+            envCleanup = setEnv({
+              OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+            })
+
+            mockChatGPTResponse("Mock LLM Response")
+
             table = await config.api.table.save(
               saveTableRequest({
                 schema: {
@@ -3251,7 +3251,9 @@ if (descriptions.length) {
           })
 
           afterAll(() => {
-            jest.unmock("@budibase/pro")
+            nock.cleanAll()
+            envCleanup()
+            mocks.licenses.useCloudFree()
           })
 
           it("should be able to save a row with an AI column", async () => {

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -1,4 +1,5 @@
 import {
+  AIOperationEnum,
   ArrayOperator,
   BasicOperator,
   BBReferenceFieldSubType,
@@ -42,7 +43,9 @@ import {
 } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { context, db, events, roles } from "@budibase/backend-core"
+import { context, db, events, roles, setEnv } from "@budibase/backend-core"
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
+import nock from "nock"
 
 const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
 
@@ -100,6 +103,7 @@ if (descriptions.length) {
 
       beforeAll(async () => {
         await config.init()
+        mocks.licenses.useCloudFree()
 
         const ds = await dsProvider()
         rawDatasource = ds.rawDatasource
@@ -109,7 +113,6 @@ if (descriptions.length) {
 
       beforeEach(() => {
         jest.clearAllMocks()
-        mocks.licenses.useCloudFree()
       })
 
       describe("view crud", () => {
@@ -507,7 +510,6 @@ if (descriptions.length) {
             })
 
             it("readonly fields can be used on free license", async () => {
-              mocks.licenses.useCloudFree()
               const table = await config.api.table.save(
                 saveTableRequest({
                   schema: {
@@ -933,6 +935,95 @@ if (descriptions.length) {
                 }
               )
             })
+
+          isInternal &&
+            describe("AI fields", () => {
+              let envCleanup: () => void
+              beforeAll(() => {
+                mocks.licenses.useBudibaseAI()
+                mocks.licenses.useAICustomConfigs()
+                envCleanup = setEnv({
+                  OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+                })
+
+                mockChatGPTResponse(prompt => {
+                  if (prompt.includes("elephant")) {
+                    return "big"
+                  }
+                  if (prompt.includes("mouse")) {
+                    return "small"
+                  }
+                  if (prompt.includes("whale")) {
+                    return "big"
+                  }
+                  return "unknown"
+                })
+              })
+
+              afterAll(() => {
+                nock.cleanAll()
+                envCleanup()
+                mocks.licenses.useCloudFree()
+              })
+
+              it("can use AI fields in view calculations", async () => {
+                const table = await config.api.table.save(
+                  saveTableRequest({
+                    schema: {
+                      animal: {
+                        name: "animal",
+                        type: FieldType.STRING,
+                      },
+                      bigOrSmall: {
+                        name: "bigOrSmall",
+                        type: FieldType.AI,
+                        operation: AIOperationEnum.CATEGORISE_TEXT,
+                        categories: "big,small",
+                        columns: ["animal"],
+                      },
+                    },
+                  })
+                )
+
+                const view = await config.api.viewV2.create({
+                  tableId: table._id!,
+                  name: generator.guid(),
+                  type: ViewV2Type.CALCULATION,
+                  schema: {
+                    bigOrSmall: {
+                      visible: true,
+                    },
+                    count: {
+                      visible: true,
+                      calculationType: CalculationType.COUNT,
+                      field: "animal",
+                    },
+                  },
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "elephant",
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "mouse",
+                })
+
+                await config.api.row.save(table._id!, {
+                  animal: "whale",
+                })
+
+                const { rows } = await config.api.row.search(view.id, {
+                  sort: "bigOrSmall",
+                  sortOrder: SortOrder.ASCENDING,
+                })
+                expect(rows).toHaveLength(2)
+                expect(rows[0].bigOrSmall).toEqual("big")
+                expect(rows[1].bigOrSmall).toEqual("small")
+                expect(rows[0].count).toEqual(2)
+                expect(rows[1].count).toEqual(1)
+              })
+            })
         })
 
         describe("update", () => {
@@ -1836,7 +1927,6 @@ if (descriptions.length) {
               },
             })
 
-            mocks.licenses.useCloudFree()
             const view = await getDelegate(res)
             expect(view.schema?.one).toEqual(
               expect.objectContaining({ visible: true, readonly: true })

packages/server/src/automations/actions.ts

@@ -27,11 +27,9 @@ import {
   Hosting,
   ActionImplementation,
   AutomationStepDefinition,
-  FeatureFlag,
 } from "@budibase/types"
 import sdk from "../sdk"
 import { getAutomationPlugin } from "../utilities/fileSystem"
-import { features } from "@budibase/backend-core"
 
 type ActionImplType = ActionImplementations<
   typeof env.SELF_HOSTED extends "true" ? Hosting.SELF : Hosting.CLOUD
@@ -78,6 +76,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   LOOP: loop.definition,
   COLLECT: collect.definition,
   TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
+  BRANCH: branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,
@@ -105,14 +104,7 @@ if (env.SELF_HOSTED) {
 export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
-  if (await features.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
+  if (env.SELF_HOSTED) {
-    BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
-  }
-  if (
-    env.SELF_HOSTED ||
-    (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
-    (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
-  ) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
   }
 

packages/server/src/automations/steps/openai.ts

@@ -7,9 +7,8 @@ import {
   AutomationIOType,
   OpenAIStepInputs,
   OpenAIStepOutputs,
-  FeatureFlag,
 } from "@budibase/types"
-import { env, features } from "@budibase/backend-core"
+import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
@@ -99,12 +98,8 @@ export async function run({
 
   try {
     let response
-    const customConfigsEnabled =
+    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
-      (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
+    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
-      (await pro.features.isAICustomConfigsEnabled())
-    const budibaseAIEnabled =
-      (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
-      (await pro.features.isBudibaseAIEnabled())
 
     let llmWrapper
     if (budibaseAIEnabled || customConfigsEnabled) {

packages/server/src/sdk/app/views/index.ts

@@ -432,6 +432,21 @@ export async function enrichSchema(
       ...tableSchema[key],
       ...ui,
       order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
+      // When this was written, the only column types in FieldSchema to have columns
+      // field were the relationship columns. We blank this out here to make sure it's
+      // not set on non-relationship columns, then below we populate it by calling
+      // populateRelSchema.
+      //
+      // For Budibase 3.0 we introduced the FieldType.AI fields. Some of these fields
+      // have `columns: string[]` and it flew under the radar here because the
+      // AIFieldMetadata type isn't a union on its subtypes, it has a collection of
+      // optional fields. So columns is `columns?: string[]` which allows undefined,
+      // and doesn't fail this type check.
+      //
+      // What this means in practice is when FieldType.AI fields get enriched, we
+      // delete their `columns`. At the time of writing, I don't believe anything in
+      // the frontend depends on this, but it is odd and will probably bite us at
+      // some point.
       columns: undefined,
     }
 

packages/server/src/tests/utilities/mocks/openai.ts

@@ -0,0 +1,46 @@
+import nock from "nock"
+
+let chatID = 1
+
+export function mockChatGPTResponse(
+  response: string | ((prompt: string) => string)
+) {
+  return nock("https://api.openai.com")
+    .post("/v1/chat/completions")
+    .reply(200, (uri, requestBody) => {
+      let content = response
+      if (typeof response === "function") {
+        const messages = (requestBody as any).messages
+        content = response(messages[0].content)
+      }
+
+      chatID++
+
+      return {
+        id: `chatcmpl-${chatID}`,
+        object: "chat.completion",
+        created: Math.floor(Date.now() / 1000),
+        model: "gpt-4o-mini",
+        system_fingerprint: `fp_${chatID}`,
+        choices: [
+          {
+            index: 0,
+            message: { role: "assistant", content },
+            logprobs: null,
+            finish_reason: "stop",
+          },
+        ],
+        usage: {
+          prompt_tokens: 0,
+          completion_tokens: 0,
+          total_tokens: 0,
+          completion_tokens_details: {
+            reasoning_tokens: 0,
+            accepted_prediction_tokens: 0,
+            rejected_prediction_tokens: 0,
+          },
+        },
+      }
+    })
+    .persist()
+}

packages/server/src/utilities/rowProcessor/utils.ts

@@ -160,7 +160,7 @@ export async function processAIColumns<T extends Row | Row[]>(
 
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })
-            const llmResponse = await llmWrapper.run(prompt!)
+            const llmResponse = await llmWrapper.run(prompt)
             return {
               ...row,
               [column]: llmResponse,

packages/types/src/documents/app/row.ts

@@ -154,6 +154,7 @@ export const GroupByTypes = [
   FieldType.BOOLEAN,
   FieldType.DATETIME,
   FieldType.BIGINT,
+  FieldType.AI,
 ]
 
 export function canGroupBy(type: FieldType) {

packages/types/src/documents/app/table/schema.ts

@@ -123,7 +123,7 @@ export interface AIFieldMetadata extends BaseFieldSchema {
   operation: AIOperationEnum
   columns?: string[]
   column?: string
-  categories?: string[]
+  categories?: string
   prompt?: string
   language?: string
 }

packages/types/src/sdk/featureFlag.ts

@@ -1,16 +1,8 @@
 export enum FeatureFlag {
-  AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
-  AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
-  DEFAULT_VALUES = "DEFAULT_VALUES",
-  BUDIBASE_AI = "BUDIBASE_AI",
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
 }
 
 export const FeatureFlagDefaults = {
-  [FeatureFlag.DEFAULT_VALUES]: true,
-  [FeatureFlag.AUTOMATION_BRANCHING]: true,
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: true,
-  [FeatureFlag.BUDIBASE_AI]: true,
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
 }
 

packages/types/src/ui/stores/grid/columns.ts

@@ -14,6 +14,7 @@ export type UIColumn = FieldSchema & {
     type: FieldType
     readonly: boolean
     autocolumn: boolean
+    cellRenderType?: FieldType | "role"
   }
   calculationType: CalculationType
   __idx: number