Merge branch 'v3-ui' of github.com:Budibase/budibase into views-openapi

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.33.13",
+  "version": "2.33.14",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/sql.ts

@@ -316,11 +316,9 @@ class InternalBuilder {
         const columnSchema = schema[column]
 
         if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
-          // TODO: figure out how to express this safely without string
-          // interpolation.
-          return this.knex.raw(`??::money::numeric as "${field}"`, [
+          return this.knex.raw(`??::money::numeric as ??`, [
             this.rawQuotedIdentifier([table, column].join(".")),
-            field,
+            this.knex.raw(this.quote(field)),
           ])
         }
 
@@ -330,8 +328,9 @@ class InternalBuilder {
 
           // TODO: figure out how to express this safely without string
           // interpolation.
-          return this.knex.raw(`CONVERT(varchar, ??, 108) as "${field}"`, [
+          return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
             this.rawQuotedIdentifier(field),
+            this.knex.raw(this.quote(field)),
           ])
         }
 

packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte

@@ -9,7 +9,7 @@
   } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { flags } from "stores/builder"
-  import { licensing } from "stores/portal"
+  import { featureFlags } from "stores/portal"
   import { API } from "api"
   import MagicWand from "../../../../assets/MagicWand.svelte"
 
@@ -26,8 +26,7 @@
   let aiCronPrompt = ""
   let loadingAICronExpression = false
 
-  $: aiEnabled =
-    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
+  $: aiEnabled = $featureFlags.AI_CUSTOM_CONFIGS || $featureFlags.BUDIBASE_AI
   $: {
     if (cronExpression) {
       try {

packages/builder/src/components/backend/DataTable/buttons/grid/GridViewCalculationButton.svelte

@@ -90,8 +90,15 @@
   const getFieldOptions = (self, calculations, schema) => {
     return Object.entries(schema)
       .filter(([field, fieldSchema]) => {
-        // Only allow numeric fields that are not calculations themselves
-        if (fieldSchema.calculationType || !isNumeric(fieldSchema.type)) {
+        // Don't allow other calculation columns
+        if (fieldSchema.calculationType) {
+          return false
+        }
+        // Only allow numeric columns for most calculation types
+        if (
+          self.type !== CalculationType.COUNT &&
+          !isNumeric(fieldSchema.type)
+        ) {
           return false
         }
         // Don't allow duplicates
@@ -234,7 +241,7 @@
   <InfoDisplay
     icon="Help"
     quiet
-    body="Calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
+    body="Most calculations only work with numeric columns and a maximum of 5 calculations can be added at once."
   />
 
   <div>
@@ -257,6 +264,4 @@
   .group-by {
     grid-column: 2 / 5;
   }
-  span {
-  }
 </style>

packages/builder/src/components/backend/DataTable/formula.js

@@ -8,6 +8,7 @@ const MAX_DEPTH = 1
 
 const TYPES_TO_SKIP = [
   FieldType.FORMULA,
+  FieldType.AI,
   FieldType.LONGFORM,
   FieldType.SIGNATURE_SINGLE,
   FieldType.ATTACHMENTS,

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -26,7 +26,7 @@
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
-  import { licensing } from "stores/portal"
+  import { featureFlags } from "stores/portal"
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
@@ -101,8 +101,7 @@
   let optionsValid = true
 
   $: rowGoldenSample = RowUtils.generateGoldenSample($rows)
-  $: aiEnabled =
-    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
+  $: aiEnabled = $featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS
   $: if (primaryDisplay) {
     editableColumn.constraints.presence = { allowEmpty: false }
   }

packages/builder/src/components/design/settings/controls/Explanation/lines/Column.svelte

@@ -80,6 +80,9 @@
     if (columnType === FieldType.FORMULA) {
       return "https://docs.budibase.com/docs/formula"
     }
+    if (columnType === FieldType.AI) {
+      return "https://docs.budibase.com/docs/ai"
+    }
     if (columnType === FieldType.OPTIONS) {
       return "https://docs.budibase.com/docs/options"
     }

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2, rowActions } from "stores/builder"
-  import { admin, themeStore, licensing } from "stores/portal"
+  import { admin, themeStore, featureFlags } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import { notifications } from "@budibase/bbui"
@@ -53,7 +53,7 @@
   {buttons}
   allowAddRows
   allowDeleteRows
-  aiEnabled={$licensing.budibaseAIEnabled || $licensing.customAIConfigsEnabled}
+  aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
   showAvatars={false}
   on:updatedatasource={handleGridViewUpdate}
   isCloud={$admin.cloud}

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -8,7 +8,7 @@
     rowActions,
     roles,
   } from "stores/builder"
-  import { themeStore, admin, licensing } from "stores/portal"
+  import { themeStore, admin, featureFlags } from "stores/portal"
   import { TableNames } from "constants"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
@@ -130,8 +130,7 @@
     schemaOverrides={isUsersTable ? userSchemaOverrides : null}
     showAvatars={false}
     isCloud={$admin.cloud}
-    aiEnabled={$licensing.budibaseAIEnabled ||
-      $licensing.customAIConfigsEnabled}
+    aiEnabled={$featureFlags.BUDIBASE_AI || $featureFlags.AI_CUSTOM_CONFIGS}
     {buttons}
     buttonsCollapsed
     on:updatedatasource={handleGridTableUpdate}

packages/builder/src/stores/portal/featureFlags.js

@@ -4,6 +4,8 @@ import { auth } from "stores/portal"
 export const INITIAL_FEATUREFLAG_STATE = {
   SQS: false,
   DEFAULT_VALUES: false,
+  BUDIBASE_AI: false,
+  AI_CUSTOM_CONFIGS: false,
 }
 
 export const featureFlags = derived([auth], ([$auth]) => {

packages/client/src/components/app/GridBlock.svelte

@@ -60,8 +60,11 @@
 
   // Provide additional data context for live binding eval
   export const getAdditionalDataContext = () => {
-    const rows = get(grid?.getContext()?.rows)
-    const goldenRow = generateGoldenSample(rows)
+    const gridContext = grid?.getContext()
+    const rows = get(gridContext?.rows) || []
+    const clean = gridContext?.rows.actions.cleanRow || (x => x)
+    const cleaned = rows.map(clean)
+    const goldenRow = generateGoldenSample(cleaned)
     const id = get(component).id
     return {
       // Not sure what this one is for...

packages/frontend-core/src/components/grid/lib/constants.js

@@ -10,6 +10,8 @@ export const DefaultColumnWidth = 200
 export const MinColumnWidth = 80
 export const NewRowID = "new"
 export const BlankRowID = "blank"
+export const GeneratedIDPrefix = "‽‽"
+export const CellIDSeparator = "‽‽"
 export const RowPageSize = 100
 export const FocusedCellMinOffset = ScrollBarSize * 3
 export const ControlsHeight = 50

packages/frontend-core/src/components/grid/lib/utils.js

@@ -1,18 +1,17 @@
-// We can't use "-" as a separator as this can be present in the ID
-// or column name, so we use something very unusual to avoid this problem
-const JOINING_CHARACTER = "‽‽"
+import { GeneratedIDPrefix, CellIDSeparator } from "./constants"
+import { Helpers } from "@budibase/bbui"
 
 export const parseCellID = cellId => {
   if (!cellId) {
     return { rowId: undefined, field: undefined }
   }
-  const parts = cellId.split(JOINING_CHARACTER)
+  const parts = cellId.split(CellIDSeparator)
   const field = parts.pop()
-  return { rowId: parts.join(JOINING_CHARACTER), field }
+  return { rowId: parts.join(CellIDSeparator), field }
 }
 
 export const getCellID = (rowId, fieldName) => {
-  return `${rowId}${JOINING_CHARACTER}${fieldName}`
+  return `${rowId}${CellIDSeparator}${fieldName}`
 }
 
 export const parseEventLocation = e => {
@@ -21,3 +20,11 @@ export const parseEventLocation = e => {
     y: e.clientY ?? e.touches?.[0]?.clientY,
   }
 }
+
+export const generateRowID = () => {
+  return `${GeneratedIDPrefix}${Helpers.uuid()}`
+}
+
+export const isGeneratedRowID = id => {
+  return id?.startsWith(GeneratedIDPrefix)
+}

packages/frontend-core/src/components/grid/stores/rows.js

@@ -1,7 +1,12 @@
 import { writable, derived, get } from "svelte/store"
 import { fetchData } from "../../../fetch"
 import { NewRowID, RowPageSize } from "../lib/constants"
-import { getCellID, parseCellID } from "../lib/utils"
+import {
+  generateRowID,
+  getCellID,
+  isGeneratedRowID,
+  parseCellID,
+} from "../lib/utils"
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
@@ -634,10 +639,10 @@ export const createActions = context => {
       newRow = newRows[i]
 
       // Ensure we have a unique _id.
-      // This means generating one for non DS+, overwriting any that may already
-      // exist as we cannot allow duplicates.
-      if (!$hasBudibaseIdentifiers) {
-        newRow._id = Helpers.uuid()
+      // We generate one for non DS+ where required, but trust that any existing
+      // _id values are unique (e.g. Mongo)
+      if (!$hasBudibaseIdentifiers && !newRow._id?.length) {
+        newRow._id = generateRowID()
       }
 
       if (!rowCacheMap[newRow._id]) {
@@ -674,7 +679,7 @@ export const createActions = context => {
     let clone = { ...row }
     delete clone.__idx
     delete clone.__metadata
-    if (!get(hasBudibaseIdentifiers)) {
+    if (!get(hasBudibaseIdentifiers) && isGeneratedRowID(clone._id)) {
       delete clone._id
     }
     return clone

packages/server/src/api/controllers/table/tests/utils.spec.ts

@@ -1,4 +1,4 @@
-import { AutoFieldSubType, FieldType } from "@budibase/types"
+import { AIOperationEnum, AutoFieldSubType, FieldType } from "@budibase/types"
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
 import { importToRows } from "../utils"
 
@@ -92,5 +92,65 @@ describe("utils", () => {
         expect(result).toHaveLength(3)
       })
     })
+
+    it("Imports write as expected with AI columns", async () => {
+      await config.doInContext(config.appId, async () => {
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            autoId: {
+              name: "autoId",
+              type: FieldType.NUMBER,
+              subtype: AutoFieldSubType.AUTO_ID,
+              autocolumn: true,
+              constraints: {
+                type: FieldType.NUMBER,
+                presence: true,
+              },
+            },
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+              constraints: {
+                type: FieldType.STRING,
+                presence: true,
+              },
+            },
+            aicol: {
+              name: "aicol",
+              type: FieldType.AI,
+              operation: AIOperationEnum.PROMPT,
+              prompt: "Test prompt",
+            },
+          },
+        })
+
+        const data = [
+          { name: "Alice", aicol: "test" },
+          { name: "Bob", aicol: "test" },
+          { name: "Claire", aicol: "test" },
+        ]
+
+        const result = await importToRows(data, table, config.user?._id)
+        expect(result).toEqual([
+          expect.objectContaining({
+            autoId: 1,
+            name: "Alice",
+            aicol: "test",
+          }),
+          expect.objectContaining({
+            autoId: 2,
+            name: "Bob",
+            aicol: "test",
+          }),
+          expect.objectContaining({
+            autoId: 3,
+            name: "Claire",
+            aicol: "test",
+          }),
+        ])
+      })
+    })
   })
 })

packages/server/src/api/controllers/table/utils.ts

@@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { context, events, features } from "@budibase/backend-core"
+import { context, events, features, HTTPError } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   Database,
@@ -145,14 +145,21 @@ export async function importToRows(
     // the real schema of the table passed in, not the clone used for
     // incrementing auto IDs
     for (const [fieldName, schema] of Object.entries(originalTable.schema)) {
-      const rowVal = Array.isArray(row[fieldName])
-        ? row[fieldName]
-        : [row[fieldName]]
+      if (schema.type === FieldType.LINK) {
+        throw new HTTPError(
+          `Can't bulk import relationship fields for internal databases, found value in field "${fieldName}"`,
+          400
+        )
+      }
+
       if (
         (schema.type === FieldType.OPTIONS ||
           schema.type === FieldType.ARRAY) &&
         row[fieldName]
       ) {
+        const rowVal = Array.isArray(row[fieldName])
+          ? row[fieldName]
+          : [row[fieldName]]
         let merged = [...schema.constraints!.inclusion!, ...rowVal]
         let superSet = new Set(merged)
         schema.constraints!.inclusion = Array.from(superSet)

packages/server/src/api/routes/tests/row.spec.ts

@@ -1823,6 +1823,39 @@ describe.each([
         expect(row.autoId).toEqual(3)
       })
 
+    isInternal &&
+      it("should reject bulkImporting relationship fields", async () => {
+        const table1 = await config.api.table.save(saveTableRequest())
+        const table2 = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              relationship: {
+                name: "relationship",
+                type: FieldType.LINK,
+                tableId: table1._id!,
+                relationshipType: RelationshipType.ONE_TO_MANY,
+                fieldName: "relationship",
+              },
+            },
+          })
+        )
+
+        const table1Row1 = await config.api.row.save(table1._id!, {})
+        await config.api.row.bulkImport(
+          table2._id!,
+          {
+            rows: [{ relationship: [table1Row1._id!] }],
+          },
+          {
+            status: 400,
+            body: {
+              message:
+                'Can\'t bulk import relationship fields for internal databases, found value in field "relationship"',
+            },
+          }
+        )
+      })
+
     it("should be able to bulkImport rows", async () => {
       const table = await config.api.table.save(
         saveTableRequest({
@@ -2462,7 +2495,7 @@ describe.each([
           [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
           [FieldType.FORMULA]: undefined, // generated field
           [FieldType.AUTO]: undefined, // generated field
-          [FieldType.AI]: undefined, // generated field
+          [FieldType.AI]: "LLM Output",
           [FieldType.JSON]: { name: generator.guid() },
           [FieldType.INTERNAL]: generator.guid(),
           [FieldType.BARCODEQR]: generator.guid(),
@@ -2494,6 +2527,7 @@ describe.each([
           }),
           [FieldType.FORMULA]: fullSchema[FieldType.FORMULA].formula,
           [FieldType.AUTO]: expect.any(Number),
+          [FieldType.AI]: expect.any(String),
           [FieldType.JSON]: rowValues[FieldType.JSON],
           [FieldType.INTERNAL]: rowValues[FieldType.INTERNAL],
           [FieldType.BARCODEQR]: rowValues[FieldType.BARCODEQR],
@@ -2566,7 +2600,7 @@ describe.each([
               expectedRowData["bb_reference_single"].sample,
               false
             ),
-            ai: null,
+            ai: "LLM Output",
           },
         ])
       })

packages/server/src/integration-test/postgres.spec.ts

@@ -1,5 +1,5 @@
 import * as setup from "../api/routes/tests/utilities"
-import { Datasource, FieldType } from "@budibase/types"
+import { Datasource, FieldType, Table } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import {
@@ -229,4 +229,54 @@ describe("postgres integrations", () => {
       ).toBeUndefined()
     })
   })
+
+  describe("money field 💰", () => {
+    const tableName = "moneytable"
+    let table: Table
+
+    beforeAll(async () => {
+      await client.raw(`
+        CREATE TABLE ${tableName} (
+          id serial PRIMARY KEY,
+          price money
+        )   
+      `)
+      const response = await config.api.datasource.fetchSchema({
+        datasourceId: datasource._id!,
+      })
+      table = response.datasource.entities![tableName]
+    })
+
+    it("should be able to import a money field", async () => {
+      expect(table).toBeDefined()
+      expect(table?.schema.price.type).toBe(FieldType.NUMBER)
+    })
+
+    it("should be able to search a money field", async () => {
+      await config.api.row.bulkImport(table._id!, {
+        rows: [{ price: 200 }, { price: 300 }],
+      })
+
+      const { rows } = await config.api.row.search(table._id!, {
+        query: {
+          equal: {
+            price: 200,
+          },
+        },
+      })
+      expect(rows).toHaveLength(1)
+      expect(rows[0].price).toBe("200.00")
+    })
+
+    it("should be able to update a money field", async () => {
+      let row = await config.api.row.save(table._id!, { price: 200 })
+      expect(row.price).toBe("200.00")
+
+      row = await config.api.row.save(table._id!, { ...row, price: 300 })
+      expect(row.price).toBe("300.00")
+
+      row = await config.api.row.save(table._id!, { ...row, price: "400.00" })
+      expect(row.price).toBe("400.00")
+    })
+  })
 })

packages/server/src/utilities/rowProcessor/index.ts

@@ -216,10 +216,6 @@ export async function inputProcessing(
     if (field.type === FieldType.FORMULA) {
       delete clonedRow[key]
     }
-    // remove any AI values, they are to be generated
-    if (field.type === FieldType.AI) {
-      delete clonedRow[key]
-    }
     // otherwise coerce what is there to correct types
     else {
       clonedRow[key] = coerce(value, field.type)