Merge remote-tracking branch 'origin/v3-ui' into feature/automation-branching-ux

.github/workflows/deploy-featurebranch.yml

@@ -23,6 +23,7 @@ jobs:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
           PAYLOAD_LICENSE_TYPE: "free"
+          PAYLOAD_DEPLOY: "true"
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.10",
+  "version": "2.32.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/index.ts

@@ -1,5 +1,5 @@
 export * as utils from "./utils"
 
-export { default as Sql } from "./sql"
+export { default as Sql, COUNT_FIELD_NAME } from "./sql"
 export { default as SqlTable } from "./sqlTable"
 export * as designDoc from "./designDoc"

packages/backend-core/src/sql/sql.ts

@@ -43,6 +43,8 @@ import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
+export const COUNT_FIELD_NAME = "__bb_total"
+
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
     ? parseInt(environment.SQL_MAX_ROWS)
@@ -846,7 +848,7 @@ class InternalBuilder {
       throw new Error("SQL counting requires primary key to be supplied")
     }
     return query.countDistinct(
-      `${this.getTableName()}.${this.table.primary[0]} as __bb_total`
+      `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}`
     )
   }
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -4,6 +4,7 @@
     Button,
     Label,
     Select,
+    Multiselect,
     Toggle,
     Icon,
     DatePicker,
@@ -137,9 +138,10 @@
   }
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
-  $: required = hasDefault
+  $: required =
-    ? false
+    primaryDisplay ||
-    : !!editableColumn?.constraints?.presence || primaryDisplay
+    editableColumn?.constraints?.presence === true ||
+    editableColumn?.constraints?.presence?.allowEmpty === false
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@@ -168,8 +170,8 @@
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
-  $: canHaveDefault =
+  $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
-    isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
+  $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
@@ -188,7 +190,6 @@
     (originalName &&
       SWITCHABLE_TYPES[field.type] &&
       !editableColumn?.autocolumn)
-
   $: allowedTypes = getAllowedTypes(datasource).map(t => ({
     fieldId: makeFieldId(t.type, t.subtype),
     ...t,
@@ -206,6 +207,11 @@
     },
     ...getUserBindings(),
   ]
+  $: sanitiseDefaultValue(
+    editableColumn.type,
+    editableColumn.constraints?.inclusion || [],
+    editableColumn.default
+  )
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -295,6 +301,22 @@
       delete saveColumn.fieldName
     }
 
+    // Ensure we don't have a default value if we can't have one
+    if (!canHaveDefault || !defaultValuesEnabled) {
+      delete saveColumn.default
+    }
+
+    // Ensure primary display columns are always required and don't have default values
+    if (primaryDisplay) {
+      saveColumn.constraints.presence = { allowEmpty: false }
+      delete saveColumn.default
+    }
+
+    // Ensure the field is not required if we have a default value
+    if (saveColumn.default) {
+      saveColumn.constraints.presence = false
+    }
+
     try {
       await tables.saveField({
         originalName,
@@ -541,6 +563,20 @@
     return newError
   }
 
+  const sanitiseDefaultValue = (type, options, defaultValue) => {
+    if (!defaultValue?.length) {
+      return
+    }
+    // Delete default value for options fields if the option is no longer available
+    if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
+      delete editableColumn.default
+    }
+    // Filter array default values to only valid options
+    if (type === FieldType.ARRAY) {
+      editableColumn.default = defaultValue.filter(x => options.includes(x))
+    }
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -748,9 +784,9 @@
       </div>
     </div>
   {:else if editableColumn.type === JSON_TYPE}
-    <Button primary text on:click={openJsonSchemaEditor}
+    <Button primary text on:click={openJsonSchemaEditor}>
-      >Open schema editor</Button
+      Open schema editor
-    >
+    </Button>
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select
@@ -779,27 +815,39 @@
     </div>
   {/if}
 
-  {#if canHaveDefault}
+  {#if defaultValuesEnabled}
-    <div>
+    {#if editableColumn.type === FieldType.OPTIONS}
-      <ModalBindableInput
+      <Select
-        panel={ServerBindingPanel}
+        disabled={!canHaveDefault}
-        title="Default"
+        options={editableColumn.constraints?.inclusion || []}
-        label="Default"
+        label="Default value"
         value={editableColumn.default}
-        on:change={e => {
+        on:change={e => (editableColumn.default = e.detail)}
-          editableColumn = {
+        placeholder="None"
-            ...editableColumn,
+      />
-            default: e.detail,
+    {:else if editableColumn.type === FieldType.ARRAY}
-          }
+      <Multiselect
-
+        disabled={!canHaveDefault}
-          if (e.detail) {
+        options={editableColumn.constraints?.inclusion || []}
-            setRequired(false)
+        label="Default value"
-          }
+        value={editableColumn.default}
-        }}
+        on:change={e =>
+          (editableColumn.default = e.detail?.length ? e.detail : undefined)}
+        placeholder="None"
+      />
+    {:else}
+      <ModalBindableInput
+        disabled={!canHaveDefault}
+        panel={ServerBindingPanel}
+        title="Default value"
+        label="Default value"
+        placeholder="None"
+        value={editableColumn.default}
+        on:change={e => (editableColumn.default = e.detail)}
         bindings={defaultValueBindings}
         allowJS
       />
-    </div>
+    {/if}
   {/if}
 </Layout>
 

packages/builder/src/stores/builder/viewsV2.js

@@ -1,6 +1,24 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
+
+function convertToSearchFilterGroup(view) {
+  if (view.queryUI) {
+    view.query = view.queryUI
+    delete view.queryUI
+  }
+  return view
+}
 
 export function createViewsV2Store() {
   const store = writable({
@@ -12,7 +30,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views)
+      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
     })
     return {
       ...$store,
@@ -34,6 +52,7 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
+    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
@@ -41,6 +60,7 @@ export function createViewsV2Store() {
   }
 
   const save = async view => {
+    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
@@ -51,6 +71,7 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
+    view = convertToSearchFilterGroup(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId

packages/client/src/components/app/forms/Form.svelte

@@ -63,7 +63,7 @@
     // Look up the component tree and find something that is provided by an
     // ancestor that matches our datasource. This is for backwards compatibility
     // as previously we could use the "closest" context.
-    for (let id of path.reverse().slice(1)) {
+    for (let id of path.toReversed().slice(1)) {
       // Check for matching view datasource
       if (
         dataSource.type === "viewV2" &&

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -1,6 +1,7 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
+import { cloneDeep } from "lodash"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -164,10 +165,18 @@ export const createActions = context => {
 
   // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
-    return await saveDefinition({
+    let newDefinition = cloneDeep(get(definition))
-      ...get(definition),
+
-      primaryDisplay: column,
+    // Update primary display
-    })
+    newDefinition.primaryDisplay = column
+
+    // Sanitise schema to ensure field is required and has no default value
+    if (!newDefinition.schema[column].constraints) {
+      newDefinition.schema[column].constraints = {}
+    }
+    newDefinition.schema[column].constraints.presence = { allowEmpty: false }
+    delete newDefinition.schema[column].default
+    return await saveDefinition(newDefinition)
   }
 
   // Adds a schema mutation for a single field

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,4 +1,14 @@
 import { get } from "svelte/store"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
 
 const SuppressErrors = true
 
@@ -6,7 +16,7 @@ export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(newDefinition)
+    await API.viewV2.update(convertToSearchFilters(newDefinition))
   }
 
   const saveRow = async row => {
@@ -125,7 +135,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.query)
+          filter.set($definition.queryUI || $definition.query)
         }
       })
     )

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -124,9 +124,11 @@ export async function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === columnName)
+          !existing.find(
+            (field: string) => field === `${table.name}.${columnName}`
           )
-      .map(column => `${table.name}.${column[0]}`)
+      )
+      .map(([columnName]) => `${table.name}.${columnName}`)
   }
 
   let fields: string[] = []

packages/server/src/api/controllers/row/views.ts

@@ -3,6 +3,8 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
+  RequiredKeys,
+  RowSearchParams,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
@@ -20,30 +22,34 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
   await context.ensureSnippetContext(true)
 
-  const result = await sdk.rows.search(
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    {
+    RequiredKeys<
-      viewId: view.id,
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
+    > = {
     tableId: view.tableId,
+    viewId: view.id,
     query: body.query,
+    fields: viewFields,
     ...getSortOptions(body, view),
     limit: body.limit,
     bookmark: body.bookmark,
     paginate: body.paginate,
     countRows: body.countRows,
-    },
-    {
-      user: sdk.users.getUserContextBindings(ctx.user),
   }
-  )
 
+  const result = await sdk.rows.search(searchOptions, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -103,6 +103,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
@@ -138,6 +139,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     version: view.version,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,

packages/server/src/api/routes/tests/row.spec.ts

@@ -695,6 +695,69 @@ describe.each([
         })
       })
 
+      describe("options column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                status: {
+                  name: "status",
+                  type: FieldType.OPTIONS,
+                  default: "requested",
+                  constraints: {
+                    inclusion: ["requested", "approved"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.status).toEqual("requested")
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            status: "approved",
+          })
+          expect(row.status).toEqual("approved")
+        })
+      })
+
+      describe("array column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                food: {
+                  name: "food",
+                  type: FieldType.ARRAY,
+                  default: ["apple", "orange"],
+                  constraints: {
+                    type: JsonFieldSubType.ARRAY,
+                    inclusion: ["apple", "orange", "banana"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.food).toEqual(["apple", "orange"])
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            food: ["orange"],
+          })
+          expect(row.food).toEqual(["orange"])
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -28,11 +28,13 @@ import {
   RowSearchParams,
   SearchFilters,
   SearchResponse,
+  SearchRowRequest,
   SortOrder,
   SortType,
   Table,
   TableSchema,
   User,
+  ViewV2Schema,
 } from "@budibase/types"
 import _ from "lodash"
 import tk from "timekeeper"
@@ -64,18 +66,14 @@ describe.each([
   let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let client: Knex | undefined
-  let table: Table
+  let tableOrViewId: string
   let rows: Row[]
 
   async function basicRelationshipTables(type: RelationshipType) {
-    const relatedTable = await createTable(
+    const relatedTable = await createTable({
-      {
       name: { name: "name", type: FieldType.STRING },
-      },
+    })
-      generator.guid().substring(0, 10)
+    const tableId = await createTable({
-    )
-    table = await createTable(
-      {
       name: { name: "name", type: FieldType.STRING },
       //@ts-ignore - API accepts this structure, will build out rest of definition
       productCat: {
@@ -83,17 +81,15 @@ describe.each([
         relationshipType: type,
         name: "productCat",
         fieldName: "product",
-          tableId: relatedTable._id!,
+        tableId: relatedTable,
         constraints: {
           type: "array",
         },
       },
-      },
+    })
-      generator.guid().substring(0, 10)
-    )
     return {
-      relatedTable: await config.api.table.get(relatedTable._id!),
+      relatedTable: await config.api.table.get(relatedTable),
-      table,
+      tableId,
     }
   }
 
@@ -136,32 +132,69 @@ describe.each([
     }
   })
 
-  async function createTable(schema: TableSchema, name?: string) {
+  async function createTable(schema: TableSchema) {
-    return await config.api.table.save(
+    const table = await config.api.table.save(
-      tableForDatasource(datasource, { schema, name })
+      tableForDatasource(datasource, { schema })
     )
+    return table._id!
+  }
+
+  async function createView(tableId: string, schema: ViewV2Schema) {
+    const view = await config.api.viewV2.create({
+      tableId: tableId,
+      name: generator.guid(),
+      schema,
+    })
+    return view.id
   }
 
   async function createRows(arr: Record<string, any>[]) {
     // Shuffling to avoid false positives given a fixed order
-    await config.api.row.bulkImport(table._id!, {
+    for (const row of _.shuffle(arr)) {
-      rows: _.shuffle(arr),
+      await config.api.row.save(tableOrViewId, row)
-    })
+    }
-    rows = await config.api.row.fetch(table._id!)
+    rows = await config.api.row.fetch(tableOrViewId)
+  }
+
+  describe.each([
+    ["table", createTable],
+    [
+      "view",
+      async (schema: TableSchema) => {
+        const tableId = await createTable(schema)
+        const viewId = await createView(
+          tableId,
+          Object.keys(schema).reduce<ViewV2Schema>((viewSchema, fieldName) => {
+            const field = schema[fieldName]
+            viewSchema[fieldName] = {
+              visible: field.visible ?? true,
+              readonly: false,
+            }
+            return viewSchema
+          }, {})
+        )
+        return viewId
+      },
+    ],
+  ])("from %s", (sourceType, createTableOrView) => {
+    const isView = sourceType === "view"
+
+    if (isView && isLucene) {
+      // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests
+      return
     }
 
     class SearchAssertion {
-    constructor(private readonly query: RowSearchParams) {}
+      constructor(private readonly query: SearchRowRequest) {}
 
       private async performSearch(): Promise<SearchResponse<Row>> {
         if (isInMemory) {
-        return dataFilters.search(_.cloneDeep(rows), this.query)
+          return dataFilters.search(_.cloneDeep(rows), {
+            ...this.query,
+            tableId: tableOrViewId,
+          })
         } else {
-        const sourceId = this.query.viewId || this.query.tableId
+          return config.api.row.search(tableOrViewId, this.query)
-        if (!sourceId) {
-          throw new Error("No source ID provided")
-        }
-        return config.api.row.search(sourceId, this.query)
         }
       }
 
@@ -331,8 +364,8 @@ describe.each([
       }
     }
 
-  function expectSearch(query: Omit<RowSearchParams, "tableId">) {
+    function expectSearch(query: SearchRowRequest) {
-    return new SearchAssertion({ ...query, tableId: table._id! })
+      return new SearchAssertion(query)
     }
 
     function expectQuery(query: SearchFilters) {
@@ -341,7 +374,7 @@ describe.each([
 
     describe("boolean", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
         })
         await createRows([{ isTrue: true }, { isTrue: false }])
@@ -429,38 +462,35 @@ describe.each([
             { name: "serverDate", appointment: serverTime.toISOString() },
             {
               name: "single user, session user",
-            single_user: JSON.stringify(currentUser),
+              single_user: currentUser,
             },
             {
               name: "single user",
-            single_user: JSON.stringify(globalUsers[0]),
+              single_user: globalUsers[0],
             },
             {
               name: "deprecated single user, session user",
-            deprecated_single_user: JSON.stringify([currentUser]),
+              deprecated_single_user: [currentUser],
             },
             {
               name: "deprecated single user",
-            deprecated_single_user: JSON.stringify([globalUsers[0]]),
+              deprecated_single_user: [globalUsers[0]],
             },
             {
               name: "multi user",
-            multi_user: JSON.stringify(globalUsers),
+              multi_user: globalUsers,
             },
             {
               name: "multi user with session user",
-            multi_user: JSON.stringify([...globalUsers, currentUser]),
+              multi_user: [...globalUsers, currentUser],
             },
             {
               name: "deprecated multi user",
-            deprecated_multi_user: JSON.stringify(globalUsers),
+              deprecated_multi_user: globalUsers,
             },
             {
               name: "deprecated multi user with session user",
-            deprecated_multi_user: JSON.stringify([
+              deprecated_multi_user: [...globalUsers, currentUser],
-              ...globalUsers,
-              currentUser,
-            ]),
             },
           ]
         }
@@ -482,7 +512,7 @@ describe.each([
               })
           )
 
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: { name: "name", type: FieldType.STRING },
             appointment: { name: "appointment", type: FieldType.DATETIME },
             single_user: {
@@ -632,9 +662,11 @@ describe.each([
         })
 
         it("should match the session user id in a multi user field", async () => {
-        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          const allUsers = [...globalUsers, config.getUser()].map(
+            (user: any) => {
               return { _id: user._id }
-        })
+            }
+          )
 
           await expectQuery({
             contains: { multi_user: ["{{ [user]._id }}"] },
@@ -647,9 +679,11 @@ describe.each([
         })
 
         it("should match the session user id in a deprecated multi user field", async () => {
-        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          const allUsers = [...globalUsers, config.getUser()].map(
+            (user: any) => {
               return { _id: user._id }
-        })
+            }
+          )
 
           await expectQuery({
             contains: { deprecated_multi_user: ["{{ [user]._id }}"] },
@@ -764,7 +798,7 @@ describe.each([
 
     describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           name: { name: "name", type: FieldType.STRING },
         })
         await createRows([{ name: "foo" }, { name: "bar" }])
@@ -791,6 +825,8 @@ describe.each([
           }).toContainExactly([{ name: "foo" }, { name: "bar" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("should return nothing if onEmptyFilter is RETURN_NONE", async () => {
             await expectQuery({
               onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@@ -891,6 +927,8 @@ describe.each([
           }).toContainExactly([{ name: "foo" }, { name: "bar" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
             await expectQuery({
               onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@@ -1055,7 +1093,7 @@ describe.each([
                 datasourceId: datasource!._id!,
               })
 
-            table = resp.datasource.entities![tableName]
+              tableOrViewId = resp.datasource.entities![tableName]._id!
 
               await createRows([{ name: "foo" }, { name: "bar" }])
             })
@@ -1079,7 +1117,7 @@ describe.each([
 
     describe("numbers", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           age: { name: "age", type: FieldType.NUMBER },
         })
         await createRows([{ age: 1 }, { age: 10 }])
@@ -1087,7 +1125,9 @@ describe.each([
 
       describe("equal", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])
+          await expectQuery({ equal: { age: 1 } }).toContainExactly([
+            { age: 1 },
+          ])
         })
 
         it("fails to find nonexistent row", async () => {
@@ -1252,7 +1292,7 @@ describe.each([
       const JAN_10TH = "2020-01-10T00:00:00.000Z"
 
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           dob: { name: "dob", type: FieldType.DATETIME },
         })
 
@@ -1324,25 +1364,33 @@ describe.each([
 
         it("greater than equal to", async () => {
           await expectQuery({
-          range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } },
+            range: {
+              dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_10TH }])
         })
 
         it("greater than", async () => {
           await expectQuery({
-          range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } },
+            range: {
+              dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_10TH }])
         })
 
         it("less than equal to", async () => {
           await expectQuery({
-          range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } },
+            range: {
+              dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_1ST }])
         })
 
         it("less than", async () => {
           await expectQuery({
-          range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } },
+            range: {
+              dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_1ST }])
         })
       })
@@ -1399,7 +1447,7 @@ describe.each([
         const NULL_TIME__ID = `null_time__id`
 
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             timeid: { name: "timeid", type: FieldType.STRING },
             time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
           })
@@ -1560,7 +1608,7 @@ describe.each([
 
     describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           numbers: {
             name: "numbers",
             type: FieldType.ARRAY,
@@ -1575,9 +1623,9 @@ describe.each([
 
       describe("contains", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([
+          await expectQuery({
-          { numbers: ["one", "two"] },
+            contains: { numbers: ["one"] },
-        ])
+          }).toContainExactly([{ numbers: ["one", "two"] }])
         })
 
         it("fails to find nonexistent row", async () => {
@@ -1657,7 +1705,7 @@ describe.each([
       let BIG = "9223372036854775807"
 
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           num: { name: "num", type: FieldType.BIGINT },
         })
         await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@@ -1762,7 +1810,7 @@ describe.each([
     isInternal &&
       describe("auto", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             auto: {
               name: "auto",
               type: FieldType.AUTO,
@@ -1913,9 +1961,9 @@ describe.each([
             // to specify an order for pagination to work.
             it("is stable without a sort specified", async () => {
               let { rows: fullRowList } = await config.api.row.search(
-              table._id!,
+                tableOrViewId,
                 {
-                tableId: table._id!,
+                  tableId: tableOrViewId,
                   query: {},
                 }
               )
@@ -1925,8 +1973,8 @@ describe.each([
                 hasNextPage: boolean | undefined = true,
                 rowCount: number = 0
               do {
-              const response = await config.api.row.search(table._id!, {
+                const response = await config.api.row.search(tableOrViewId, {
-                tableId: table._id!,
+                  tableId: tableOrViewId,
                   limit: 1,
                   paginate: true,
                   query: {},
@@ -1949,8 +1997,8 @@ describe.each([
 
             // eslint-disable-next-line no-constant-condition
             while (true) {
-            const response = await config.api.row.search(table._id!, {
+              const response = await config.api.row.search(tableOrViewId, {
-              tableId: table._id!,
+                tableId: tableOrViewId,
                 limit: 3,
                 query: {},
                 bookmark,
@@ -1973,7 +2021,7 @@ describe.each([
 
     describe("field name 1:name", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           "1:name": { name: "1:name", type: FieldType.STRING },
         })
         await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@@ -1993,8 +2041,7 @@ describe.each([
     isSql &&
       describe("related formulas", () => {
         beforeAll(async () => {
-        const arrayTable = await createTable(
+          const arrayTable = await createTable({
-          {
             name: { name: "name", type: FieldType.STRING },
             array: {
               name: "array",
@@ -2004,17 +2051,14 @@ describe.each([
                 inclusion: ["option 1", "option 2"],
               },
             },
-          },
+          })
-          "array"
+          tableOrViewId = await createTableOrView({
-        )
-        table = await createTable(
-          {
             relationship: {
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_ONE,
               name: "relationship",
               fieldName: "relate",
-              tableId: arrayTable._id!,
+              tableId: arrayTable,
               constraints: {
                 type: "array",
               },
@@ -2026,21 +2070,19 @@ describe.each([
                 `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
               ),
             },
-          },
+          })
-          "main"
-        )
           const arrayRows = await Promise.all([
-          config.api.row.save(arrayTable._id!, {
+            config.api.row.save(arrayTable, {
               name: "foo",
               array: ["option 1"],
             }),
-          config.api.row.save(arrayTable._id!, {
+            config.api.row.save(arrayTable, {
               name: "bar",
               array: ["option 2"],
             }),
           ])
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               relationship: [arrayRows[0]._id, arrayRows[1]._id],
             }),
           ])
@@ -2059,7 +2101,7 @@ describe.each([
         user1 = await config.createUser({ _id: `us_${utils.newid()}` })
         user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           user: {
             name: "user",
             type: FieldType.BB_REFERENCE_SINGLE,
@@ -2067,11 +2109,7 @@ describe.each([
           },
         })
 
-      await createRows([
+        await createRows([{ user: user1 }, { user: user2 }, { user: null }])
-        { user: JSON.stringify(user1) },
-        { user: JSON.stringify(user2) },
-        { user: null },
-      ])
       })
 
       describe("equal", () => {
@@ -2088,18 +2126,15 @@ describe.each([
 
       describe("notEqual", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
+          await expectQuery({ notEqual: { user: user1._id } }).toContainExactly(
-          { user: { _id: user2._id } },
+            [{ user: { _id: user2._id } }, {}]
-          {},
+          )
-        ])
         })
 
         it("fails to find nonexistent row", async () => {
-        await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
+          await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly(
-          { user: { _id: user1._id } },
+            [{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}]
-          { user: { _id: user2._id } },
+          )
-          {},
-        ])
         })
       })
 
@@ -2139,7 +2174,7 @@ describe.each([
         user1 = await config.createUser({ _id: `us_${utils.newid()}` })
         user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           users: {
             name: "users",
             type: FieldType.BB_REFERENCE,
@@ -2153,10 +2188,10 @@ describe.each([
         })
 
         await createRows([
-        { number: 1, users: JSON.stringify([user1]) },
+          { number: 1, users: [user1] },
-        { number: 2, users: JSON.stringify([user2]) },
+          { number: 2, users: [user2] },
-        { number: 3, users: JSON.stringify([user1, user2]) },
+          { number: 3, users: [user1, user2] },
-        { number: 4, users: JSON.stringify([]) },
+          { number: 4, users: [] },
         ])
       })
 
@@ -2182,7 +2217,9 @@ describe.each([
         })
 
         it("fails to find nonexistent row", async () => {
-        await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()
+          await expectQuery({
+            contains: { users: ["us_none"] },
+          }).toFindNothing()
         })
       })
 
@@ -2249,9 +2286,10 @@ describe.each([
         let productCategoryTable: Table, productCatRows: Row[]
 
         beforeAll(async () => {
-        const { relatedTable } = await basicRelationshipTables(
+          const { relatedTable, tableId } = await basicRelationshipTables(
             RelationshipType.ONE_TO_MANY
           )
+          tableOrViewId = tableId
           productCategoryTable = relatedTable
 
           productCatRows = await Promise.all([
@@ -2260,15 +2298,15 @@ describe.each([
           ])
 
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "foo",
               productCat: [productCatRows[0]._id],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "bar",
               productCat: [productCatRows[1]._id],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "baz",
               productCat: [],
             }),
@@ -2301,10 +2339,11 @@ describe.each([
     isSql &&
       describe("big relations", () => {
         beforeAll(async () => {
-        const { relatedTable } = await basicRelationshipTables(
+          const { relatedTable, tableId } = await basicRelationshipTables(
             RelationshipType.MANY_TO_ONE
           )
-        const mainRow = await config.api.row.save(table._id!, {
+          tableOrViewId = tableId
+          const mainRow = await config.api.row.save(tableOrViewId, {
             name: "foo",
           })
           for (let i = 0; i < 11; i++) {
@@ -2329,45 +2368,42 @@ describe.each([
       })
     ;(isSqs || isLucene) &&
       describe("relations to same table", () => {
-      let relatedTable: Table, relatedRows: Row[]
+        let relatedTable: string, relatedRows: Row[]
 
         beforeAll(async () => {
-        relatedTable = await createTable(
+          relatedTable = await createTable({
-          {
             name: { name: "name", type: FieldType.STRING },
-          },
+          })
-          "productCategory"
+          tableOrViewId = await createTableOrView({
-        )
-        table = await createTable({
             name: { name: "name", type: FieldType.STRING },
             related1: {
               type: FieldType.LINK,
               name: "related1",
               fieldName: "main1",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
             related2: {
               type: FieldType.LINK,
               name: "related2",
               fieldName: "main2",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
           })
           relatedRows = await Promise.all([
-          config.api.row.save(relatedTable._id!, { name: "foo" }),
+            config.api.row.save(relatedTable, { name: "foo" }),
-          config.api.row.save(relatedTable._id!, { name: "bar" }),
+            config.api.row.save(relatedTable, { name: "bar" }),
-          config.api.row.save(relatedTable._id!, { name: "baz" }),
+            config.api.row.save(relatedTable, { name: "baz" }),
-          config.api.row.save(relatedTable._id!, { name: "boo" }),
+            config.api.row.save(relatedTable, { name: "boo" }),
           ])
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "test",
               related1: [relatedRows[0]._id!],
               related2: [relatedRows[1]._id!],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "test2",
               related1: [relatedRows[2]._id!],
               related2: [relatedRows[3]._id!],
@@ -2430,7 +2466,7 @@ describe.each([
     isInternal &&
       describe("no column error backwards compat", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2453,7 +2489,7 @@ describe.each([
     !isLucene &&
       describe("row counting", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2488,7 +2524,7 @@ describe.each([
     describe("Invalid column definitions", () => {
       beforeAll(async () => {
         // need to create an invalid table - means ignoring typescript
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           // @ts-ignore
           invalid: {
             type: FieldType.STRING,
@@ -2518,7 +2554,7 @@ describe.each([
       "special (%s) case",
       column => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             [column]: {
               name: column,
               type: FieldType.STRING,
@@ -2543,8 +2579,8 @@ describe.each([
       describe("sample data", () => {
         beforeAll(async () => {
           await config.api.application.addSampleData(config.appId!)
-        table = DEFAULT_EMPLOYEE_TABLE_SCHEMA
+          tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id!
-        rows = await config.api.row.fetch(table._id!)
+          rows = await config.api.row.fetch(tableOrViewId)
         })
 
         it("should be able to search sample data", async () => {
@@ -2567,7 +2603,7 @@ describe.each([
       const earlyDate = "2024-07-03T10:00:00.000Z",
         laterDate = "2024-07-03T11:00:00.000Z"
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           date: {
             name: "date",
             type: FieldType.DATETIME,
@@ -2610,7 +2646,7 @@ describe.each([
       "ชื่อผู้ใช้", // Thai for "username"
     ])("non-ascii column name: %s", name => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           [name]: {
             name,
             type: FieldType.STRING,
@@ -2637,7 +2673,7 @@ describe.each([
     isInternal &&
       describe("space at end of column name", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             "name ": {
               name: "name ",
               type: FieldType.STRING,
@@ -2672,7 +2708,7 @@ describe.each([
     ;(isSqs || isInMemory) &&
       describe("space at start of column name", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             " name": {
               name: " name",
               type: FieldType.STRING,
@@ -2703,9 +2739,10 @@ describe.each([
       })
 
     isSqs &&
+      !isView &&
       describe("duplicate columns", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2713,7 +2750,7 @@ describe.each([
           })
           await context.doInAppContext(config.getAppId(), async () => {
             const db = context.getAppDB()
-          const tableDoc = await db.get<Table>(table._id!)
+            const tableDoc = await db.get<Table>(tableOrViewId)
             tableDoc.schema.Name = {
               name: "Name",
               type: FieldType.STRING,
@@ -2747,7 +2784,7 @@ describe.each([
               type: FieldType.STRING,
             },
           })
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2756,15 +2793,15 @@ describe.each([
               name: "rel",
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_MANY,
-            tableId: toRelateTable._id!,
+              tableId: toRelateTable,
               fieldName: "rel",
             },
           })
           const [row1, row2] = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
+            config.api.row.save(toRelateTable, { name: "tag 1" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
+            config.api.row.save(toRelateTable, { name: "tag 2" }),
           ])
-        row = await config.api.row.save(table._id!, {
+          row = await config.api.row.save(tableOrViewId, {
             name: "product 1",
             rel: [row1._id, row2._id],
           })
@@ -2783,7 +2820,7 @@ describe.each([
     !isInternal &&
       describe("search by composite key", () => {
         beforeAll(async () => {
-        table = await config.api.table.save(
+          const table = await config.api.table.save(
             tableForDatasource(datasource, {
               schema: {
                 idColumn1: {
@@ -2798,6 +2835,7 @@ describe.each([
               primary: ["idColumn1", "idColumn2"],
             })
           )
+          tableOrViewId = table._id!
           await createRows([{ idColumn1: 1, idColumn2: 2 }])
         })
 
@@ -2819,15 +2857,13 @@ describe.each([
     isSql &&
       describe("primaryDisplay", () => {
         beforeAll(async () => {
-        let toRelateTable = await createTable({
+          let toRelateTableId = await createTable({
             name: {
               name: "name",
               type: FieldType.STRING,
             },
           })
-        table = await config.api.table.save(
+          tableOrViewId = await createTableOrView({
-          tableForDatasource(datasource, {
-            schema: {
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2836,33 +2872,30 @@ describe.each([
               name: "link",
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_ONE,
-                tableId: toRelateTable._id!,
+              tableId: toRelateTableId,
               fieldName: "link",
             },
-            },
           })
-        )
+
-        toRelateTable = await config.api.table.get(toRelateTable._id!)
+          const toRelateTable = await config.api.table.get(toRelateTableId)
           await config.api.table.save({
             ...toRelateTable,
             primaryDisplay: "link",
           })
           const relatedRows = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "test" }),
+            config.api.row.save(toRelateTable._id!, { name: "related" }),
           ])
-        await Promise.all([
+          await config.api.row.save(tableOrViewId, {
-          config.api.row.save(table._id!, {
             name: "test",
             link: relatedRows.map(row => row._id),
-          }),
+          })
-        ])
         })
 
         it("should be able to query, primary display on related table shouldn't be used", async () => {
           // this test makes sure that if a relationship has been specified as the primary display on a table
           // it is ignored and another column is used instead
           await expectQuery({}).toContain([
-          { name: "test", link: [{ primaryDisplay: "test" }] },
+            { name: "test", link: [{ primaryDisplay: "related" }] },
           ])
         })
       })
@@ -2870,7 +2903,7 @@ describe.each([
     !isLucene &&
       describe("$and", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             age: { name: "age", type: FieldType.NUMBER },
             name: { name: "name", type: FieldType.STRING },
           })
@@ -2945,7 +2978,10 @@ describe.each([
             await expect(
               expectQuery({
                 $and: {
-                conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
+                  conditions: [
+                    { equal: { age: 10 } },
+                    "invalidCondition" as any,
+                  ],
                 },
               }).toFindNothing()
             ).rejects.toThrow(
@@ -2973,6 +3009,8 @@ describe.each([
             )
           })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("returns no rows when onEmptyFilter set to none", async () => {
             await expectSearch({
               query: {
@@ -2999,7 +3037,7 @@ describe.each([
     !isLucene &&
       describe("$or", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             age: { name: "age", type: FieldType.NUMBER },
             name: { name: "name", type: FieldType.STRING },
           })
@@ -3123,6 +3161,8 @@ describe.each([
           }).toContainExactly([{ age: 1, name: "Jane" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("returns no rows when onEmptyFilter set to none", async () => {
             await expectSearch({
               query: {
@@ -3159,20 +3199,20 @@ describe.each([
             row[name] = i
           }
           const relatedTable = await createTable(relatedSchema)
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: { name: "name", type: FieldType.STRING },
             related1: {
               type: FieldType.LINK,
               name: "related1",
               fieldName: "main1",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
           })
           relatedRows = await Promise.all([
-          config.api.row.save(relatedTable._id!, row),
+            config.api.row.save(relatedTable, row),
           ])
-        await config.api.row.save(table._id!, {
+          await config.api.row.save(tableOrViewId, {
             name: "foo",
             related1: [relatedRows[0]._id],
           })
@@ -3187,4 +3227,5 @@ describe.each([
           ])
         })
       })
+  })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,9 +22,10 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  ViewFieldMetadata,
   FeatureFlag,
   BBReferenceFieldSubType,
+  ViewV2Schema,
+  ViewCalculationFieldMetadata,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -154,7 +155,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<CreateViewRequest> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -540,6 +541,33 @@ describe.each([
           status: 201,
         })
       })
+
+      it("can create a view with calculation fields", async () => {
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "Price",
+            },
+          },
+        })
+
+        expect(Object.keys(view.schema!)).toHaveLength(1)
+
+        let sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+
+        view = await config.api.viewV2.get(view.id)
+        sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+      })
     })
 
     describe("update", () => {
@@ -584,7 +612,7 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<UpdateViewRequest> = {
+        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
           version: view.version,
           id: view.id,
           tableId,
@@ -1152,10 +1180,7 @@ describe.each([
           return table
         }
 
-        const createView = async (
+        const createView = async (tableId: string, schema: ViewV2Schema) =>
-          tableId: string,
-          schema: Record<string, ViewFieldMetadata>
-        ) =>
           await config.api.viewV2.create({
             name: generator.guid(),
             tableId,
@@ -2546,6 +2571,51 @@ describe.each([
             }
           })
         })
+
+      !isLucene &&
+        it("should not need required fields to be present", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                name: {
+                  name: "name",
+                  type: FieldType.STRING,
+                  constraints: {
+                    presence: true,
+                  },
+                },
+                age: {
+                  name: "age",
+                  type: FieldType.NUMBER,
+                },
+              },
+            })
+          )
+
+          await Promise.all([
+            config.api.row.save(table._id!, { name: "Steve", age: 30 }),
+            config.api.row.save(table._id!, { name: "Jane", age: 31 }),
+          ])
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "age",
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {},
+          })
+
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows[0].sum).toEqual(61)
+        })
     })
 
     describe("permissions", () => {

packages/server/src/db/linkedRows/index.ts

@@ -23,8 +23,8 @@ import {
   Row,
   Table,
   TableSchema,
-  ViewFieldMetadata,
   ViewV2,
+  ViewV2Schema,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { helpers } from "@budibase/shared-core"
@@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
     FeatureFlag.ENRICHED_RELATIONSHIPS
   )
 
-  let viewSchema: Record<string, ViewFieldMetadata> = {}
+  let viewSchema: ViewV2Schema = {}
   if (sdk.views.isView(source)) {
     if (helpers.views.isCalculationView(source)) {
       return enriched

packages/server/src/sdk/app/rows/search.ts

@@ -1,12 +1,10 @@
 import {
   EmptyFilterOption,
+  LegacyFilter,
   LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilter,
-  SearchFilterGroup,
   SearchFilterKey,
-  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -63,7 +61,6 @@ export async function search(
     if (options.viewId) {
       source = await sdk.views.get(options.viewId)
       table = await sdk.views.getTable(source)
-      options = searchInputMapping(table, options)
     } else if (options.tableId) {
       source = await sdk.tables.getTable(options.tableId)
       table = source
@@ -76,7 +73,7 @@ export async function search(
     if (options.query) {
       const visibleFields = (
         options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
+      ).filter(field => table.schema[field]?.visible !== false)
 
       const queryableFields = await getQueryableFields(table, visibleFields)
       options.query = removeInvalidFilters(options.query, queryableFields)
@@ -84,40 +81,59 @@ export async function search(
       options.query = {}
     }
 
+    // need to make sure filters in correct shape before checking for view
+    options = searchInputMapping(table, options)
+
     if (options.viewId) {
-      const view = await sdk.views.get(options.viewId)
+      // Delete extraneous search params that cannot be overridden
+      delete options.query.onEmptyFilter
+
+      const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQuery(view.query || [])
+      let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
+      delete viewQuery?.onEmptyFilter
 
-      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
+      const sqsEnabled = await features.flags.isEnabled("SQS")
-        // Lucene does not accept conditional filters, so we need to keep the old logic
+      const supportsLogicalOperators =
-        const query: SearchFilters = viewQuery || {}
+        isExternalTableID(view.tableId) || sqsEnabled
-        const viewFilters = view.query as SearchFilter[]
+      if (!supportsLogicalOperators) {
+        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+        // It needs to be ignored entirely
+        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+          ? view.query
+          : []
+
+        delete options.query.onEmptyFilter
 
         // Extract existing fields
         const existingFields =
-          viewFilters
+          queryFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
+        viewQuery ??= {}
         // Carry over filters for unused fields
-        Object.keys(options.query || {}).forEach(key => {
+        Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              query[operator]![field] = options.query[operator]![field]
+              viewQuery![operator]![field] = options.query[operator]![field]
             }
           })
         })
-        options.query = query
+        options.query = viewQuery
       } else {
+        const conditions = viewQuery ? [viewQuery] : []
         options.query = {
           $and: {
-            conditions: [viewQuery as SearchFilterGroup, options.query],
+            conditions: [...conditions, options.query],
           },
         }
+        if (viewQuery.onEmptyFilter) {
+          options.query.onEmptyFilter = viewQuery.onEmptyFilter
+        }
       }
     }
 
@@ -146,8 +162,6 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    options = searchInputMapping(table, options)
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/utils.ts

@@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
     }
     return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
+  if (options.query) {
     options.query = checkFilters(options.query)
+  }
   return options
 }
 

packages/server/src/sdk/app/rows/utils.ts

@@ -20,7 +20,7 @@ import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
-import { docIds } from "@budibase/backend-core"
+import { docIds, sql } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
@@ -57,8 +57,12 @@ export function getSQLClient(datasource: Datasource): SqlClient {
 export function processRowCountResponse(
   response: DatasourcePlusQueryResponse
 ): number {
-  if (response && response.length === 1 && "__bb_total" in response[0]) {
+  if (
-    const total = response[0].__bb_total
+    response &&
+    response.length === 1 &&
+    sql.COUNT_FIELD_NAME in response[0]
+  ) {
+    const total = response[0][sql.COUNT_FIELD_NAME]
     return typeof total === "number" ? total : parseInt(total)
   } else {
     throw new Error("Unable to count rows in query - no count response")

packages/server/src/sdk/app/views/index.ts

@@ -255,19 +255,12 @@ export async function enrichSchema(
   view: ViewV2,
   tableSchema: TableSchema
 ): Promise<ViewV2Enriched> {
-  const tableCache: Record<string, Table> = {}
-
   async function populateRelTableSchema(
     tableId: string,
     viewFields: Record<string, RelationSchemaField>
   ) {
-    if (!tableCache[tableId]) {
+    const relTable = await sdk.tables.getTable(tableId)
-      tableCache[tableId] = await sdk.tables.getTable(tableId)
-    }
-    const relTable = tableCache[tableId]
-
     const result: Record<string, ViewV2ColumnEnriched> = {}
-
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
       if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
@@ -296,15 +289,24 @@ export async function enrichSchema(
 
   const viewSchema = view.schema || {}
   const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
-  for (const key of Object.keys(tableSchema).filter(
+
-    k => tableSchema[k].visible !== false
+  const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
-  )) {
+    if (helpers.views.isCalculationField(viewSchema[key])) {
+      return viewSchema[key].visible !== false
+    }
+    return key in tableSchema && tableSchema[key].visible !== false
+  })
+  const visibleTableFields = Object.keys(tableSchema).filter(
+    key => tableSchema[key].visible !== false
+  )
+  const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
+  for (const key of visibleFields) {
     // if nothing specified in view, then it is not visible
     const ui = viewSchema[key] || { visible: false }
     schema[key] = {
       ...tableSchema[key],
       ...ui,
-      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order,
+      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
       columns: undefined,
     }
 
@@ -316,10 +318,7 @@ export async function enrichSchema(
     }
   }
 
-  return {
+  return { ...view, schema }
-    ...view,
-    schema: schema,
-  }
 }
 
 export function syncSchema(

packages/server/src/tests/utilities/api/row.ts

@@ -7,11 +7,11 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   SearchRowResponse,
-  RowSearchParams,
   DeleteRows,
   DeleteRow,
   PaginatedSearchRowResponse,
   RowExportFormat,
+  SearchRowRequest,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
     )
   }
 
-  search = async <T extends RowSearchParams>(
+  search = async <T extends SearchRowRequest>(
     sourceId: string,
     params?: T,
     expectations?: Expectations

packages/server/src/utilities/rowProcessor/index.ts

@@ -134,8 +134,10 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed = await processString(schema.default, ctx)
+      const processed =
-
+        typeof schema.default === "string"
+          ? await processString(schema.default, ctx)
+          : schema.default
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {

packages/shared-core/src/filters.ts

@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -163,9 +163,6 @@ export function recurseSearchFilters(
  * https://github.com/Budibase/budibase/issues/10118
  */
 export const cleanupQuery = (query: SearchFilters) => {
-  if (!query) {
-    return query
-  }
   for (let filterField of NoEmptyFilterStrings) {
     if (!query[filterField]) {
       continue
@@ -311,7 +308,7 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: SearchFilter) => {
+const buildCondition = (expression: LegacyFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
 }
 
 export const buildQueryLegacy = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: LegacyFilter[] | SearchFilters
 ): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -572,7 +574,7 @@ export const buildQueryLegacy = (
  */
 
 export const buildQuery = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: SearchFilterGroup | LegacyFilter[]
 ): SearchFilters | undefined => {
   const parsedFilter: SearchFilterGroup | undefined =
     processSearchFilters(filter)
@@ -594,7 +596,7 @@ export const buildQuery = (
   const globalOperator: LogicalOperator =
     operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
-  const coreRequest: SearchFilters = {
+  return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@@ -608,7 +610,6 @@ export const buildQuery = (
       }),
     },
   }
-  return coreRequest
 }
 
 // The frontend can send single values for array fields sometimes, so to handle

packages/shared-core/src/table.ts

@@ -53,8 +53,9 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.DATETIME]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.ARRAY]: true,
 
-  [FieldType.OPTIONS]: false,
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,
@@ -64,7 +65,6 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
-  [FieldType.ARRAY]: false,
   [FieldType.LINK]: false,
   [FieldType.BB_REFERENCE]: false,
   [FieldType.BB_REFERENCE_SINGLE]: false,

packages/shared-core/src/utils.ts

@@ -1,5 +1,5 @@
 import {
-  SearchFilter,
+  LegacyFilter,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
   SearchFilters,
@@ -9,6 +9,10 @@ import {
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
 
+// an array of keys from filter type to properties that are in the type
+// this can then be converted using .fromEntries to an object
+type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+
 export function unreachable(
   value: never,
   message = `No such case in exhaustive switch: ${value}`
@@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to SearchFilterGroup
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {SearchFilter[] | SearchFilterGroup} filters
- */
-export const processSearchFilters = (
-  filters: SearchFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
-  }
-
-  // Base search config.
-  const defaultCfg: SearchFilterGroup = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
-    groups: [],
-  }
-
-  const filterWhitelistKeys = [
-    "field",
-    "operator",
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  if (Array.isArray(filters)) {
-    let baseGroup: SearchFilterGroup = {
-      filters: [],
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-    }
-
-    const migratedSetting: SearchFilterGroup = filters.reduce(
-      (acc: SearchFilterGroup, filter: SearchFilter) => {
-        // Sort the properties for easier debugging
-        const filterEntries = Object.entries(filter)
-          .sort((a, b) => {
-            return a[0].localeCompare(b[0])
-          })
-          .filter(x => x[1] ?? false)
-
-        if (filterEntries.length == 1) {
-          const [key, value] = filterEntries[0]
-          // Global
-          if (key === "onEmptyFilter") {
-            // unset otherwise
-            acc.onEmptyFilter = value
-          } else if (key === "operator" && value === "allOr") {
-            // Group 1 logical operator
-            baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-          }
-
-          return acc
-        }
-
-        const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
-          (acc: [string, any][], entry: [string, any]) => {
-            const [key, value] = entry
-
-            if (filterWhitelistKeys.includes(key)) {
-              if (key === "field") {
-                acc.push([key, removeKeyNumbering(value)])
-              } else {
-                acc.push([key, value])
-              }
-            }
-            return acc
-          },
-          []
-        )
-
-        const migratedFilter: SearchFilter = Object.fromEntries(
-          whiteListedFilterSettings
-        ) as SearchFilter
-
-        baseGroup.filters!.push(migratedFilter)
-
-        if (!acc.groups || !acc.groups.length) {
-          // init the base group
-          acc.groups = [baseGroup]
-        }
-
-        return acc
-      },
-      defaultCfg
-    )
-
-    return migratedSetting
-  } else if (!filters?.groups) {
-    return
-  }
-  return filters
-}
-
 export function isSupportedUserSearch(query: SearchFilters) {
   const allowed = [
     { op: BasicOperator.STRING, key: "email" },
@@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
   }
   return true
 }
+
+/**
+ * Processes the filter config. Filters are migrated from
+ * SearchFilter[] to SearchFilterGroup
+ *
+ * If filters is not an array, the migration is skipped
+ *
+ * @param {LegacyFilter[] | SearchFilterGroup} filters
+ */
+export const processSearchFilters = (
+  filters: LegacyFilter[] | SearchFilterGroup | undefined
+): SearchFilterGroup | undefined => {
+  if (!filters) {
+    return
+  }
+
+  // Base search config.
+  const defaultCfg: SearchFilterGroup = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    groups: [],
+  }
+
+  const filterAllowedKeys = [
+    "field",
+    "operator",
+    "value",
+    "type",
+    "externalType",
+    "valueType",
+    "noValue",
+    "formulaType",
+  ]
+
+  if (Array.isArray(filters)) {
+    let baseGroup: SearchFilterGroup = {
+      filters: [],
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+    }
+
+    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+      // Sort the properties for easier debugging
+      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+        .sort((a, b) => {
+          return a.localeCompare(b)
+        })
+        .filter(key => key in filter)
+
+      if (filterPropertyKeys.length == 1) {
+        const key = filterPropertyKeys[0],
+          value = filter[key]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+        }
+
+        return acc
+      }
+
+      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
+        (acc: AllowedFilters, key) => {
+          const value = filter[key]
+          if (filterAllowedKeys.includes(key)) {
+            if (key === "field") {
+              acc.push([key, removeKeyNumbering(value)])
+            } else {
+              acc.push([key, value])
+            }
+          }
+          return acc
+        },
+        []
+      )
+
+      const migratedFilter: LegacyFilter = Object.fromEntries(
+        allowedFilterSettings
+      ) as LegacyFilter
+
+      baseGroup.filters!.push(migratedFilter)
+
+      if (!acc.groups || !acc.groups.length) {
+        // init the base group
+        acc.groups = [baseGroup]
+      }
+
+      return acc
+    }, defaultCfg)
+  } else if (!filters?.groups) {
+    return
+  }
+  return filters
+}

packages/types/src/api/web/searchFilter.ts

@@ -5,7 +5,7 @@ import {
   SearchFilters,
 } from "../../sdk"
 
-export type SearchFilter = {
+export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
   field: string
@@ -14,9 +14,10 @@ export type SearchFilter = {
   externalType?: string
 }
 
+// this is a type purely used by the UI
 export type SearchFilterGroup = {
   logicalOperator: FilterGroupLogicalOperator
   onEmptyFilter?: EmptyFilterOption
   groups?: SearchFilterGroup[]
-  filters?: SearchFilter[]
+  filters?: LegacyFilter[]
 }

packages/types/src/documents/app/table/schema.ts

@@ -161,6 +161,7 @@ export interface OptionsFieldMetadata extends BaseFieldSchema {
   constraints: FieldConstraints & {
     inclusion: string[]
   }
+  default?: string
 }
 
 export interface ArrayFieldMetadata extends BaseFieldSchema {
@@ -169,6 +170,7 @@ export interface ArrayFieldMetadata extends BaseFieldSchema {
     type: JsonFieldSubType.ARRAY
     inclusion: string[]
   }
+  default?: string[]
 }
 
 interface BaseFieldSchema extends UIFieldMetadata {

packages/types/src/documents/app/view.ts

@@ -1,7 +1,7 @@
-import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
-import { DBView } from "../../sdk"
+import { DBView, SearchFilters } from "../../sdk"
 
 export type ViewTemplateOpts = {
   field: string
@@ -65,15 +65,19 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[] | SearchFilterGroup
+  query?: LegacyFilter[] | SearchFilters
+  // duplicate to store UI information about filters
+  queryUI?: SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder
     type?: SortType
   }
-  schema?: Record<string, ViewFieldMetadata>
+  schema?: ViewV2Schema
 }
 
+export type ViewV2Schema = Record<string, ViewFieldMetadata>
+
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
 
 export interface ViewCountOrSumSchema {