Merge branch 'master' of github.com:Budibase/budibase into v3-ui

.github/workflows/deploy-featurebranch.yml

@@ -23,6 +23,7 @@ jobs:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
           PAYLOAD_LICENSE_TYPE: "free"
+          PAYLOAD_DEPLOY: "true"
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.10",
+  "version": "2.32.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/index.ts

@@ -1,5 +1,5 @@
 export * as utils from "./utils"
 
-export { default as Sql } from "./sql"
+export { default as Sql, COUNT_FIELD_NAME } from "./sql"
 export { default as SqlTable } from "./sqlTable"
 export * as designDoc from "./designDoc"

packages/backend-core/src/sql/sql.ts

@@ -43,6 +43,8 @@ import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
+export const COUNT_FIELD_NAME = "__bb_total"
+
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
     ? parseInt(environment.SQL_MAX_ROWS)
@@ -846,7 +848,7 @@ class InternalBuilder {
       throw new Error("SQL counting requires primary key to be supplied")
     }
     return query.countDistinct(
-      `${this.getTableName()}.${this.table.primary[0]} as __bb_total`
+      `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}`
     )
   }
 

packages/client/src/components/app/forms/Form.svelte

@@ -63,7 +63,7 @@
     // Look up the component tree and find something that is provided by an
     // ancestor that matches our datasource. This is for backwards compatibility
     // as previously we could use the "closest" context.
-    for (let id of path.reverse().slice(1)) {
+    for (let id of path.toReversed().slice(1)) {
       // Check for matching view datasource
       if (
         dataSource.type === "viewV2" &&

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -124,9 +124,11 @@ export async function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === columnName)
+          !existing.find(
+            (field: string) => field === `${table.name}.${columnName}`
           )
-      .map(column => `${table.name}.${column[0]}`)
+      )
+      .map(([columnName]) => `${table.name}.${columnName}`)
   }
 
   let fields: string[] = []

packages/server/src/api/routes/tests/row.spec.ts

@@ -695,6 +695,69 @@ describe.each([
         })
       })
 
+      describe("options column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                status: {
+                  name: "status",
+                  type: FieldType.OPTIONS,
+                  default: "requested",
+                  constraints: {
+                    inclusion: ["requested", "approved"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.status).toEqual("requested")
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            status: "approved",
+          })
+          expect(row.status).toEqual("approved")
+        })
+      })
+
+      describe("array column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                food: {
+                  name: "food",
+                  type: FieldType.ARRAY,
+                  default: ["apple", "orange"],
+                  constraints: {
+                    type: JsonFieldSubType.ARRAY,
+                    inclusion: ["apple", "orange", "banana"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.food).toEqual(["apple", "orange"])
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            food: ["orange"],
+          })
+          expect(row.food).toEqual(["orange"])
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {

packages/server/src/api/routes/tests/search.spec.ts

@@ -28,11 +28,13 @@ import {
   RowSearchParams,
   SearchFilters,
   SearchResponse,
+  SearchRowRequest,
   SortOrder,
   SortType,
   Table,
   TableSchema,
   User,
+  ViewV2Schema,
 } from "@budibase/types"
 import _ from "lodash"
 import tk from "timekeeper"
@@ -64,18 +66,14 @@ describe.each([
   let envCleanup: (() => void) | undefined
   let datasource: Datasource | undefined
   let client: Knex | undefined
-  let table: Table
+  let tableOrViewId: string
   let rows: Row[]
 
   async function basicRelationshipTables(type: RelationshipType) {
-    const relatedTable = await createTable(
+    const relatedTable = await createTable({
-      {
       name: { name: "name", type: FieldType.STRING },
-      },
+    })
-      generator.guid().substring(0, 10)
+    const tableId = await createTable({
-    )
-    table = await createTable(
-      {
       name: { name: "name", type: FieldType.STRING },
       //@ts-ignore - API accepts this structure, will build out rest of definition
       productCat: {
@@ -83,17 +81,15 @@ describe.each([
         relationshipType: type,
         name: "productCat",
         fieldName: "product",
-          tableId: relatedTable._id!,
+        tableId: relatedTable,
         constraints: {
           type: "array",
         },
       },
-      },
+    })
-      generator.guid().substring(0, 10)
-    )
     return {
-      relatedTable: await config.api.table.get(relatedTable._id!),
+      relatedTable: await config.api.table.get(relatedTable),
-      table,
+      tableId,
     }
   }
 
@@ -136,32 +132,69 @@ describe.each([
     }
   })
 
-  async function createTable(schema: TableSchema, name?: string) {
+  async function createTable(schema: TableSchema) {
-    return await config.api.table.save(
+    const table = await config.api.table.save(
-      tableForDatasource(datasource, { schema, name })
+      tableForDatasource(datasource, { schema })
     )
+    return table._id!
+  }
+
+  async function createView(tableId: string, schema: ViewV2Schema) {
+    const view = await config.api.viewV2.create({
+      tableId: tableId,
+      name: generator.guid(),
+      schema,
+    })
+    return view.id
   }
 
   async function createRows(arr: Record<string, any>[]) {
     // Shuffling to avoid false positives given a fixed order
-    await config.api.row.bulkImport(table._id!, {
+    for (const row of _.shuffle(arr)) {
-      rows: _.shuffle(arr),
+      await config.api.row.save(tableOrViewId, row)
-    })
+    }
-    rows = await config.api.row.fetch(table._id!)
+    rows = await config.api.row.fetch(tableOrViewId)
+  }
+
+  describe.each([
+    ["table", createTable],
+    [
+      "view",
+      async (schema: TableSchema) => {
+        const tableId = await createTable(schema)
+        const viewId = await createView(
+          tableId,
+          Object.keys(schema).reduce<ViewV2Schema>((viewSchema, fieldName) => {
+            const field = schema[fieldName]
+            viewSchema[fieldName] = {
+              visible: field.visible ?? true,
+              readonly: false,
+            }
+            return viewSchema
+          }, {})
+        )
+        return viewId
+      },
+    ],
+  ])("from %s", (sourceType, createTableOrView) => {
+    const isView = sourceType === "view"
+
+    if (isView && isLucene) {
+      // Some tests don't have the expected result in views via lucene, and given that it is getting deprecated, we exclude them from the tests
+      return
     }
 
     class SearchAssertion {
-    constructor(private readonly query: RowSearchParams) {}
+      constructor(private readonly query: SearchRowRequest) {}
 
       private async performSearch(): Promise<SearchResponse<Row>> {
         if (isInMemory) {
-        return dataFilters.search(_.cloneDeep(rows), this.query)
+          return dataFilters.search(_.cloneDeep(rows), {
+            ...this.query,
+            tableId: tableOrViewId,
+          })
         } else {
-        const sourceId = this.query.viewId || this.query.tableId
+          return config.api.row.search(tableOrViewId, this.query)
-        if (!sourceId) {
-          throw new Error("No source ID provided")
-        }
-        return config.api.row.search(sourceId, this.query)
         }
       }
 
@@ -331,8 +364,8 @@ describe.each([
       }
     }
 
-  function expectSearch(query: Omit<RowSearchParams, "tableId">) {
+    function expectSearch(query: SearchRowRequest) {
-    return new SearchAssertion({ ...query, tableId: table._id! })
+      return new SearchAssertion(query)
     }
 
     function expectQuery(query: SearchFilters) {
@@ -341,7 +374,7 @@ describe.each([
 
     describe("boolean", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
         })
         await createRows([{ isTrue: true }, { isTrue: false }])
@@ -429,38 +462,35 @@ describe.each([
             { name: "serverDate", appointment: serverTime.toISOString() },
             {
               name: "single user, session user",
-            single_user: JSON.stringify(currentUser),
+              single_user: currentUser,
             },
             {
               name: "single user",
-            single_user: JSON.stringify(globalUsers[0]),
+              single_user: globalUsers[0],
             },
             {
               name: "deprecated single user, session user",
-            deprecated_single_user: JSON.stringify([currentUser]),
+              deprecated_single_user: [currentUser],
             },
             {
               name: "deprecated single user",
-            deprecated_single_user: JSON.stringify([globalUsers[0]]),
+              deprecated_single_user: [globalUsers[0]],
             },
             {
               name: "multi user",
-            multi_user: JSON.stringify(globalUsers),
+              multi_user: globalUsers,
             },
             {
               name: "multi user with session user",
-            multi_user: JSON.stringify([...globalUsers, currentUser]),
+              multi_user: [...globalUsers, currentUser],
             },
             {
               name: "deprecated multi user",
-            deprecated_multi_user: JSON.stringify(globalUsers),
+              deprecated_multi_user: globalUsers,
             },
             {
               name: "deprecated multi user with session user",
-            deprecated_multi_user: JSON.stringify([
+              deprecated_multi_user: [...globalUsers, currentUser],
-              ...globalUsers,
-              currentUser,
-            ]),
             },
           ]
         }
@@ -482,7 +512,7 @@ describe.each([
               })
           )
 
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: { name: "name", type: FieldType.STRING },
             appointment: { name: "appointment", type: FieldType.DATETIME },
             single_user: {
@@ -632,9 +662,11 @@ describe.each([
         })
 
         it("should match the session user id in a multi user field", async () => {
-        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          const allUsers = [...globalUsers, config.getUser()].map(
+            (user: any) => {
               return { _id: user._id }
-        })
+            }
+          )
 
           await expectQuery({
             contains: { multi_user: ["{{ [user]._id }}"] },
@@ -647,9 +679,11 @@ describe.each([
         })
 
         it("should match the session user id in a deprecated multi user field", async () => {
-        const allUsers = [...globalUsers, config.getUser()].map((user: any) => {
+          const allUsers = [...globalUsers, config.getUser()].map(
+            (user: any) => {
               return { _id: user._id }
-        })
+            }
+          )
 
           await expectQuery({
             contains: { deprecated_multi_user: ["{{ [user]._id }}"] },
@@ -764,7 +798,7 @@ describe.each([
 
     describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           name: { name: "name", type: FieldType.STRING },
         })
         await createRows([{ name: "foo" }, { name: "bar" }])
@@ -791,6 +825,8 @@ describe.each([
           }).toContainExactly([{ name: "foo" }, { name: "bar" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("should return nothing if onEmptyFilter is RETURN_NONE", async () => {
             await expectQuery({
               onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@@ -891,6 +927,8 @@ describe.each([
           }).toContainExactly([{ name: "foo" }, { name: "bar" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
             await expectQuery({
               onEmptyFilter: EmptyFilterOption.RETURN_NONE,
@@ -1055,7 +1093,7 @@ describe.each([
                 datasourceId: datasource!._id!,
               })
 
-            table = resp.datasource.entities![tableName]
+              tableOrViewId = resp.datasource.entities![tableName]._id!
 
               await createRows([{ name: "foo" }, { name: "bar" }])
             })
@@ -1079,7 +1117,7 @@ describe.each([
 
     describe("numbers", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           age: { name: "age", type: FieldType.NUMBER },
         })
         await createRows([{ age: 1 }, { age: 10 }])
@@ -1087,7 +1125,9 @@ describe.each([
 
       describe("equal", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])
+          await expectQuery({ equal: { age: 1 } }).toContainExactly([
+            { age: 1 },
+          ])
         })
 
         it("fails to find nonexistent row", async () => {
@@ -1252,7 +1292,7 @@ describe.each([
       const JAN_10TH = "2020-01-10T00:00:00.000Z"
 
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           dob: { name: "dob", type: FieldType.DATETIME },
         })
 
@@ -1324,25 +1364,33 @@ describe.each([
 
         it("greater than equal to", async () => {
           await expectQuery({
-          range: { dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() } },
+            range: {
+              dob: { low: JAN_10TH, high: MAX_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_10TH }])
         })
 
         it("greater than", async () => {
           await expectQuery({
-          range: { dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() } },
+            range: {
+              dob: { low: JAN_5TH, high: MAX_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_10TH }])
         })
 
         it("less than equal to", async () => {
           await expectQuery({
-          range: { dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() } },
+            range: {
+              dob: { high: JAN_1ST, low: MIN_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_1ST }])
         })
 
         it("less than", async () => {
           await expectQuery({
-          range: { dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() } },
+            range: {
+              dob: { high: JAN_5TH, low: MIN_VALID_DATE.toISOString() },
+            },
           }).toContainExactly([{ dob: JAN_1ST }])
         })
       })
@@ -1399,7 +1447,7 @@ describe.each([
         const NULL_TIME__ID = `null_time__id`
 
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             timeid: { name: "timeid", type: FieldType.STRING },
             time: { name: "time", type: FieldType.DATETIME, timeOnly: true },
           })
@@ -1560,7 +1608,7 @@ describe.each([
 
     describe.each([FieldType.ARRAY, FieldType.OPTIONS])("%s", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           numbers: {
             name: "numbers",
             type: FieldType.ARRAY,
@@ -1575,9 +1623,9 @@ describe.each([
 
       describe("contains", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([
+          await expectQuery({
-          { numbers: ["one", "two"] },
+            contains: { numbers: ["one"] },
-        ])
+          }).toContainExactly([{ numbers: ["one", "two"] }])
         })
 
         it("fails to find nonexistent row", async () => {
@@ -1657,7 +1705,7 @@ describe.each([
       let BIG = "9223372036854775807"
 
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           num: { name: "num", type: FieldType.BIGINT },
         })
         await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
@@ -1762,7 +1810,7 @@ describe.each([
     isInternal &&
       describe("auto", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             auto: {
               name: "auto",
               type: FieldType.AUTO,
@@ -1913,9 +1961,9 @@ describe.each([
             // to specify an order for pagination to work.
             it("is stable without a sort specified", async () => {
               let { rows: fullRowList } = await config.api.row.search(
-              table._id!,
+                tableOrViewId,
                 {
-                tableId: table._id!,
+                  tableId: tableOrViewId,
                   query: {},
                 }
               )
@@ -1925,8 +1973,8 @@ describe.each([
                 hasNextPage: boolean | undefined = true,
                 rowCount: number = 0
               do {
-              const response = await config.api.row.search(table._id!, {
+                const response = await config.api.row.search(tableOrViewId, {
-                tableId: table._id!,
+                  tableId: tableOrViewId,
                   limit: 1,
                   paginate: true,
                   query: {},
@@ -1949,8 +1997,8 @@ describe.each([
 
             // eslint-disable-next-line no-constant-condition
             while (true) {
-            const response = await config.api.row.search(table._id!, {
+              const response = await config.api.row.search(tableOrViewId, {
-              tableId: table._id!,
+                tableId: tableOrViewId,
                 limit: 3,
                 query: {},
                 bookmark,
@@ -1973,7 +2021,7 @@ describe.each([
 
     describe("field name 1:name", () => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           "1:name": { name: "1:name", type: FieldType.STRING },
         })
         await createRows([{ "1:name": "bar" }, { "1:name": "foo" }])
@@ -1993,8 +2041,7 @@ describe.each([
     isSql &&
       describe("related formulas", () => {
         beforeAll(async () => {
-        const arrayTable = await createTable(
+          const arrayTable = await createTable({
-          {
             name: { name: "name", type: FieldType.STRING },
             array: {
               name: "array",
@@ -2004,17 +2051,14 @@ describe.each([
                 inclusion: ["option 1", "option 2"],
               },
             },
-          },
+          })
-          "array"
+          tableOrViewId = await createTableOrView({
-        )
-        table = await createTable(
-          {
             relationship: {
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_ONE,
               name: "relationship",
               fieldName: "relate",
-              tableId: arrayTable._id!,
+              tableId: arrayTable,
               constraints: {
                 type: "array",
               },
@@ -2026,21 +2070,19 @@ describe.each([
                 `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
               ),
             },
-          },
+          })
-          "main"
-        )
           const arrayRows = await Promise.all([
-          config.api.row.save(arrayTable._id!, {
+            config.api.row.save(arrayTable, {
               name: "foo",
               array: ["option 1"],
             }),
-          config.api.row.save(arrayTable._id!, {
+            config.api.row.save(arrayTable, {
               name: "bar",
               array: ["option 2"],
             }),
           ])
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               relationship: [arrayRows[0]._id, arrayRows[1]._id],
             }),
           ])
@@ -2059,7 +2101,7 @@ describe.each([
         user1 = await config.createUser({ _id: `us_${utils.newid()}` })
         user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           user: {
             name: "user",
             type: FieldType.BB_REFERENCE_SINGLE,
@@ -2067,11 +2109,7 @@ describe.each([
           },
         })
 
-      await createRows([
+        await createRows([{ user: user1 }, { user: user2 }, { user: null }])
-        { user: JSON.stringify(user1) },
-        { user: JSON.stringify(user2) },
-        { user: null },
-      ])
       })
 
       describe("equal", () => {
@@ -2088,18 +2126,15 @@ describe.each([
 
       describe("notEqual", () => {
         it("successfully finds a row", async () => {
-        await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([
+          await expectQuery({ notEqual: { user: user1._id } }).toContainExactly(
-          { user: { _id: user2._id } },
+            [{ user: { _id: user2._id } }, {}]
-          {},
+          )
-        ])
         })
 
         it("fails to find nonexistent row", async () => {
-        await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([
+          await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly(
-          { user: { _id: user1._id } },
+            [{ user: { _id: user1._id } }, { user: { _id: user2._id } }, {}]
-          { user: { _id: user2._id } },
+          )
-          {},
-        ])
         })
       })
 
@@ -2139,7 +2174,7 @@ describe.each([
         user1 = await config.createUser({ _id: `us_${utils.newid()}` })
         user2 = await config.createUser({ _id: `us_${utils.newid()}` })
 
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           users: {
             name: "users",
             type: FieldType.BB_REFERENCE,
@@ -2153,10 +2188,10 @@ describe.each([
         })
 
         await createRows([
-        { number: 1, users: JSON.stringify([user1]) },
+          { number: 1, users: [user1] },
-        { number: 2, users: JSON.stringify([user2]) },
+          { number: 2, users: [user2] },
-        { number: 3, users: JSON.stringify([user1, user2]) },
+          { number: 3, users: [user1, user2] },
-        { number: 4, users: JSON.stringify([]) },
+          { number: 4, users: [] },
         ])
       })
 
@@ -2182,7 +2217,9 @@ describe.each([
         })
 
         it("fails to find nonexistent row", async () => {
-        await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()
+          await expectQuery({
+            contains: { users: ["us_none"] },
+          }).toFindNothing()
         })
       })
 
@@ -2249,9 +2286,10 @@ describe.each([
         let productCategoryTable: Table, productCatRows: Row[]
 
         beforeAll(async () => {
-        const { relatedTable } = await basicRelationshipTables(
+          const { relatedTable, tableId } = await basicRelationshipTables(
             RelationshipType.ONE_TO_MANY
           )
+          tableOrViewId = tableId
           productCategoryTable = relatedTable
 
           productCatRows = await Promise.all([
@@ -2260,15 +2298,15 @@ describe.each([
           ])
 
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "foo",
               productCat: [productCatRows[0]._id],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "bar",
               productCat: [productCatRows[1]._id],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "baz",
               productCat: [],
             }),
@@ -2301,10 +2339,11 @@ describe.each([
     isSql &&
       describe("big relations", () => {
         beforeAll(async () => {
-        const { relatedTable } = await basicRelationshipTables(
+          const { relatedTable, tableId } = await basicRelationshipTables(
             RelationshipType.MANY_TO_ONE
           )
-        const mainRow = await config.api.row.save(table._id!, {
+          tableOrViewId = tableId
+          const mainRow = await config.api.row.save(tableOrViewId, {
             name: "foo",
           })
           for (let i = 0; i < 11; i++) {
@@ -2329,45 +2368,42 @@ describe.each([
       })
     ;(isSqs || isLucene) &&
       describe("relations to same table", () => {
-      let relatedTable: Table, relatedRows: Row[]
+        let relatedTable: string, relatedRows: Row[]
 
         beforeAll(async () => {
-        relatedTable = await createTable(
+          relatedTable = await createTable({
-          {
             name: { name: "name", type: FieldType.STRING },
-          },
+          })
-          "productCategory"
+          tableOrViewId = await createTableOrView({
-        )
-        table = await createTable({
             name: { name: "name", type: FieldType.STRING },
             related1: {
               type: FieldType.LINK,
               name: "related1",
               fieldName: "main1",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
             related2: {
               type: FieldType.LINK,
               name: "related2",
               fieldName: "main2",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
           })
           relatedRows = await Promise.all([
-          config.api.row.save(relatedTable._id!, { name: "foo" }),
+            config.api.row.save(relatedTable, { name: "foo" }),
-          config.api.row.save(relatedTable._id!, { name: "bar" }),
+            config.api.row.save(relatedTable, { name: "bar" }),
-          config.api.row.save(relatedTable._id!, { name: "baz" }),
+            config.api.row.save(relatedTable, { name: "baz" }),
-          config.api.row.save(relatedTable._id!, { name: "boo" }),
+            config.api.row.save(relatedTable, { name: "boo" }),
           ])
           await Promise.all([
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "test",
               related1: [relatedRows[0]._id!],
               related2: [relatedRows[1]._id!],
             }),
-          config.api.row.save(table._id!, {
+            config.api.row.save(tableOrViewId, {
               name: "test2",
               related1: [relatedRows[2]._id!],
               related2: [relatedRows[3]._id!],
@@ -2430,7 +2466,7 @@ describe.each([
     isInternal &&
       describe("no column error backwards compat", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2453,7 +2489,7 @@ describe.each([
     !isLucene &&
       describe("row counting", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2488,7 +2524,7 @@ describe.each([
     describe("Invalid column definitions", () => {
       beforeAll(async () => {
         // need to create an invalid table - means ignoring typescript
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           // @ts-ignore
           invalid: {
             type: FieldType.STRING,
@@ -2518,7 +2554,7 @@ describe.each([
       "special (%s) case",
       column => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             [column]: {
               name: column,
               type: FieldType.STRING,
@@ -2543,8 +2579,8 @@ describe.each([
       describe("sample data", () => {
         beforeAll(async () => {
           await config.api.application.addSampleData(config.appId!)
-        table = DEFAULT_EMPLOYEE_TABLE_SCHEMA
+          tableOrViewId = DEFAULT_EMPLOYEE_TABLE_SCHEMA._id!
-        rows = await config.api.row.fetch(table._id!)
+          rows = await config.api.row.fetch(tableOrViewId)
         })
 
         it("should be able to search sample data", async () => {
@@ -2567,7 +2603,7 @@ describe.each([
       const earlyDate = "2024-07-03T10:00:00.000Z",
         laterDate = "2024-07-03T11:00:00.000Z"
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           date: {
             name: "date",
             type: FieldType.DATETIME,
@@ -2610,7 +2646,7 @@ describe.each([
       "ชื่อผู้ใช้", // Thai for "username"
     ])("non-ascii column name: %s", name => {
       beforeAll(async () => {
-      table = await createTable({
+        tableOrViewId = await createTableOrView({
           [name]: {
             name,
             type: FieldType.STRING,
@@ -2637,7 +2673,7 @@ describe.each([
     isInternal &&
       describe("space at end of column name", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             "name ": {
               name: "name ",
               type: FieldType.STRING,
@@ -2672,7 +2708,7 @@ describe.each([
     ;(isSqs || isInMemory) &&
       describe("space at start of column name", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             " name": {
               name: " name",
               type: FieldType.STRING,
@@ -2703,9 +2739,10 @@ describe.each([
       })
 
     isSqs &&
+      !isView &&
       describe("duplicate columns", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2713,7 +2750,7 @@ describe.each([
           })
           await context.doInAppContext(config.getAppId(), async () => {
             const db = context.getAppDB()
-          const tableDoc = await db.get<Table>(table._id!)
+            const tableDoc = await db.get<Table>(tableOrViewId)
             tableDoc.schema.Name = {
               name: "Name",
               type: FieldType.STRING,
@@ -2747,7 +2784,7 @@ describe.each([
               type: FieldType.STRING,
             },
           })
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2756,15 +2793,15 @@ describe.each([
               name: "rel",
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_MANY,
-            tableId: toRelateTable._id!,
+              tableId: toRelateTable,
               fieldName: "rel",
             },
           })
           const [row1, row2] = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
+            config.api.row.save(toRelateTable, { name: "tag 1" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
+            config.api.row.save(toRelateTable, { name: "tag 2" }),
           ])
-        row = await config.api.row.save(table._id!, {
+          row = await config.api.row.save(tableOrViewId, {
             name: "product 1",
             rel: [row1._id, row2._id],
           })
@@ -2783,7 +2820,7 @@ describe.each([
     !isInternal &&
       describe("search by composite key", () => {
         beforeAll(async () => {
-        table = await config.api.table.save(
+          const table = await config.api.table.save(
             tableForDatasource(datasource, {
               schema: {
                 idColumn1: {
@@ -2798,6 +2835,7 @@ describe.each([
               primary: ["idColumn1", "idColumn2"],
             })
           )
+          tableOrViewId = table._id!
           await createRows([{ idColumn1: 1, idColumn2: 2 }])
         })
 
@@ -2819,15 +2857,13 @@ describe.each([
     isSql &&
       describe("primaryDisplay", () => {
         beforeAll(async () => {
-        let toRelateTable = await createTable({
+          let toRelateTableId = await createTable({
             name: {
               name: "name",
               type: FieldType.STRING,
             },
           })
-        table = await config.api.table.save(
+          tableOrViewId = await createTableOrView({
-          tableForDatasource(datasource, {
-            schema: {
             name: {
               name: "name",
               type: FieldType.STRING,
@@ -2836,33 +2872,30 @@ describe.each([
               name: "link",
               type: FieldType.LINK,
               relationshipType: RelationshipType.MANY_TO_ONE,
-                tableId: toRelateTable._id!,
+              tableId: toRelateTableId,
               fieldName: "link",
             },
-            },
           })
-        )
+
-        toRelateTable = await config.api.table.get(toRelateTable._id!)
+          const toRelateTable = await config.api.table.get(toRelateTableId)
           await config.api.table.save({
             ...toRelateTable,
             primaryDisplay: "link",
           })
           const relatedRows = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "test" }),
+            config.api.row.save(toRelateTable._id!, { name: "related" }),
           ])
-        await Promise.all([
+          await config.api.row.save(tableOrViewId, {
-          config.api.row.save(table._id!, {
             name: "test",
             link: relatedRows.map(row => row._id),
-          }),
+          })
-        ])
         })
 
         it("should be able to query, primary display on related table shouldn't be used", async () => {
           // this test makes sure that if a relationship has been specified as the primary display on a table
           // it is ignored and another column is used instead
           await expectQuery({}).toContain([
-          { name: "test", link: [{ primaryDisplay: "test" }] },
+            { name: "test", link: [{ primaryDisplay: "related" }] },
           ])
         })
       })
@@ -2870,7 +2903,7 @@ describe.each([
     !isLucene &&
       describe("$and", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             age: { name: "age", type: FieldType.NUMBER },
             name: { name: "name", type: FieldType.STRING },
           })
@@ -2945,7 +2978,10 @@ describe.each([
             await expect(
               expectQuery({
                 $and: {
-                conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
+                  conditions: [
+                    { equal: { age: 10 } },
+                    "invalidCondition" as any,
+                  ],
                 },
               }).toFindNothing()
             ).rejects.toThrow(
@@ -2973,6 +3009,8 @@ describe.each([
             )
           })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("returns no rows when onEmptyFilter set to none", async () => {
             await expectSearch({
               query: {
@@ -2999,7 +3037,7 @@ describe.each([
     !isLucene &&
       describe("$or", () => {
         beforeAll(async () => {
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             age: { name: "age", type: FieldType.NUMBER },
             name: { name: "name", type: FieldType.STRING },
           })
@@ -3123,6 +3161,8 @@ describe.each([
           }).toContainExactly([{ age: 1, name: "Jane" }])
         })
 
+        // onEmptyFilter cannot be sent to view searches
+        !isView &&
           it("returns no rows when onEmptyFilter set to none", async () => {
             await expectSearch({
               query: {
@@ -3159,20 +3199,20 @@ describe.each([
             row[name] = i
           }
           const relatedTable = await createTable(relatedSchema)
-        table = await createTable({
+          tableOrViewId = await createTableOrView({
             name: { name: "name", type: FieldType.STRING },
             related1: {
               type: FieldType.LINK,
               name: "related1",
               fieldName: "main1",
-            tableId: relatedTable._id!,
+              tableId: relatedTable,
               relationshipType: RelationshipType.MANY_TO_MANY,
             },
           })
           relatedRows = await Promise.all([
-          config.api.row.save(relatedTable._id!, row),
+            config.api.row.save(relatedTable, row),
           ])
-        await config.api.row.save(table._id!, {
+          await config.api.row.save(tableOrViewId, {
             name: "foo",
             related1: [relatedRows[0]._id],
           })
@@ -3187,4 +3227,5 @@ describe.each([
           ])
         })
       })
+  })
 })

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,9 +22,10 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  ViewFieldMetadata,
   FeatureFlag,
   BBReferenceFieldSubType,
+  ViewV2Schema,
+  ViewCalculationFieldMetadata,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -540,6 +541,33 @@ describe.each([
           status: 201,
         })
       })
+
+      it("can create a view with calculation fields", async () => {
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "Price",
+            },
+          },
+        })
+
+        expect(Object.keys(view.schema!)).toHaveLength(1)
+
+        let sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+
+        view = await config.api.viewV2.get(view.id)
+        sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+      })
     })
 
     describe("update", () => {
@@ -1152,10 +1180,7 @@ describe.each([
           return table
         }
 
-        const createView = async (
+        const createView = async (tableId: string, schema: ViewV2Schema) =>
-          tableId: string,
-          schema: Record<string, ViewFieldMetadata>
-        ) =>
           await config.api.viewV2.create({
             name: generator.guid(),
             tableId,
@@ -2546,6 +2571,51 @@ describe.each([
             }
           })
         })
+
+      !isLucene &&
+        it("should not need required fields to be present", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                name: {
+                  name: "name",
+                  type: FieldType.STRING,
+                  constraints: {
+                    presence: true,
+                  },
+                },
+                age: {
+                  name: "age",
+                  type: FieldType.NUMBER,
+                },
+              },
+            })
+          )
+
+          await Promise.all([
+            config.api.row.save(table._id!, { name: "Steve", age: 30 }),
+            config.api.row.save(table._id!, { name: "Jane", age: 31 }),
+          ])
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "age",
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {},
+          })
+
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows[0].sum).toEqual(61)
+        })
     })
 
     describe("permissions", () => {

packages/server/src/db/linkedRows/index.ts

@@ -23,8 +23,8 @@ import {
   Row,
   Table,
   TableSchema,
-  ViewFieldMetadata,
   ViewV2,
+  ViewV2Schema,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { helpers } from "@budibase/shared-core"
@@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
     FeatureFlag.ENRICHED_RELATIONSHIPS
   )
 
-  let viewSchema: Record<string, ViewFieldMetadata> = {}
+  let viewSchema: ViewV2Schema = {}
   if (sdk.views.isView(source)) {
     if (helpers.views.isCalculationView(source)) {
       return enriched

packages/server/src/sdk/app/rows/search.ts

@@ -96,6 +96,8 @@ export async function search(
         const query: SearchFilters = viewQuery || {}
         const viewFilters = view.query as SearchFilter[]
 
+        delete options.query.onEmptyFilter
+
         // Extract existing fields
         const existingFields =
           viewFilters
@@ -118,6 +120,9 @@ export async function search(
             conditions: [viewQuery as SearchFilterGroup, options.query],
           },
         }
+        if (viewQuery.onEmptyFilter) {
+          options.query.onEmptyFilter = viewQuery.onEmptyFilter
+        }
       }
     }
 

packages/server/src/sdk/app/rows/utils.ts

@@ -20,7 +20,7 @@ import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
-import { docIds } from "@budibase/backend-core"
+import { docIds, sql } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
@@ -57,8 +57,12 @@ export function getSQLClient(datasource: Datasource): SqlClient {
 export function processRowCountResponse(
   response: DatasourcePlusQueryResponse
 ): number {
-  if (response && response.length === 1 && "__bb_total" in response[0]) {
+  if (
-    const total = response[0].__bb_total
+    response &&
+    response.length === 1 &&
+    sql.COUNT_FIELD_NAME in response[0]
+  ) {
+    const total = response[0][sql.COUNT_FIELD_NAME]
     return typeof total === "number" ? total : parseInt(total)
   } else {
     throw new Error("Unable to count rows in query - no count response")

packages/server/src/sdk/app/views/index.ts

@@ -255,19 +255,12 @@ export async function enrichSchema(
   view: ViewV2,
   tableSchema: TableSchema
 ): Promise<ViewV2Enriched> {
-  const tableCache: Record<string, Table> = {}
-
   async function populateRelTableSchema(
     tableId: string,
     viewFields: Record<string, RelationSchemaField>
   ) {
-    if (!tableCache[tableId]) {
+    const relTable = await sdk.tables.getTable(tableId)
-      tableCache[tableId] = await sdk.tables.getTable(tableId)
-    }
-    const relTable = tableCache[tableId]
-
     const result: Record<string, ViewV2ColumnEnriched> = {}
-
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
       if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
@@ -296,15 +289,24 @@ export async function enrichSchema(
 
   const viewSchema = view.schema || {}
   const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
-  for (const key of Object.keys(tableSchema).filter(
+
-    k => tableSchema[k].visible !== false
+  const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
-  )) {
+    if (helpers.views.isCalculationField(viewSchema[key])) {
+      return viewSchema[key].visible !== false
+    }
+    return key in tableSchema && tableSchema[key].visible !== false
+  })
+  const visibleTableFields = Object.keys(tableSchema).filter(
+    key => tableSchema[key].visible !== false
+  )
+  const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
+  for (const key of visibleFields) {
     // if nothing specified in view, then it is not visible
     const ui = viewSchema[key] || { visible: false }
     schema[key] = {
       ...tableSchema[key],
       ...ui,
-      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order,
+      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
       columns: undefined,
     }
 
@@ -316,10 +318,7 @@ export async function enrichSchema(
     }
   }
 
-  return {
+  return { ...view, schema }
-    ...view,
-    schema: schema,
-  }
 }
 
 export function syncSchema(

packages/server/src/tests/utilities/api/row.ts

@@ -7,11 +7,11 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   SearchRowResponse,
-  RowSearchParams,
   DeleteRows,
   DeleteRow,
   PaginatedSearchRowResponse,
   RowExportFormat,
+  SearchRowRequest,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
     )
   }
 
-  search = async <T extends RowSearchParams>(
+  search = async <T extends SearchRowRequest>(
     sourceId: string,
     params?: T,
     expectations?: Expectations

packages/server/src/utilities/rowProcessor/index.ts

@@ -134,8 +134,10 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed = await processString(schema.default, ctx)
+      const processed =
-
+        typeof schema.default === "string"
+          ? await processString(schema.default, ctx)
+          : schema.default
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {

packages/shared-core/src/table.ts

@@ -53,8 +53,9 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.DATETIME]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.ARRAY]: true,
 
-  [FieldType.OPTIONS]: false,
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,
@@ -64,7 +65,6 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
-  [FieldType.ARRAY]: false,
   [FieldType.LINK]: false,
   [FieldType.BB_REFERENCE]: false,
   [FieldType.BB_REFERENCE_SINGLE]: false,

packages/types/src/documents/app/table/schema.ts

@@ -161,6 +161,7 @@ export interface OptionsFieldMetadata extends BaseFieldSchema {
   constraints: FieldConstraints & {
     inclusion: string[]
   }
+  default?: string
 }
 
 export interface ArrayFieldMetadata extends BaseFieldSchema {
@@ -169,6 +170,7 @@ export interface ArrayFieldMetadata extends BaseFieldSchema {
     type: JsonFieldSubType.ARRAY
     inclusion: string[]
   }
+  default?: string[]
 }
 
 interface BaseFieldSchema extends UIFieldMetadata {

packages/types/src/documents/app/view.ts

@@ -71,9 +71,11 @@ export interface ViewV2 {
     order?: SortOrder
     type?: SortType
   }
-  schema?: Record<string, ViewFieldMetadata>
+  schema?: ViewV2Schema
 }
 
+export type ViewV2Schema = Record<string, ViewFieldMetadata>
+
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
 
 export interface ViewCountOrSumSchema {