Merge remote-tracking branch 'origin/v3-ui' into feature/automation-branching-ux

.github/workflows/deploy-featurebranch.yml

@@ -23,6 +23,7 @@ jobs:
           PAYLOAD_BRANCH: ${{ github.head_ref }}
           PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
           PAYLOAD_LICENSE_TYPE: "free"
+          PAYLOAD_DEPLOY: "true"
         with:
           repository: budibase/budibase-deploys
           event: featurebranch-qa-deploy

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.10",
+  "version": "2.32.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/sql/index.ts

@@ -1,5 +1,5 @@
 export * as utils from "./utils"
 
-export { default as Sql } from "./sql"
+export { default as Sql, COUNT_FIELD_NAME } from "./sql"
 export { default as SqlTable } from "./sqlTable"
 export * as designDoc from "./designDoc"

packages/backend-core/src/sql/sql.ts

@@ -43,6 +43,8 @@ import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
 
+export const COUNT_FIELD_NAME = "__bb_total"
+
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
     ? parseInt(environment.SQL_MAX_ROWS)
@@ -846,7 +848,7 @@ class InternalBuilder {
       throw new Error("SQL counting requires primary key to be supplied")
     }
     return query.countDistinct(
-      `${this.getTableName()}.${this.table.primary[0]} as __bb_total`
+      `${this.getTableName()}.${this.table.primary[0]} as ${COUNT_FIELD_NAME}`
     )
   }
 

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -4,6 +4,7 @@
     Button,
     Label,
     Select,
+    Multiselect,
     Toggle,
     Icon,
     DatePicker,
@@ -137,9 +138,10 @@
   }
   $: initialiseField(field, savingColumn)
   $: checkConstraints(editableColumn)
-  $: required = hasDefault
+  $: required =
-    ? false
+    primaryDisplay ||
-    : !!editableColumn?.constraints?.presence || primaryDisplay
+    editableColumn?.constraints?.presence === true ||
+    editableColumn?.constraints?.presence?.allowEmpty === false
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
     UNEDITABLE_USER_FIELDS.includes(editableColumn.name)
@@ -168,8 +170,8 @@
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
-  $: canHaveDefault =
+  $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
-    isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
+  $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== LINK_TYPE &&
     !uneditable &&
@@ -188,7 +190,6 @@
     (originalName &&
       SWITCHABLE_TYPES[field.type] &&
       !editableColumn?.autocolumn)
-
   $: allowedTypes = getAllowedTypes(datasource).map(t => ({
     fieldId: makeFieldId(t.type, t.subtype),
     ...t,
@@ -206,6 +207,11 @@
     },
     ...getUserBindings(),
   ]
+  $: sanitiseDefaultValue(
+    editableColumn.type,
+    editableColumn.constraints?.inclusion || [],
+    editableColumn.default
+  )
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -295,6 +301,22 @@
       delete saveColumn.fieldName
     }
 
+    // Ensure we don't have a default value if we can't have one
+    if (!canHaveDefault || !defaultValuesEnabled) {
+      delete saveColumn.default
+    }
+
+    // Ensure primary display columns are always required and don't have default values
+    if (primaryDisplay) {
+      saveColumn.constraints.presence = { allowEmpty: false }
+      delete saveColumn.default
+    }
+
+    // Ensure the field is not required if we have a default value
+    if (saveColumn.default) {
+      saveColumn.constraints.presence = false
+    }
+
     try {
       await tables.saveField({
         originalName,
@@ -541,6 +563,20 @@
     return newError
   }
 
+  const sanitiseDefaultValue = (type, options, defaultValue) => {
+    if (!defaultValue?.length) {
+      return
+    }
+    // Delete default value for options fields if the option is no longer available
+    if (type === FieldType.OPTIONS && !options.includes(defaultValue)) {
+      delete editableColumn.default
+    }
+    // Filter array default values to only valid options
+    if (type === FieldType.ARRAY) {
+      editableColumn.default = defaultValue.filter(x => options.includes(x))
+    }
+  }
+
   onMount(() => {
     mounted = true
   })
@@ -748,9 +784,9 @@
       </div>
     </div>
   {:else if editableColumn.type === JSON_TYPE}
-    <Button primary text on:click={openJsonSchemaEditor}
+    <Button primary text on:click={openJsonSchemaEditor}>
-      >Open schema editor</Button
+      Open schema editor
-    >
+    </Button>
   {/if}
   {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn}
     <Select
@@ -779,27 +815,39 @@
     </div>
   {/if}
 
-  {#if canHaveDefault}
+  {#if defaultValuesEnabled}
-    <div>
+    {#if editableColumn.type === FieldType.OPTIONS}
-      <ModalBindableInput
+      <Select
-        panel={ServerBindingPanel}
+        disabled={!canHaveDefault}
-        title="Default"
+        options={editableColumn.constraints?.inclusion || []}
-        label="Default"
+        label="Default value"
         value={editableColumn.default}
-        on:change={e => {
+        on:change={e => (editableColumn.default = e.detail)}
-          editableColumn = {
+        placeholder="None"
-            ...editableColumn,
+      />
-            default: e.detail,
+    {:else if editableColumn.type === FieldType.ARRAY}
-          }
+      <Multiselect
-
+        disabled={!canHaveDefault}
-          if (e.detail) {
+        options={editableColumn.constraints?.inclusion || []}
-            setRequired(false)
+        label="Default value"
-          }
+        value={editableColumn.default}
-        }}
+        on:change={e =>
+          (editableColumn.default = e.detail?.length ? e.detail : undefined)}
+        placeholder="None"
+      />
+    {:else}
+      <ModalBindableInput
+        disabled={!canHaveDefault}
+        panel={ServerBindingPanel}
+        title="Default value"
+        label="Default value"
+        placeholder="None"
+        value={editableColumn.default}
+        on:change={e => (editableColumn.default = e.detail)}
         bindings={defaultValueBindings}
         allowJS
       />
-    </div>
+    {/if}
   {/if}
 </Layout>
 

packages/builder/src/stores/builder/viewsV2.js

@@ -1,6 +1,24 @@
 import { writable, derived, get } from "svelte/store"
 import { tables } from "./tables"
 import { API } from "api"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
+
+function convertToSearchFilterGroup(view) {
+  if (view.queryUI) {
+    view.query = view.queryUI
+    delete view.queryUI
+  }
+  return view
+}
 
 export function createViewsV2Store() {
   const store = writable({
@@ -12,7 +30,7 @@ export function createViewsV2Store() {
       const views = Object.values(table?.views || {}).filter(view => {
         return view.version === 2
       })
-      list = list.concat(views)
+      list = list.concat(views.map(view => convertToSearchFilterGroup(view)))
     })
     return {
       ...$store,
@@ -34,6 +52,7 @@ export function createViewsV2Store() {
   }
 
   const create = async view => {
+    view = convertToSearchFilters(view)
     const savedViewResponse = await API.viewV2.create(view)
     const savedView = savedViewResponse.data
     replaceView(savedView.id, savedView)
@@ -41,6 +60,7 @@ export function createViewsV2Store() {
   }
 
   const save = async view => {
+    view = convertToSearchFilters(view)
     const res = await API.viewV2.update(view)
     const savedView = res?.data
     replaceView(view.id, savedView)
@@ -51,6 +71,7 @@ export function createViewsV2Store() {
     if (!viewId) {
       return
     }
+    view = convertToSearchFilterGroup(view)
     const existingView = get(derivedStore).list.find(view => view.id === viewId)
     const tableIndex = get(tables).list.findIndex(table => {
       return table._id === view?.tableId || table._id === existingView?.tableId

packages/client/src/components/app/forms/Form.svelte

@@ -63,7 +63,7 @@
     // Look up the component tree and find something that is provided by an
     // ancestor that matches our datasource. This is for backwards compatibility
     // as previously we could use the "closest" context.
-    for (let id of path.reverse().slice(1)) {
+    for (let id of path.toReversed().slice(1)) {
       // Check for matching view datasource
       if (
         dataSource.type === "viewV2" &&

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -1,6 +1,7 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { enrichSchemaWithRelColumns, memo } from "../../../utils"
+import { cloneDeep } from "lodash"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -164,10 +165,18 @@ export const createActions = context => {
 
   // Updates the datasources primary display column
   const changePrimaryDisplay = async column => {
-    return await saveDefinition({
+    let newDefinition = cloneDeep(get(definition))
-      ...get(definition),
+
-      primaryDisplay: column,
+    // Update primary display
-    })
+    newDefinition.primaryDisplay = column
+
+    // Sanitise schema to ensure field is required and has no default value
+    if (!newDefinition.schema[column].constraints) {
+      newDefinition.schema[column].constraints = {}
+    }
+    newDefinition.schema[column].constraints.presence = { allowEmpty: false }
+    delete newDefinition.schema[column].default
+    return await saveDefinition(newDefinition)
   }
 
   // Adds a schema mutation for a single field

packages/frontend-core/src/components/grid/stores/datasources/viewV2.js

@@ -1,4 +1,14 @@
 import { get } from "svelte/store"
+import { dataFilters } from "@budibase/shared-core"
+
+function convertToSearchFilters(view) {
+  // convert from SearchFilterGroup type
+  if (view.query) {
+    view.queryUI = view.query
+    view.query = dataFilters.buildQuery(view.query)
+  }
+  return view
+}
 
 const SuppressErrors = true
 
@@ -6,7 +16,7 @@ export const createActions = context => {
   const { API, datasource, columns } = context
 
   const saveDefinition = async newDefinition => {
-    await API.viewV2.update(newDefinition)
+    await API.viewV2.update(convertToSearchFilters(newDefinition))
   }
 
   const saveRow = async row => {
@@ -125,7 +135,7 @@ export const initialise = context => {
         }
         // Only override filter state if we don't have an initial filter
         if (!get(initialFilter)) {
-          filter.set($definition.query)
+          filter.set($definition.queryUI || $definition.query)
         }
       })
     )

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -124,9 +124,11 @@ export async function buildSqlFieldList(
         ([columnName, column]) =>
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
-          !existing.find((field: string) => field === columnName)
+          !existing.find(
+            (field: string) => field === `${table.name}.${columnName}`
+          )
       )
-      .map(column => `${table.name}.${column[0]}`)
+      .map(([columnName]) => `${table.name}.${columnName}`)
   }
 
   let fields: string[] = []

packages/server/src/api/controllers/row/views.ts

@@ -3,6 +3,8 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
+  RequiredKeys,
+  RowSearchParams,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
@@ -20,30 +22,34 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
   await context.ensureSnippetContext(true)
 
-  const result = await sdk.rows.search(
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    {
+    RequiredKeys<
-      viewId: view.id,
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
-      tableId: view.tableId,
+    > = {
-      query: body.query,
+    tableId: view.tableId,
-      ...getSortOptions(body, view),
+    viewId: view.id,
-      limit: body.limit,
+    query: body.query,
-      bookmark: body.bookmark,
+    fields: viewFields,
-      paginate: body.paginate,
+    ...getSortOptions(body, view),
-      countRows: body.countRows,
+    limit: body.limit,
-    },
+    bookmark: body.bookmark,
-    {
+    paginate: body.paginate,
-      user: sdk.users.getUserContextBindings(ctx.user),
+    countRows: body.countRows,
-    }
+  }
-  )
 
+  const result = await sdk.rows.search(searchOptions, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -103,6 +103,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,
@@ -138,6 +139,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
     version: view.version,
     tableId: view.tableId,
     query: view.query,
+    queryUI: view.queryUI,
     sort: view.sort,
     schema,
     primaryDisplay: view.primaryDisplay,

packages/server/src/api/routes/tests/row.spec.ts

@@ -695,6 +695,69 @@ describe.each([
         })
       })
 
+      describe("options column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                status: {
+                  name: "status",
+                  type: FieldType.OPTIONS,
+                  default: "requested",
+                  constraints: {
+                    inclusion: ["requested", "approved"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.status).toEqual("requested")
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            status: "approved",
+          })
+          expect(row.status).toEqual("approved")
+        })
+      })
+
+      describe("array column", () => {
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                food: {
+                  name: "food",
+                  type: FieldType.ARRAY,
+                  default: ["apple", "orange"],
+                  constraints: {
+                    type: JsonFieldSubType.ARRAY,
+                    inclusion: ["apple", "orange", "banana"],
+                  },
+                },
+              },
+            })
+          )
+        })
+
+        it("creates a new row with a default value successfully", async () => {
+          const row = await config.api.row.save(table._id!, {})
+          expect(row.food).toEqual(["apple", "orange"])
+        })
+
+        it("does not use default value if value specified", async () => {
+          const row = await config.api.row.save(table._id!, {
+            food: ["orange"],
+          })
+          expect(row.food).toEqual(["orange"])
+        })
+      })
+
       describe("bindings", () => {
         describe("string column", () => {
           beforeAll(async () => {

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,9 +22,10 @@ import {
   RelationshipType,
   TableSchema,
   RenameColumn,
-  ViewFieldMetadata,
   FeatureFlag,
   BBReferenceFieldSubType,
+  ViewV2Schema,
+  ViewCalculationFieldMetadata,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -154,7 +155,7 @@ describe.each([
       })
 
       it("can persist views with all fields", async () => {
-        const newView: Required<CreateViewRequest> = {
+        const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
           name: generator.name(),
           tableId: table._id!,
           primaryDisplay: "id",
@@ -540,6 +541,33 @@ describe.each([
           status: 201,
         })
       })
+
+      it("can create a view with calculation fields", async () => {
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "Price",
+            },
+          },
+        })
+
+        expect(Object.keys(view.schema!)).toHaveLength(1)
+
+        let sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+
+        view = await config.api.viewV2.get(view.id)
+        sum = view.schema!.sum as ViewCalculationFieldMetadata
+        expect(sum).toBeDefined()
+        expect(sum.calculationType).toEqual(CalculationType.SUM)
+        expect(sum.field).toEqual("Price")
+      })
     })
 
     describe("update", () => {
@@ -584,7 +612,7 @@ describe.each([
       it("can update all fields", async () => {
         const tableId = table._id!
 
-        const updatedData: Required<UpdateViewRequest> = {
+        const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
           version: view.version,
           id: view.id,
           tableId,
@@ -1152,10 +1180,7 @@ describe.each([
           return table
         }
 
-        const createView = async (
+        const createView = async (tableId: string, schema: ViewV2Schema) =>
-          tableId: string,
-          schema: Record<string, ViewFieldMetadata>
-        ) =>
           await config.api.viewV2.create({
             name: generator.guid(),
             tableId,
@@ -2546,6 +2571,51 @@ describe.each([
             }
           })
         })
+
+      !isLucene &&
+        it("should not need required fields to be present", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                name: {
+                  name: "name",
+                  type: FieldType.STRING,
+                  constraints: {
+                    presence: true,
+                  },
+                },
+                age: {
+                  name: "age",
+                  type: FieldType.NUMBER,
+                },
+              },
+            })
+          )
+
+          await Promise.all([
+            config.api.row.save(table._id!, { name: "Steve", age: 30 }),
+            config.api.row.save(table._id!, { name: "Jane", age: 31 }),
+          ])
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              sum: {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "age",
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {},
+          })
+
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows[0].sum).toEqual(61)
+        })
     })
 
     describe("permissions", () => {

packages/server/src/db/linkedRows/index.ts

@@ -23,8 +23,8 @@ import {
   Row,
   Table,
   TableSchema,
-  ViewFieldMetadata,
   ViewV2,
+  ViewV2Schema,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { helpers } from "@budibase/shared-core"
@@ -262,7 +262,7 @@ export async function squashLinks<T = Row[] | Row>(
     FeatureFlag.ENRICHED_RELATIONSHIPS
   )
 
-  let viewSchema: Record<string, ViewFieldMetadata> = {}
+  let viewSchema: ViewV2Schema = {}
   if (sdk.views.isView(source)) {
     if (helpers.views.isCalculationView(source)) {
       return enriched

packages/server/src/sdk/app/rows/search.ts

@@ -1,12 +1,10 @@
 import {
   EmptyFilterOption,
+  LegacyFilter,
   LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilter,
-  SearchFilterGroup,
   SearchFilterKey,
-  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -63,7 +61,6 @@ export async function search(
     if (options.viewId) {
       source = await sdk.views.get(options.viewId)
       table = await sdk.views.getTable(source)
-      options = searchInputMapping(table, options)
     } else if (options.tableId) {
       source = await sdk.tables.getTable(options.tableId)
       table = source
@@ -76,7 +73,7 @@ export async function search(
     if (options.query) {
       const visibleFields = (
         options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
+      ).filter(field => table.schema[field]?.visible !== false)
 
       const queryableFields = await getQueryableFields(table, visibleFields)
       options.query = removeInvalidFilters(options.query, queryableFields)
@@ -84,40 +81,59 @@ export async function search(
       options.query = {}
     }
 
+    // need to make sure filters in correct shape before checking for view
+    options = searchInputMapping(table, options)
+
     if (options.viewId) {
-      const view = await sdk.views.get(options.viewId)
+      // Delete extraneous search params that cannot be overridden
+      delete options.query.onEmptyFilter
+
+      const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQuery(view.query || [])
+      let viewQuery = dataFilters.buildQueryLegacy(view.query) || {}
+      delete viewQuery?.onEmptyFilter
 
-      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
+      const sqsEnabled = await features.flags.isEnabled("SQS")
-        // Lucene does not accept conditional filters, so we need to keep the old logic
+      const supportsLogicalOperators =
-        const query: SearchFilters = viewQuery || {}
+        isExternalTableID(view.tableId) || sqsEnabled
-        const viewFilters = view.query as SearchFilter[]
+      if (!supportsLogicalOperators) {
+        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+        // It needs to be ignored entirely
+        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+          ? view.query
+          : []
+
+        delete options.query.onEmptyFilter
 
         // Extract existing fields
         const existingFields =
-          viewFilters
+          queryFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
+        viewQuery ??= {}
         // Carry over filters for unused fields
-        Object.keys(options.query || {}).forEach(key => {
+        Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              query[operator]![field] = options.query[operator]![field]
+              viewQuery![operator]![field] = options.query[operator]![field]
             }
           })
         })
-        options.query = query
+        options.query = viewQuery
       } else {
+        const conditions = viewQuery ? [viewQuery] : []
         options.query = {
           $and: {
-            conditions: [viewQuery as SearchFilterGroup, options.query],
+            conditions: [...conditions, options.query],
           },
         }
+        if (viewQuery.onEmptyFilter) {
+          options.query.onEmptyFilter = viewQuery.onEmptyFilter
+        }
       }
     }
 
@@ -146,8 +162,6 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    options = searchInputMapping(table, options)
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/utils.ts

@@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
     }
     return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
-  options.query = checkFilters(options.query)
+  if (options.query) {
+    options.query = checkFilters(options.query)
+  }
   return options
 }
 

packages/server/src/sdk/app/rows/utils.ts

@@ -20,7 +20,7 @@ import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
-import { docIds } from "@budibase/backend-core"
+import { docIds, sql } from "@budibase/backend-core"
 import { getTableFromSource } from "../../../api/controllers/row/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
@@ -57,8 +57,12 @@ export function getSQLClient(datasource: Datasource): SqlClient {
 export function processRowCountResponse(
   response: DatasourcePlusQueryResponse
 ): number {
-  if (response && response.length === 1 && "__bb_total" in response[0]) {
+  if (
-    const total = response[0].__bb_total
+    response &&
+    response.length === 1 &&
+    sql.COUNT_FIELD_NAME in response[0]
+  ) {
+    const total = response[0][sql.COUNT_FIELD_NAME]
     return typeof total === "number" ? total : parseInt(total)
   } else {
     throw new Error("Unable to count rows in query - no count response")

packages/server/src/sdk/app/views/index.ts

@@ -255,19 +255,12 @@ export async function enrichSchema(
   view: ViewV2,
   tableSchema: TableSchema
 ): Promise<ViewV2Enriched> {
-  const tableCache: Record<string, Table> = {}
-
   async function populateRelTableSchema(
     tableId: string,
     viewFields: Record<string, RelationSchemaField>
   ) {
-    if (!tableCache[tableId]) {
+    const relTable = await sdk.tables.getTable(tableId)
-      tableCache[tableId] = await sdk.tables.getTable(tableId)
-    }
-    const relTable = tableCache[tableId]
-
     const result: Record<string, ViewV2ColumnEnriched> = {}
-
     for (const relTableFieldName of Object.keys(relTable.schema)) {
       const relTableField = relTable.schema[relTableFieldName]
       if ([FieldType.LINK, FieldType.FORMULA].includes(relTableField.type)) {
@@ -296,15 +289,24 @@ export async function enrichSchema(
 
   const viewSchema = view.schema || {}
   const anyViewOrder = Object.values(viewSchema).some(ui => ui.order != null)
-  for (const key of Object.keys(tableSchema).filter(
+
-    k => tableSchema[k].visible !== false
+  const visibleSchemaFields = Object.keys(viewSchema).filter(key => {
-  )) {
+    if (helpers.views.isCalculationField(viewSchema[key])) {
+      return viewSchema[key].visible !== false
+    }
+    return key in tableSchema && tableSchema[key].visible !== false
+  })
+  const visibleTableFields = Object.keys(tableSchema).filter(
+    key => tableSchema[key].visible !== false
+  )
+  const visibleFields = new Set([...visibleSchemaFields, ...visibleTableFields])
+  for (const key of visibleFields) {
     // if nothing specified in view, then it is not visible
     const ui = viewSchema[key] || { visible: false }
     schema[key] = {
       ...tableSchema[key],
       ...ui,
-      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order,
+      order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
       columns: undefined,
     }
 
@@ -316,10 +318,7 @@ export async function enrichSchema(
     }
   }
 
-  return {
+  return { ...view, schema }
-    ...view,
-    schema: schema,
-  }
 }
 
 export function syncSchema(

packages/server/src/tests/utilities/api/row.ts

@@ -7,11 +7,11 @@ import {
   BulkImportRequest,
   BulkImportResponse,
   SearchRowResponse,
-  RowSearchParams,
   DeleteRows,
   DeleteRow,
   PaginatedSearchRowResponse,
   RowExportFormat,
+  SearchRowRequest,
 } from "@budibase/types"
 import { Expectations, TestAPI } from "./base"
 
@@ -136,7 +136,7 @@ export class RowAPI extends TestAPI {
     )
   }
 
-  search = async <T extends RowSearchParams>(
+  search = async <T extends SearchRowRequest>(
     sourceId: string,
     params?: T,
     expectations?: Expectations

packages/server/src/utilities/rowProcessor/index.ts

@@ -134,8 +134,10 @@ async function processDefaultValues(table: Table, row: Row) {
 
   for (const [key, schema] of Object.entries(table.schema)) {
     if ("default" in schema && schema.default != null && row[key] == null) {
-      const processed = await processString(schema.default, ctx)
+      const processed =
-
+        typeof schema.default === "string"
+          ? await processString(schema.default, ctx)
+          : schema.default
       try {
         row[key] = coerce(processed, schema.type)
       } catch (err: any) {

packages/shared-core/src/filters.ts

@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -163,9 +163,6 @@ export function recurseSearchFilters(
  * https://github.com/Budibase/budibase/issues/10118
  */
 export const cleanupQuery = (query: SearchFilters) => {
-  if (!query) {
-    return query
-  }
   for (let filterField of NoEmptyFilterStrings) {
     if (!query[filterField]) {
       continue
@@ -311,7 +308,7 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: SearchFilter) => {
+const buildCondition = (expression: LegacyFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
 }
 
 export const buildQueryLegacy = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: LegacyFilter[] | SearchFilters
 ): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -572,7 +574,7 @@ export const buildQueryLegacy = (
  */
 
 export const buildQuery = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: SearchFilterGroup | LegacyFilter[]
 ): SearchFilters | undefined => {
   const parsedFilter: SearchFilterGroup | undefined =
     processSearchFilters(filter)
@@ -594,7 +596,7 @@ export const buildQuery = (
   const globalOperator: LogicalOperator =
     operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
-  const coreRequest: SearchFilters = {
+  return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@@ -608,7 +610,6 @@ export const buildQuery = (
       }),
     },
   }
-  return coreRequest
 }
 
 // The frontend can send single values for array fields sometimes, so to handle

packages/shared-core/src/table.ts

@@ -53,8 +53,9 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.DATETIME]: true,
   [FieldType.LONGFORM]: true,
   [FieldType.STRING]: true,
+  [FieldType.OPTIONS]: true,
+  [FieldType.ARRAY]: true,
 
-  [FieldType.OPTIONS]: false,
   [FieldType.AUTO]: false,
   [FieldType.INTERNAL]: false,
   [FieldType.BARCODEQR]: false,
@@ -64,7 +65,6 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
   [FieldType.ATTACHMENTS]: false,
   [FieldType.ATTACHMENT_SINGLE]: false,
   [FieldType.SIGNATURE_SINGLE]: false,
-  [FieldType.ARRAY]: false,
   [FieldType.LINK]: false,
   [FieldType.BB_REFERENCE]: false,
   [FieldType.BB_REFERENCE_SINGLE]: false,

packages/shared-core/src/utils.ts

@@ -1,5 +1,5 @@
 import {
-  SearchFilter,
+  LegacyFilter,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
   SearchFilters,
@@ -9,6 +9,10 @@ import {
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
 
+// an array of keys from filter type to properties that are in the type
+// this can then be converted using .fromEntries to an object
+type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+
 export function unreachable(
   value: never,
   message = `No such case in exhaustive switch: ${value}`
@@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to SearchFilterGroup
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {SearchFilter[] | SearchFilterGroup} filters
- */
-export const processSearchFilters = (
-  filters: SearchFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
-  }
-
-  // Base search config.
-  const defaultCfg: SearchFilterGroup = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
-    groups: [],
-  }
-
-  const filterWhitelistKeys = [
-    "field",
-    "operator",
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  if (Array.isArray(filters)) {
-    let baseGroup: SearchFilterGroup = {
-      filters: [],
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-    }
-
-    const migratedSetting: SearchFilterGroup = filters.reduce(
-      (acc: SearchFilterGroup, filter: SearchFilter) => {
-        // Sort the properties for easier debugging
-        const filterEntries = Object.entries(filter)
-          .sort((a, b) => {
-            return a[0].localeCompare(b[0])
-          })
-          .filter(x => x[1] ?? false)
-
-        if (filterEntries.length == 1) {
-          const [key, value] = filterEntries[0]
-          // Global
-          if (key === "onEmptyFilter") {
-            // unset otherwise
-            acc.onEmptyFilter = value
-          } else if (key === "operator" && value === "allOr") {
-            // Group 1 logical operator
-            baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-          }
-
-          return acc
-        }
-
-        const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
-          (acc: [string, any][], entry: [string, any]) => {
-            const [key, value] = entry
-
-            if (filterWhitelistKeys.includes(key)) {
-              if (key === "field") {
-                acc.push([key, removeKeyNumbering(value)])
-              } else {
-                acc.push([key, value])
-              }
-            }
-            return acc
-          },
-          []
-        )
-
-        const migratedFilter: SearchFilter = Object.fromEntries(
-          whiteListedFilterSettings
-        ) as SearchFilter
-
-        baseGroup.filters!.push(migratedFilter)
-
-        if (!acc.groups || !acc.groups.length) {
-          // init the base group
-          acc.groups = [baseGroup]
-        }
-
-        return acc
-      },
-      defaultCfg
-    )
-
-    return migratedSetting
-  } else if (!filters?.groups) {
-    return
-  }
-  return filters
-}
-
 export function isSupportedUserSearch(query: SearchFilters) {
   const allowed = [
     { op: BasicOperator.STRING, key: "email" },
@@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
   }
   return true
 }
+
+/**
+ * Processes the filter config. Filters are migrated from
+ * SearchFilter[] to SearchFilterGroup
+ *
+ * If filters is not an array, the migration is skipped
+ *
+ * @param {LegacyFilter[] | SearchFilterGroup} filters
+ */
+export const processSearchFilters = (
+  filters: LegacyFilter[] | SearchFilterGroup | undefined
+): SearchFilterGroup | undefined => {
+  if (!filters) {
+    return
+  }
+
+  // Base search config.
+  const defaultCfg: SearchFilterGroup = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    groups: [],
+  }
+
+  const filterAllowedKeys = [
+    "field",
+    "operator",
+    "value",
+    "type",
+    "externalType",
+    "valueType",
+    "noValue",
+    "formulaType",
+  ]
+
+  if (Array.isArray(filters)) {
+    let baseGroup: SearchFilterGroup = {
+      filters: [],
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+    }
+
+    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+      // Sort the properties for easier debugging
+      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+        .sort((a, b) => {
+          return a.localeCompare(b)
+        })
+        .filter(key => key in filter)
+
+      if (filterPropertyKeys.length == 1) {
+        const key = filterPropertyKeys[0],
+          value = filter[key]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+        }
+
+        return acc
+      }
+
+      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
+        (acc: AllowedFilters, key) => {
+          const value = filter[key]
+          if (filterAllowedKeys.includes(key)) {
+            if (key === "field") {
+              acc.push([key, removeKeyNumbering(value)])
+            } else {
+              acc.push([key, value])
+            }
+          }
+          return acc
+        },
+        []
+      )
+
+      const migratedFilter: LegacyFilter = Object.fromEntries(
+        allowedFilterSettings
+      ) as LegacyFilter
+
+      baseGroup.filters!.push(migratedFilter)
+
+      if (!acc.groups || !acc.groups.length) {
+        // init the base group
+        acc.groups = [baseGroup]
+      }
+
+      return acc
+    }, defaultCfg)
+  } else if (!filters?.groups) {
+    return
+  }
+  return filters
+}

packages/types/src/api/web/searchFilter.ts

@@ -5,7 +5,7 @@ import {
   SearchFilters,
 } from "../../sdk"
 
-export type SearchFilter = {
+export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
   field: string
@@ -14,9 +14,10 @@ export type SearchFilter = {
   externalType?: string
 }
 
+// this is a type purely used by the UI
 export type SearchFilterGroup = {
   logicalOperator: FilterGroupLogicalOperator
   onEmptyFilter?: EmptyFilterOption
   groups?: SearchFilterGroup[]
-  filters?: SearchFilter[]
+  filters?: LegacyFilter[]
 }

packages/types/src/documents/app/table/schema.ts

@@ -161,6 +161,7 @@ export interface OptionsFieldMetadata extends BaseFieldSchema {
   constraints: FieldConstraints & {
     inclusion: string[]
   }
+  default?: string
 }
 
 export interface ArrayFieldMetadata extends BaseFieldSchema {
@@ -169,6 +170,7 @@ export interface ArrayFieldMetadata extends BaseFieldSchema {
     type: JsonFieldSubType.ARRAY
     inclusion: string[]
   }
+  default?: string[]
 }
 
 interface BaseFieldSchema extends UIFieldMetadata {

packages/types/src/documents/app/view.ts

@@ -1,7 +1,7 @@
-import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
-import { DBView } from "../../sdk"
+import { DBView, SearchFilters } from "../../sdk"
 
 export type ViewTemplateOpts = {
   field: string
@@ -65,15 +65,19 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[] | SearchFilterGroup
+  query?: LegacyFilter[] | SearchFilters
+  // duplicate to store UI information about filters
+  queryUI?: SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder
     type?: SortType
   }
-  schema?: Record<string, ViewFieldMetadata>
+  schema?: ViewV2Schema
 }
 
+export type ViewV2Schema = Record<string, ViewFieldMetadata>
+
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema
 
 export interface ViewCountOrSumSchema {