Merging v3-backport branch

packages/server/src/api/controllers/row/views.ts

@@ -3,6 +3,8 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
+  RequiredKeys,
+  RowSearchParams,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import { context } from "@budibase/backend-core"
@@ -20,30 +22,34 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
+  const viewFields = Object.entries(view.schema || {})
+    .filter(([_, value]) => value.visible)
+    .map(([key]) => key)
   const { body } = ctx.request
 
   await context.ensureSnippetContext(true)
 
-  const result = await sdk.rows.search(
-    {
-      viewId: view.id,
-      tableId: view.tableId,
-      query: body.query,
-      ...getSortOptions(body, view),
-      limit: body.limit,
-      bookmark: body.bookmark,
-      paginate: body.paginate,
-      countRows: body.countRows,
-    },
-    {
-      user: sdk.users.getUserContextBindings(ctx.user),
-    }
-  )
+  const searchOptions: RequiredKeys<SearchViewRowRequest> &
+    RequiredKeys<
+      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
+    > = {
+    tableId: view.tableId,
+    viewId: view.id,
+    query: body.query,
+    fields: viewFields,
+    ...getSortOptions(body, view),
+    limit: body.limit,
+    bookmark: body.bookmark,
+    paginate: body.paginate,
+    countRows: body.countRows,
+  }
 
+  const result = await sdk.rows.search(searchOptions, {
+    user: sdk.users.getUserContextBindings(ctx.user),
+  })
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }
-
 function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
   if (request.sort) {
     return {

packages/server/src/api/controllers/view/viewsV2.ts

@@ -99,7 +99,7 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
 
   const schema = await parseSchema(view)
 
-  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
+  const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version" | "queryUI"> = {
     name: view.name,
     tableId: view.tableId,
     query: view.query,
@@ -132,7 +132,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
   const { tableId } = view
 
   const schema = await parseSchema(view)
-  const parsedView: RequiredKeys<ViewV2> = {
+  const parsedView: RequiredKeys<Omit<ViewV2, "queryUI">> = {
     id: view.id,
     name: view.name,
     version: view.version,

packages/server/src/sdk/app/rows/search.ts

@@ -1,12 +1,10 @@
 import {
   EmptyFilterOption,
+  LegacyFilter,
   LogicalOperator,
   Row,
   RowSearchParams,
-  SearchFilter,
-  SearchFilterGroup,
   SearchFilterKey,
-  SearchFilters,
   SearchResponse,
   SortOrder,
   Table,
@@ -63,7 +61,6 @@ export async function search(
     if (options.viewId) {
       source = await sdk.views.get(options.viewId)
       table = await sdk.views.getTable(source)
-      options = searchInputMapping(table, options)
     } else if (options.tableId) {
       source = await sdk.tables.getTable(options.tableId)
       table = source
@@ -76,7 +73,7 @@ export async function search(
     if (options.query) {
       const visibleFields = (
         options.fields || Object.keys(table.schema)
-      ).filter(field => table.schema[field].visible !== false)
+      ).filter(field => table.schema[field]?.visible !== false)
 
       const queryableFields = await getQueryableFields(table, visibleFields)
       options.query = removeInvalidFilters(options.query, queryableFields)
@@ -84,38 +81,52 @@ export async function search(
       options.query = {}
     }
 
+    // need to make sure filters in correct shape before checking for view
+    options = searchInputMapping(table, options)
+
     if (options.viewId) {
-      const view = await sdk.views.get(options.viewId)
+      // Delete extraneous search params that cannot be overridden
+      delete options.query.onEmptyFilter
+
+      const view = source as ViewV2
       // Enrich saved query with ephemeral query params.
       // We prevent searching on any fields that are saved as part of the query, as
       // that could let users find rows they should not be allowed to access.
-      let viewQuery = dataFilters.buildQuery(view.query || [])
+      let viewQuery = dataFilters.buildQueryLegacy(view.query || [])
+      delete viewQuery?.onEmptyFilter
 
-      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
-        // Lucene does not accept conditional filters, so we need to keep the old logic
-        const query: SearchFilters = viewQuery || {}
-        const viewFilters = view.query as SearchFilter[]
+      const sqsEnabled = await features.flags.isEnabled("SQS")
+      const supportsLogicalOperators =
+        isExternalTableID(view.tableId) || sqsEnabled
+      if (!supportsLogicalOperators) {
+        // In the unlikely event that a Grouped Filter is in a non-SQS environment
+        // It needs to be ignored entirely
+        let queryFilters: LegacyFilter[] = Array.isArray(view.query)
+          ? view.query
+          : []
 
         // Extract existing fields
         const existingFields =
-          viewFilters
+          queryFilters
             ?.filter(filter => filter.field)
             .map(filter => db.removeKeyNumbering(filter.field)) || []
 
+        viewQuery ??= {}
         // Carry over filters for unused fields
-        Object.keys(options.query || {}).forEach(key => {
+        Object.keys(options.query).forEach(key => {
           const operator = key as Exclude<SearchFilterKey, LogicalOperator>
           Object.keys(options.query[operator] || {}).forEach(field => {
             if (!existingFields.includes(db.removeKeyNumbering(field))) {
-              query[operator]![field] = options.query[operator]![field]
+              viewQuery![operator]![field] = options.query[operator]![field]
             }
           })
         })
-        options.query = query
+        options.query = viewQuery
       } else {
+        const conditions = viewQuery ? [viewQuery] : []
         options.query = {
           $and: {
-            conditions: [viewQuery as SearchFilterGroup, options.query],
+            conditions: [...conditions, options.query],
           },
         }
       }
@@ -146,8 +157,6 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    options = searchInputMapping(table, options)
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })

packages/server/src/sdk/app/rows/search/utils.ts

@@ -107,7 +107,9 @@ export function searchInputMapping(table: Table, options: RowSearchParams) {
     }
     return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
-  options.query = checkFilters(options.query)
+  if (options.query) {
+    options.query = checkFilters(options.query)
+  }
   return options
 }
 

packages/shared-core/src/filters.ts

@@ -3,7 +3,7 @@ import {
   BBReferenceFieldSubType,
   FieldType,
   FormulaType,
-  SearchFilter,
+  LegacyFilter,
   SearchFilters,
   SearchQueryFields,
   ArrayOperator,
@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters?.[logical]) {
+    if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )
@@ -163,9 +163,6 @@ export function recurseSearchFilters(
  * https://github.com/Budibase/budibase/issues/10118
  */
 export const cleanupQuery = (query: SearchFilters) => {
-  if (!query) {
-    return query
-  }
   for (let filterField of NoEmptyFilterStrings) {
     if (!query[filterField]) {
       continue
@@ -311,7 +308,7 @@ export class ColumnSplitter {
  * @param filter the builder filter structure
  */
 
-const buildCondition = (expression: SearchFilter) => {
+const buildCondition = (expression: LegacyFilter) => {
   // Filter body
   let query: SearchFilters = {
     string: {},
@@ -437,8 +434,13 @@ const buildCondition = (expression: SearchFilter) => {
 }
 
 export const buildQueryLegacy = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: LegacyFilter[] | SearchFilters
 ): SearchFilters | undefined => {
+  // this is of type SearchFilters or is undefined
+  if (!Array.isArray(filter)) {
+    return filter
+  }
+
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -572,7 +574,7 @@ export const buildQueryLegacy = (
  */
 
 export const buildQuery = (
-  filter?: SearchFilterGroup | SearchFilter[]
+  filter?: SearchFilterGroup | LegacyFilter[]
 ): SearchFilters | undefined => {
   const parsedFilter: SearchFilterGroup | undefined =
     processSearchFilters(filter)
@@ -594,7 +596,7 @@ export const buildQuery = (
   const globalOperator: LogicalOperator =
     operatorMap[parsedFilter.logicalOperator as FilterGroupLogicalOperator]
 
-  const coreRequest: SearchFilters = {
+  return {
     ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
     [globalOperator]: {
       conditions: parsedFilter.groups?.map((group: SearchFilterGroup) => {
@@ -608,7 +610,6 @@ export const buildQuery = (
       }),
     },
   }
-  return coreRequest
 }
 
 // The frontend can send single values for array fields sometimes, so to handle

packages/shared-core/src/utils.ts

@@ -1,5 +1,5 @@
 import {
-  SearchFilter,
+  LegacyFilter,
   SearchFilterGroup,
   FilterGroupLogicalOperator,
   SearchFilters,
@@ -9,6 +9,10 @@ import {
 import * as Constants from "./constants"
 import { removeKeyNumbering } from "./filters"
 
+// an array of keys from filter type to properties that are in the type
+// this can then be converted using .fromEntries to an object
+type AllowedFilters = [keyof LegacyFilter, LegacyFilter[keyof LegacyFilter]][]
+
 export function unreachable(
   value: never,
   message = `No such case in exhaustive switch: ${value}`
@@ -87,106 +91,6 @@ export function trimOtherProps(object: any, allowedProps: string[]) {
   return result
 }
 
-/**
- * Processes the filter config. Filters are migrated from
- * SearchFilter[] to SearchFilterGroup
- *
- * If filters is not an array, the migration is skipped
- *
- * @param {SearchFilter[] | SearchFilterGroup} filters
- */
-export const processSearchFilters = (
-  filters: SearchFilter[] | SearchFilterGroup | undefined
-): SearchFilterGroup | undefined => {
-  if (!filters) {
-    return
-  }
-
-  // Base search config.
-  const defaultCfg: SearchFilterGroup = {
-    logicalOperator: FilterGroupLogicalOperator.ALL,
-    groups: [],
-  }
-
-  const filterWhitelistKeys = [
-    "field",
-    "operator",
-    "value",
-    "type",
-    "externalType",
-    "valueType",
-    "noValue",
-    "formulaType",
-  ]
-
-  if (Array.isArray(filters)) {
-    let baseGroup: SearchFilterGroup = {
-      filters: [],
-      logicalOperator: FilterGroupLogicalOperator.ALL,
-    }
-
-    const migratedSetting: SearchFilterGroup = filters.reduce(
-      (acc: SearchFilterGroup, filter: SearchFilter) => {
-        // Sort the properties for easier debugging
-        const filterEntries = Object.entries(filter)
-          .sort((a, b) => {
-            return a[0].localeCompare(b[0])
-          })
-          .filter(x => x[1] ?? false)
-
-        if (filterEntries.length == 1) {
-          const [key, value] = filterEntries[0]
-          // Global
-          if (key === "onEmptyFilter") {
-            // unset otherwise
-            acc.onEmptyFilter = value
-          } else if (key === "operator" && value === "allOr") {
-            // Group 1 logical operator
-            baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
-          }
-
-          return acc
-        }
-
-        const whiteListedFilterSettings: [string, any][] = filterEntries.reduce(
-          (acc: [string, any][], entry: [string, any]) => {
-            const [key, value] = entry
-
-            if (filterWhitelistKeys.includes(key)) {
-              if (key === "field") {
-                acc.push([key, removeKeyNumbering(value)])
-              } else {
-                acc.push([key, value])
-              }
-            }
-            return acc
-          },
-          []
-        )
-
-        const migratedFilter: SearchFilter = Object.fromEntries(
-          whiteListedFilterSettings
-        ) as SearchFilter
-
-        baseGroup.filters!.push(migratedFilter)
-
-        if (!acc.groups || !acc.groups.length) {
-          // init the base group
-          acc.groups = [baseGroup]
-        }
-
-        return acc
-      },
-      defaultCfg
-    )
-
-    return migratedSetting
-  } else if (!filters?.groups) {
-    return
-  }
-  return filters
-}
-
 export function isSupportedUserSearch(query: SearchFilters) {
   const allowed = [
     { op: BasicOperator.STRING, key: "email" },
@@ -212,3 +116,98 @@ export function isSupportedUserSearch(query: SearchFilters) {
   }
   return true
 }
+
+/**
+ * Processes the filter config. Filters are migrated from
+ * SearchFilter[] to SearchFilterGroup
+ *
+ * If filters is not an array, the migration is skipped
+ *
+ * @param {LegacyFilter[] | SearchFilterGroup} filters
+ */
+export const processSearchFilters = (
+  filters: LegacyFilter[] | SearchFilterGroup | undefined
+): SearchFilterGroup | undefined => {
+  if (!filters) {
+    return
+  }
+
+  // Base search config.
+  const defaultCfg: SearchFilterGroup = {
+    logicalOperator: FilterGroupLogicalOperator.ALL,
+    groups: [],
+  }
+
+  const filterAllowedKeys = [
+    "field",
+    "operator",
+    "value",
+    "type",
+    "externalType",
+    "valueType",
+    "noValue",
+    "formulaType",
+  ]
+
+  if (Array.isArray(filters)) {
+    let baseGroup: SearchFilterGroup = {
+      filters: [],
+      logicalOperator: FilterGroupLogicalOperator.ALL,
+    }
+
+    return filters.reduce((acc: SearchFilterGroup, filter: LegacyFilter) => {
+      // Sort the properties for easier debugging
+      const filterPropertyKeys = (Object.keys(filter) as (keyof LegacyFilter)[])
+        .sort((a, b) => {
+          return a.localeCompare(b)
+        })
+        .filter(key => key in filter)
+
+      if (filterPropertyKeys.length == 1) {
+        const key = filterPropertyKeys[0],
+          value = filter[key]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = FilterGroupLogicalOperator.ANY
+        }
+
+        return acc
+      }
+
+      const allowedFilterSettings: AllowedFilters = filterPropertyKeys.reduce(
+        (acc: AllowedFilters, key) => {
+          const value = filter[key]
+          if (filterAllowedKeys.includes(key)) {
+            if (key === "field") {
+              acc.push([key, removeKeyNumbering(value)])
+            } else {
+              acc.push([key, value])
+            }
+          }
+          return acc
+        },
+        []
+      )
+
+      const migratedFilter: LegacyFilter = Object.fromEntries(
+        allowedFilterSettings
+      ) as LegacyFilter
+
+      baseGroup.filters!.push(migratedFilter)
+
+      if (!acc.groups || !acc.groups.length) {
+        // init the base group
+        acc.groups = [baseGroup]
+      }
+
+      return acc
+    }, defaultCfg)
+  } else if (!filters?.groups) {
+    return
+  }
+  return filters
+}

packages/types/src/api/web/app/view.ts

@@ -9,6 +9,7 @@ export interface ViewResponseEnriched {
   data: ViewV2Enriched
 }
 
-export interface CreateViewRequest extends Omit<ViewV2, "version" | "id"> {}
+export interface CreateViewRequest
+  extends Omit<ViewV2, "version" | "id" | "queryUI"> {}
 
-export interface UpdateViewRequest extends ViewV2 {}
+export interface UpdateViewRequest extends Omit<ViewV2, "queryUI"> {}

packages/types/src/api/web/searchFilter.ts

@@ -5,7 +5,7 @@ import {
   SearchFilters,
 } from "../../sdk"
 
-export type SearchFilter = {
+export type LegacyFilter = {
   operator: keyof SearchFilters | "rangeLow" | "rangeHigh"
   onEmptyFilter?: EmptyFilterOption
   field: string
@@ -14,9 +14,10 @@ export type SearchFilter = {
   externalType?: string
 }
 
+// this is a type purely used by the UI
 export type SearchFilterGroup = {
   logicalOperator: FilterGroupLogicalOperator
   onEmptyFilter?: EmptyFilterOption
   groups?: SearchFilterGroup[]
-  filters?: SearchFilter[]
+  filters?: LegacyFilter[]
 }

packages/types/src/documents/app/view.ts

@@ -1,7 +1,7 @@
-import { SearchFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
+import { LegacyFilter, SearchFilterGroup, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
 import { Document } from "../document"
-import { DBView } from "../../sdk"
+import { DBView, SearchFilters } from "../../sdk"
 
 export type ViewTemplateOpts = {
   field: string
@@ -65,7 +65,9 @@ export interface ViewV2 {
   name: string
   primaryDisplay?: string
   tableId: string
-  query?: SearchFilter[] | SearchFilterGroup
+  query?: LegacyFilter[] | SearchFilters
+  // duplicate to store UI information about filters
+  queryUI?: SearchFilterGroup
   sort?: {
     field: string
     order?: SortOrder