Merge commit

packages/builder/src/stores/builder/components.js

@@ -29,7 +29,7 @@ import {
   DB_TYPE_EXTERNAL,
 } from "constants/backend"
 import BudiStore from "../BudiStore"
-import { Utils } from "@budibase/frontend-core"
+import { Utils, Constants } from "@budibase/frontend-core"
 import { FieldType } from "@budibase/types"
 
 export const INITIAL_COMPONENTS_STATE = {
@@ -196,6 +196,26 @@ export class ComponentStore extends BudiStore {
       }
     }
 
+    if (!enrichedComponent?._component) {
+      return migrated
+    }
+
+    const def = this.getDefinition(enrichedComponent?._component)
+    const filterableTypes = def.settings.filter(setting =>
+      setting?.type?.startsWith("filter")
+    )
+
+    // Map this?
+    for (let setting of filterableTypes) {
+      const updateFilter = Utils.migrateSearchFilters(
+        enrichedComponent[setting.key]
+      )
+      // DEAN - switch to real value when complete
+      if (updateFilter) {
+        enrichedComponent[setting.key + "_x"] = updateFilter
+      }
+    }
+
     return migrated
   }
 
@@ -405,7 +425,13 @@ export class ComponentStore extends BudiStore {
       screen: get(selectedScreen),
       useDefaultValues: true,
     })
+
+    try {
       this.migrateSettings(instance)
+    } catch (e) {
+      console.error(e)
+      throw e
+    }
 
     // Custom post processing for creation only
     let extras = {}
@@ -555,7 +581,10 @@ export class ComponentStore extends BudiStore {
       const patchResult = patchFn(component, screen)
 
       // Post processing
-      const migrated = this.migrateSettings(component)
+      // DEAN - SKIP ON SAVE FOR THE MOMENT
+      const migrated = null
+
+      this.migrateSettings(component)
 
       // Returning an explicit false signifies that we should skip this
       // update. If we migrated something, ensure we never skip.

packages/frontend-core/src/utils/utils.js

@@ -1,5 +1,6 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { Helpers } from "@budibase/bbui"
+import * as Constants from "../constants"
 
 export const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
 
@@ -350,3 +351,99 @@ export const buildMultiStepFormBlockDefaultProps = props => {
     title,
   }
 }
+
+/**
+ * Processes the filter config. Filters are migrated
+ * SearchFilter[] to SearchFilterGroup
+ *
+ *  This is supposed to be in shared-core
+ * @param {Object[]} filter
+ */
+export const migrateSearchFilters = filters => {
+  const defaultCfg = {
+    logicalOperator: Constants.FilterOperator.ALL,
+    groups: [],
+  }
+
+  const filterWhitelistKeys = [
+    "field",
+    "operator",
+    "valueType", // bb
+    "value",
+    "type",
+    "noValue", // bb
+  ]
+
+  /**
+   * Review these
+   * externalType, formulaType, subtype
+   */
+
+  if (Array.isArray(filters)) {
+    const baseGroup = {
+      filters: [],
+      logicalOperator: Constants.FilterOperator.ALL,
+    }
+
+    const migratedSetting = filters.reduce((acc, filter) => {
+      // Sort the properties for easier debugging
+      // Remove unset values
+      const filterEntries = Object.entries(filter)
+        .sort((a, b) => {
+          return a[0].localeCompare(b[0])
+        })
+        .filter(x => x[1] ?? false)
+
+      // Scrub invalid filters
+      const { operator, onEmptyFilter, field, valueType } = filter
+      if (!field || !valueType) {
+        // THIS SCRUBS THE 2 GLOBALS
+        // return acc
+      }
+
+      if (filterEntries.length == 1) {
+        console.log("### one entry ")
+        const [key, value] = filterEntries[0]
+        // Global
+        if (key === "onEmptyFilter") {
+          // unset otherwise, seems to be the default
+          acc.onEmptyFilter = value
+        } else if (key === "operator" && value === "allOr") {
+          // Group 1 logical operator
+          baseGroup.logicalOperator = Constants.FilterOperator.ANY
+        }
+
+        return acc
+      }
+
+      // Process settings??
+      const whiteListedFilterSettings = filterEntries.reduce((acc, entry) => {
+        const [key, value] = entry
+
+        if (filterWhitelistKeys.includes(key)) {
+          if (key === "field") {
+            acc.push([key, value.replace(/^[0-9]+:/, "")])
+          } else {
+            acc.push([key, value])
+          }
+        }
+        return acc
+      }, [])
+
+      const migratedFilter = Object.fromEntries(whiteListedFilterSettings)
+
+      baseGroup.filters.push(migratedFilter)
+
+      if (!acc.groups.length) {
+        // init the base group
+        acc.groups.push(baseGroup)
+      }
+
+      return acc
+    }, defaultCfg)
+
+    console.log("MIGRATED ", migratedSetting)
+    return migratedSetting
+  }
+  return false
+}

packages/shared-core/src/filters.ts

@@ -19,12 +19,15 @@ import {
   RangeOperator,
   LogicalOperator,
   isLogicalSearchOperator,
+  SearchFilterGroup,
+  FilterGroupLogicalOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
 import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
+// import { Constants } from "@budibase/frontend-core"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -263,11 +266,134 @@ export class ColumnSplitter {
   }
 }
 
+/*
+  separate into buildQuery and build filter?
+*/
+
 /**
  * Builds a JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
  */
+
+const builderFilter = (expression: SearchFilter) => {
+  // Filter body
+  let query: SearchFilters = {
+    // string: {},
+    // fuzzy: {},
+    // range: {},
+    // equal: {},
+    // notEqual: {},
+    // empty: {},
+    // notEmpty: {},
+    // contains: {},
+    // notContains: {},
+    // oneOf: {},
+    // containsAny: {},
+  }
+
+  // DEAN -
+  // This is the chattiest service we have, pruning the requests
+  // of bloat should have meaningful impact
+  // Further validation in this area is a must
+
+  let { operator, field, type, value, externalType, onEmptyFilter } = expression
+  const queryOperator = operator as SearchFilterOperator
+  const isHbs =
+    typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+  // Parse all values into correct types
+  if (operator === "allOr") {
+    query.allOr = true
+    return
+  }
+  if (onEmptyFilter) {
+    query.onEmptyFilter = onEmptyFilter
+    return
+  }
+  if (
+    type === "datetime" &&
+    !isHbs &&
+    queryOperator !== "empty" &&
+    queryOperator !== "notEmpty"
+  ) {
+    // Ensure date value is a valid date and parse into correct format
+    if (!value) {
+      return
+    }
+    try {
+      value = new Date(value).toISOString()
+    } catch (error) {
+      return
+    }
+  }
+  if (type === "number" && typeof value === "string" && !isHbs) {
+    if (queryOperator === "oneOf") {
+      value = value.split(",").map(item => parseFloat(item))
+    } else {
+      value = parseFloat(value)
+    }
+  }
+  if (type === "boolean") {
+    value = `${value}`?.toLowerCase() === "true"
+  }
+  if (
+    ["contains", "notContains", "containsAny"].includes(operator) &&
+    type === "array" &&
+    typeof value === "string"
+  ) {
+    value = value.split(",")
+  }
+  if (operator.startsWith("range") && query.range) {
+    const minint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.min || Number.MIN_SAFE_INTEGER
+    const maxint =
+      SqlNumberTypeRangeMap[externalType as keyof typeof SqlNumberTypeRangeMap]
+        ?.max || Number.MAX_SAFE_INTEGER
+    if (!query.range[field]) {
+      query.range[field] = {
+        low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+        high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+      }
+    }
+    if (operator === "rangeLow" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        low: value,
+      }
+    } else if (operator === "rangeHigh" && value != null && value !== "") {
+      query.range[field] = {
+        ...query.range[field],
+        high: value,
+      }
+    }
+  } else if (isLogicalSearchOperator(queryOperator)) {
+    // TODO
+  } else if (query[queryOperator] && operator !== "onEmptyFilter") {
+    if (type === "boolean") {
+      // Transform boolean filters to cope with null.
+      // "equals false" needs to be "not equals true"
+      // "not equals false" needs to be "equals true"
+      if (queryOperator === "equal" && value === false) {
+        query.notEqual = query.notEqual || {}
+        query.notEqual[field] = true
+      } else if (queryOperator === "notEqual" && value === false) {
+        query.equal = query.equal || {}
+        query.equal[field] = true
+      } else {
+        query[queryOperator] ??= {}
+        query[queryOperator]![field] = value
+      }
+    } else {
+      query[queryOperator] ??= {}
+      query[queryOperator]![field] = value
+    }
+  }
+
+  return query
+}
+
 export const buildQuery = (filter: SearchFilter[]) => {
+  //
   let query: SearchFilters = {
     string: {},
     fuzzy: {},
@@ -383,10 +509,36 @@ export const buildQuery = (filter: SearchFilter[]) => {
       }
     }
   })
-
   return query
 }
 
+/**
+ * FOR TESTING - this still isnt type safe
+ */
+export const buildQueryX = (filter: SearchFilterGroup) => {
+  const operatorMap = {
+    [FilterGroupLogicalOperator.ALL]: LogicalOperator.AND,
+    [FilterGroupLogicalOperator.ANY]: LogicalOperator.OR,
+  }
+
+  const globalOnEmpty = filter.onEmptyFilter ? filter.onEmptyFilter : null
+  const globalOperator = operatorMap[filter.logicalOperator]
+
+  const coreRequest: SearchFilters = {
+    ...(globalOnEmpty ? { onEmptyFilter: globalOnEmpty } : {}),
+    [globalOperator]: {
+      conditions: filter.groups?.map((group: SearchFilterGroup) => {
+        return {
+          [operatorMap[group.logicalOperator]]: {
+            conditions: group.filters?.map(x => builderFilter(x)), //buildFilters
+          },
+        }
+      }),
+    },
+  }
+  return coreRequest
+}
+
 // The frontend can send single values for array fields sometimes, so to handle
 // this we convert them to arrays at the controller level so that nothing below
 // this has to worry about the non-array values.