Add data-utils to compiled code

packages/builder/package.json

@@ -61,6 +61,7 @@
     "@budibase/bbui": "2.3.18-alpha.29",
     "@budibase/client": "2.3.18-alpha.29",
     "@budibase/frontend-core": "2.3.18-alpha.29",
+    "@budibase/data-utils": "0.0.1",
     "@budibase/string-templates": "2.3.18-alpha.29",
     "@fortawesome/fontawesome-svg-core": "^6.2.1",
     "@fortawesome/free-brands-svg-icons": "^6.2.1",

packages/builder/vite.config.js

@@ -97,6 +97,10 @@ export default defineConfig(({ mode }) => {
           find: "helpers",
           replacement: path.resolve("./src/helpers"),
         },
+        {
+          find: "@budibase/data-utils",
+          replacement: path.resolve("../data-utils/src"),
+        },
       ],
     },
   }

packages/client/package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@budibase/bbui": "2.3.18-alpha.29",
     "@budibase/frontend-core": "2.3.18-alpha.29",
+    "@budibase/data-utils": "0.0.1",
     "@budibase/string-templates": "2.3.18-alpha.29",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",

packages/client/rollup.config.js

@@ -69,6 +69,10 @@ export default {
           find: "sdk",
           replacement: path.resolve("./src/sdk"),
         },
+        {
+          find: "@budibase/data-utils",
+          replacement: path.resolve("../data-utils/dist/src"),
+        },
       ],
     }),
     svelte({

packages/data-utils/package.json

@@ -3,7 +3,8 @@
   "private": true,
   "version": "0.0.1",
   "description": "Shared data utils",
-  "main": "src/index.ts",
+  "main": "dist/src/index.js",
+  "types": "dist/src/index.d.ts",
   "author": "Budibase",
   "license": "GPL-3.0",
   "scripts": {

packages/data-utils/tsconfig.json

@@ -1,3 +1,6 @@
 {
-  "extends": "./tsconfig.build.json"
+  "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "composite": true
+  }
 }

packages/frontend-core/src/utils/lucene.js

@@ -1,360 +1 @@
-import { Helpers } from "@budibase/bbui"
+export * from "@budibase/data-utils"
-import { OperatorOptions, SqlNumberTypeRangeMap } from "../constants"
-
-const HBS_REGEX = /{{([^{].*?)}}/g
-
-/**
- * Returns the valid operator options for a certain data type
- * @param type the data type
- */
-export const getValidOperatorsForType = (type, field, datasource) => {
-  const Op = OperatorOptions
-  const stringOps = [
-    Op.Equals,
-    Op.NotEquals,
-    Op.StartsWith,
-    Op.Like,
-    Op.Empty,
-    Op.NotEmpty,
-    Op.In,
-  ]
-  const numOps = [
-    Op.Equals,
-    Op.NotEquals,
-    Op.MoreThan,
-    Op.LessThan,
-    Op.Empty,
-    Op.NotEmpty,
-    Op.In,
-  ]
-  let ops = []
-  if (type === "string") {
-    ops = stringOps
-  } else if (type === "number") {
-    ops = numOps
-  } else if (type === "options") {
-    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
-  } else if (type === "array") {
-    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
-  } else if (type === "boolean") {
-    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
-  } else if (type === "longform") {
-    ops = stringOps
-  } else if (type === "datetime") {
-    ops = numOps
-  } else if (type === "formula") {
-    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
-  }
-
-  // Filter out "like" for internal tables
-  const externalTable = datasource?.tableId?.includes("datasource_plus")
-  if (datasource?.type === "table" && !externalTable) {
-    ops = ops.filter(x => x !== Op.Like)
-  }
-
-  // Only allow equal/not equal for _id in SQL tables
-  if (field === "_id" && externalTable) {
-    ops = [Op.Equals, Op.NotEquals]
-  }
-
-  return ops
-}
-
-/**
- * Operators which do not support empty strings as values
- */
-export const NoEmptyFilterStrings = [
-  OperatorOptions.StartsWith.value,
-  OperatorOptions.Like.value,
-  OperatorOptions.Equals.value,
-  OperatorOptions.NotEquals.value,
-  OperatorOptions.Contains.value,
-  OperatorOptions.NotContains.value,
-]
-
-/**
- * Removes any fields that contain empty strings that would cause inconsistent
- * behaviour with how backend tables are filtered (no value means no filter).
- */
-const cleanupQuery = query => {
-  if (!query) {
-    return query
-  }
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!query[filterField]) {
-      continue
-    }
-    for (let [key, value] of Object.entries(query[filterField])) {
-      if (value == null || value === "") {
-        delete query[filterField][key]
-      }
-    }
-  }
-  return query
-}
-
-/**
- * Removes a numeric prefix on field names designed to give fields uniqueness
- */
-const removeKeyNumbering = key => {
-  if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
-    const parts = key.split(":")
-    parts.shift()
-    return parts.join(":")
-  } else {
-    return key
-  }
-}
-
-/**
- * Builds a lucene JSON query from the filter structure generated in the builder
- * @param filter the builder filter structure
- */
-export const buildLuceneQuery = filter => {
-  let query = {
-    string: {},
-    fuzzy: {},
-    range: {},
-    equal: {},
-    notEqual: {},
-    empty: {},
-    notEmpty: {},
-    contains: {},
-    notContains: {},
-    oneOf: {},
-    containsAny: {},
-  }
-  if (Array.isArray(filter)) {
-    filter.forEach(expression => {
-      let { operator, field, type, value, externalType } = expression
-      const isHbs =
-        typeof value === "string" && value.match(HBS_REGEX)?.length > 0
-      // Parse all values into correct types
-      if (operator === "allOr") {
-        query.allOr = true
-        return
-      }
-      if (
-        type === "datetime" &&
-        !isHbs &&
-        operator !== "empty" &&
-        operator !== "notEmpty"
-      ) {
-        // Ensure date value is a valid date and parse into correct format
-        if (!value) {
-          return
-        }
-        try {
-          value = new Date(value).toISOString()
-        } catch (error) {
-          return
-        }
-      }
-      if (type === "number" && typeof value === "string") {
-        if (operator === "oneOf") {
-          value = value.split(",").map(item => parseFloat(item))
-        } else if (!isHbs) {
-          value = parseFloat(value)
-        }
-      }
-      if (type === "boolean") {
-        value = `${value}`?.toLowerCase() === "true"
-      }
-      if (
-        ["contains", "notContains", "containsAny"].includes(operator) &&
-        type === "array" &&
-        typeof value === "string"
-      ) {
-        value = value.split(",")
-      }
-      if (operator.startsWith("range")) {
-        const minint =
-          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
-        const maxint =
-          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
-        if (!query.range[field]) {
-          query.range[field] = {
-            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-          }
-        }
-        if (operator === "rangeLow" && value != null && value !== "") {
-          query.range[field].low = value
-        } else if (operator === "rangeHigh" && value != null && value !== "") {
-          query.range[field].high = value
-        }
-      } else if (query[operator]) {
-        if (type === "boolean") {
-          // Transform boolean filters to cope with null.
-          // "equals false" needs to be "not equals true"
-          // "not equals false" needs to be "equals true"
-          if (operator === "equal" && value === false) {
-            query.notEqual[field] = true
-          } else if (operator === "notEqual" && value === false) {
-            query.equal[field] = true
-          } else {
-            query[operator][field] = value
-          }
-        } else {
-          query[operator][field] = value
-        }
-      }
-    })
-  }
-  return query
-}
-
-/**
- * Performs a client-side lucene search on an array of data
- * @param docs the data
- * @param query the JSON lucene query
- */
-export const runLuceneQuery = (docs, query) => {
-  if (!docs || !Array.isArray(docs)) {
-    return []
-  }
-  if (!query) {
-    return docs
-  }
-
-  // Make query consistent first
-  query = cleanupQuery(query)
-
-  // Iterates over a set of filters and evaluates a fail function against a doc
-  const match = (type, failFn) => doc => {
-    const filters = Object.entries(query[type] || {})
-    for (let i = 0; i < filters.length; i++) {
-      const [key, testValue] = filters[i]
-      const docValue = Helpers.deepGet(doc, removeKeyNumbering(key))
-      if (failFn(docValue, testValue)) {
-        return false
-      }
-    }
-    return true
-  }
-
-  // Process a string match (fails if the value does not start with the string)
-  const stringMatch = match("string", (docValue, testValue) => {
-    return (
-      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
-    )
-  })
-
-  // Process a fuzzy match (treat the same as starts with when running locally)
-  const fuzzyMatch = match("fuzzy", (docValue, testValue) => {
-    return (
-      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
-    )
-  })
-
-  // Process a range match
-  const rangeMatch = match("range", (docValue, testValue) => {
-    return (
-      docValue == null ||
-      docValue === "" ||
-      docValue < testValue.low ||
-      docValue > testValue.high
-    )
-  })
-
-  // Process an equal match (fails if the value is different)
-  const equalMatch = match("equal", (docValue, testValue) => {
-    return testValue != null && testValue !== "" && docValue !== testValue
-  })
-
-  // Process a not-equal match (fails if the value is the same)
-  const notEqualMatch = match("notEqual", (docValue, testValue) => {
-    return testValue != null && testValue !== "" && docValue === testValue
-  })
-
-  // Process an empty match (fails if the value is not empty)
-  const emptyMatch = match("empty", docValue => {
-    return docValue != null && docValue !== ""
-  })
-
-  // Process a not-empty match (fails is the value is empty)
-  const notEmptyMatch = match("notEmpty", docValue => {
-    return docValue == null || docValue === ""
-  })
-
-  // Process an includes match (fails if the value is not included)
-  const oneOf = match("oneOf", (docValue, testValue) => {
-    if (typeof testValue === "string") {
-      testValue = testValue.split(",")
-      if (typeof docValue === "number") {
-        testValue = testValue.map(item => parseFloat(item))
-      }
-    }
-    return !testValue?.includes(docValue)
-  })
-
-  const containsAny = match("containsAny", (docValue, testValue) => {
-    return !docValue?.includes(...testValue)
-  })
-
-  const contains = match("contains", (docValue, testValue) => {
-    return !testValue?.every(item => docValue?.includes(item))
-  })
-
-  const notContains = match("notContains", (docValue, testValue) => {
-    return testValue?.every(item => docValue?.includes(item))
-  })
-
-  // Match a document against all criteria
-  const docMatch = doc => {
-    return (
-      stringMatch(doc) &&
-      fuzzyMatch(doc) &&
-      rangeMatch(doc) &&
-      equalMatch(doc) &&
-      notEqualMatch(doc) &&
-      emptyMatch(doc) &&
-      notEmptyMatch(doc) &&
-      oneOf(doc) &&
-      contains(doc) &&
-      containsAny(doc) &&
-      notContains(doc)
-    )
-  }
-
-  // Process all docs
-  return docs.filter(docMatch)
-}
-
-/**
- * Performs a client-side sort from the equivalent server-side lucene sort
- * parameters.
- * @param docs the data
- * @param sort the sort column
- * @param sortOrder the sort order ("ascending" or "descending")
- * @param sortType the type of sort ("string" or "number")
- */
-export const luceneSort = (docs, sort, sortOrder, sortType = "string") => {
-  if (!sort || !sortOrder || !sortType) {
-    return docs
-  }
-  const parse = sortType === "string" ? x => `${x}` : x => parseFloat(x)
-  return docs.slice().sort((a, b) => {
-    const colA = parse(a[sort])
-    const colB = parse(b[sort])
-    if (sortOrder === "Descending") {
-      return colA > colB ? -1 : 1
-    } else {
-      return colA > colB ? 1 : -1
-    }
-  })
-}
-
-/**
- * Limits the specified docs to the specified number of rows from the equivalent
- * server-side lucene limit parameters.
- * @param docs the data
- * @param limit the number of docs to limit to
- */
-export const luceneLimit = (docs, limit) => {
-  const numLimit = parseFloat(limit)
-  if (isNaN(numLimit)) {
-    return docs
-  }
-  return docs.slice(0, numLimit)
-}