Add data-utils with filters

packages/data-utils/.gitignore

@@ -0,0 +1 @@
+node_modules/

packages/data-utils/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@budibase/data-utils",
+  "version": "0.0.1",
+  "description": "Shared data utils",
+  "main": "src/index.ts",
+  "author": "Budibase",
+  "license": "GPL-3.0",
+  "private": true,
+  "devDependencies": {
+    "typescript": "4.7.3"
+  }
+}

packages/data-utils/src/constants.ts

@@ -0,0 +1,69 @@
+export const OperatorOptions = {
+  Equals: {
+    value: "equal",
+    label: "Equals",
+  },
+  NotEquals: {
+    value: "notEqual",
+    label: "Not equals",
+  },
+  Empty: {
+    value: "empty",
+    label: "Is empty",
+  },
+  NotEmpty: {
+    value: "notEmpty",
+    label: "Is not empty",
+  },
+  StartsWith: {
+    value: "string",
+    label: "Starts with",
+  },
+  Like: {
+    value: "fuzzy",
+    label: "Like",
+  },
+  MoreThan: {
+    value: "rangeLow",
+    label: "More than or equal to",
+  },
+  LessThan: {
+    value: "rangeHigh",
+    label: "Less than or equal to",
+  },
+  Contains: {
+    value: "contains",
+    label: "Contains",
+  },
+  NotContains: {
+    value: "notContains",
+    label: "Does not contain",
+  },
+  In: {
+    value: "oneOf",
+    label: "Is in",
+  },
+  ContainsAny: {
+    value: "containsAny",
+    label: "Has any",
+  },
+}
+
+export const SqlNumberTypeRangeMap = {
+  integer: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  int: {
+    max: 2147483647,
+    min: -2147483648,
+  },
+  smallint: {
+    max: 32767,
+    min: -32768,
+  },
+  mediumint: {
+    max: 8388607,
+    min: -8388608,
+  },
+}

packages/data-utils/src/filters.ts

@@ -0,0 +1,427 @@
+import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
+
+const HBS_REGEX = /{{([^{].*?)}}/g
+
+/**
+ * Returns the valid operator options for a certain data type
+ * @param type the data type
+ */
+export const getValidOperatorsForType = (
+  type: string,
+  field: string,
+  datasource: { tableId: string | string[]; type: string }
+) => {
+  const Op = OperatorOptions
+  const stringOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.StartsWith,
+    Op.Like,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  const numOps = [
+    Op.Equals,
+    Op.NotEquals,
+    Op.MoreThan,
+    Op.LessThan,
+    Op.Empty,
+    Op.NotEmpty,
+    Op.In,
+  ]
+  let ops: any[] = []
+  if (type === "string") {
+    ops = stringOps
+  } else if (type === "number") {
+    ops = numOps
+  } else if (type === "options") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty, Op.In]
+  } else if (type === "array") {
+    ops = [Op.Contains, Op.NotContains, Op.Empty, Op.NotEmpty, Op.ContainsAny]
+  } else if (type === "boolean") {
+    ops = [Op.Equals, Op.NotEquals, Op.Empty, Op.NotEmpty]
+  } else if (type === "longform") {
+    ops = stringOps
+  } else if (type === "datetime") {
+    ops = numOps
+  } else if (type === "formula") {
+    ops = stringOps.concat([Op.MoreThan, Op.LessThan])
+  }
+
+  // Filter out "like" for internal tables
+  const externalTable = datasource?.tableId?.includes("datasource_plus")
+  if (datasource?.type === "table" && !externalTable) {
+    ops = ops.filter(x => x !== Op.Like)
+  }
+
+  // Only allow equal/not equal for _id in SQL tables
+  if (field === "_id" && externalTable) {
+    ops = [Op.Equals, Op.NotEquals]
+  }
+
+  return ops
+}
+
+/**
+ * Operators which do not support empty strings as values
+ */
+export const NoEmptyFilterStrings = [
+  OperatorOptions.StartsWith.value,
+  OperatorOptions.Like.value,
+  OperatorOptions.Equals.value,
+  OperatorOptions.NotEquals.value,
+  OperatorOptions.Contains.value,
+  OperatorOptions.NotContains.value,
+]
+
+/**
+ * Removes any fields that contain empty strings that would cause inconsistent
+ * behaviour with how backend tables are filtered (no value means no filter).
+ */
+const cleanupQuery = (query: { [x: string]: { [x: string]: any } }) => {
+  if (!query) {
+    return query
+  }
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!query[filterField]) {
+      continue
+    }
+    for (let [key, value] of Object.entries(query[filterField])) {
+      if (value == null || value === "") {
+        delete query[filterField][key]
+      }
+    }
+  }
+  return query
+}
+
+/**
+ * Removes a numeric prefix on field names designed to give fields uniqueness
+ */
+const removeKeyNumbering = (key: string) => {
+  if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
+    const parts = key.split(":")
+    parts.shift()
+    return parts.join(":")
+  } else {
+    return key
+  }
+}
+
+/**
+ * Builds a lucene JSON query from the filter structure generated in the builder
+ * @param filter the builder filter structure
+ */
+export const buildLuceneQuery = (filter: any[]) => {
+  let query = {
+    string: {},
+    fuzzy: {},
+    range: {},
+    equal: {},
+    notEqual: {},
+    empty: {},
+    notEmpty: {},
+    contains: {},
+    notContains: {},
+    oneOf: {},
+    containsAny: {},
+  }
+  if (Array.isArray(filter)) {
+    filter.forEach(expression => {
+      let { operator, field, type, value, externalType } = expression
+      const isHbs =
+        typeof value === "string" && value.match(HBS_REGEX)?.length > 0
+      // Parse all values into correct types
+      if (operator === "allOr") {
+        query.allOr = true
+        return
+      }
+      if (
+        type === "datetime" &&
+        !isHbs &&
+        operator !== "empty" &&
+        operator !== "notEmpty"
+      ) {
+        // Ensure date value is a valid date and parse into correct format
+        if (!value) {
+          return
+        }
+        try {
+          value = new Date(value).toISOString()
+        } catch (error) {
+          return
+        }
+      }
+      if (type === "number" && typeof value === "string") {
+        if (operator === "oneOf") {
+          value = value.split(",").map(item => parseFloat(item))
+        } else if (!isHbs) {
+          value = parseFloat(value)
+        }
+      }
+      if (type === "boolean") {
+        value = `${value}`?.toLowerCase() === "true"
+      }
+      if (
+        ["contains", "notContains", "containsAny"].includes(operator) &&
+        type === "array" &&
+        typeof value === "string"
+      ) {
+        value = value.split(",")
+      }
+      if (operator.startsWith("range")) {
+        const minint =
+          SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
+        const maxint =
+          SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
+        if (!query.range[field]) {
+          query.range[field] = {
+            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+          }
+        }
+        if (operator === "rangeLow" && value != null && value !== "") {
+          query.range[field].low = value
+        } else if (operator === "rangeHigh" && value != null && value !== "") {
+          query.range[field].high = value
+        }
+      } else if (query[operator]) {
+        if (type === "boolean") {
+          // Transform boolean filters to cope with null.
+          // "equals false" needs to be "not equals true"
+          // "not equals false" needs to be "equals true"
+          if (operator === "equal" && value === false) {
+            query.notEqual[field] = true
+          } else if (operator === "notEqual" && value === false) {
+            query.equal[field] = true
+          } else {
+            query[operator][field] = value
+          }
+        } else {
+          query[operator][field] = value
+        }
+      }
+    })
+  }
+  return query
+}
+
+const deepGet = (obj: { [x: string]: any }, key: string) => {
+  if (!obj || !key) {
+    return null
+  }
+  if (Object.prototype.hasOwnProperty.call(obj, key)) {
+    return obj[key]
+  }
+  const split = key.split(".")
+  for (let i = 0; i < split.length; i++) {
+    obj = obj?.[split[i]]
+  }
+  return obj
+}
+
+/**
+ * Performs a client-side lucene search on an array of data
+ * @param docs the data
+ * @param query the JSON lucene query
+ */
+export const runLuceneQuery = (
+  docs: any[],
+  query?: { [x: string]: any; sheet?: string }
+) => {
+  if (!docs || !Array.isArray(docs)) {
+    return []
+  }
+  if (!query) {
+    return docs
+  }
+
+  // Make query consistent first
+  query = cleanupQuery(query)
+
+  // Iterates over a set of filters and evaluates a fail function against a doc
+  const match =
+    (
+      type: string,
+      failFn: {
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any): boolean
+        (docValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): boolean
+        (docValue: any, testValue: any): any
+        (arg0: any, arg1: unknown): any
+      }
+    ) =>
+    (doc: any) => {
+      const filters = Object.entries(query[type] || {})
+      for (let i = 0; i < filters.length; i++) {
+        const [key, testValue] = filters[i]
+        const docValue = deepGet(doc, removeKeyNumbering(key))
+        if (failFn(docValue, testValue)) {
+          return false
+        }
+      }
+      return true
+    }
+
+  // Process a string match (fails if the value does not start with the string)
+  const stringMatch = match("string", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a fuzzy match (treat the same as starts with when running locally)
+  const fuzzyMatch = match("fuzzy", (docValue: string, testValue: string) => {
+    return (
+      !docValue || !docValue?.toLowerCase().startsWith(testValue?.toLowerCase())
+    )
+  })
+
+  // Process a range match
+  const rangeMatch = match(
+    "range",
+    (
+      docValue: string | number | null,
+      testValue: { low: number; high: number }
+    ) => {
+      return (
+        docValue == null ||
+        docValue === "" ||
+        docValue < testValue.low ||
+        docValue > testValue.high
+      )
+    }
+  )
+
+  // Process an equal match (fails if the value is different)
+  const equalMatch = match(
+    "equal",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue !== testValue
+    }
+  )
+
+  // Process a not-equal match (fails if the value is the same)
+  const notEqualMatch = match(
+    "notEqual",
+    (docValue: any, testValue: string | null) => {
+      return testValue != null && testValue !== "" && docValue === testValue
+    }
+  )
+
+  // Process an empty match (fails if the value is not empty)
+  const emptyMatch = match("empty", (docValue: string | null) => {
+    return docValue != null && docValue !== ""
+  })
+
+  // Process a not-empty match (fails is the value is empty)
+  const notEmptyMatch = match("notEmpty", (docValue: string | null) => {
+    return docValue == null || docValue === ""
+  })
+
+  // Process an includes match (fails if the value is not included)
+  const oneOf = match("oneOf", (docValue: any, testValue: string[]) => {
+    if (typeof testValue === "string") {
+      testValue = testValue.split(",")
+      if (typeof docValue === "number") {
+        testValue = testValue.map((item: string) => parseFloat(item))
+      }
+    }
+    return !testValue?.includes(docValue)
+  })
+
+  const containsAny = match(
+    "containsAny",
+    (docValue: string | any[], testValue: any) => {
+      return !docValue?.includes(...testValue)
+    }
+  )
+
+  const contains = match(
+    "contains",
+    (docValue: string | any[], testValue: any[]) => {
+      return !testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  const notContains = match(
+    "notContains",
+    (docValue: string | any[], testValue: any[]) => {
+      return testValue?.every((item: any) => docValue?.includes(item))
+    }
+  )
+
+  // Match a document against all criteria
+  const docMatch = (doc: any) => {
+    return (
+      stringMatch(doc) &&
+      fuzzyMatch(doc) &&
+      rangeMatch(doc) &&
+      equalMatch(doc) &&
+      notEqualMatch(doc) &&
+      emptyMatch(doc) &&
+      notEmptyMatch(doc) &&
+      oneOf(doc) &&
+      contains(doc) &&
+      containsAny(doc) &&
+      notContains(doc)
+    )
+  }
+
+  // Process all docs
+  return docs.filter(docMatch)
+}
+
+/**
+ * Performs a client-side sort from the equivalent server-side lucene sort
+ * parameters.
+ * @param docs the data
+ * @param sort the sort column
+ * @param sortOrder the sort order ("ascending" or "descending")
+ * @param sortType the type of sort ("string" or "number")
+ */
+export const luceneSort = (
+  docs: any[],
+  sort: string | number,
+  sortOrder: string,
+  sortType = "string"
+) => {
+  if (!sort || !sortOrder || !sortType) {
+    return docs
+  }
+  const parse =
+    sortType === "string" ? (x: any) => `${x}` : (x: string) => parseFloat(x)
+  return docs
+    .slice()
+    .sort((a: { [x: string]: any }, b: { [x: string]: any }) => {
+      const colA = parse(a[sort])
+      const colB = parse(b[sort])
+      if (sortOrder === "Descending") {
+        return colA > colB ? -1 : 1
+      } else {
+        return colA > colB ? 1 : -1
+      }
+    })
+}
+
+/**
+ * Limits the specified docs to the specified number of rows from the equivalent
+ * server-side lucene limit parameters.
+ * @param docs the data
+ * @param limit the number of docs to limit to
+ */
+export const luceneLimit = (docs: string | any[], limit: string) => {
+  const numLimit = parseFloat(limit)
+  if (isNaN(numLimit)) {
+    return docs
+  }
+  return docs.slice(0, numLimit)
+}

packages/data-utils/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./constants"
+export * from "./filters"

packages/data-utils/tsconfig.build.json

@@ -0,0 +1,29 @@
+{
+  "compilerOptions": {
+    "target": "es6",
+    "module": "commonjs",
+    "lib": ["es2020"],
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": true,
+    "sourceMap": true,
+    "declaration": true,
+    "types": [ "node", "jest" ],
+    "outDir": "dist",
+    "skipLibCheck": true
+  },
+  "include": [
+    "**/*.js",
+    "**/*.ts",
+    "package.json"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "**/*.spec.ts",
+    "**/*.spec.js",
+    "__mocks__"
+  ]
+}

packages/data-utils/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.build.json",
+  "compilerOptions": {
+    "composite": true,
+    "baseUrl": "."
+  },
+  "exclude": ["node_modules", "dist"]
+}

packages/data-utils/yarn.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+typescript@4.7.3:
+  version "4.7.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.3.tgz#8364b502d5257b540f9de4c40be84c98e23a129d"
+  integrity sha512-WOkT3XYvrpXx4vMMqlD+8R8R37fZkjyLGlxavMc4iB8lrl8L0DeTcHbYgw/v0N/z9wAFsgBhcsF0ruoySS22mA==

packages/server/package.json

@@ -45,6 +45,7 @@
     "@apidevtools/swagger-parser": "10.0.3",
     "@budibase/backend-core": "2.3.18-alpha.29",
     "@budibase/client": "2.3.18-alpha.29",
+    "@budibase/data-utils": "0.0.1",
     "@budibase/pro": "2.3.18-alpha.29",
     "@budibase/string-templates": "2.3.18-alpha.29",
     "@budibase/types": "2.3.18-alpha.29",

packages/server/src/integrations/googlesheets.ts

@@ -2,8 +2,11 @@ import {
   DatasourceFieldType,
   DatasourcePlus,
   Integration,
+  PaginationJson,
   QueryJson,
   QueryType,
+  SearchFilters,
+  SortJson,
   Table,
   TableSchema,
 } from "@budibase/types"
@@ -13,6 +16,7 @@ import { DataSourceOperation, FieldTypes } from "../constants"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 import fetch from "node-fetch"
 import { configs, HTTPError } from "@budibase/backend-core"
+import { runLuceneQuery } from "@budibase/data-utils"
 
 interface GoogleSheetsConfig {
   spreadsheetId: string
@@ -237,7 +241,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     const handlers = {
       [DataSourceOperation.CREATE]: () =>
         this.create({ sheet, row: json.body }),
-      [DataSourceOperation.READ]: () => this.read({ sheet }),
+      [DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
       [DataSourceOperation.UPDATE]: () =>
         this.update({
           // exclude the header row and zero index
@@ -345,14 +349,20 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
   }
 
-  async read(query: { sheet: string }) {
+  async read(query: {
+    sheet: string
+    filters?: SearchFilters
+    sort?: SortJson
+    paginate?: PaginationJson
+  }) {
     try {
       await this.connect()
       const sheet = this.client.sheetsByTitle[query.sheet]
       const rows = await sheet.getRows()
+      const filtered = runLuceneQuery(rows, query.filters)
       const headerValues = sheet.headerValues
       const response = []
-      for (let row of rows) {
+      for (let row of filtered) {
         response.push(
           this.buildRowObject(headerValues, row._rawData, row._rowNumber)
         )

packages/server/tsconfig.json

@@ -9,6 +9,7 @@
       "@budibase/types": ["../types/src"],
       "@budibase/backend-core": ["../backend-core/src"],
       "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/data-utils": ["../data-utils/src"],
       "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
     }
   },
@@ -19,15 +20,9 @@
   "references": [
     { "path": "../types" },
     { "path": "../backend-core" },
+    { "path": "../data-utils" },
     { "path": "../../../budibase-pro/packages/pro" }
   ],
-  "include": [
+  "include": ["src/**/*", "specs", "package.json"],
-    "src/**/*",
+  "exclude": ["node_modules", "dist"]
-    "specs",
+}
-    "package.json"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist"
-  ]
-}