From f35e3cb99c1a0e47bceb7fc7741f1c4e4bf96ed0 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Tue, 21 Nov 2023 16:08:20 +0000
Subject: [PATCH] Handle dates in query range filter (#12413)
* Handle dates in query range filter
* Add unit tests for runLuceneQuery
* Fix build errors
---
packages/shared-core/src/filters.ts | 20 +-
.../shared-core/src/tests/filters.test.ts | 174 ++++++++++++++++++
packages/shared-core/tsconfig.build.json | 3 +-
3 files changed, 190 insertions(+), 7 deletions(-)
create mode 100644 packages/shared-core/src/tests/filters.test.ts
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 1839a53525..564e8a52c9 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -9,6 +9,7 @@ import {
SortDirection,
SortType,
} from "@budibase/types"
+import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet } from "./helpers"
@@ -302,12 +303,19 @@ export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
docValue: string | number | null,
testValue: { low: number; high: number }
) => {
- return (
- docValue == null ||
- docValue === "" ||
- +docValue < testValue.low ||
- +docValue > testValue.high
- )
+ if (docValue == null || docValue === "") {
+ return true
+ }
+ if (!isNaN(+docValue)) {
+ return +docValue < testValue.low || +docValue > testValue.high
+ }
+ if (dayjs(docValue).isValid()) {
+ return (
+ new Date(docValue).getTime() < new Date(testValue.low).getTime() ||
+ new Date(docValue).getTime() > new Date(testValue.high).getTime()
+ )
+ }
+ throw "Cannot perform range filter - invalid type."
}
)
diff --git a/packages/shared-core/src/tests/filters.test.ts b/packages/shared-core/src/tests/filters.test.ts
new file mode 100644
index 0000000000..6f488cffbd
--- /dev/null
+++ b/packages/shared-core/src/tests/filters.test.ts
@@ -0,0 +1,174 @@
+import { SearchQuery, SearchQueryOperators } from "@budibase/types"
+import { runLuceneQuery } from "../filters"
+import { expect, describe, it } from "vitest"
+
+describe("runLuceneQuery", () => {
+ const docs = [
+ {
+ order_id: 1,
+ customer_id: 259,
+ order_status: 4,
+ order_date: "2016-01-01T00:00:00.000Z",
+ required_date: "2016-01-03T00:00:00.000Z",
+ shipped_date: "2016-01-03T00:00:00.000Z",
+ store_id: 1,
+ staff_id: 2,
+ description: "Large box",
+ label: undefined,
+ },
+ {
+ order_id: 2,
+ customer_id: 1212,
+ order_status: 4,
+ order_date: "2016-01-05T00:00:00.000Z",
+ required_date: "2016-01-04T00:00:00.000Z",
+ shipped_date: "2016-01-03T00:00:00.000Z",
+ store_id: 2,
+ staff_id: 6,
+ description: "Small box",
+ label: "FRAGILE",
+ },
+ {
+ order_id: 3,
+ customer_id: 523,
+ order_status: 5,
+ order_date: "2016-01-12T00:00:00.000Z",
+ required_date: "2016-01-05T00:00:00.000Z",
+ shipped_date: "2016-01-03T00:00:00.000Z",
+ store_id: 2,
+ staff_id: 7,
+ description: "Heavy box",
+ label: "HEAVY",
+ },
+ ]
+
+ function buildQuery(
+ filterKey: string,
+ value: { [key: string]: any }
+ ): SearchQuery {
+ const query: SearchQuery = {
+ string: {},
+ fuzzy: {},
+ range: {},
+ equal: {},
+ notEqual: {},
+ empty: {},
+ notEmpty: {},
+ contains: {},
+ notContains: {},
+ oneOf: {},
+ containsAny: {},
+ }
+ query[filterKey as SearchQueryOperators] = value
+ return query
+ }
+
+ it("should return input docs if no search query is provided", () => {
+ expect(runLuceneQuery(docs)).toBe(docs)
+ })
+
+ it("should return matching rows for equal filter", () => {
+ const query = buildQuery("equal", {
+ order_status: 4,
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1, 2])
+ })
+
+ it("should return matching row for notEqual filter", () => {
+ const query = buildQuery("notEqual", {
+ order_status: 4,
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
+ })
+
+ it("should return starts with matching rows for fuzzy and string filters", () => {
+ expect(
+ runLuceneQuery(
+ docs,
+ buildQuery("fuzzy", {
+ description: "sm",
+ })
+ ).map(row => row.description)
+ ).toEqual(["Small box"])
+ expect(
+ runLuceneQuery(
+ docs,
+ buildQuery("string", {
+ description: "SM",
+ })
+ ).map(row => row.description)
+ ).toEqual(["Small box"])
+ })
+
+ it("should return rows within a range filter", () => {
+ const query = buildQuery("range", {
+ customer_id: {
+ low: 500,
+ high: 1000,
+ },
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
+ })
+
+ it("should return rows with numeric strings within a range filter", () => {
+ const query = buildQuery("range", {
+ customer_id: {
+ low: "500",
+ high: "1000",
+ },
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([3])
+ })
+
+ it("should return rows with ISO date strings within a range filter", () => {
+ const query = buildQuery("range", {
+ order_date: {
+ low: "2016-01-04T00:00:00.000Z",
+ high: "2016-01-11T00:00:00.000Z",
+ },
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2])
+ })
+
+ it("should throw an error is an invalid doc value is passed into a range filter", async () => {
+ const query = buildQuery("range", {
+ order_date: {
+ low: "2016-01-04T00:00:00.000Z",
+ high: "2016-01-11T00:00:00.000Z",
+ },
+ })
+ expect(() =>
+ runLuceneQuery(
+ [
+ {
+ order_id: 4,
+ customer_id: 1758,
+ order_status: 5,
+ order_date: "INVALID",
+ required_date: "2017-03-05T00:00:00.000Z",
+ shipped_date: "2017-03-03T00:00:00.000Z",
+ store_id: 2,
+ staff_id: 7,
+ description: undefined,
+ label: "",
+ },
+ ],
+ query
+ )
+ ).toThrowError("Cannot perform range filter - invalid type.")
+ })
+
+ it("should return rows with matches on empty filter", () => {
+ const query = buildQuery("empty", {
+ label: null,
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([1])
+ })
+
+ it("should return rows with matches on notEmpty filter", () => {
+ const query = buildQuery("notEmpty", {
+ label: null,
+ })
+ expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
+ })
+})
diff --git a/packages/shared-core/tsconfig.build.json b/packages/shared-core/tsconfig.build.json
index 1969c286b1..8a7f0ea216 100644
--- a/packages/shared-core/tsconfig.build.json
+++ b/packages/shared-core/tsconfig.build.json
@@ -24,6 +24,7 @@
"dist",
"**/*.spec.ts",
"**/*.spec.js",
- "__mocks__"
+ "__mocks__",
+ "src/tests"
]
}