diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index 288a0462e7..d63596f08f 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -108,7 +108,7 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
- docker pull budibase/couchdb:v3.2.1-sql &
+ docker pull budibase/couchdb:v3.2.1-sqs &
docker pull redis &
wait $(jobs -p)
diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts
index 7d62a6ef39..0c994d8287 100644
--- a/packages/backend-core/src/users/users.ts
+++ b/packages/backend-core/src/users/users.ts
@@ -18,9 +18,10 @@ import {
CouchFindOptions,
DatabaseQueryOpts,
SearchFilters,
- SearchFilterOperator,
SearchUsersRequest,
User,
+ BasicOperator,
+ ArrayOperator,
} from "@budibase/types"
import * as context from "../context"
import { getGlobalDB } from "../context"
@@ -46,9 +47,9 @@ function removeUserPassword(users: User | User[]) {
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
- { op: SearchFilterOperator.STRING, key: "email" },
- { op: SearchFilterOperator.EQUAL, key: "_id" },
- { op: SearchFilterOperator.ONE_OF, key: "_id" },
+ { op: BasicOperator.STRING, key: "email" },
+ { op: BasicOperator.EQUAL, key: "_id" },
+ { op: ArrayOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {
diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte
index c0bc328a4e..5b6b6b4c86 100644
--- a/packages/frontend-core/src/components/FilterBuilder.svelte
+++ b/packages/frontend-core/src/components/FilterBuilder.svelte
@@ -11,7 +11,7 @@
Label,
Multiselect,
} from "@budibase/bbui"
- import { FieldType, SearchFilterOperator } from "@budibase/types"
+ import { ArrayOperator, FieldType } from "@budibase/types"
import { generate } from "shortid"
import { QueryUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
@@ -268,7 +268,7 @@
{:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
- {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)}
+ {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
{
await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()
})
+
+ it("can have multiple values for same column", async () => {
+ await expectQuery({
+ oneOf: {
+ name: ["foo", "bar"],
+ },
+ }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+ })
+
+ it("splits comma separated strings", async () => {
+ await expectQuery({
+ oneOf: {
+ // @ts-ignore
+ name: "foo,bar",
+ },
+ }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+ })
+
+ it("trims whitespace", async () => {
+ await expectQuery({
+ oneOf: {
+ // @ts-ignore
+ name: "foo, bar",
+ },
+ }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+ })
})
describe("fuzzy", () => {
@@ -1002,6 +1028,32 @@ describe.each([
it("fails to find nonexistent row", async () => {
await expectQuery({ oneOf: { age: [2] } }).toFindNothing()
})
+
+ // I couldn't find a way to make this work in Lucene and given that
+ // we're getting rid of Lucene soon I wasn't inclined to spend time on
+ // it.
+ !isLucene &&
+ it("can convert from a string", async () => {
+ await expectQuery({
+ oneOf: {
+ // @ts-ignore
+ age: "1",
+ },
+ }).toContainExactly([{ age: 1 }])
+ })
+
+ // I couldn't find a way to make this work in Lucene and given that
+ // we're getting rid of Lucene soon I wasn't inclined to spend time on
+ // it.
+ !isLucene &&
+ it("can find multiple values for same column", async () => {
+ await expectQuery({
+ oneOf: {
+ // @ts-ignore
+ age: "1,10",
+ },
+ }).toContainExactly([{ age: 1 }, { age: 10 }])
+ })
})
describe("range", () => {
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 43a6d39172..93ce912472 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -9,7 +9,6 @@ import {
QuotaUsageType,
Row,
SaveTableRequest,
- SearchFilterOperator,
SortOrder,
SortType,
StaticQuotaName,
@@ -19,6 +18,7 @@ import {
ViewUIFieldMetadata,
ViewV2,
SearchResponse,
+ BasicOperator,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -149,7 +149,7 @@ describe.each([
primaryDisplay: "id",
query: [
{
- operator: SearchFilterOperator.EQUAL,
+ operator: BasicOperator.EQUAL,
field: "field",
value: "value",
},
@@ -561,7 +561,7 @@ describe.each([
...view,
query: [
{
- operator: SearchFilterOperator.EQUAL,
+ operator: BasicOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@@ -589,7 +589,7 @@ describe.each([
primaryDisplay: "Price",
query: [
{
- operator: SearchFilterOperator.EQUAL,
+ operator: BasicOperator.EQUAL,
field: generator.word(),
value: generator.word(),
},
@@ -673,7 +673,7 @@ describe.each([
tableId: generator.guid(),
query: [
{
- operator: SearchFilterOperator.EQUAL,
+ operator: BasicOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@@ -1194,7 +1194,7 @@ describe.each([
name: generator.guid(),
query: [
{
- operator: SearchFilterOperator.EQUAL,
+ operator: BasicOperator.EQUAL,
field: "two",
value: "bar2",
},
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 286a88054c..7bcd26806c 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -2,7 +2,6 @@ import {
EmptyFilterOption,
Row,
RowSearchParams,
- SearchFilterOperator,
SearchFilters,
SearchResponse,
SortOrder,
@@ -66,37 +65,12 @@ export function removeEmptyFilters(filters: SearchFilters) {
return filters
}
-// The frontend can send single values for array fields sometimes, so to handle
-// this we convert them to arrays at the controller level so that nothing below
-// this has to worry about the non-array values.
-function fixupFilterArrays(filters: SearchFilters) {
- const arrayFields = [
- SearchFilterOperator.ONE_OF,
- SearchFilterOperator.CONTAINS,
- SearchFilterOperator.NOT_CONTAINS,
- SearchFilterOperator.CONTAINS_ANY,
- ]
- for (const searchField of arrayFields) {
- const field = filters[searchField]
- if (field == null) {
- continue
- }
-
- for (const key of Object.keys(field)) {
- if (!Array.isArray(field[key])) {
- field[key] = [field[key]]
- }
- }
- }
- return filters
-}
-
export async function search(
options: RowSearchParams
): Promise> {
const isExternalTable = isExternalTableID(options.tableId)
options.query = removeEmptyFilters(options.query || {})
- options.query = fixupFilterArrays(options.query)
+ options.query = dataFilters.fixupFilterArrays(options.query)
if (
!dataFilters.hasFilters(options.query) &&
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 28f0b28425..d3fcae11e2 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -6,6 +6,7 @@ import {
SearchFilter,
SearchFilters,
SearchQueryFields,
+ ArrayOperator,
SearchFilterOperator,
SortType,
FieldConstraints,
@@ -14,11 +15,13 @@ import {
EmptyFilterOption,
SearchResponse,
Table,
+ BasicOperator,
+ RangeOperator,
} from "@budibase/types"
import dayjs from "dayjs"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet, schema } from "./helpers"
-import _ from "lodash"
+import { isPlainObject, isEmpty } from "lodash"
const HBS_REGEX = /{{([^{].*?)}}/g
@@ -323,6 +326,32 @@ export const buildQuery = (filter: SearchFilter[]) => {
return query
}
+// The frontend can send single values for array fields sometimes, so to handle
+// this we convert them to arrays at the controller level so that nothing below
+// this has to worry about the non-array values.
+export function fixupFilterArrays(filters: SearchFilters) {
+ for (const searchField of Object.values(ArrayOperator)) {
+ const field = filters[searchField]
+ if (field == null || !isPlainObject(field)) {
+ continue
+ }
+
+ for (const key of Object.keys(field)) {
+ if (Array.isArray(field[key])) {
+ continue
+ }
+
+ const value = field[key] as any
+ if (typeof value === "string") {
+ field[key] = value.split(",").map((x: string) => x.trim())
+ } else {
+ field[key] = [value]
+ }
+ }
+ }
+ return filters
+}
+
export const search = (
docs: Record[],
query: RowSearchParams
@@ -356,6 +385,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
}
query = cleanupQuery(query)
+ query = fixupFilterArrays(query)
if (
!hasFilters(query) &&
@@ -382,7 +412,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
}
const stringMatch = match(
- SearchFilterOperator.STRING,
+ BasicOperator.STRING,
(docValue: any, testValue: any) => {
if (!(typeof docValue === "string")) {
return false
@@ -395,7 +425,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
)
const fuzzyMatch = match(
- SearchFilterOperator.FUZZY,
+ BasicOperator.FUZZY,
(docValue: any, testValue: any) => {
if (!(typeof docValue === "string")) {
return false
@@ -408,17 +438,17 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
)
const rangeMatch = match(
- SearchFilterOperator.RANGE,
+ RangeOperator.RANGE,
(docValue: any, testValue: any) => {
if (docValue == null || docValue === "") {
return false
}
- if (_.isObject(testValue.low) && _.isEmpty(testValue.low)) {
+ if (isPlainObject(testValue.low) && isEmpty(testValue.low)) {
testValue.low = undefined
}
- if (_.isObject(testValue.high) && _.isEmpty(testValue.high)) {
+ if (isPlainObject(testValue.high) && isEmpty(testValue.high)) {
testValue.high = undefined
}
@@ -497,11 +527,8 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
(...args: T): boolean =>
!f(...args)
- const equalMatch = match(SearchFilterOperator.EQUAL, _valueMatches)
- const notEqualMatch = match(
- SearchFilterOperator.NOT_EQUAL,
- not(_valueMatches)
- )
+ const equalMatch = match(BasicOperator.EQUAL, _valueMatches)
+ const notEqualMatch = match(BasicOperator.NOT_EQUAL, not(_valueMatches))
const _empty = (docValue: any) => {
if (typeof docValue === "string") {
@@ -516,26 +543,24 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
return docValue == null
}
- const emptyMatch = match(SearchFilterOperator.EMPTY, _empty)
- const notEmptyMatch = match(SearchFilterOperator.NOT_EMPTY, not(_empty))
+ const emptyMatch = match(BasicOperator.EMPTY, _empty)
+ const notEmptyMatch = match(BasicOperator.NOT_EMPTY, not(_empty))
- const oneOf = match(
- SearchFilterOperator.ONE_OF,
- (docValue: any, testValue: any) => {
- if (typeof testValue === "string") {
- testValue = testValue.split(",")
- if (typeof docValue === "number") {
- testValue = testValue.map((item: string) => parseFloat(item))
- }
- }
-
- if (!Array.isArray(testValue)) {
- return false
- }
-
- return testValue.some(item => _valueMatches(docValue, item))
+ const oneOf = match(ArrayOperator.ONE_OF, (docValue: any, testValue: any) => {
+ if (typeof testValue === "string") {
+ testValue = testValue.split(",")
}
- )
+
+ if (typeof docValue === "number") {
+ testValue = testValue.map((item: string) => parseFloat(item))
+ }
+
+ if (!Array.isArray(testValue)) {
+ return false
+ }
+
+ return testValue.some(item => _valueMatches(docValue, item))
+ })
const _contains =
(f: "some" | "every") => (docValue: any, testValue: any) => {
@@ -562,7 +587,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
}
const contains = match(
- SearchFilterOperator.CONTAINS,
+ ArrayOperator.CONTAINS,
(docValue: any, testValue: any) => {
if (Array.isArray(testValue) && testValue.length === 0) {
return true
@@ -571,7 +596,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
}
)
const notContains = match(
- SearchFilterOperator.NOT_CONTAINS,
+ ArrayOperator.NOT_CONTAINS,
(docValue: any, testValue: any) => {
// Not sure if this is logically correct, but at the time this code was
// written the search endpoint behaved this way and we wanted to make this
@@ -582,10 +607,7 @@ export const runQuery = (docs: Record[], query: SearchFilters) => {
return not(_contains("every"))(docValue, testValue)
}
)
- const containsAny = match(
- SearchFilterOperator.CONTAINS_ANY,
- _contains("some")
- )
+ const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
const docMatch = (doc: Record) => {
const filterFunctions = {
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index ccb73a7fba..5607efece8 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -3,20 +3,28 @@ import { Row, Table, DocumentType } from "../documents"
import { SortOrder, SortType } from "../api"
import { Knex } from "knex"
-export enum SearchFilterOperator {
- STRING = "string",
- FUZZY = "fuzzy",
- RANGE = "range",
+export enum BasicOperator {
EQUAL = "equal",
NOT_EQUAL = "notEqual",
EMPTY = "empty",
NOT_EMPTY = "notEmpty",
- ONE_OF = "oneOf",
+ FUZZY = "fuzzy",
+ STRING = "string",
+}
+
+export enum ArrayOperator {
CONTAINS = "contains",
NOT_CONTAINS = "notContains",
CONTAINS_ANY = "containsAny",
+ ONE_OF = "oneOf",
}
+export enum RangeOperator {
+ RANGE = "range",
+}
+
+export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
+
export enum InternalSearchFilterOperator {
COMPLEX_ID_OPERATOR = "_complexIdOperator",
}
@@ -52,17 +60,17 @@ export interface SearchFilters {
// allows just fuzzy to be or - all the fuzzy/like parameters
fuzzyOr?: boolean
onEmptyFilter?: EmptyFilterOption
- [SearchFilterOperator.STRING]?: BasicFilter
- [SearchFilterOperator.FUZZY]?: BasicFilter
- [SearchFilterOperator.RANGE]?: RangeFilter
- [SearchFilterOperator.EQUAL]?: BasicFilter
- [SearchFilterOperator.NOT_EQUAL]?: BasicFilter
- [SearchFilterOperator.EMPTY]?: BasicFilter
- [SearchFilterOperator.NOT_EMPTY]?: BasicFilter
- [SearchFilterOperator.ONE_OF]?: ArrayFilter
- [SearchFilterOperator.CONTAINS]?: ArrayFilter
- [SearchFilterOperator.NOT_CONTAINS]?: ArrayFilter
- [SearchFilterOperator.CONTAINS_ANY]?: ArrayFilter
+ [BasicOperator.STRING]?: BasicFilter
+ [BasicOperator.FUZZY]?: BasicFilter
+ [RangeOperator.RANGE]?: RangeFilter
+ [BasicOperator.EQUAL]?: BasicFilter
+ [BasicOperator.NOT_EQUAL]?: BasicFilter
+ [BasicOperator.EMPTY]?: BasicFilter
+ [BasicOperator.NOT_EMPTY]?: BasicFilter
+ [ArrayOperator.ONE_OF]?: ArrayFilter
+ [ArrayOperator.CONTAINS]?: ArrayFilter
+ [ArrayOperator.NOT_CONTAINS]?: ArrayFilter
+ [ArrayOperator.CONTAINS_ANY]?: ArrayFilter
// specific to SQS/SQLite search on internal tables this can be used
// to make sure the documents returned are always filtered down to a
// specific document type (such as just rows)