Merge remote-tracking branch 'origin/master' into dean-fixes
This commit is contained in:
commit
b5485fac0e
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
|
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
|
||||||
"version": "2.29.29",
|
"version": "2.29.30",
|
||||||
"npmClient": "yarn",
|
"npmClient": "yarn",
|
||||||
"packages": [
|
"packages": [
|
||||||
"packages/*",
|
"packages/*",
|
||||||
|
|
|
@ -463,6 +463,24 @@ class InternalBuilder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (filters.$and) {
|
||||||
|
const { $and } = filters
|
||||||
|
query = query.where(x => {
|
||||||
|
for (const condition of $and.conditions) {
|
||||||
|
x = this.addFilters(x, condition, opts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if (filters.$or) {
|
||||||
|
const { $or } = filters
|
||||||
|
query = query.where(x => {
|
||||||
|
for (const condition of $or.conditions) {
|
||||||
|
x = this.addFilters(x, { ...condition, allOr: true }, opts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
if (filters.oneOf) {
|
if (filters.oneOf) {
|
||||||
const fnc = allOr ? "orWhereIn" : "whereIn"
|
const fnc = allOr ? "orWhereIn" : "whereIn"
|
||||||
iterate(
|
iterate(
|
||||||
|
|
|
@ -6,11 +6,13 @@ import {
|
||||||
RequiredKeys,
|
RequiredKeys,
|
||||||
RowSearchParams,
|
RowSearchParams,
|
||||||
SearchFilterKey,
|
SearchFilterKey,
|
||||||
|
LogicalOperator,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import { dataFilters } from "@budibase/shared-core"
|
import { dataFilters } from "@budibase/shared-core"
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
import { db, context } from "@budibase/backend-core"
|
import { db, context } from "@budibase/backend-core"
|
||||||
import { enrichSearchContext } from "./utils"
|
import { enrichSearchContext } from "./utils"
|
||||||
|
import { isExternalTableID } from "../../../integrations/utils"
|
||||||
|
|
||||||
export async function searchView(
|
export async function searchView(
|
||||||
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
|
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
|
||||||
|
@ -35,25 +37,33 @@ export async function searchView(
|
||||||
// that could let users find rows they should not be allowed to access.
|
// that could let users find rows they should not be allowed to access.
|
||||||
let query = dataFilters.buildQuery(view.query || [])
|
let query = dataFilters.buildQuery(view.query || [])
|
||||||
if (body.query) {
|
if (body.query) {
|
||||||
// Extract existing fields
|
|
||||||
const existingFields =
|
|
||||||
view.query
|
|
||||||
?.filter(filter => filter.field)
|
|
||||||
.map(filter => db.removeKeyNumbering(filter.field)) || []
|
|
||||||
|
|
||||||
// Delete extraneous search params that cannot be overridden
|
// Delete extraneous search params that cannot be overridden
|
||||||
delete body.query.allOr
|
delete body.query.allOr
|
||||||
delete body.query.onEmptyFilter
|
delete body.query.onEmptyFilter
|
||||||
|
|
||||||
// Carry over filters for unused fields
|
if (!isExternalTableID(view.tableId) && !db.isSqsEnabledForTenant()) {
|
||||||
Object.keys(body.query).forEach(key => {
|
// Extract existing fields
|
||||||
const operator = key as SearchFilterKey
|
const existingFields =
|
||||||
Object.keys(body.query[operator] || {}).forEach(field => {
|
view.query
|
||||||
if (!existingFields.includes(db.removeKeyNumbering(field))) {
|
?.filter(filter => filter.field)
|
||||||
query[operator]![field] = body.query[operator]![field]
|
.map(filter => db.removeKeyNumbering(filter.field)) || []
|
||||||
}
|
|
||||||
|
// Carry over filters for unused fields
|
||||||
|
Object.keys(body.query).forEach(key => {
|
||||||
|
const operator = key as Exclude<SearchFilterKey, LogicalOperator>
|
||||||
|
Object.keys(body.query[operator] || {}).forEach(field => {
|
||||||
|
if (!existingFields.includes(db.removeKeyNumbering(field))) {
|
||||||
|
query[operator]![field] = body.query[operator]![field]
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
} else {
|
||||||
|
query = {
|
||||||
|
$and: {
|
||||||
|
conditions: [query, body.query],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await context.ensureSnippetContext(true)
|
await context.ensureSnippetContext(true)
|
||||||
|
|
|
@ -2696,4 +2696,239 @@ describe.each([
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
!isLucene &&
|
||||||
|
describe("$and", () => {
|
||||||
|
beforeAll(async () => {
|
||||||
|
table = await createTable({
|
||||||
|
age: { name: "age", type: FieldType.NUMBER },
|
||||||
|
name: { name: "name", type: FieldType.STRING },
|
||||||
|
})
|
||||||
|
await createRows([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 10, name: "Jack" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds a row for one level condition", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
|
||||||
|
},
|
||||||
|
}).toContainExactly([{ age: 10, name: "Jack" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds a row for one level with multiple conditions", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [{ equal: { age: 10 } }, { equal: { name: "Jack" } }],
|
||||||
|
},
|
||||||
|
}).toContainExactly([{ age: 10, name: "Jack" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds multiple rows for one level with multiple conditions", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{ range: { age: { low: 1, high: 9 } } },
|
||||||
|
{ string: { name: "Ja" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds rows for nested filters", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
range: { age: { low: 1, high: 10 } },
|
||||||
|
},
|
||||||
|
{ string: { name: "Ja" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
equal: { name: "Jane" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([{ age: 1, name: "Jane" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("returns nothing when filtering out all data", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
|
||||||
|
},
|
||||||
|
}).toFindNothing()
|
||||||
|
})
|
||||||
|
|
||||||
|
!isInMemory &&
|
||||||
|
it("validates conditions that are not objects", async () => {
|
||||||
|
await expect(
|
||||||
|
expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [{ equal: { age: 10 } }, "invalidCondition" as any],
|
||||||
|
},
|
||||||
|
}).toFindNothing()
|
||||||
|
).rejects.toThrow(
|
||||||
|
'Invalid body - "query.$and.conditions[1]" must be of type object'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
!isInMemory &&
|
||||||
|
it("validates $and without conditions", async () => {
|
||||||
|
await expect(
|
||||||
|
expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{ equal: { age: 10 } },
|
||||||
|
{
|
||||||
|
$and: {
|
||||||
|
conditions: undefined as any,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toFindNothing()
|
||||||
|
).rejects.toThrow(
|
||||||
|
'Invalid body - "query.$and.conditions[1].$and.conditions" is required'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
!isLucene &&
|
||||||
|
describe("$or", () => {
|
||||||
|
beforeAll(async () => {
|
||||||
|
table = await createTable({
|
||||||
|
age: { name: "age", type: FieldType.NUMBER },
|
||||||
|
name: { name: "name", type: FieldType.STRING },
|
||||||
|
})
|
||||||
|
await createRows([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 10, name: "Jack" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds a row for one level condition", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 10, name: "Jack" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds a row for one level with multiple conditions", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [{ equal: { age: 7 } }, { equal: { name: "Jack" } }],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 10, name: "Jack" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds multiple rows for one level with multiple conditions", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [
|
||||||
|
{ range: { age: { low: 1, high: 9 } } },
|
||||||
|
{ string: { name: "Jan" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("successfully finds rows for nested filters", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
$or: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
range: { age: { low: 1, high: 7 } },
|
||||||
|
},
|
||||||
|
{ string: { name: "Jan" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
equal: { name: "Jane" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 7, name: "Hanna" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("returns nothing when filtering out all data", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [{ equal: { age: 6 } }, { equal: { name: "John" } }],
|
||||||
|
},
|
||||||
|
}).toFindNothing()
|
||||||
|
})
|
||||||
|
|
||||||
|
it("can nest $and under $or filters", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$or: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
range: { age: { low: 1, high: 8 } },
|
||||||
|
},
|
||||||
|
{ equal: { name: "Jan" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
equal: { name: "Jane" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([
|
||||||
|
{ age: 1, name: "Jane" },
|
||||||
|
{ age: 8, name: "Jan" },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("can nest $or under $and filters", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
$and: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
$or: {
|
||||||
|
conditions: [
|
||||||
|
{
|
||||||
|
range: { age: { low: 1, high: 8 } },
|
||||||
|
},
|
||||||
|
{ equal: { name: "Jan" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
equal: { name: "Jane" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}).toContainExactly([{ age: 1, name: "Jane" }])
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
|
@ -1485,6 +1485,119 @@ describe.each([
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
isLucene &&
|
||||||
|
it("in lucene, cannot override a view filter", async () => {
|
||||||
|
await config.api.row.save(table._id!, {
|
||||||
|
one: "foo",
|
||||||
|
two: "bar",
|
||||||
|
})
|
||||||
|
const two = await config.api.row.save(table._id!, {
|
||||||
|
one: "foo2",
|
||||||
|
two: "bar2",
|
||||||
|
})
|
||||||
|
|
||||||
|
const view = await config.api.viewV2.create({
|
||||||
|
tableId: table._id!,
|
||||||
|
name: generator.guid(),
|
||||||
|
query: [
|
||||||
|
{
|
||||||
|
operator: BasicOperator.EQUAL,
|
||||||
|
field: "two",
|
||||||
|
value: "bar2",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
schema: {
|
||||||
|
id: { visible: true },
|
||||||
|
one: { visible: false },
|
||||||
|
two: { visible: true },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const response = await config.api.viewV2.search(view.id, {
|
||||||
|
query: {
|
||||||
|
equal: {
|
||||||
|
two: "bar",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(response.rows).toHaveLength(1)
|
||||||
|
expect(response.rows).toEqual([
|
||||||
|
expect.objectContaining({ _id: two._id }),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
!isLucene &&
|
||||||
|
it("can filter a view without a view filter", async () => {
|
||||||
|
const one = await config.api.row.save(table._id!, {
|
||||||
|
one: "foo",
|
||||||
|
two: "bar",
|
||||||
|
})
|
||||||
|
await config.api.row.save(table._id!, {
|
||||||
|
one: "foo2",
|
||||||
|
two: "bar2",
|
||||||
|
})
|
||||||
|
|
||||||
|
const view = await config.api.viewV2.create({
|
||||||
|
tableId: table._id!,
|
||||||
|
name: generator.guid(),
|
||||||
|
schema: {
|
||||||
|
id: { visible: true },
|
||||||
|
one: { visible: false },
|
||||||
|
two: { visible: true },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const response = await config.api.viewV2.search(view.id, {
|
||||||
|
query: {
|
||||||
|
equal: {
|
||||||
|
two: "bar",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(response.rows).toHaveLength(1)
|
||||||
|
expect(response.rows).toEqual([
|
||||||
|
expect.objectContaining({ _id: one._id }),
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
!isLucene &&
|
||||||
|
it("cannot bypass a view filter", async () => {
|
||||||
|
await config.api.row.save(table._id!, {
|
||||||
|
one: "foo",
|
||||||
|
two: "bar",
|
||||||
|
})
|
||||||
|
await config.api.row.save(table._id!, {
|
||||||
|
one: "foo2",
|
||||||
|
two: "bar2",
|
||||||
|
})
|
||||||
|
|
||||||
|
const view = await config.api.viewV2.create({
|
||||||
|
tableId: table._id!,
|
||||||
|
name: generator.guid(),
|
||||||
|
query: [
|
||||||
|
{
|
||||||
|
operator: BasicOperator.EQUAL,
|
||||||
|
field: "two",
|
||||||
|
value: "bar2",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
schema: {
|
||||||
|
id: { visible: true },
|
||||||
|
one: { visible: false },
|
||||||
|
two: { visible: true },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const response = await config.api.viewV2.search(view.id, {
|
||||||
|
query: {
|
||||||
|
equal: {
|
||||||
|
two: "bar",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(response.rows).toHaveLength(0)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("permissions", () => {
|
describe("permissions", () => {
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
import { auth, permissions } from "@budibase/backend-core"
|
import { auth, permissions } from "@budibase/backend-core"
|
||||||
import { DataSourceOperation } from "../../../constants"
|
import { DataSourceOperation } from "../../../constants"
|
||||||
import { Table, WebhookActionType } from "@budibase/types"
|
import {
|
||||||
|
EmptyFilterOption,
|
||||||
|
SearchFilters,
|
||||||
|
Table,
|
||||||
|
WebhookActionType,
|
||||||
|
} from "@budibase/types"
|
||||||
import Joi, { CustomValidator } from "joi"
|
import Joi, { CustomValidator } from "joi"
|
||||||
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
|
import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
|
@ -84,7 +89,12 @@ export function datasourceValidator() {
|
||||||
}
|
}
|
||||||
|
|
||||||
function filterObject() {
|
function filterObject() {
|
||||||
return Joi.object({
|
const conditionalFilteringObject = () =>
|
||||||
|
Joi.object({
|
||||||
|
conditions: Joi.array().items(Joi.link("#schema")).required(),
|
||||||
|
})
|
||||||
|
|
||||||
|
const filtersValidators: Record<keyof SearchFilters, any> = {
|
||||||
string: Joi.object().optional(),
|
string: Joi.object().optional(),
|
||||||
fuzzy: Joi.object().optional(),
|
fuzzy: Joi.object().optional(),
|
||||||
range: Joi.object().optional(),
|
range: Joi.object().optional(),
|
||||||
|
@ -95,8 +105,17 @@ function filterObject() {
|
||||||
oneOf: Joi.object().optional(),
|
oneOf: Joi.object().optional(),
|
||||||
contains: Joi.object().optional(),
|
contains: Joi.object().optional(),
|
||||||
notContains: Joi.object().optional(),
|
notContains: Joi.object().optional(),
|
||||||
|
containsAny: Joi.object().optional(),
|
||||||
allOr: Joi.boolean().optional(),
|
allOr: Joi.boolean().optional(),
|
||||||
}).unknown(true)
|
onEmptyFilter: Joi.string()
|
||||||
|
.optional()
|
||||||
|
.valid(...Object.values(EmptyFilterOption)),
|
||||||
|
$and: conditionalFilteringObject(),
|
||||||
|
$or: conditionalFilteringObject(),
|
||||||
|
fuzzyOr: Joi.forbidden(),
|
||||||
|
documentType: Joi.forbidden(),
|
||||||
|
}
|
||||||
|
return Joi.object(filtersValidators).unknown(true).id("schema")
|
||||||
}
|
}
|
||||||
|
|
||||||
export function internalSearchValidator() {
|
export function internalSearchValidator() {
|
||||||
|
|
|
@ -11,13 +11,10 @@ import {
|
||||||
AutomationStepSchema,
|
AutomationStepSchema,
|
||||||
AutomationStepType,
|
AutomationStepType,
|
||||||
EmptyFilterOption,
|
EmptyFilterOption,
|
||||||
SearchFilters,
|
|
||||||
Table,
|
|
||||||
SortOrder,
|
SortOrder,
|
||||||
QueryRowsStepInputs,
|
QueryRowsStepInputs,
|
||||||
QueryRowsStepOutputs,
|
QueryRowsStepOutputs,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import { db as dbCore } from "@budibase/backend-core"
|
|
||||||
|
|
||||||
const SortOrderPretty = {
|
const SortOrderPretty = {
|
||||||
[SortOrder.ASCENDING]: "Ascending",
|
[SortOrder.ASCENDING]: "Ascending",
|
||||||
|
@ -95,38 +92,6 @@ async function getTable(appId: string, tableId: string) {
|
||||||
return ctx.body
|
return ctx.body
|
||||||
}
|
}
|
||||||
|
|
||||||
function typeCoercion(filters: SearchFilters, table: Table) {
|
|
||||||
if (!filters || !table) {
|
|
||||||
return filters
|
|
||||||
}
|
|
||||||
for (let key of Object.keys(filters)) {
|
|
||||||
const searchParam = filters[key as keyof SearchFilters]
|
|
||||||
if (typeof searchParam === "object") {
|
|
||||||
for (let [property, value] of Object.entries(searchParam)) {
|
|
||||||
// We need to strip numerical prefixes here, so that we can look up
|
|
||||||
// the correct field name in the schema
|
|
||||||
const columnName = dbCore.removeKeyNumbering(property)
|
|
||||||
const column = table.schema[columnName]
|
|
||||||
|
|
||||||
// convert string inputs
|
|
||||||
if (!column || typeof value !== "string") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (column.type === FieldType.NUMBER) {
|
|
||||||
if (key === "oneOf") {
|
|
||||||
searchParam[property] = value
|
|
||||||
.split(",")
|
|
||||||
.map(item => parseFloat(item))
|
|
||||||
} else {
|
|
||||||
searchParam[property] = parseFloat(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return filters
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasNullFilters(filters: any[]) {
|
function hasNullFilters(filters: any[]) {
|
||||||
return (
|
return (
|
||||||
filters.length === 0 ||
|
filters.length === 0 ||
|
||||||
|
@ -157,7 +122,7 @@ export async function run({
|
||||||
sortType =
|
sortType =
|
||||||
fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
|
fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
|
||||||
}
|
}
|
||||||
const ctx: any = buildCtx(appId, null, {
|
const ctx = buildCtx(appId, null, {
|
||||||
params: {
|
params: {
|
||||||
tableId,
|
tableId,
|
||||||
},
|
},
|
||||||
|
@ -165,7 +130,7 @@ export async function run({
|
||||||
sortType,
|
sortType,
|
||||||
limit,
|
limit,
|
||||||
sort: sortColumn,
|
sort: sortColumn,
|
||||||
query: typeCoercion(filters || {}, table),
|
query: filters || {},
|
||||||
// default to ascending, like data tab
|
// default to ascending, like data tab
|
||||||
sortOrder: sortOrder || SortOrder.ASCENDING,
|
sortOrder: sortOrder || SortOrder.ASCENDING,
|
||||||
},
|
},
|
||||||
|
|
|
@ -13,6 +13,7 @@ import { dataFilters } from "@budibase/shared-core"
|
||||||
import sdk from "../../index"
|
import sdk from "../../index"
|
||||||
import { searchInputMapping } from "./search/utils"
|
import { searchInputMapping } from "./search/utils"
|
||||||
import { db as dbCore } from "@budibase/backend-core"
|
import { db as dbCore } from "@budibase/backend-core"
|
||||||
|
import tracer from "dd-trace"
|
||||||
|
|
||||||
export { isValidFilter } from "../../../integrations/utils"
|
export { isValidFilter } from "../../../integrations/utils"
|
||||||
|
|
||||||
|
@ -32,32 +33,65 @@ function pickApi(tableId: any) {
|
||||||
export async function search(
|
export async function search(
|
||||||
options: RowSearchParams
|
options: RowSearchParams
|
||||||
): Promise<SearchResponse<Row>> {
|
): Promise<SearchResponse<Row>> {
|
||||||
const isExternalTable = isExternalTableID(options.tableId)
|
return await tracer.trace("search", async span => {
|
||||||
options.query = dataFilters.cleanupQuery(options.query || {})
|
span?.addTags({
|
||||||
options.query = dataFilters.fixupFilterArrays(options.query)
|
tableId: options.tableId,
|
||||||
if (
|
query: options.query,
|
||||||
!dataFilters.hasFilters(options.query) &&
|
sort: options.sort,
|
||||||
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
|
sortOrder: options.sortOrder,
|
||||||
) {
|
sortType: options.sortType,
|
||||||
return {
|
limit: options.limit,
|
||||||
rows: [],
|
bookmark: options.bookmark,
|
||||||
|
paginate: options.paginate,
|
||||||
|
fields: options.fields,
|
||||||
|
countRows: options.countRows,
|
||||||
|
})
|
||||||
|
|
||||||
|
const isExternalTable = isExternalTableID(options.tableId)
|
||||||
|
options.query = dataFilters.cleanupQuery(options.query || {})
|
||||||
|
options.query = dataFilters.fixupFilterArrays(options.query)
|
||||||
|
|
||||||
|
span?.addTags({
|
||||||
|
cleanedQuery: options.query,
|
||||||
|
isExternalTable,
|
||||||
|
})
|
||||||
|
|
||||||
|
if (
|
||||||
|
!dataFilters.hasFilters(options.query) &&
|
||||||
|
options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
|
||||||
|
) {
|
||||||
|
span?.addTags({ emptyQuery: true })
|
||||||
|
return {
|
||||||
|
rows: [],
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (options.sortOrder) {
|
if (options.sortOrder) {
|
||||||
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
|
options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
|
||||||
}
|
}
|
||||||
|
|
||||||
const table = await sdk.tables.getTable(options.tableId)
|
const table = await sdk.tables.getTable(options.tableId)
|
||||||
options = searchInputMapping(table, options)
|
options = searchInputMapping(table, options)
|
||||||
|
|
||||||
if (isExternalTable) {
|
let result: SearchResponse<Row>
|
||||||
return external.search(options, table)
|
if (isExternalTable) {
|
||||||
} else if (dbCore.isSqsEnabledForTenant()) {
|
span?.addTags({ searchType: "external" })
|
||||||
return internal.sqs.search(options, table)
|
result = await external.search(options, table)
|
||||||
} else {
|
} else if (dbCore.isSqsEnabledForTenant()) {
|
||||||
return internal.lucene.search(options, table)
|
span?.addTags({ searchType: "sqs" })
|
||||||
}
|
result = await internal.sqs.search(options, table)
|
||||||
|
} else {
|
||||||
|
span?.addTags({ searchType: "lucene" })
|
||||||
|
result = await internal.lucene.search(options, table)
|
||||||
|
}
|
||||||
|
|
||||||
|
span?.addTags({
|
||||||
|
foundRows: result.rows.length,
|
||||||
|
totalRows: result.totalRows,
|
||||||
|
})
|
||||||
|
|
||||||
|
return result
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function exportRows(
|
export async function exportRows(
|
||||||
|
|
|
@ -2,6 +2,7 @@ import {
|
||||||
Datasource,
|
Datasource,
|
||||||
DocumentType,
|
DocumentType,
|
||||||
FieldType,
|
FieldType,
|
||||||
|
isLogicalSearchOperator,
|
||||||
Operation,
|
Operation,
|
||||||
QueryJson,
|
QueryJson,
|
||||||
RelationshipFieldMetadata,
|
RelationshipFieldMetadata,
|
||||||
|
@ -137,20 +138,33 @@ function cleanupFilters(
|
||||||
allTables.some(table => table.schema[key])
|
allTables.some(table => table.schema[key])
|
||||||
|
|
||||||
const splitter = new dataFilters.ColumnSplitter(allTables)
|
const splitter = new dataFilters.ColumnSplitter(allTables)
|
||||||
for (const filter of Object.values(filters)) {
|
|
||||||
for (const key of Object.keys(filter)) {
|
const prefixFilters = (filters: SearchFilters) => {
|
||||||
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
|
for (const filterKey of Object.keys(filters) as (keyof SearchFilters)[]) {
|
||||||
if (keyInAnyTable(column)) {
|
if (isLogicalSearchOperator(filterKey)) {
|
||||||
filter[
|
for (const condition of filters[filterKey]!.conditions) {
|
||||||
`${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn(
|
prefixFilters(condition)
|
||||||
column
|
}
|
||||||
)}`
|
} else {
|
||||||
] = filter[key]
|
const filter = filters[filterKey]!
|
||||||
delete filter[key]
|
if (typeof filter !== "object") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for (const key of Object.keys(filter)) {
|
||||||
|
const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
|
||||||
|
if (keyInAnyTable(column)) {
|
||||||
|
filter[
|
||||||
|
`${numberPrefix || ""}${
|
||||||
|
relationshipPrefix || ""
|
||||||
|
}${mapToUserColumn(column)}`
|
||||||
|
] = filter[key]
|
||||||
|
delete filter[key]
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
prefixFilters(filters)
|
||||||
return filters
|
return filters
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -17,6 +17,8 @@ import {
|
||||||
Table,
|
Table,
|
||||||
BasicOperator,
|
BasicOperator,
|
||||||
RangeOperator,
|
RangeOperator,
|
||||||
|
LogicalOperator,
|
||||||
|
isLogicalSearchOperator,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import dayjs from "dayjs"
|
import dayjs from "dayjs"
|
||||||
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
|
||||||
|
@ -358,6 +360,8 @@ export const buildQuery = (filter: SearchFilter[]) => {
|
||||||
high: value,
|
high: value,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else if (isLogicalSearchOperator(queryOperator)) {
|
||||||
|
// TODO
|
||||||
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
|
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
|
||||||
if (type === "boolean") {
|
if (type === "boolean") {
|
||||||
// Transform boolean filters to cope with null.
|
// Transform boolean filters to cope with null.
|
||||||
|
@ -465,14 +469,17 @@ export const runQuery = (
|
||||||
) =>
|
) =>
|
||||||
(doc: Record<string, any>) => {
|
(doc: Record<string, any>) => {
|
||||||
for (const [key, testValue] of Object.entries(query[type] || {})) {
|
for (const [key, testValue] of Object.entries(query[type] || {})) {
|
||||||
const result = test(findInDoc(doc, removeKeyNumbering(key)), testValue)
|
const valueToCheck = isLogicalSearchOperator(type)
|
||||||
|
? doc
|
||||||
|
: findInDoc(doc, removeKeyNumbering(key))
|
||||||
|
const result = test(valueToCheck, testValue)
|
||||||
if (query.allOr && result) {
|
if (query.allOr && result) {
|
||||||
return true
|
return true
|
||||||
} else if (!query.allOr && !result) {
|
} else if (!query.allOr && !result) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return true
|
return !query.allOr
|
||||||
}
|
}
|
||||||
|
|
||||||
const stringMatch = match(
|
const stringMatch = match(
|
||||||
|
@ -673,8 +680,45 @@ export const runQuery = (
|
||||||
)
|
)
|
||||||
const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
|
const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
|
||||||
|
|
||||||
|
const and = match(
|
||||||
|
LogicalOperator.AND,
|
||||||
|
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
|
||||||
|
if (!conditions.length) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for (const condition of conditions) {
|
||||||
|
const matchesCondition = runQuery([docValue], condition)
|
||||||
|
if (!matchesCondition.length) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
)
|
||||||
|
const or = match(
|
||||||
|
LogicalOperator.OR,
|
||||||
|
(docValue: Record<string, any>, conditions: SearchFilters[]) => {
|
||||||
|
if (!conditions.length) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for (const condition of conditions) {
|
||||||
|
const matchesCondition = runQuery([docValue], {
|
||||||
|
...condition,
|
||||||
|
allOr: true,
|
||||||
|
})
|
||||||
|
if (matchesCondition.length) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
const docMatch = (doc: Record<string, any>) => {
|
const docMatch = (doc: Record<string, any>) => {
|
||||||
const filterFunctions = {
|
const filterFunctions: Record<
|
||||||
|
SearchFilterOperator,
|
||||||
|
(doc: Record<string, any>) => boolean
|
||||||
|
> = {
|
||||||
string: stringMatch,
|
string: stringMatch,
|
||||||
fuzzy: fuzzyMatch,
|
fuzzy: fuzzyMatch,
|
||||||
range: rangeMatch,
|
range: rangeMatch,
|
||||||
|
@ -686,6 +730,8 @@ export const runQuery = (
|
||||||
contains: contains,
|
contains: contains,
|
||||||
containsAny: containsAny,
|
containsAny: containsAny,
|
||||||
notContains: notContains,
|
notContains: notContains,
|
||||||
|
[LogicalOperator.AND]: and,
|
||||||
|
[LogicalOperator.OR]: or,
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = Object.entries(query || {})
|
const results = Object.entries(query || {})
|
||||||
|
|
|
@ -18,6 +18,6 @@
|
||||||
},
|
},
|
||||||
"tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
|
"tsBuildInfoFile": "dist/tsconfig.tsbuildinfo"
|
||||||
},
|
},
|
||||||
"include": ["src/**/*"],
|
"include": ["src/**/*.ts"],
|
||||||
"exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"]
|
"exclude": ["**/*.spec.ts", "**/*.spec.js", "__mocks__", "src/tests"]
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,6 @@
|
||||||
{
|
{
|
||||||
"extends": "./tsconfig.build.json",
|
"extends": "./tsconfig.build.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"baseUrl": "..",
|
|
||||||
"rootDir": "src",
|
|
||||||
"composite": true,
|
|
||||||
"types": ["node", "jest"]
|
"types": ["node", "jest"]
|
||||||
},
|
},
|
||||||
"exclude": ["node_modules", "dist"]
|
"exclude": ["node_modules", "dist"]
|
||||||
|
|
|
@ -23,7 +23,22 @@ export enum RangeOperator {
|
||||||
RANGE = "range",
|
RANGE = "range",
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
|
export enum LogicalOperator {
|
||||||
|
AND = "$and",
|
||||||
|
OR = "$or",
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isLogicalSearchOperator(
|
||||||
|
value: string
|
||||||
|
): value is LogicalOperator {
|
||||||
|
return value === LogicalOperator.AND || value === LogicalOperator.OR
|
||||||
|
}
|
||||||
|
|
||||||
|
export type SearchFilterOperator =
|
||||||
|
| BasicOperator
|
||||||
|
| ArrayOperator
|
||||||
|
| RangeOperator
|
||||||
|
| LogicalOperator
|
||||||
|
|
||||||
export enum InternalSearchFilterOperator {
|
export enum InternalSearchFilterOperator {
|
||||||
COMPLEX_ID_OPERATOR = "_complexIdOperator",
|
COMPLEX_ID_OPERATOR = "_complexIdOperator",
|
||||||
|
@ -75,6 +90,13 @@ export interface SearchFilters {
|
||||||
// to make sure the documents returned are always filtered down to a
|
// to make sure the documents returned are always filtered down to a
|
||||||
// specific document type (such as just rows)
|
// specific document type (such as just rows)
|
||||||
documentType?: DocumentType
|
documentType?: DocumentType
|
||||||
|
|
||||||
|
[LogicalOperator.AND]?: {
|
||||||
|
conditions: SearchFilters[]
|
||||||
|
}
|
||||||
|
[LogicalOperator.OR]?: {
|
||||||
|
conditions: SearchFilters[]
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export type SearchFilterKey = keyof Omit<
|
export type SearchFilterKey = keyof Omit<
|
||||||
|
|
Loading…
Reference in New Issue