Fix boolean searching for Lucene/SQS

This commit is contained in:
Sam Rose 2024-05-03 17:29:20 +01:00
parent f49e36d0dd
commit 1777f1f8fe
No known key found for this signature in database
5 changed files with 115 additions and 19 deletions

View File

@ -283,7 +283,7 @@ export class QueryBuilder<T> {
const equal = (key: string, value: any) => { const equal = (key: string, value: any) => {
// 0 evaluates to false, which means we would return all rows if we don't check it // 0 evaluates to false, which means we would return all rows if we don't check it
if (!value && value !== 0) { if (value === null || value === undefined) {
return null return null
} }
return `${key}:${builder.preprocess(value, allPreProcessingOpts)}` return `${key}:${builder.preprocess(value, allPreProcessingOpts)}`
@ -421,7 +421,7 @@ export class QueryBuilder<T> {
} }
if (this.#query.notEqual) { if (this.#query.notEqual) {
build(this.#query.notEqual, (key: string, value: any) => { build(this.#query.notEqual, (key: string, value: any) => {
if (!value) { if (value === null || value === undefined) {
return null return null
} }
if (typeof value === "boolean") { if (typeof value === "boolean") {

View File

@ -117,6 +117,19 @@ export async function validate(
}) })
} }
function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.BOOLEAN) {
if (row[col.name] === 1) {
row[col.name] = true
} else if (row[col.name] === 0) {
row[col.name] = false
}
}
}
return row
}
export async function sqlOutputProcessing( export async function sqlOutputProcessing(
rows: DatasourcePlusQueryResponse, rows: DatasourcePlusQueryResponse,
table: Table, table: Table,
@ -161,7 +174,13 @@ export async function sqlOutputProcessing(
if (thisRow._id == null) { if (thisRow._id == null) {
throw new Error("Unable to generate row ID for SQL rows") throw new Error("Unable to generate row ID for SQL rows")
} }
finalRows[thisRow._id] = thisRow
if (opts?.sqs) {
finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
} else {
finalRows[thisRow._id] = thisRow
}
// do this at end once its been added to the final rows // do this at end once its been added to the final rows
finalRows = await updateRelationshipColumns( finalRows = await updateRelationshipColumns(
table, table,

View File

@ -15,16 +15,17 @@ import {
TableSchema, TableSchema,
} from "@budibase/types" } from "@budibase/types"
import _ from "lodash" import _ from "lodash"
import exp from "constants"
jest.unmock("mssql") jest.unmock("mssql")
describe.each([ describe.each([
["lucene", undefined], // ["lucene", undefined],
["sqs", undefined], ["sqs", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/api/:sourceId/search (%s)", (name, dsProvider) => { ])("/api/:sourceId/search (%s)", (name, dsProvider) => {
const isSqs = name === "sqs" const isSqs = name === "sqs"
const isLucene = name === "lucene" const isLucene = name === "lucene"
@ -67,6 +68,22 @@ describe.each([
class SearchAssertion { class SearchAssertion {
constructor(private readonly query: RowSearchParams) {} constructor(private readonly query: RowSearchParams) {}
private findRow(expectedRow: any, foundRows: any[]) {
const row = foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
if (!row) {
const fields = Object.keys(expectedRow)
// To make the error message more readable, we only include the fields
// that are present in the expected row.
const searchedObjects = foundRows.map(row => _.pick(row, fields))
throw new Error(
`Failed to find row: ${JSON.stringify(
expectedRow
)} in ${JSON.stringify(searchedObjects)}`
)
}
return row
}
// Asserts that the query returns rows matching exactly the set of rows // Asserts that the query returns rows matching exactly the set of rows
// passed in. The order of the rows matters. Rows returned in an order // passed in. The order of the rows matters. Rows returned in an order
// different to the one passed in will cause the assertion to fail. Extra // different to the one passed in will cause the assertion to fail. Extra
@ -82,9 +99,7 @@ describe.each([
// eslint-disable-next-line jest/no-standalone-expect // eslint-disable-next-line jest/no-standalone-expect
expect(foundRows).toEqual( expect(foundRows).toEqual(
expectedRows.map((expectedRow: any) => expectedRows.map((expectedRow: any) =>
expect.objectContaining( expect.objectContaining(this.findRow(expectedRow, foundRows))
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
) )
) )
} }
@ -104,9 +119,7 @@ describe.each([
expect(foundRows).toEqual( expect(foundRows).toEqual(
expect.arrayContaining( expect.arrayContaining(
expectedRows.map((expectedRow: any) => expectedRows.map((expectedRow: any) =>
expect.objectContaining( expect.objectContaining(this.findRow(expectedRow, foundRows))
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
) )
) )
) )
@ -125,9 +138,7 @@ describe.each([
expect(foundRows).toEqual( expect(foundRows).toEqual(
expect.arrayContaining( expect.arrayContaining(
expectedRows.map((expectedRow: any) => expectedRows.map((expectedRow: any) =>
expect.objectContaining( expect.objectContaining(this.findRow(expectedRow, foundRows))
foundRows.find(foundRow => _.isMatch(foundRow, expectedRow))
)
) )
) )
) )
@ -156,6 +167,67 @@ describe.each([
return expectSearch({ query }) return expectSearch({ query })
} }
describe.only("boolean", () => {
beforeAll(async () => {
await createTable({
isTrue: { name: "isTrue", type: FieldType.BOOLEAN },
})
await createRows([{ isTrue: true }, { isTrue: false }])
})
describe("equal", () => {
it("successfully finds true row", () =>
expectQuery({ equal: { isTrue: true } }).toMatchExactly([
{ isTrue: true },
]))
it("successfully finds false row", () =>
expectQuery({ equal: { isTrue: false } }).toMatchExactly([
{ isTrue: false },
]))
})
describe("notEqual", () => {
it("successfully finds false row", () =>
expectQuery({ notEqual: { isTrue: true } }).toContainExactly([
{ isTrue: false },
]))
it("successfully finds true row", () =>
expectQuery({ notEqual: { isTrue: false } }).toContainExactly([
{ isTrue: true },
]))
})
describe("oneOf", () => {
it("successfully finds true row", () =>
expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([
{ isTrue: true },
]))
it("successfully finds false row", () =>
expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([
{ isTrue: false },
]))
})
describe("sort", () => {
it("sorts ascending", () =>
expectSearch({
query: {},
sort: "isTrue",
sortOrder: SortOrder.ASCENDING,
}).toMatchExactly([{ isTrue: false }, { isTrue: true }]))
it("sorts descending", () =>
expectSearch({
query: {},
sort: "isTrue",
sortOrder: SortOrder.DESCENDING,
}).toMatchExactly([{ isTrue: true }, { isTrue: false }]))
})
})
describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => { describe.each([FieldType.STRING, FieldType.LONGFORM])("%s", () => {
beforeAll(async () => { beforeAll(async () => {
await createTable({ await createTable({

View File

@ -164,8 +164,13 @@ export async function search(
throw new Error("SQS cannot currently handle multiple queries") throw new Error("SQS cannot currently handle multiple queries")
} }
let sql = query.sql, let sql = query.sql
bindings = query.bindings let bindings = query.bindings?.map(b => {
if (typeof b === "boolean") {
return b ? 1 : 0
}
return b
})
// quick hack for docIds // quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`") sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")