Merge branch 'develop' into BUDI-7189/change_query_format

This commit is contained in:
Adria Navarro 2023-08-07 15:32:55 +03:00
commit f1739d9aa4
14 changed files with 409 additions and 346 deletions

View File

@ -1,5 +1,5 @@
{ {
"version": "2.8.32-alpha.3", "version": "2.8.32-alpha.5",
"npmClient": "yarn", "npmClient": "yarn",
"packages": [ "packages": [
"packages/*" "packages/*"

View File

@ -11,6 +11,9 @@ import {
Row, Row,
PatchRowRequest, PatchRowRequest,
PatchRowResponse, PatchRowResponse,
SearchRowResponse,
SearchRowRequest,
SearchParams,
} from "@budibase/types" } from "@budibase/types"
import * as utils from "./utils" import * as utils from "./utils"
import { gridSocket } from "../../../websockets" import { gridSocket } from "../../../websockets"
@ -197,10 +200,10 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
ctx.body = response ctx.body = response
} }
export async function search(ctx: any) { export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const tableId = utils.getTableId(ctx) const tableId = utils.getTableId(ctx)
const searchParams = { const searchParams: SearchParams = {
...ctx.request.body, ...ctx.request.body,
tableId, tableId,
} }

View File

@ -1,15 +1,18 @@
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { import {
UserCtx, UserCtx,
SearchResponse,
SortOrder,
SortType,
ViewV2, ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
SearchParams,
} from "@budibase/types" } from "@budibase/types"
import { dataFilters } from "@budibase/shared-core" import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk" import sdk from "../../../sdk"
export async function searchView(ctx: UserCtx<void, SearchResponse>) { export async function searchView(
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
) {
const { viewId } = ctx.params const { viewId } = ctx.params
const view = await sdk.views.get(viewId) const view = await sdk.views.get(viewId)
@ -30,50 +33,35 @@ export async function searchView(ctx: UserCtx<void, SearchResponse>) {
undefined undefined
ctx.status = 200 ctx.status = 200
const { body } = ctx.request
const query = dataFilters.buildLuceneQuery(view.query || []) const query = dataFilters.buildLuceneQuery(view.query || [])
const result = await quotas.addQuery(
() => const searchOptions: RequiredKeys<SearchViewRowRequest> &
sdk.rows.search({ RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
tableId: view.tableId, tableId: view.tableId,
query, query,
fields: viewFields, fields: viewFields,
...getSortOptions(ctx, view), ...getSortOptions(body, view),
}), limit: body.limit,
{ bookmark: body.bookmark,
datasourceId: view.tableId, paginate: body.paginate,
} }
)
const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), {
datasourceId: view.tableId,
})
result.rows.forEach(r => (r._viewId = view.id)) result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result ctx.body = result
} }
function getSortOptions( function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
ctx: UserCtx, if (request.sort) {
view: ViewV2
):
| {
sort: string
sortOrder?: SortOrder
sortType?: SortType
}
| undefined {
const { sort_column, sort_order, sort_type } = ctx.query
if (Array.isArray(sort_column)) {
ctx.throw(400, "sort_column cannot be an array")
}
if (Array.isArray(sort_order)) {
ctx.throw(400, "sort_order cannot be an array")
}
if (Array.isArray(sort_type)) {
ctx.throw(400, "sort_type cannot be an array")
}
if (sort_column) {
return { return {
sort: sort_column, sort: request.sort,
sortOrder: sort_order as SortOrder, sortOrder: request.sortOrder,
sortType: sort_type as SortType, sortType: request.sortType,
} }
} }
if (view.sort) { if (view.sort) {
@ -84,5 +72,9 @@ function getSortOptions(
} }
} }
return return {
sort: undefined,
sortOrder: undefined,
sortType: undefined,
}
} }

View File

@ -269,7 +269,7 @@ router
) )
router router
.get( .post(
"/api/v2/views/:viewId/search", "/api/v2/views/:viewId/search",
authorized(PermissionType.VIEW, PermissionLevel.READ), authorized(PermissionType.VIEW, PermissionLevel.READ),
rowController.views.searchView rowController.views.searchView

View File

@ -813,259 +813,6 @@ describe("/rows", () => {
}) })
}) })
describe("view search", () => {
function userTable(): Table {
return {
name: "user",
type: "user",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: { type: "string" },
},
age: {
type: FieldType.NUMBER,
name: "age",
constraints: {},
},
},
}
}
it("returns table rows from view", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body).toEqual({
rows: expect.arrayContaining(rows.map(expect.objectContaining)),
})
})
it("searching respects the view filters", async () => {
const table = await config.createTable(userTable())
const expectedRows = []
for (let i = 0; i < 10; i++)
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.integer({ min: 10, max: 30 }),
})
for (let i = 0; i < 5; i++)
expectedRows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: 40,
})
)
const createViewResponse = await config.api.viewV2.create({
query: [
{
operator: "equal",
field: "age",
value: "40",
type: FieldType.NUMBER,
},
],
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(5)
expect(response.body).toEqual({
rows: expect.arrayContaining(expectedRows.map(expect.objectContaining)),
})
})
const sortTestOptions: [
{
field: string
order?: SortOrder
type?: SortType
},
string[]
][] = [
[
{
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
type: SortType.STRING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
type: SortType.number,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
},
["Bob", "Charly", "Alice", "Danny"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
type: SortType.number,
},
["Bob", "Charly", "Alice", "Danny"],
],
]
it.each(sortTestOptions)(
"allow sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: sortParams,
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it.each(sortTestOptions)(
"allow override the default view sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: {
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
})
const response = await config.api.viewV2.search(createViewResponse.id, {
sort: {
column: sortParams.field,
order: sortParams.order,
type: sortParams.type,
},
})
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it("when schema is defined, defined columns and row attributes are returned", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.age(),
})
)
}
const view = await config.api.viewV2.create({
schema: { name: {} },
})
const response = await config.api.viewV2.search(view.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyInternalColsAttributes,
_viewId: view.id,
name: r.name,
}))
)
)
})
it("views without data can be returned", async () => {
const table = await config.createTable(userTable())
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(0)
})
})
describe("view 2.0", () => { describe("view 2.0", () => {
function userTable(): Table { function userTable(): Table {
return { return {
@ -1268,5 +1015,327 @@ describe("/rows", () => {
await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 }) await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
}) })
}) })
describe("view search", () => {
function userTable(): Table {
return {
name: "user",
type: "user",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: { type: "string" },
},
age: {
type: FieldType.NUMBER,
name: "age",
constraints: {},
},
},
}
}
it("returns table rows from view", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body).toEqual({
rows: expect.arrayContaining(rows.map(expect.objectContaining)),
})
})
it("searching respects the view filters", async () => {
const table = await config.createTable(userTable())
const expectedRows = []
for (let i = 0; i < 10; i++)
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.integer({ min: 10, max: 30 }),
})
for (let i = 0; i < 5; i++)
expectedRows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: 40,
})
)
const createViewResponse = await config.api.viewV2.create({
query: { equal: { age: 40 } },
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(5)
expect(response.body).toEqual({
rows: expect.arrayContaining(
expectedRows.map(expect.objectContaining)
),
})
})
const sortTestOptions: [
{
field: string
order?: SortOrder
type?: SortType
},
string[]
][] = [
[
{
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
type: SortType.STRING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
type: SortType.number,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
},
["Bob", "Charly", "Alice", "Danny"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
type: SortType.number,
},
["Bob", "Charly", "Alice", "Danny"],
],
]
it.each(sortTestOptions)(
"allow sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: sortParams,
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it.each(sortTestOptions)(
"allow override the default view sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: {
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
})
const response = await config.api.viewV2.search(
createViewResponse.id,
{
sort: sortParams.field,
sortOrder: sortParams.order,
sortType: sortParams.type,
}
)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it("when schema is defined, defined columns and row attributes are returned", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.age(),
})
)
}
const view = await config.api.viewV2.create({
schema: { name: {} },
})
const response = await config.api.viewV2.search(view.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyInternalColsAttributes,
_viewId: view.id,
name: r.name,
}))
)
)
})
it("views without data can be returned", async () => {
const table = await config.createTable(userTable())
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(0)
})
it("respects the limit parameter", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const limit = generator.integer({ min: 1, max: 8 })
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id, {
limit,
})
expect(response.body.rows).toHaveLength(limit)
})
it("can handle pagination", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
// rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
const createViewResponse = await config.api.viewV2.create()
const allRows = (await config.api.viewV2.search(createViewResponse.id))
.body.rows
const firstPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
}
)
expect(firstPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(0, 4)),
totalRows: 10,
hasNextPage: true,
bookmark: expect.any(String),
})
const secondPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
bookmark: firstPageResponse.body.bookmark,
}
)
expect(secondPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(4, 8)),
totalRows: 10,
hasNextPage: true,
bookmark: expect.any(String),
})
const lastPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
bookmark: secondPageResponse.body.bookmark,
}
)
expect(lastPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(8)),
totalRows: 10,
hasNextPage: false,
bookmark: expect.any(String),
})
})
})
}) })
}) })

View File

@ -1,23 +1,9 @@
import { SearchFilters, SortOrder, SortType } from "@budibase/types" import { SearchFilters, SearchParams } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils" import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal" import * as internal from "./search/internal"
import * as external from "./search/external" import * as external from "./search/external"
import { Format } from "../../../api/controllers/view/exporters" import { Format } from "../../../api/controllers/view/exporters"
export interface SearchParams {
tableId: string
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: SortOrder
sortType?: SortType
version?: string
disableEscaping?: boolean
fields?: string[]
}
export interface ViewParams { export interface ViewParams {
calculation: string calculation: string
group: string group: string

View File

@ -6,6 +6,7 @@ import {
IncludeRelationship, IncludeRelationship,
Row, Row,
SearchFilters, SearchFilters,
SearchParams,
} from "@budibase/types" } from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters" import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk" import sdk from "../../../../sdk"
@ -13,7 +14,7 @@ import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils" import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils" import { cleanExportRows } from "../utils"
import { utils } from "@budibase/shared-core" import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" import { ExportRowsParams, ExportRowsResult } from "../search"
import { HTTPError, db } from "@budibase/backend-core" import { HTTPError, db } from "@budibase/backend-core"
import pick from "lodash/pick" import pick from "lodash/pick"

View File

@ -12,7 +12,7 @@ import {
} from "../../../../db/utils" } from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global" import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor" import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Database, Row, Table } from "@budibase/types" import { Database, Row, Table, SearchParams } from "@budibase/types"
import { cleanExportRows } from "../utils" import { cleanExportRows } from "../utils"
import { import {
Format, Format,
@ -28,7 +28,7 @@ import {
getFromMemoryDoc, getFromMemoryDoc,
} from "../../../../api/controllers/view/utils" } from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk" import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search" import { ExportRowsParams, ExportRowsResult } from "../search"
import pick from "lodash/pick" import pick from "lodash/pick"
export async function search(options: SearchParams) { export async function search(options: SearchParams) {

View File

@ -1,8 +1,15 @@
import { GenericContainer } from "testcontainers" import { GenericContainer } from "testcontainers"
import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types" import {
Datasource,
FieldType,
Row,
SourceName,
Table,
SearchParams,
} from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../external" import { search } from "../external"
import { import {
expectAnyExternalColsAttributes, expectAnyExternalColsAttributes,

View File

@ -1,6 +1,5 @@
import { FieldType, Row, Table } from "@budibase/types" import { FieldType, Row, Table, SearchParams } from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../internal" import { search } from "../internal"
import { import {
expectAnyInternalColsAttributes, expectAnyInternalColsAttributes,

View File

@ -1,13 +1,12 @@
import { import {
CreateViewRequest, CreateViewRequest,
SortOrder,
SortType,
UpdateViewRequest, UpdateViewRequest,
DeleteRowRequest, DeleteRowRequest,
PatchRowRequest, PatchRowRequest,
PatchRowResponse, PatchRowResponse,
Row, Row,
ViewV2, ViewV2,
SearchViewRowRequest,
} from "@budibase/types" } from "@budibase/types"
import TestConfiguration from "../TestConfiguration" import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base" import { TestAPI } from "./base"
@ -81,31 +80,12 @@ export class ViewV2API extends TestAPI {
search = async ( search = async (
viewId: string, viewId: string,
options?: { params?: SearchViewRowRequest,
sort: {
column: string
order?: SortOrder
type?: SortType
}
},
{ expectStatus } = { expectStatus: 200 } { expectStatus } = { expectStatus: 200 }
) => { ) => {
const qs: [string, any][] = []
if (options?.sort.column) {
qs.push(["sort_column", options.sort.column])
}
if (options?.sort.order) {
qs.push(["sort_order", options.sort.order])
}
if (options?.sort.type) {
qs.push(["sort_type", options.sort.type])
}
let url = `/api/v2/views/${viewId}/search`
if (qs.length) {
url += "?" + qs.map(q => q.join("=")).join("&")
}
return this.request return this.request
.get(url) .post(`/api/v2/views/${viewId}/search`)
.send(params)
.set(this.config.defaultHeaders()) .set(this.config.defaultHeaders())
.expect("Content-Type", /json/) .expect("Content-Type", /json/)
.expect(expectStatus) .expect(expectStatus)

View File

@ -1,3 +1,4 @@
import { SearchParams } from "../../../sdk"
import { Row } from "../../../documents" import { Row } from "../../../documents"
export interface PatchRowRequest extends Row { export interface PatchRowRequest extends Row {
@ -8,6 +9,14 @@ export interface PatchRowRequest extends Row {
export interface PatchRowResponse extends Row {} export interface PatchRowResponse extends Row {}
export interface SearchResponse { export interface SearchRowRequest extends Omit<SearchParams, "tableId"> {}
export interface SearchViewRowRequest
extends Pick<
SearchRowRequest,
"sort" | "sortOrder" | "sortType" | "limit" | "bookmark" | "paginate"
> {}
export interface SearchRowResponse {
rows: any[] rows: any[]
} }

View File

@ -19,3 +19,4 @@ export * from "./user"
export * from "./cli" export * from "./cli"
export * from "./websocket" export * from "./websocket"
export * from "./permissions" export * from "./permissions"
export * from "./row"

View File

@ -0,0 +1,16 @@
import { SortOrder, SortType } from "../api"
import { SearchFilters } from "./search"
export interface SearchParams {
tableId: string
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: SortOrder
sortType?: SortType
version?: string
disableEscaping?: boolean
fields?: string[]
}