MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11239 from Budibase/BUDI-7189/extract_search_functionality 

Moving files and functions
This commit is contained in:
Adria Navarro 2023-07-17 14:17:26 +01:00 committed by GitHub
parent 5361a6d12b 20635ae2ff
commit 832483442f
20 changed files with 824 additions and 690 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.tool-versions
View File

@ -1,2 +1,2 @@
nodejs 14.20.1
nodejs 14.21.3
python 3.10.0

3
packages/server/src/api/controllers/datasource.ts
View File

@ -9,7 +9,6 @@ import {
import { destroy as tableDestroy } from "./table/internal"
import { BuildSchemaErrors, InvalidColumns } from "../../constants"
import { getIntegration } from "../../integrations"
import { getDatasourceAndQuery } from "./row/utils"
import { invalidateDynamicVariables } from "../../threads/utils"
import { db as dbCore, context, events } from "@budibase/backend-core"
import {
@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) {
export async function query(ctx: UserCtx) {
const queryJson = ctx.request.body
try {
ctx.body = await getDatasourceAndQuery(queryJson)
ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
} catch (err: any) {
ctx.throw(400, err)
}

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -23,14 +23,13 @@ import {
isRowId,
isSQL,
} from "../../../integrations/utils"
import { getDatasourceAndQuery } from "./utils"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { FieldTypes } from "../../../constants"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { processDates, processFormulas } from "../../../utilities/rowProcessor"
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { isEditableColumn } from "../../../sdk/app/tables/validation"
export interface ManyRelationship {
tableId?: string

150
packages/server/src/api/controllers/row/external.ts
View File

@ -1,30 +1,20 @@
import {
FieldTypes,
NoEmptyFilterStrings,
SortDirection,
} from "../../../constants"
import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
import {
breakExternalTableId,
breakRowIdField,
} from "../../../integrations/utils"
import { ExternalRequest, RunConfig } from "./ExternalRequest"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Datasource,
IncludeRelationship,
Operation,
PaginationJson,
Row,
SortJson,
Table,
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils")
async function getRow(
tableId: string,
rowId: string,
@ -114,21 +104,6 @@ export async function save(ctx: UserCtx) {
}
}
export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
}
export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function find(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
@ -161,129 +136,6 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) }
}
export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(ctx: UserCtx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
}
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
// @ts-ignore
const exporter = exporters[format]
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId

10
packages/server/src/api/controllers/row/index.ts
View File

@ -5,6 +5,7 @@ import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
import { gridSocket } from "../../../websockets"
import sdk from "../../../sdk"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -64,14 +65,14 @@ export const save = async (ctx: any) => {
}
export async function fetchView(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), {
datasourceId: tableId,
})
}
export async function fetch(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), {
datasourceId: tableId,
})
}
@ -120,7 +121,8 @@ export async function destroy(ctx: any) {
export async function search(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.status = 200
ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), {
datasourceId: tableId,
})
}
@ -150,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) {
export const exportRows = async (ctx: any) => {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), {
datasourceId: tableId,
})
}

234
packages/server/src/api/controllers/row/internal.ts
View File

@ -1,7 +1,6 @@
import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getRowParams,
getTableIDFromRowID,
DocumentType,
InternalTables,
@ -14,8 +13,6 @@ import {
} from "../../../utilities/rowProcessor"
import { FieldTypes } from "../../../constants"
import * as utils from "./utils"
import { fullSearch, paginatedSearch } from "./internalSearch"
import { getGlobalUsersFromMetadata } from "../../../utilities/global"
import * as inMemoryViews from "../../../db/inMemoryView"
import env from "../../../environment"
import {
@ -27,65 +24,7 @@ import {
import { cloneDeep } from "lodash/fp"
import { context, db as dbCore } from "@budibase/backend-core"
import { finaliseRow, updateRelatedFormula } from "./staticFormula"
import { csv, json, jsonWithSchema, Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
UserCtx,
Database,
LinkDocumentValue,
Row,
Table,
} from "@budibase/types"
import { cleanExportRows } from "./utils"
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}
async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
export async function patch(ctx: UserCtx) {
const db = context.getAppDB()
@ -195,82 +134,6 @@ export async function save(ctx: UserCtx) {
})
}
export async function fetchView(ctx: UserCtx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(ctx, db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table
try {
table = await db.get(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(ctx, db, tableId)
return outputProcessing(table, rows)
}
export async function find(ctx: UserCtx) {
const db = dbCore.getDB(ctx.appId)
const table = await db.get(ctx.params.tableId)
@ -354,101 +217,6 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: processedRows }
}
export async function search(ctx: UserCtx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(ctx: UserCtx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId

101
packages/server/src/api/controllers/row/utils.ts
View File

@ -1,14 +1,19 @@
import { InternalTables } from "../../../db/utils"
import * as userController from "../user"
import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { FieldType, Row, Table, UserCtx } from "@budibase/types"
import { Format } from "../view/exporters"
import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
import { FieldTypes } from "../../../constants"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import validateJs from "validate.js"
import { cloneDeep } from "lodash/fp"
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
},
})
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row
@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
return row
}
export function getTableId(ctx: Ctx) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}
export async function validate({
tableId,
row,
@ -81,8 +85,8 @@ export async function validate({
continue
}
// special case for options, need to always allow unselected (empty)
if (type === FieldTypes.OPTIONS && constraints.inclusion) {
constraints.inclusion.push(null, "")
if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
constraints.inclusion.push(null as any, "")
}
let res
@ -94,13 +98,13 @@ export async function validate({
}
row[fieldName].map((val: any) => {
if (
!constraints.inclusion.includes(val) &&
constraints.inclusion.length !== 0
!constraints?.inclusion?.includes(val) &&
constraints?.inclusion?.length !== 0
) {
errors[fieldName] = "Field not in list"
}
})
} else if (constraints.presence && row[fieldName].length === 0) {
} else if (constraints?.presence && row[fieldName].length === 0) {
// non required MultiSelect creates an empty array, which should not throw errors
errors[fieldName] = [`${fieldName} is required`]
}
@ -128,52 +132,3 @@ export async function validate({
}
return { valid: Object.keys(errors).length === 0, errors }
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}
export function getTableId(ctx: any) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}

4
packages/server/src/api/controllers/user.ts
View File

@ -3,10 +3,10 @@ import { InternalTables } from "../../db/utils"
import { getGlobalUsers } from "../../utilities/global"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { UserCtx } from "@budibase/types"
import { Ctx, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function fetchMetadata(ctx: UserCtx) {
export async function fetchMetadata(ctx: Ctx) {
const global = await getGlobalUsers()
const metadata = await sdk.users.rawUserMetadata()
const users = []

3
packages/server/src/api/controllers/view/index.ts
View File

@ -15,7 +15,6 @@ import {
TableSchema,
View,
} from "@budibase/types"
import { cleanExportRows } from "../row/utils"
import { builderSocket } from "../../../websockets"
const { cloneDeep, isEqual } = require("lodash")
@ -169,7 +168,7 @@ export async function exportView(ctx: Ctx) {
schema = table.schema
}
let exportRows = cleanExportRows(rows, schema, format, [])
let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
if (format === Format.CSV) {
ctx.attachment(`${viewName}.csv`)

197
packages/server/src/api/routes/tests/internalSearch.spec.js
View File

@ -1,197 +0,0 @@
const fetch = require("node-fetch")
fetch.mockSearch()
const search = require("../../controllers/row/internalSearch")
// this will be mocked out for _search endpoint
const PARAMS = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: null,
sort: null,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp, expected, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({
}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch({
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch({
string: {
"column": "2",
},
}, updatedParams)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch({
range: {
"column": { low: 1, high: 2 },
},
}, PARAMS)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch({
empty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch({
notEmpty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch({
oneOf: {
"column": ["a", "b"],
},
}, PARAMS)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch({
contains: {
"column": "a",
"colArr": [1, 2, 3],
},
}, PARAMS)
checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
}, PARAMS)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch({
equal: {
"1:1:column": "a",
},
}, PARAMS)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch({
containsAny: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch({
notContains: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = null
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

248
packages/server/src/api/routes/tests/internalSearch.spec.ts Normal file
View File

@ -0,0 +1,248 @@
const nodeFetch = require("node-fetch")
nodeFetch.mockSearch()
import { SearchParams } from "@budibase/backend-core"
import * as search from "../../../sdk/app/rows/search/internalSearch"
import { Row } from "@budibase/types"
// this will be mocked out for _search endpoint
const PARAMS: SearchParams<Row> = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: undefined,
sort: undefined,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp: any, expected: any, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch(
{
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch(
{
string: {
column: "2",
},
},
updatedParams
)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch(
{
range: {
column: { low: 1, high: 2 },
},
},
PARAMS
)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch(
{
empty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch(
{
notEmpty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch(
{
oneOf: {
column: ["a", "b"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch(
{
contains: {
column: "a",
colArr: [1, 2, 3],
},
},
PARAMS
)
checkLucene(
response,
`(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
PARAMS
)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
},
PARAMS
)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch(
{
equal: {
"1:1:column": "a",
},
},
PARAMS
)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch(
{
containsAny: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch(
{
notContains: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = undefined
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

13
packages/server/src/automations/tests/queryRows.spec.js → packages/server/src/automations/tests/queryRows.spec.ts
View File

@ -1,6 +1,6 @@
// lucene searching not supported in test due to use of PouchDB
let rows = []
jest.mock("../../api/controllers/row/internalSearch", () => ({
let rows: Row[] = []
jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
fullSearch: jest.fn(() => {
return {
rows,
@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
}),
paginatedSearch: jest.fn(),
}))
const setup = require("./utilities")
import { Row, Table } from "@budibase/types"
import * as setup from "./utilities"
const NAME = "Test"
describe("Test a query step automation", () => {
let table
let table: Table
let config = setup.getConfig()
beforeAll(async () => {
@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
filters: {},
"filters-def": [
{
value: null
}
value: null,
},
],
sortColumn: "name",
sortOrder: "ascending",

4
packages/server/src/sdk/app/rows/index.ts
View File

@ -1,7 +1,11 @@
import * as attachments from "./attachments"
import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
export default {
...attachments,
...rows,
...search,
utils: utils,
}

38
packages/server/src/sdk/app/rows/search.ts Normal file
View File

@ -0,0 +1,38 @@
import { Ctx, SearchFilters } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
export interface SearchParams {
tableId: string
paginate: boolean
query?: SearchFilters
bookmark?: number
limit: number
sort?: string
sortOrder?: string
sortType?: string
}
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
return external
}
return internal
}
export async function search(tableId: string, ctx: Ctx) {
return pickApi(tableId).search(ctx)
}
export async function exportRows(tableId: string, ctx: Ctx) {
return pickApi(tableId).exportRows(ctx)
}
export async function fetch(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetch(ctx)
}
export async function fetchView(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetchView(ctx)
}

177
packages/server/src/sdk/app/rows/search/external.ts Normal file
View File

@ -0,0 +1,177 @@
import {
SortJson,
SortDirection,
Operation,
PaginationJson,
IncludeRelationship,
Row,
Ctx,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import { utils } from "@budibase/shared-core"
export async function search(ctx: Ctx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(ctx: Ctx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format as string
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
if (!exporters.isFormat(format)) {
ctx.throw(
400,
`Format ${format} not valid. Valid values: ${Object.values(
exporters.Format
)}`
)
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
}
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
let content
switch (format) {
case exporters.Format.CSV:
content = exporters.csv(headers, exportRows)
break
case exporters.Format.JSON:
content = exporters.json(exportRows)
break
case exporters.Format.JSON_WITH_SCHEMA:
content = exporters.jsonWithSchema(schema, exportRows)
break
default:
utils.unreachable(format)
break
}
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(content)
}
export async function fetch(ctx: Ctx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function fetchView(ctx: Ctx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
}

246
packages/server/src/sdk/app/rows/search/internal.ts Normal file
View File

@ -0,0 +1,246 @@
import { context } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import {
InternalTables,
getRowParams,
DocumentType,
} from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Ctx, Database, Row, UserCtx } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
Format,
csv,
json,
jsonWithSchema,
} from "../../../../api/controllers/view/exporters"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import * as userController from "../../../../api/controllers/user"
import * as inMemoryViews from "../../../../db/inMemoryView"
import {
migrateToInMemoryView,
migrateToDesignView,
getFromDesignDoc,
getFromMemoryDoc,
} from "../../../../api/controllers/view/utils"
export async function search(ctx: Ctx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(ctx: Ctx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}
}
export async function fetch(ctx: Ctx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(ctx, db, tableId)
return outputProcessing(table, rows)
}
async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
export async function fetchView(ctx: Ctx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(ctx, db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table
try {
table = await db.get(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}

0
packages/server/src/api/controllers/row/internalSearch.ts → packages/server/src/sdk/app/rows/search/internalSearch.ts
View File

47
packages/server/src/sdk/app/rows/utils.ts Normal file
View File

@ -0,0 +1,47 @@
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}

32
packages/server/src/api/controllers/tests/row.spec.ts → packages/server/src/sdk/tests/rows/row.spec.ts
View File

@ -1,14 +1,14 @@
import { exportRows } from "../row/external"
import sdk from "../../../sdk"
import { ExternalRequest } from "../row/ExternalRequest"
import { exportRows } from "../../app/rows/search/external"
import sdk from "../.."
import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
// @ts-ignore
sdk.datasources = {
get: jest.fn(),
}
const mockDatasourcesGet = jest.fn()
sdk.datasources.get = mockDatasourcesGet
jest.mock("../row/ExternalRequest")
jest.mock("../view/exporters", () => ({
jest.mock("../../../api/controllers/row/ExternalRequest")
jest.mock("../../../api/controllers/view/exporters", () => ({
...jest.requireActual("../../../api/controllers/view/exporters"),
csv: jest.fn(),
Format: {
CSV: "csv",
@ -31,14 +31,15 @@ function getUserCtx() {
throw "Err"
}),
attachment: jest.fn(),
}
} as any
}
describe("external row controller", () => {
describe("exportRows", () => {
beforeAll(() => {
//@ts-ignore
jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
jest
.spyOn(ExternalRequest.prototype, "run")
.mockImplementation(() => Promise.resolve([]))
})
afterEach(() => {
@ -48,7 +49,6 @@ describe("external row controller", () => {
it("should throw a 400 if no datasource entities are present", async () => {
let userCtx = getUserCtx()
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
@ -59,8 +59,7 @@ describe("external row controller", () => {
})
it("should handle single quotes from a row ID", async () => {
//@ts-ignore
sdk.datasources.get.mockImplementation(() => ({
mockDatasourcesGet.mockImplementation(async () => ({
entities: {
tablename: {
schema: {},
@ -72,7 +71,6 @@ describe("external row controller", () => {
rows: ["['d001']"],
}
//@ts-ignore
await exportRows(userCtx)
expect(userCtx.request.body).toEqual({
@ -90,7 +88,6 @@ describe("external row controller", () => {
rows: ["[123]", "['d001'%2C'10111']"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
@ -107,7 +104,6 @@ describe("external row controller", () => {
rows: ["[123]"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(

2
packages/types/src/sdk/search.ts
View File

@ -32,7 +32,7 @@ export interface SearchFilters {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
[key: string]: any[] | any
}
notContains?: {
[key: string]: any[]