MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

viewV2.spec.ts passsing in full

This commit is contained in:
Sam Rose 2024-09-26 15:22:10 +01:00
parent 0ef633b87a
commit c4c524c6ff
No known key found for this signature in database
10 changed files with 151 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/backend-core/src/sql/sql.ts
View File

@ -859,7 +859,7 @@ class InternalBuilder {
} }
addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
let { sort } = this.query let { sort, resource } = this.query
const primaryKey = this.table.primary const primaryKey = this.table.primary
const tableName = getTableName(this.table) const tableName = getTableName(this.table)
const aliases = this.query.tableAliases const aliases = this.query.tableAliases
@ -896,7 +896,8 @@ class InternalBuilder {
// add sorting by the primary key if the result isn't already sorted by it, // add sorting by the primary key if the result isn't already sorted by it,
// to make sure result is deterministic // to make sure result is deterministic
if (!sort || sort[primaryKey[0]] === undefined) { const hasAggregations = (resource?.aggregations?.length ?? 0) > 0
if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) {
query = query.orderBy(`${aliased}.${primaryKey[0]}`) query = query.orderBy(`${aliased}.${primaryKey[0]}`)
} }
return query return query

22
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -1,5 +1,6 @@
import dayjs from "dayjs" import dayjs from "dayjs"
import { import {
Aggregation,
AutoFieldSubType, AutoFieldSubType,
AutoReason, AutoReason,
Datasource, Datasource,
@ -47,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import env from "../../../environment" import env from "../../../environment"
import { makeExternalQuery } from "../../../integrations/base/query" import { makeExternalQuery } from "../../../integrations/base/query"
import { dataFilters } from "@budibase/shared-core" import { dataFilters, helpers } from "@budibase/shared-core"
export interface ManyRelationship { export interface ManyRelationship {
tableId?: string tableId?: string
@ -682,12 +683,26 @@ export class ExternalRequest<T extends Operation> {
} }
} }
} }
if ( if (
operation === Operation.DELETE && operation === Operation.DELETE &&
(filters == null || Object.keys(filters).length === 0) (filters == null || Object.keys(filters).length === 0)
) { ) {
throw "Deletion must be filtered" throw "Deletion must be filtered"
} }
let aggregations: Aggregation[] = []
if (sdk.views.isView(this.source)) {
const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({
name: key,
field: field.field,
calculationType: field.calculationType,
})
}
}
let json: QueryJson = { let json: QueryJson = {
endpoint: { endpoint: {
datasourceId: this.datasource._id!, datasourceId: this.datasource._id!,
@ -697,10 +712,11 @@ export class ExternalRequest<T extends Operation> {
resource: { resource: {
// have to specify the fields to avoid column overlap (for SQL) // have to specify the fields to avoid column overlap (for SQL)
fields: isSql fields: isSql
? buildSqlFieldList(table, this.tables, { ? await buildSqlFieldList(this.source, this.tables, {
relationships: incRelationships, relationships: incRelationships,
}) })
: [], : [],
aggregations,
}, },
filters, filters,
sort, sort,
@ -748,7 +764,7 @@ export class ExternalRequest<T extends Operation> {
} }
const output = await sqlOutputProcessing( const output = await sqlOutputProcessing(
response, response,
table, this.source,
this.tables, this.tables,
relationships relationships
) )

32
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -100,8 +100,10 @@ export async function basicProcessing({
sqs?: boolean sqs?: boolean
}): Promise<Row> { }): Promise<Row> {
let table: Table let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) { if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id) table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else { } else {
table = source table = source
} }
@ -132,20 +134,22 @@ export async function basicProcessing({
} }
let columns: string[] = Object.keys(table.schema) let columns: string[] = Object.keys(table.schema)
if (!sqs) { if (!isCalculationView) {
thisRow._id = generateIdForRow(row, table, isLinked) if (!sqs) {
thisRow.tableId = table._id thisRow._id = generateIdForRow(row, table, isLinked)
thisRow._rev = "rev" thisRow.tableId = table._id
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) thisRow._rev = "rev"
} else { columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS) } else {
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
thisRow[internalColumn] = extractFieldValue({ for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
row, thisRow[internalColumn] = extractFieldValue({
tableName: table._id!, row,
fieldName: internalColumn, tableName: table._id!,
isLinked, fieldName: internalColumn,
}) isLinked,
})
}
} }
} }
for (let col of columns) { for (let col of columns) {

41
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -9,9 +9,12 @@ import {
RelationshipsJson, RelationshipsJson,
Row, Row,
Table, Table,
ViewV2,
} from "@budibase/types" } from "@budibase/types"
import { breakExternalTableId } from "../../../../integrations/utils" import { breakExternalTableId } from "../../../../integrations/utils"
import { generateJunctionTableID } from "../../../../db/utils" import { generateJunctionTableID } from "../../../../db/utils"
import sdk from "../../../../sdk"
import { helpers } from "@budibase/shared-core"
type TableMap = Record<string, Table> type TableMap = Record<string, Table>
@ -109,11 +112,12 @@ export function buildInternalRelationships(
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
* is more performant and has the added benefit of protecting against this scenario. * is more performant and has the added benefit of protecting against this scenario.
*/ */
export function buildSqlFieldList( export async function buildSqlFieldList(
table: Table, source: Table | ViewV2,
tables: TableMap, tables: TableMap,
opts?: { relationships: boolean } opts?: { relationships: boolean }
) { ) {
const { relationships } = opts || {}
function extractRealFields(table: Table, existing: string[] = []) { function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema) return Object.entries(table.schema)
.filter( .filter(
@ -124,22 +128,33 @@ export function buildSqlFieldList(
) )
.map(column => `${table.name}.${column[0]}`) .map(column => `${table.name}.${column[0]}`)
} }
let fields = extractRealFields(table)
let fields: string[] = []
if (sdk.views.isView(source)) {
fields = Object.keys(helpers.views.basicFields(source)).filter(
key => source.schema?.[key]?.visible !== false
)
} else {
fields = extractRealFields(source)
}
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
for (let field of Object.values(table.schema)) { for (let field of Object.values(table.schema)) {
if ( if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
field.type !== FieldType.LINK ||
!opts?.relationships ||
!field.tableId
) {
continue continue
} }
const { tableName: linkTableName } = breakExternalTableId(field.tableId) const { tableName } = breakExternalTableId(field.tableId)
const linkTable = tables[linkTableName] if (tables[tableName]) {
if (linkTable) { fields = fields.concat(extractRealFields(tables[tableName], fields))
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
} }
} }
return fields return fields
} }

5
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -19,6 +19,7 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
import sdk from "../../../../sdk" import sdk from "../../../../sdk"
import { processStringSync } from "@budibase/string-templates" import { processStringSync } from "@budibase/string-templates"
import validateJs from "validate.js" import validateJs from "validate.js"
import { helpers } from "@budibase/shared-core"
validateJs.extend(validateJs.validators.datetime, { validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) { parse: function (value: string) {
@ -121,8 +122,10 @@ export async function sqlOutputProcessing(
} }
let table: Table let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) { if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id) table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else { } else {
table = source table = source
} }
@ -131,7 +134,7 @@ export async function sqlOutputProcessing(
for (let row of rows) { for (let row of rows) {
if (opts?.sqs) { if (opts?.sqs) {
row._id = getInternalRowId(row, table) row._id = getInternalRowId(row, table)
} else if (row._id == null) { } else if (row._id == null && !isCalculationView) {
row._id = generateIdForRow(row, table) row._id = generateIdForRow(row, table)
} }

114
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -37,16 +37,15 @@ import {
setEnv as setCoreEnv, setEnv as setCoreEnv,
env, env,
} from "@budibase/backend-core" } from "@budibase/backend-core"
import sdk from "../../../sdk"
describe.each([ describe.each([
// ["lucene", undefined], ["lucene", undefined],
["sqs", undefined], ["sqs", undefined],
// [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
// [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
// [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
// [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
// [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/v2/views (%s)", (name, dsProvider) => { ])("/v2/views (%s)", (name, dsProvider) => {
const config = setup.getConfig() const config = setup.getConfig()
const isSqs = name === "sqs" const isSqs = name === "sqs"
@ -2362,63 +2361,70 @@ describe.each([
}) })
}) })
describe("calculations", () => { !isLucene &&
let table: Table describe("calculations", () => {
let rows: Row[] let table: Table
let rows: Row[]
beforeAll(async () => { beforeAll(async () => {
table = await config.api.table.save( table = await config.api.table.save(
saveTableRequest({ saveTableRequest({
schema: { schema: {
quantity: { quantity: {
type: FieldType.NUMBER, type: FieldType.NUMBER,
name: "quantity", name: "quantity",
},
price: {
type: FieldType.NUMBER,
name: "price",
},
}, },
price: { })
type: FieldType.NUMBER, )
name: "price",
rows = await Promise.all(
Array.from({ length: 10 }, () =>
config.api.row.save(table._id!, {
quantity: generator.natural({ min: 1, max: 10 }),
price: generator.natural({ min: 1, max: 10 }),
})
)
)
})
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
"Quantity Sum": {
visible: true,
calculationType: CalculationType.SUM,
field: "quantity",
}, },
}, },
}) })
)
rows = await Promise.all( const response = await config.api.viewV2.search(view.id, {
Array.from({ length: 10 }, () => query: {},
config.api.row.save(table._id!, { })
quantity: generator.natural({ min: 1, max: 10 }),
price: generator.natural({ min: 1, max: 10 }), expect(response.rows).toHaveLength(1)
}) expect(response.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
}),
])
) )
)
})
it("should be able to search by calculations", async () => { // Calculation views do not return rows that can be linked back to
const view = await config.api.viewV2.create({ // the source table, and so should not have an _id field.
tableId: table._id!, for (const row of response.rows) {
name: generator.guid(), expect("_id" in row).toBe(false)
schema: { }
"Quantity Sum": {
visible: true,
calculationType: CalculationType.SUM,
field: "quantity",
},
},
}) })
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
}),
])
)
}) })
})
}) })
describe("permissions", () => { describe("permissions", () => {

11
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -127,10 +127,13 @@ export async function search(
} }
} }
if (options.fields) { const visibleFields =
const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS] options.fields ||
processed = processed.map((r: any) => pick(r, fields)) Object.keys(source.schema || {}).filter(
} key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS]
processed = processed.map((r: any) => pick(r, allowedFields))
// need wrapper object for bookmarks etc when paginating // need wrapper object for bookmarks etc when paginating
const response: SearchResponse<Row> = { rows: processed, hasNextPage } const response: SearchResponse<Row> = { rows: processed, hasNextPage }

11
packages/server/src/sdk/app/rows/search/internal/lucene.ts
View File

@ -62,10 +62,13 @@ export async function search(
response.rows = await getGlobalUsersFromMetadata(response.rows as User[]) response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
} }
if (options.fields) { const visibleFields =
const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] options.fields ||
response.rows = response.rows.map((r: any) => pick(r, fields)) Object.keys(source.schema || {}).filter(
} key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, allowedFields))
response.rows = await outputProcessing(source, response.rows, { response.rows = await outputProcessing(source, response.rows, {
squash: true, squash: true,

1
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -460,7 +460,6 @@ export async function search(
let finalRows = await outputProcessing(source, processed, { let finalRows = await outputProcessing(source, processed, {
preserveLinks: true, preserveLinks: true,
squash: true, squash: true,
aggregations,
}) })
const visibleFields = const visibleFields =

9
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -11,7 +11,6 @@ import {
import { InternalTables } from "../../db/utils" import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map" import { TYPE_TRANSFORM_MAP } from "./map"
import { import {
Aggregation,
AutoFieldSubType, AutoFieldSubType,
FieldType, FieldType,
IdentityType, IdentityType,
@ -263,7 +262,6 @@ export async function outputProcessing<T extends Row[] | Row>(
preserveLinks?: boolean preserveLinks?: boolean
fromRow?: Row fromRow?: Row
skipBBReferences?: boolean skipBBReferences?: boolean
aggregations?: Aggregation[]
} = { } = {
squash: true, squash: true,
preserveLinks: false, preserveLinks: false,
@ -411,8 +409,11 @@ export async function outputProcessing<T extends Row[] | Row>(
f.toLowerCase() f.toLowerCase()
) )
for (const aggregation of opts.aggregations || []) { if (sdk.views.isView(source)) {
fields.push(aggregation.name.toLowerCase()) const aggregations = helpers.views.calculationFields(source)
for (const key of Object.keys(aggregations)) {
fields.push(key.toLowerCase())
}
} }
for (const row of enriched) { for (const row of enriched) {