MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

viewV2.spec.ts passsing in full

This commit is contained in:
Sam Rose 2024-09-26 15:22:10 +01:00
parent 0ef633b87a
commit c4c524c6ff
No known key found for this signature in database
10 changed files with 151 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/backend-core/src/sql/sql.ts
View File

@ -859,7 +859,7 @@ class InternalBuilder {
}
addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
let { sort } = this.query
let { sort, resource } = this.query
const primaryKey = this.table.primary
const tableName = getTableName(this.table)
const aliases = this.query.tableAliases
@ -896,7 +896,8 @@ class InternalBuilder {
// add sorting by the primary key if the result isn't already sorted by it,
// to make sure result is deterministic
if (!sort || sort[primaryKey[0]] === undefined) {
const hasAggregations = (resource?.aggregations?.length ?? 0) > 0
if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) {
query = query.orderBy(`${aliased}.${primaryKey[0]}`)
}
return query

22
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -1,5 +1,6 @@
import dayjs from "dayjs"
import {
Aggregation,
AutoFieldSubType,
AutoReason,
Datasource,
@ -47,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import env from "../../../environment"
import { makeExternalQuery } from "../../../integrations/base/query"
import { dataFilters } from "@budibase/shared-core"
import { dataFilters, helpers } from "@budibase/shared-core"
export interface ManyRelationship {
tableId?: string
@ -682,12 +683,26 @@ export class ExternalRequest<T extends Operation> {
}
}
}
if (
operation === Operation.DELETE &&
(filters == null || Object.keys(filters).length === 0)
) {
throw "Deletion must be filtered"
}
let aggregations: Aggregation[] = []
if (sdk.views.isView(this.source)) {
const calculationFields = helpers.views.calculationFields(this.source)
for (const [key, field] of Object.entries(calculationFields)) {
aggregations.push({
name: key,
field: field.field,
calculationType: field.calculationType,
})
}
}
let json: QueryJson = {
endpoint: {
datasourceId: this.datasource._id!,
@ -697,10 +712,11 @@ export class ExternalRequest<T extends Operation> {
resource: {
// have to specify the fields to avoid column overlap (for SQL)
fields: isSql
? buildSqlFieldList(table, this.tables, {
? await buildSqlFieldList(this.source, this.tables, {
relationships: incRelationships,
})
: [],
aggregations,
},
filters,
sort,
@ -748,7 +764,7 @@ export class ExternalRequest<T extends Operation> {
}
const output = await sqlOutputProcessing(
response,
table,
this.source,
this.tables,
relationships
)

32
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -100,8 +100,10 @@ export async function basicProcessing({
sqs?: boolean
}): Promise<Row> {
let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else {
table = source
}
@ -132,20 +134,22 @@ export async function basicProcessing({
}
let columns: string[] = Object.keys(table.schema)
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
} else {
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,
fieldName: internalColumn,
isLinked,
})
if (!isCalculationView) {
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
} else {
columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,
fieldName: internalColumn,
isLinked,
})
}
}
}
for (let col of columns) {

41
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -9,9 +9,12 @@ import {
RelationshipsJson,
Row,
Table,
ViewV2,
} from "@budibase/types"
import { breakExternalTableId } from "../../../../integrations/utils"
import { generateJunctionTableID } from "../../../../db/utils"
import sdk from "../../../../sdk"
import { helpers } from "@budibase/shared-core"
type TableMap = Record<string, Table>
@ -109,11 +112,12 @@ export function buildInternalRelationships(
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us
* is more performant and has the added benefit of protecting against this scenario.
*/
export function buildSqlFieldList(
table: Table,
export async function buildSqlFieldList(
source: Table | ViewV2,
tables: TableMap,
opts?: { relationships: boolean }
) {
const { relationships } = opts || {}
function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema)
.filter(
@ -124,22 +128,33 @@ export function buildSqlFieldList(
)
.map(column => `${table.name}.${column[0]}`)
}
let fields = extractRealFields(table)
let fields: string[] = []
if (sdk.views.isView(source)) {
fields = Object.keys(helpers.views.basicFields(source)).filter(
key => source.schema?.[key]?.visible !== false
)
} else {
fields = extractRealFields(source)
}
let table: Table
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
} else {
table = source
}
for (let field of Object.values(table.schema)) {
if (
field.type !== FieldType.LINK ||
!opts?.relationships ||
!field.tableId
) {
if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
const linkTable = tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
const { tableName } = breakExternalTableId(field.tableId)
if (tables[tableName]) {
fields = fields.concat(extractRealFields(tables[tableName], fields))
}
}
return fields
}

5
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -19,6 +19,7 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
import sdk from "../../../../sdk"
import { processStringSync } from "@budibase/string-templates"
import validateJs from "validate.js"
import { helpers } from "@budibase/shared-core"
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -121,8 +122,10 @@ export async function sqlOutputProcessing(
}
let table: Table
let isCalculationView = false
if (sdk.views.isView(source)) {
table = await sdk.views.getTable(source.id)
isCalculationView = helpers.views.isCalculationView(source)
} else {
table = source
}
@ -131,7 +134,7 @@ export async function sqlOutputProcessing(
for (let row of rows) {
if (opts?.sqs) {
row._id = getInternalRowId(row, table)
} else if (row._id == null) {
} else if (row._id == null && !isCalculationView) {
row._id = generateIdForRow(row, table)
}

114
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -37,16 +37,15 @@ import {
setEnv as setCoreEnv,
env,
} from "@budibase/backend-core"
import sdk from "../../../sdk"
describe.each([
// ["lucene", undefined],
["lucene", undefined],
["sqs", undefined],
// [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
// [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
// [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
// [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
// [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
[DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
])("/v2/views (%s)", (name, dsProvider) => {
const config = setup.getConfig()
const isSqs = name === "sqs"
@ -2362,63 +2361,70 @@ describe.each([
})
})
describe("calculations", () => {
let table: Table
let rows: Row[]
!isLucene &&
describe("calculations", () => {
let table: Table
let rows: Row[]
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
quantity: {
type: FieldType.NUMBER,
name: "quantity",
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
quantity: {
type: FieldType.NUMBER,
name: "quantity",
},
price: {
type: FieldType.NUMBER,
name: "price",
},
},
price: {
type: FieldType.NUMBER,
name: "price",
})
)
rows = await Promise.all(
Array.from({ length: 10 }, () =>
config.api.row.save(table._id!, {
quantity: generator.natural({ min: 1, max: 10 }),
price: generator.natural({ min: 1, max: 10 }),
})
)
)
})
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
"Quantity Sum": {
visible: true,
calculationType: CalculationType.SUM,
field: "quantity",
},
},
})
)
rows = await Promise.all(
Array.from({ length: 10 }, () =>
config.api.row.save(table._id!, {
quantity: generator.natural({ min: 1, max: 10 }),
price: generator.natural({ min: 1, max: 10 }),
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
}),
])
)
)
})
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
"Quantity Sum": {
visible: true,
calculationType: CalculationType.SUM,
field: "quantity",
},
},
// Calculation views do not return rows that can be linked back to
// the source table, and so should not have an _id field.
for (const row of response.rows) {
expect("_id" in row).toBe(false)
}
})
const response = await config.api.viewV2.search(view.id, {
query: {},
})
expect(response.rows).toHaveLength(1)
expect(response.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({
"Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
}),
])
)
})
})
})
describe("permissions", () => {

11
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -127,10 +127,13 @@ export async function search(
}
}
if (options.fields) {
const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS]
processed = processed.map((r: any) => pick(r, fields))
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS]
processed = processed.map((r: any) => pick(r, allowedFields))
// need wrapper object for bookmarks etc when paginating
const response: SearchResponse<Row> = { rows: processed, hasNextPage }

11
packages/server/src/sdk/app/rows/search/internal/lucene.ts
View File

@ -62,10 +62,13 @@ export async function search(
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
if (options.fields) {
const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, fields))
}
const visibleFields =
options.fields ||
Object.keys(source.schema || {}).filter(
key => source.schema?.[key].visible !== false
)
const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
response.rows = response.rows.map((r: any) => pick(r, allowedFields))
response.rows = await outputProcessing(source, response.rows, {
squash: true,

1
packages/server/src/sdk/app/rows/search/internal/sqs.ts
View File

@ -460,7 +460,6 @@ export async function search(
let finalRows = await outputProcessing(source, processed, {
preserveLinks: true,
squash: true,
aggregations,
})
const visibleFields =

9
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -11,7 +11,6 @@ import {
import { InternalTables } from "../../db/utils"
import { TYPE_TRANSFORM_MAP } from "./map"
import {
Aggregation,
AutoFieldSubType,
FieldType,
IdentityType,
@ -263,7 +262,6 @@ export async function outputProcessing<T extends Row[] | Row>(
preserveLinks?: boolean
fromRow?: Row
skipBBReferences?: boolean
aggregations?: Aggregation[]
} = {
squash: true,
preserveLinks: false,
@ -411,8 +409,11 @@ export async function outputProcessing<T extends Row[] | Row>(
f.toLowerCase()
)
for (const aggregation of opts.aggregations || []) {
fields.push(aggregation.name.toLowerCase())
if (sdk.views.isView(source)) {
const aggregations = helpers.views.calculationFields(source)
for (const key of Object.keys(aggregations)) {
fields.push(key.toLowerCase())
}
}
for (const row of enriched) {