From c5db1d1da305e350804685e59877780aafa45011 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 20 Sep 2024 16:37:23 +0100 Subject: [PATCH] Got a test passing but I hate it a bit. --- .../src/api/controllers/row/utils/basic.ts | 16 +++++++++++++++- .../src/api/controllers/row/utils/utils.ts | 4 +++- .../server/src/api/routes/tests/viewV2.spec.ts | 16 ++++++++-------- packages/server/src/db/linkedRows/index.ts | 16 ++++++++++++---- packages/server/src/sdk/app/rows/search.ts | 1 + .../src/sdk/app/rows/search/internal/sqs.ts | 11 ++++++++--- .../server/src/utilities/rowProcessor/index.ts | 9 +++++++++ 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 9c49f5636a..20a1cd0742 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,12 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types" +import { + FieldSchema, + FieldType, + Row, + Table, + JsonTypes, + Aggregation, +} from "@budibase/types" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -84,12 +91,14 @@ export function basicProcessing({ tables, isLinked, sqs, + aggregations, }: { row: Row table: Table tables: Table[] isLinked: boolean sqs?: boolean + aggregations?: Aggregation[] }): Row { const thisRow: Row = {} // filter the row down to what is actually the row (not joined) @@ -108,6 +117,11 @@ export function basicProcessing({ thisRow[fieldName] = value } } + + for (let aggregation of aggregations || []) { + thisRow[aggregation.name] = row[aggregation.name] + } + let columns: string[] = Object.keys(table.schema) if (!sqs) { thisRow._id = generateIdForRow(row, table, isLinked) diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index ac305e70b6..1fc7221294 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -2,6 +2,7 @@ import * as utils from "../../../../db/utils" import { context } from "@budibase/backend-core" import { + Aggregation, Ctx, DatasourcePlusQueryResponse, FieldType, @@ -129,7 +130,7 @@ export async function sqlOutputProcessing( table: Table, tables: Record, relationships: RelationshipsJson[], - opts?: { sqs?: boolean } + opts?: { sqs?: boolean; aggregations?: Aggregation[] } ): Promise { if (isKnexEmptyReadResponse(rows)) { return [] @@ -150,6 +151,7 @@ export async function sqlOutputProcessing( tables: Object.values(tables), isLinked: false, sqs: opts?.sqs, + aggregations: opts?.aggregations, }) if (thisRow._id == null) { throw new Error("Unable to generate row ID for SQL rows") diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index aaaf03c113..dc940c5ace 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -37,13 +37,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -2215,7 +2215,7 @@ describe.each([ ) }) - describe.skip("calculations", () => { + describe("calculations", () => { let table: Table let rows: Row[] @@ -2245,7 +2245,7 @@ describe.each([ ) }) - it("should be able to search by calculations", async () => { + it.only("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..4222f9b5e4 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -20,10 +20,11 @@ import { Row, Table, TableSchema, - ViewFieldMetadata, + ViewUIFieldMetadata, ViewV2, } from "@budibase/types" import sdk from "../../sdk" +import { helpers } from "@budibase/shared-core" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -264,12 +265,19 @@ export async function squashLinks( FeatureFlag.ENRICHED_RELATIONSHIPS ) - let viewSchema: Record = {} - if (options?.fromViewId && allowRelationshipSchemas) { + let viewSchema: Record = {} + if (options?.fromViewId) { const view = Object.values(table.views || {}).find( (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId ) - viewSchema = view?.schema || {} + + if (view && helpers.views.isCalculationView(view)) { + return enriched + } + + if (allowRelationshipSchemas && view) { + viewSchema = view.schema || {} + } } // will populate this as we find them diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 95cbc919a1..dbf0cefd51 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -37,6 +37,7 @@ export async function search( return await tracer.trace("search", async span => { span?.addTags({ tableId: options.tableId, + viewId: options.viewId, query: options.query, sort: options.sort, sortOrder: options.sortOrder, diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 188c95fb5c..b5bf8e752f 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -386,8 +386,9 @@ export async function search( // make sure JSON columns corrected const processed = builder.convertJsonStringColumns( table, - await sqlOutputProcessing(rows, table!, allTablesMap, relationships, { + await sqlOutputProcessing(rows, table, allTablesMap, relationships, { sqs: true, + aggregations: options.aggregations, }) ) @@ -406,11 +407,16 @@ export async function search( preserveLinks: true, squash: true, fromViewId: options.viewId, + aggregations: options.aggregations, }) // check if we need to pick specific rows out if (options.fields) { - const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS] + const fields = [ + ...options.fields, + ...PROTECTED_INTERNAL_COLUMNS, + ...(options.aggregations || []).map(a => a.name), + ] finalRows = finalRows.map((r: any) => pick(r, fields)) } @@ -440,6 +446,5 @@ export async function search( return { rows: [] } } throw err - throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index f6cf44d6d6..73768fdd57 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -11,6 +11,7 @@ import { import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { + Aggregation, AutoFieldSubType, FieldType, IdentityType, @@ -250,6 +251,7 @@ export async function outputProcessing( fromRow?: Row skipBBReferences?: boolean fromViewId?: string + aggregations?: Aggregation[] } = { squash: true, preserveLinks: false, @@ -357,6 +359,7 @@ export async function outputProcessing( fromViewId: opts?.fromViewId, }) } + // remove null properties to match internal API const isExternal = isExternalTableID(table._id!) if (isExternal || (await features.flags.isEnabled("SQS"))) { @@ -385,9 +388,15 @@ export async function outputProcessing( const tableFields = Object.keys(table.schema).filter( f => table.schema[f].visible !== false ) + const fields = [...tableFields, ...protectedColumns].map(f => f.toLowerCase() ) + + for (const aggregation of opts.aggregations || []) { + fields.push(aggregation.name.toLowerCase()) + } + for (const row of enriched) { for (const key of Object.keys(row)) { if (!fields.includes(key.toLowerCase())) {