viewV2.spec.ts passsing in full

packages/backend-core/src/sql/sql.ts

@@ -859,7 +859,7 @@ class InternalBuilder {
   }
 
   addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
-    let { sort } = this.query
+    let { sort, resource } = this.query
     const primaryKey = this.table.primary
     const tableName = getTableName(this.table)
     const aliases = this.query.tableAliases
@@ -896,7 +896,8 @@ class InternalBuilder {
 
     // add sorting by the primary key if the result isn't already sorted by it,
     // to make sure result is deterministic
-    if (!sort || sort[primaryKey[0]] === undefined) {
+    const hasAggregations = (resource?.aggregations?.length ?? 0) > 0
+    if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) {
       query = query.orderBy(`${aliased}.${primaryKey[0]}`)
     }
     return query

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,5 +1,6 @@
 import dayjs from "dayjs"
 import {
+  Aggregation,
   AutoFieldSubType,
   AutoReason,
   Datasource,
@@ -47,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, helpers } from "@budibase/shared-core"
 
 export interface ManyRelationship {
   tableId?: string
@@ -682,12 +683,26 @@ export class ExternalRequest<T extends Operation> {
         }
       }
     }
+
     if (
       operation === Operation.DELETE &&
       (filters == null || Object.keys(filters).length === 0)
     ) {
       throw "Deletion must be filtered"
     }
+
+    let aggregations: Aggregation[] = []
+    if (sdk.views.isView(this.source)) {
+      const calculationFields = helpers.views.calculationFields(this.source)
+      for (const [key, field] of Object.entries(calculationFields)) {
+        aggregations.push({
+          name: key,
+          field: field.field,
+          calculationType: field.calculationType,
+        })
+      }
+    }
+
     let json: QueryJson = {
       endpoint: {
         datasourceId: this.datasource._id!,
@@ -697,10 +712,11 @@ export class ExternalRequest<T extends Operation> {
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
         fields: isSql
-          ? buildSqlFieldList(table, this.tables, {
+          ? await buildSqlFieldList(this.source, this.tables, {
               relationships: incRelationships,
             })
           : [],
+        aggregations,
       },
       filters,
       sort,
@@ -748,7 +764,7 @@ export class ExternalRequest<T extends Operation> {
     }
     const output = await sqlOutputProcessing(
       response,
-      table,
+      this.source,
       this.tables,
       relationships
     )

packages/server/src/api/controllers/row/utils/basic.ts

@@ -100,8 +100,10 @@ export async function basicProcessing({
   sqs?: boolean
 }): Promise<Row> {
   let table: Table
+  let isCalculationView = false
   if (sdk.views.isView(source)) {
     table = await sdk.views.getTable(source.id)
+    isCalculationView = helpers.views.isCalculationView(source)
   } else {
     table = source
   }
@@ -132,6 +134,7 @@ export async function basicProcessing({
   }
 
   let columns: string[] = Object.keys(table.schema)
+  if (!isCalculationView) {
     if (!sqs) {
       thisRow._id = generateIdForRow(row, table, isLinked)
       thisRow.tableId = table._id
@@ -148,6 +151,7 @@ export async function basicProcessing({
         })
       }
     }
+  }
   for (let col of columns) {
     const schema: FieldSchema | undefined = table.schema[col]
     if (schema?.type !== FieldType.LINK) {

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -9,9 +9,12 @@ import {
   RelationshipsJson,
   Row,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
+import sdk from "../../../../sdk"
+import { helpers } from "@budibase/shared-core"
 
 type TableMap = Record<string, Table>
 
@@ -109,11 +112,12 @@ export function buildInternalRelationships(
  * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
  * is more performant and has the added benefit of protecting against this scenario.
  */
-export function buildSqlFieldList(
+export async function buildSqlFieldList(
-  table: Table,
+  source: Table | ViewV2,
   tables: TableMap,
   opts?: { relationships: boolean }
 ) {
+  const { relationships } = opts || {}
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
@@ -124,22 +128,33 @@ export function buildSqlFieldList(
       )
       .map(column => `${table.name}.${column[0]}`)
   }
-  let fields = extractRealFields(table)
+
+  let fields: string[] = []
+  if (sdk.views.isView(source)) {
+    fields = Object.keys(helpers.views.basicFields(source)).filter(
+      key => source.schema?.[key]?.visible !== false
+    )
+  } else {
+    fields = extractRealFields(source)
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   for (let field of Object.values(table.schema)) {
-    if (
+    if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
-      field.type !== FieldType.LINK ||
-      !opts?.relationships ||
-      !field.tableId
-    ) {
       continue
     }
-    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    const { tableName } = breakExternalTableId(field.tableId)
-    const linkTable = tables[linkTableName]
+    if (tables[tableName]) {
-    if (linkTable) {
+      fields = fields.concat(extractRealFields(tables[tableName], fields))
-      const linkedFields = extractRealFields(linkTable, fields)
-      fields = fields.concat(linkedFields)
     }
   }
+
   return fields
 }
 

packages/server/src/api/controllers/row/utils/utils.ts

@@ -19,6 +19,7 @@ import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
 import sdk from "../../../../sdk"
 import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
+import { helpers } from "@budibase/shared-core"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -121,8 +122,10 @@ export async function sqlOutputProcessing(
   }
 
   let table: Table
+  let isCalculationView = false
   if (sdk.views.isView(source)) {
     table = await sdk.views.getTable(source.id)
+    isCalculationView = helpers.views.isCalculationView(source)
   } else {
     table = source
   }
@@ -131,7 +134,7 @@ export async function sqlOutputProcessing(
   for (let row of rows) {
     if (opts?.sqs) {
       row._id = getInternalRowId(row, table)
-    } else if (row._id == null) {
+    } else if (row._id == null && !isCalculationView) {
       row._id = generateIdForRow(row, table)
     }
 

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -37,16 +37,15 @@ import {
   setEnv as setCoreEnv,
   env,
 } from "@budibase/backend-core"
-import sdk from "../../../sdk"
 
 describe.each([
-  // ["lucene", undefined],
+  ["lucene", undefined],
   ["sqs", undefined],
-  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/v2/views (%s)", (name, dsProvider) => {
   const config = setup.getConfig()
   const isSqs = name === "sqs"
@@ -2362,6 +2361,7 @@ describe.each([
         })
       })
 
+      !isLucene &&
         describe("calculations", () => {
           let table: Table
           let rows: Row[]
@@ -2417,6 +2417,12 @@ describe.each([
                 }),
               ])
             )
+
+            // Calculation views do not return rows that can be linked back to
+            // the source table, and so should not have an _id field.
+            for (const row of response.rows) {
+              expect("_id" in row).toBe(false)
+            }
           })
         })
     })

packages/server/src/sdk/app/rows/search/external.ts

@@ -127,10 +127,13 @@ export async function search(
       }
     }
 
-    if (options.fields) {
+    const visibleFields =
-      const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS]
+      options.fields ||
-      processed = processed.map((r: any) => pick(r, fields))
+      Object.keys(source.schema || {}).filter(
-    }
+        key => source.schema?.[key].visible !== false
+      )
+    const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS]
+    processed = processed.map((r: any) => pick(r, allowedFields))
 
     // need wrapper object for bookmarks etc when paginating
     const response: SearchResponse<Row> = { rows: processed, hasNextPage }

packages/server/src/sdk/app/rows/search/internal/lucene.ts

@@ -62,10 +62,13 @@ export async function search(
       response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
     }
 
-    if (options.fields) {
+    const visibleFields =
-      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
+      options.fields ||
-      response.rows = response.rows.map((r: any) => pick(r, fields))
+      Object.keys(source.schema || {}).filter(
-    }
+        key => source.schema?.[key].visible !== false
+      )
+    const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
+    response.rows = response.rows.map((r: any) => pick(r, allowedFields))
 
     response.rows = await outputProcessing(source, response.rows, {
       squash: true,

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -460,7 +460,6 @@ export async function search(
     let finalRows = await outputProcessing(source, processed, {
       preserveLinks: true,
       squash: true,
-      aggregations,
     })
 
     const visibleFields =

packages/server/src/utilities/rowProcessor/index.ts

@@ -11,7 +11,6 @@ import {
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
 import {
-  Aggregation,
   AutoFieldSubType,
   FieldType,
   IdentityType,
@@ -263,7 +262,6 @@ export async function outputProcessing<T extends Row[] | Row>(
     preserveLinks?: boolean
     fromRow?: Row
     skipBBReferences?: boolean
-    aggregations?: Aggregation[]
   } = {
     squash: true,
     preserveLinks: false,
@@ -411,8 +409,11 @@ export async function outputProcessing<T extends Row[] | Row>(
       f.toLowerCase()
     )
 
-    for (const aggregation of opts.aggregations || []) {
+    if (sdk.views.isView(source)) {
-      fields.push(aggregation.name.toLowerCase())
+      const aggregations = helpers.views.calculationFields(source)
+      for (const key of Object.keys(aggregations)) {
+        fields.push(key.toLowerCase())
+      }
     }
 
     for (const row of enriched) {