wip

packages/backend-core/src/sql/sql.ts

@@ -10,10 +10,12 @@ import {
 } from "./utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
+  Aggregation,
   AnySearchFilter,
   ArrayOperator,
   BasicOperator,
   BBReferenceFieldMetadata,
+  CalculationType,
   FieldSchema,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
@@ -789,6 +791,38 @@ class InternalBuilder {
     return query.countDistinct(`${aliased}.${primary[0]} as total`)
   }
 
+  addAggregations(
+    query: Knex.QueryBuilder,
+    aggregations: Aggregation[]
+  ): Knex.QueryBuilder {
+    const fields = this.query.resource?.fields || []
+    if (fields.length > 0) {
+      query = query.groupBy(fields.map(field => `${this.table.name}.${field}`))
+    }
+    for (const aggregation of aggregations) {
+      const op = aggregation.calculationType
+      const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}`
+      switch (op) {
+        case CalculationType.COUNT:
+          query = query.count(field)
+          break
+        case CalculationType.SUM:
+          query = query.sum(field)
+          break
+        case CalculationType.AVG:
+          query = query.avg(field)
+          break
+        case CalculationType.MIN:
+          query = query.min(field)
+          break
+        case CalculationType.MAX:
+          query = query.max(field)
+          break
+      }
+    }
+    return query
+  }
+
   addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
     let { sort } = this.query
     const primaryKey = this.table.primary
@@ -1172,10 +1206,18 @@ class InternalBuilder {
       }
     }
 
-    // if counting, use distinct count, else select
+    const aggregations = this.query.resource?.aggregations || []
-    query = !counting
+    if (counting) {
-      ? query.select(this.generateSelectStatement())
+      query = this.addDistinctCount(query)
-      : this.addDistinctCount(query)
+    } else if (aggregations.length > 0) {
+      query = query.select(
+        this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id")
+      )
+      query = this.addAggregations(query, aggregations)
+    } else {
+      query = query.select(this.generateSelectStatement())
+    }
+
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL && !counting) {
       query = this.addSorting(query)

packages/server/src/api/controllers/row/views.ts

@@ -7,8 +7,9 @@ import {
   RowSearchParams,
   SearchFilterKey,
   LogicalOperator,
+  Aggregation,
 } from "@budibase/types"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, helpers } from "@budibase/shared-core"
 import sdk from "../../../sdk"
 import { db, context, features } from "@budibase/backend-core"
 import { enrichSearchContext } from "./utils"
@@ -27,7 +28,7 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
-  const viewFields = Object.entries(view.schema || {})
+  const viewFields = Object.entries(helpers.views.basicFields(view))
     .filter(([_, value]) => value.visible)
     .map(([key]) => key)
   const { body } = ctx.request
@@ -74,6 +75,14 @@ export async function searchView(
     user: sdk.users.getUserContextBindings(ctx.user),
   })
 
+  const aggregations: Aggregation[] = Object.entries(
+    helpers.views.calculationFields(view)
+  ).map(([name, { field, calculationType }]) => ({
+    name,
+    calculationType,
+    field,
+  }))
+
   const searchOptions: RequiredKeys<SearchViewRowRequest> &
     RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
     tableId: view.tableId,
@@ -84,6 +93,7 @@ export async function searchView(
     bookmark: body.bookmark,
     paginate: body.paginate,
     countRows: body.countRows,
+    aggregations,
   }
 
   const result = await sdk.rows.search(searchOptions)

packages/server/src/api/controllers/view/viewsV2.ts

@@ -8,8 +8,45 @@ import {
   ViewResponse,
   ViewResponseEnriched,
   ViewV2,
+  BasicViewUIFieldMetadata,
+  ViewCalculationFieldMetadata,
 } from "@budibase/types"
 import { builderSocket, gridSocket } from "../../../websockets"
+import { helpers } from "@budibase/shared-core"
+
+function stripUnknownFields(
+  field: ViewUIFieldMetadata
+): RequiredKeys<ViewUIFieldMetadata> {
+  if (helpers.views.isCalculationField(field)) {
+    const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
+      order: field.order,
+      width: field.width,
+      visible: field.visible,
+      readonly: field.readonly,
+      icon: field.icon,
+      calculationType: field.calculationType,
+      field: field.field,
+    }
+    return strippedField
+  } else {
+    const strippedField: RequiredKeys<BasicViewUIFieldMetadata> = {
+      order: field.order,
+      width: field.width,
+      visible: field.visible,
+      readonly: field.readonly,
+      icon: field.icon,
+    }
+    return strippedField
+  }
+}
+
+function stripUndefinedFields(obj: Record<string, any>): void {
+  Object.keys(obj)
+    .filter(key => obj[key] === undefined)
+    .forEach(key => {
+      delete obj[key]
+    })
+}
 
 async function parseSchema(view: CreateViewRequest) {
   if (!view.schema) {
@@ -18,18 +55,8 @@ async function parseSchema(view: CreateViewRequest) {
   const finalViewSchema =
     view.schema &&
     Object.entries(view.schema).reduce((p, [fieldName, schemaValue]) => {
-      const fieldSchema: RequiredKeys<ViewUIFieldMetadata> = {
+      const fieldSchema = stripUnknownFields(schemaValue)
-        order: schemaValue.order,
+      stripUndefinedFields(fieldSchema)
-        width: schemaValue.width,
-        visible: schemaValue.visible,
-        readonly: schemaValue.readonly,
-        icon: schemaValue.icon,
-      }
-      Object.entries(fieldSchema)
-        .filter(([, val]) => val === undefined)
-        .forEach(([key]) => {
-          delete fieldSchema[key as keyof ViewUIFieldMetadata]
-        })
       p[fieldName] = fieldSchema
       return p
     }, {} as Record<string, RequiredKeys<ViewUIFieldMetadata>>)

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -18,6 +18,7 @@ import {
   ViewV2,
   SearchResponse,
   BasicOperator,
+  CalculationType,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -32,13 +33,13 @@ import {
 import sdk from "../../../sdk"
 
 describe.each([
-  ["lucene", undefined],
+  // ["lucene", undefined],
   ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("/v2/views (%s)", (name, dsProvider) => {
   const config = setup.getConfig()
   const isSqs = name === "sqs"
@@ -1978,6 +1979,64 @@ describe.each([
           })
         )
       })
+
+      describe("calculations", () => {
+        let table: Table
+        let rows: Row[]
+
+        beforeAll(async () => {
+          table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                quantity: {
+                  type: FieldType.NUMBER,
+                  name: "quantity",
+                },
+                price: {
+                  type: FieldType.NUMBER,
+                  name: "price",
+                },
+              },
+            })
+          )
+
+          rows = await Promise.all(
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, {
+                quantity: generator.natural({ min: 1, max: 10 }),
+                price: generator.natural({ min: 1, max: 10 }),
+              })
+            )
+          )
+        })
+
+        it.only("should be able to search by calculations", async () => {
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              "Quantity Sum": {
+                visible: true,
+                calculationType: CalculationType.SUM,
+                field: "quantity",
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id, {
+            query: {},
+          })
+
+          expect(response.rows).toHaveLength(1)
+          expect(response.rows).toEqual(
+            expect.arrayContaining([
+              expect.objectContaining({
+                "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
+              }),
+            ])
+          )
+        })
+      })
     })
 
     describe("permissions", () => {

packages/server/src/sdk/app/rows/search.ts

@@ -46,6 +46,9 @@ export async function search(
       paginate: options.paginate,
       fields: options.fields,
       countRows: options.countRows,
+      aggregations: options.aggregations
+        ?.map(a => `${a.field}:${a.calculationType}`)
+        .join(", "),
     })
 
     const isExternalTable = isExternalTableID(options.tableId)

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -48,6 +48,7 @@ import {
 } from "@budibase/shared-core"
 import { isSearchingByRowID } from "../utils"
 import tracer from "dd-trace"
+import { cloneDeep } from "lodash"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const SQLITE_COLUMN_LIMIT = 2000
@@ -285,7 +286,7 @@ export async function search(
   table: Table,
   opts?: { retrying?: boolean }
 ): Promise<SearchResponse<Row>> {
-  let { paginate, query, ...params } = options
+  let { paginate, query, ...params } = cloneDeep(options)
 
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
@@ -303,6 +304,21 @@ export async function search(
     ...cleanupFilters(query, table, allTables),
     documentType: DocumentType.ROW,
   }
+
+  let fields = options.fields
+  if (fields === undefined) {
+    fields = buildInternalFieldList(table, allTables, { relationships })
+  } else {
+    fields = fields.map(f => mapToUserColumn(f))
+  }
+
+  if (options.aggregations) {
+    options.aggregations = options.aggregations.map(a => {
+      a.field = mapToUserColumn(a.field)
+      return a
+    })
+  }
+
   const request: QueryJson = {
     endpoint: {
       // not important, we query ourselves
@@ -317,9 +333,7 @@ export async function search(
       tables: allTablesMap,
       columnPrefix: USER_COLUMN_PREFIX,
     },
-    resource: {
+    resource: { fields, aggregations: options.aggregations },
-      fields: buildInternalFieldList(table, allTables, { relationships }),
-    },
     relationships,
   }
 
@@ -426,6 +440,7 @@ export async function search(
     if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {
       return { rows: [] }
     }
+    throw err
     throw new Error(`Unable to search by SQL - ${msg}`, { cause: err })
   }
 }

packages/server/src/sdk/app/views/index.ts

@@ -1,8 +1,9 @@
 import {
+  BasicViewUIFieldMetadata,
   RenameColumn,
+  Table,
   TableSchema,
   View,
-  ViewUIFieldMetadata,
   ViewV2,
   ViewV2Enriched,
 } from "@budibase/types"
@@ -38,31 +39,84 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   return pickApi(tableId).getEnriched(viewId)
 }
 
+async function guardCalculationViewSchema(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const calculationFields = helpers.views.calculationFields(view)
+  for (const calculationFieldName of Object.keys(calculationFields)) {
+    const schema = calculationFields[calculationFieldName]
+    const targetSchema = table.schema[schema.field]
+    if (!targetSchema) {
+      throw new HTTPError(
+        `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
+        400
+      )
+    }
+
+    if (!helpers.schema.isNumeric(targetSchema)) {
+      throw new HTTPError(
+        `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
+        400
+      )
+    }
+  }
+
+  const groupByFields = helpers.views.basicFields(view)
+  for (const groupByFieldName of Object.keys(groupByFields)) {
+    const targetSchema = table.schema[groupByFieldName]
+    if (!targetSchema) {
+      throw new HTTPError(
+        `Group by field "${groupByFieldName}" does not exist in the table schema`,
+        400
+      )
+    }
+  }
+}
+
 async function guardViewSchema(
   tableId: string,
   view: Omit<ViewV2, "id" | "version">
 ) {
-  const viewSchema = view.schema || {}
   const table = await sdk.tables.getTable(tableId)
 
+  if (helpers.views.isCalculationView(view)) {
+    await guardCalculationViewSchema(table, view)
+  }
+
+  await checkReadonlyFields(table, view)
+  checkRequiredFields(table, view)
+  checkDisplayField(view)
+}
+
+async function checkReadonlyFields(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const viewSchema = view.schema || {}
   for (const field of Object.keys(viewSchema)) {
-    const tableSchemaField = table.schema[field]
+    const viewFieldSchema = viewSchema[field]
-    if (!tableSchemaField) {
+    if (helpers.views.isCalculationField(viewFieldSchema)) {
+      continue
+    }
+
+    const tableFieldSchema = table.schema[field]
+    if (!tableFieldSchema) {
       throw new HTTPError(
         `Field "${field}" is not valid for the requested table`,
         400
       )
     }
 
-    if (viewSchema[field].readonly) {
+    if (viewFieldSchema.readonly) {
       if (
         !(await features.isViewReadonlyColumnsEnabled()) &&
-        !(tableSchemaField as ViewUIFieldMetadata).readonly
+        !(tableFieldSchema as BasicViewUIFieldMetadata).readonly
       ) {
         throw new HTTPError(`Readonly fields are not enabled`, 400)
       }
 
-      if (!viewSchema[field].visible) {
+      if (!viewFieldSchema.visible) {
         throw new HTTPError(
           `Field "${field}" must be visible if you want to make it readonly`,
           400
@@ -70,18 +124,33 @@ async function guardViewSchema(
       }
     }
   }
+}
 
-  const existingView =
+function checkDisplayField(view: Omit<ViewV2, "id" | "version">) {
-    table?.views && (table.views[view.name] as ViewV2 | undefined)
+  if (view.primaryDisplay) {
+    const viewSchemaField = view.schema?.[view.primaryDisplay]
 
+    if (!viewSchemaField?.visible) {
+      throw new HTTPError(
+        `You can't hide "${view.primaryDisplay}" because it is the display column.`,
+        400
+      )
+    }
+  }
+}
+
+function checkRequiredFields(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const existingView = table.views?.[view.name] as ViewV2 | undefined
   for (const field of Object.values(table.schema)) {
     if (!helpers.schema.isRequired(field.constraints)) {
       continue
     }
 
-    const viewSchemaField = viewSchema[field.name]
+    const viewSchemaField = view.schema?.[field.name]
-    const existingViewSchema =
+    const existingViewSchema = existingView?.schema?.[field.name]
-      existingView?.schema && existingView.schema[field.name]
     if (!viewSchemaField && !existingViewSchema?.visible) {
       // Supporting existing configs with required columns but hidden in views
       continue
@@ -94,24 +163,16 @@ async function guardViewSchema(
       )
     }
 
-    if (viewSchemaField.readonly) {
+    if (
+      helpers.views.isBasicViewField(viewSchemaField) &&
+      viewSchemaField.readonly
+    ) {
       throw new HTTPError(
         `You can't make "${field.name}" readonly because it is a required field.`,
         400
       )
     }
   }
-
-  if (view.primaryDisplay) {
-    const viewSchemaField = viewSchema[view.primaryDisplay]
-
-    if (!viewSchemaField?.visible) {
-      throw new HTTPError(
-        `You can't hide "${view.primaryDisplay}" because it is the display column.`,
-        400
-      )
-    }
-  }
 }
 
 export async function create(

packages/shared-core/src/helpers/index.ts

@@ -2,3 +2,4 @@ export * from "./helpers"
 export * from "./integrations"
 export * as cron from "./cron"
 export * as schema from "./schema"
+export * as views from "./views"

packages/shared-core/src/helpers/schema.ts

@@ -45,3 +45,7 @@ export function decodeNonAscii(str: string): string {
     String.fromCharCode(parseInt(p1, 16))
   )
 }
+
+export function isNumeric(field: FieldSchema) {
+  return field.type === FieldType.NUMBER || field.type === FieldType.BIGINT
+}

packages/shared-core/src/helpers/views.ts

@@ -0,0 +1,39 @@
+import {
+  BasicViewUIFieldMetadata,
+  ViewCalculationFieldMetadata,
+  ViewUIFieldMetadata,
+  ViewV2,
+} from "@budibase/types"
+import { pickBy } from "lodash"
+
+export function isCalculationField(
+  field: ViewUIFieldMetadata
+): field is ViewCalculationFieldMetadata {
+  return "calculationType" in field
+}
+
+export function isBasicViewField(
+  field: ViewUIFieldMetadata
+): field is BasicViewUIFieldMetadata {
+  return !isCalculationField(field)
+}
+
+type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
+
+export function isCalculationView(view: UnsavedViewV2) {
+  return Object.values(view.schema || {}).some(isCalculationField)
+}
+
+export function calculationFields(view: UnsavedViewV2) {
+  if (!isCalculationView(view)) {
+    throw new Error("View is not a calculation view")
+  }
+  return pickBy(view.schema || {}, isCalculationField)
+}
+
+export function basicFields(view: UnsavedViewV2) {
+  if (!isCalculationView(view)) {
+    throw new Error("View is not a calculation view")
+  }
+  return pickBy(view.schema || {}, field => !isCalculationField(field))
+}

packages/types/src/api/web/app/rows.ts

@@ -26,6 +26,7 @@ export interface SearchViewRowRequest
     | "paginate"
     | "query"
     | "countRows"
+    | "aggregations"
   > {}
 
 export interface SearchRowResponse {

packages/types/src/documents/app/view.ts

@@ -33,10 +33,19 @@ export interface View {
   groupBy?: string
 }
 
-export type ViewUIFieldMetadata = UIFieldMetadata & {
+export interface BasicViewUIFieldMetadata extends UIFieldMetadata {
   readonly?: boolean
 }
 
+export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata {
+  calculationType: CalculationType
+  field: string
+}
+
+export type ViewUIFieldMetadata =
+  | BasicViewUIFieldMetadata
+  | ViewCalculationFieldMetadata
+
 export enum CalculationType {
   SUM = "sum",
   AVG = "avg",
@@ -45,11 +54,6 @@ export enum CalculationType {
   MAX = "max",
 }
 
-export type ViewCalculationFieldMetadata = ViewUIFieldMetadata & {
-  calculationType: CalculationType
-  field: string
-}
-
 export interface ViewV2 {
   version: 2
   id: string
@@ -62,7 +66,7 @@ export interface ViewV2 {
     order?: SortOrder
     type?: SortType
   }
-  schema?: Record<string, ViewUIFieldMetadata | ViewCalculationFieldMetadata>
+  schema?: Record<string, ViewUIFieldMetadata>
 }
 
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema

packages/types/src/sdk/row.ts

@@ -1,8 +1,14 @@
 import { SortOrder, SortType } from "../api"
 import { SearchFilters } from "./search"
-import { Row } from "../documents"
+import { CalculationType, Row } from "../documents"
 import { WithRequired } from "../shared"
 
+export interface Aggregation {
+  name: string
+  calculationType: CalculationType
+  field: string
+}
+
 export interface SearchParams {
   tableId?: string
   query?: SearchFilters
@@ -18,6 +24,7 @@ export interface SearchParams {
   indexer?: () => Promise<any>
   rows?: Row[]
   countRows?: boolean
+  aggregations?: Aggregation[]
 }
 
 // when searching for rows we want a more extensive search type that requires certain properties

packages/types/src/sdk/search.ts

@@ -2,6 +2,7 @@ import { Operation } from "./datasources"
 import { Row, Table, DocumentType } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
+import { Aggregation } from "./row"
 
 export enum BasicOperator {
   EQUAL = "equal",
@@ -143,6 +144,7 @@ export interface QueryJson {
   }
   resource?: {
     fields: string[]
+    aggregations?: Aggregation[]
   }
   filters?: SearchFilters
   sort?: SortJson

packages/types/src/shared/typeUtils.ts

@@ -4,6 +4,29 @@ export type DeepPartial<T> = {
 
 export type ISO8601 = string
 
+/**
+ * RequiredKeys make it such that you _must_ assign a value to every key in the
+ * type. It differs subtly from Required<T> in that it doesn't change the type
+ * of the fields, you can specify undefined as a value and that's fine.
+ *
+ * Example:
+ *
+ * ```ts
+ * interface Foo {
+ *   bar: string
+ *   baz?: string
+ * }
+ *
+ * type FooRequiredKeys = RequiredKeys<Foo>
+ * type FooRequired = Required<Foo>
+ *
+ * const a: FooRequiredKeys = { bar: "hello", baz: undefined }
+ * const b: FooRequired = { bar: "hello", baz: undefined }
+ * ```
+ *
+ * In this code, a passes type checking whereas b does not. This is because
+ * Required<Foo> makes baz non-optional.
+ */
 export type RequiredKeys<T> = {
   [K in keyof Required<T>]: T[K]
 }