Merge branch 'master' into view-calculation-no-deletes

packages/backend-core/src/features/features.ts

@@ -272,6 +272,7 @@ export const flags = new FlagSet({
   SQS: Flag.boolean(env.isDev()),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
   [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),
 })
 
 type UnwrapPromise<T> = T extends Promise<infer U> ? U : T

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -133,9 +133,7 @@ export async function buildSqlFieldList(
 
   let fields: string[] = []
   if (sdk.views.isView(source)) {
-    fields = Object.keys(helpers.views.basicFields(source)).filter(
+    fields = Object.keys(helpers.views.basicFields(source))
-      key => source.schema?.[key]?.visible !== false
-    )
   } else {
     fields = extractRealFields(source)
   }

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -800,6 +800,34 @@ describe.each([
             )
           }
         })
+
+      isInternal &&
+        it("shouldn't trigger a complex type check on a group by field if field is invisible", async () => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: {
+                field: {
+                  name: "field",
+                  type: FieldType.JSON,
+                },
+              },
+            })
+          )
+
+          await config.api.viewV2.create(
+            {
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                field: { visible: false },
+              },
+            },
+            {
+              status: 201,
+            }
+          )
+        })
     })
 
     describe("update", () => {
@@ -1904,6 +1932,59 @@ describe.each([
         })
       })
     })
+
+    describe("calculation views", () => {
+      it("should not remove calculation columns when modifying table schema", async () => {
+        let table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+              age: {
+                name: "age",
+                type: FieldType.NUMBER,
+              },
+            },
+          })
+        )
+
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          type: ViewV2Type.CALCULATION,
+          schema: {
+            sum: {
+              visible: true,
+              calculationType: CalculationType.SUM,
+              field: "age",
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save({
+          ...table,
+          schema: {
+            ...table.schema,
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+              constraints: { presence: true },
+            },
+          },
+        })
+
+        view = await config.api.viewV2.get(view.id)
+        expect(Object.keys(view.schema!).sort()).toEqual([
+          "age",
+          "id",
+          "name",
+          "sum",
+        ])
+      })
+    })
   })
 
   describe("row operations", () => {
@@ -3183,6 +3264,123 @@ describe.each([
               }
             )
           })
+
+          it("should be able to filter rows on the view itself", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+              schema: {
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
+
+          it("should be able to filter on group by fields", async () => {
+            const table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              type: ViewV2Type.CALCULATION,
+              schema: {
+                quantity: { visible: true },
+                sum: {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            await config.api.row.bulkImport(table._id!, {
+              rows: [
+                {
+                  quantity: 1,
+                  price: 1,
+                },
+                {
+                  quantity: 1,
+                  price: 2,
+                },
+                {
+                  quantity: 2,
+                  price: 10,
+                },
+              ],
+            })
+
+            const { rows } = await config.api.viewV2.search(view.id, {
+              query: {
+                equal: {
+                  quantity: 1,
+                },
+              },
+            })
+
+            expect(rows).toHaveLength(1)
+            expect(rows[0].sum).toEqual(3)
+          })
         })
 
       !isLucene &&

packages/server/src/api/routes/utils/validators.ts

@@ -20,31 +20,30 @@ const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
-const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
+const validateViewSchemas: CustomValidator<Table> = (table, joiHelpers) => {
-  if (table.views && Object.entries(table.views).length) {
+  if (!table.views || Object.keys(table.views).length === 0) {
-    const requiredFields = Object.entries(table.schema)
+    return table
-      .filter(([_, v]) => isRequired(v.constraints))
+  }
-      .map(([key]) => key)
+  const required = Object.keys(table.schema).filter(key =>
-    if (requiredFields.length) {
+    isRequired(table.schema[key].constraints)
+  )
+  if (required.length === 0) {
+    return table
+  }
   for (const view of Object.values(table.views)) {
-        if (!sdk.views.isV2(view)) {
+    if (!sdk.views.isV2(view) || helpers.views.isCalculationView(view)) {
       continue
     }
-
+    const editable = Object.entries(view.schema || {})
-        const editableViewFields = Object.entries(view.schema || {})
       .filter(([_, f]) => f.visible && !f.readonly)
       .map(([key]) => key)
-        const missingField = requiredFields.find(
+    const missingField = required.find(f => !editable.includes(f))
-          f => !editableViewFields.includes(f)
-        )
     if (missingField) {
-          return helpers.message({
+      return joiHelpers.message({
         custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
       })
     }
   }
-    }
-  }
   return table
 }
 

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -68,9 +68,7 @@ async function buildInternalFieldList(
   const { relationships, allowedFields } = opts || {}
   let schemaFields: string[] = []
   if (sdk.views.isView(source)) {
-    schemaFields = Object.keys(helpers.views.basicFields(source)).filter(
+    schemaFields = Object.keys(helpers.views.basicFields(source))
-      key => source.schema?.[key]?.visible !== false
-    )
   } else {
     schemaFields = Object.keys(source.schema).filter(
       key => source.schema[key].visible !== false

packages/server/src/sdk/app/tables/create.ts

@@ -1,10 +1,10 @@
-import { Row, Table } from "@budibase/types"
+import { FeatureFlag, Row, Table } from "@budibase/types"
 
 import * as external from "./external"
 import * as internal from "./internal"
 import { isExternal } from "./utils"
 import { setPermissions } from "../permissions"
-import { roles } from "@budibase/backend-core"
+import { features, roles } from "@budibase/backend-core"
 
 export async function create(
   table: Omit<Table, "_id" | "_rev">,
@@ -18,10 +18,16 @@ export async function create(
     createdTable = await internal.create(table, rows, userId)
   }
 
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+
+  if (setExplicitPermission) {
     await setPermissions(createdTable._id!, {
       writeRole: roles.BUILTIN_ROLE_IDS.ADMIN,
       readRole: roles.BUILTIN_ROLE_IDS.ADMIN,
     })
+  }
 
   return createdTable
 }

packages/server/src/sdk/app/views/index.ts

@@ -1,6 +1,7 @@
 import {
   CalculationType,
   canGroupBy,
+  FeatureFlag,
   FieldType,
   isNumeric,
   PermissionLevel,
@@ -13,7 +14,7 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { context, docIds, HTTPError } from "@budibase/backend-core"
+import { context, docIds, features, HTTPError } from "@budibase/backend-core"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -96,6 +97,13 @@ async function guardCalculationViewSchema(
       continue
     }
 
+    if (!schema.field) {
+      throw new HTTPError(
+        `Calculation field "${name}" is missing a "field" property`,
+        400
+      )
+    }
+
     const targetSchema = table.schema[schema.field]
     if (!targetSchema) {
       throw new HTTPError(
@@ -244,12 +252,17 @@ export async function create(
 
   const view = await pickApi(tableId).create(tableId, viewRequest)
 
+  const setExplicitPermission = await features.flags.isEnabled(
+    FeatureFlag.TABLES_DEFAULT_ADMIN
+  )
+  if (setExplicitPermission) {
     // Set permissions to be the same as the table
     const tablePerms = await sdk.permissions.getResourcePerms(tableId)
     await sdk.permissions.setPermissions(view.id, {
       writeRole: tablePerms[PermissionLevel.WRITE].role,
       readRole: tablePerms[PermissionLevel.READ].role,
     })
+  }
 
   return view
 }
@@ -371,7 +384,8 @@ export function syncSchema(
 
   if (view.schema) {
     for (const fieldName of Object.keys(view.schema)) {
-      if (!schema[fieldName]) {
+      const viewSchema = view.schema[fieldName]
+      if (!helpers.views.isCalculationField(viewSchema) && !schema[fieldName]) {
         delete view.schema[fieldName]
       }
     }

packages/shared-core/src/helpers/views.ts

@@ -33,6 +33,13 @@ export function calculationFields(view: UnsavedViewV2) {
   return pickBy(view.schema || {}, isCalculationField)
 }
 
-export function basicFields(view: UnsavedViewV2) {
+export function isVisible(field: ViewFieldMetadata) {
-  return pickBy(view.schema || {}, field => !isCalculationField(field))
+  return field.visible !== false
+}
+
+export function basicFields(view: UnsavedViewV2, opts?: { visible?: boolean }) {
+  const { visible = true } = opts || {}
+  return pickBy(view.schema || {}, field => {
+    return !isCalculationField(field) && (!visible || isVisible(field))
+  })
 }

packages/types/src/sdk/featureFlag.ts

@@ -3,6 +3,7 @@ export enum FeatureFlag {
   PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
   AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
   ENRICHED_RELATIONSHIPS = "ENRICHED_RELATIONSHIPS",
+  TABLES_DEFAULT_ADMIN = "TABLES_DEFAULT_ADMIN",
 }
 
 export interface TenantFeatureFlags {