Merge pull request #13833 from Budibase/BUDI-8282/validate-configuration

Validate readonly configuration

packages/backend-core/src/middleware/joi-validator.ts

@@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types"
 
 function validate(
   schema: Joi.ObjectSchema | Joi.ArraySchema,
-  property: string
+  property: string,
+  opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` }
 ) {
   // Return a Koa middleware function
   return (ctx: Ctx, next: any) => {
@@ -29,16 +30,26 @@ function validate(
 
     const { error } = schema.validate(params)
     if (error) {
-      ctx.throw(400, `Invalid ${property} - ${error.message}`)
+      let message = error.message
+      if (opts.errorPrefix) {
+        message = `Invalid ${property} - ${message}`
+      }
+      ctx.throw(400, message)
     }
     return next()
   }
 }
 
-export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function body(
-  return validate(schema, "body")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "body", opts)
 }
 
-export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function params(
-  return validate(schema, "params")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "params", opts)
 }

packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte

@@ -19,11 +19,17 @@
     const visible = permission !== PERMISSION_OPTIONS.HIDDEN
     const readonly = permission === PERMISSION_OPTIONS.READONLY
 
-    datasource.actions.addSchemaMutation(column.name, { visible, readonly })
+    await datasource.actions.addSchemaMutation(column.name, {
+      visible,
+      readonly,
+    })
     try {
       await datasource.actions.saveSchemaMutations()
     } catch (e) {
       notifications.error(e.message)
+    } finally {
+      await datasource.actions.resetSchemaMutations()
+      await datasource.actions.refreshDefinition()
     }
     dispatch(visible ? "show-column" : "hide-column")
   }

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -204,6 +204,10 @@ export const createActions = context => {
       ...$definition,
       schema: newSchema,
     })
+    resetSchemaMutations()
+  }
+
+  const resetSchemaMutations = () => {
     schemaMutations.set({})
   }
 
@@ -253,6 +257,7 @@ export const createActions = context => {
         addSchemaMutation,
         addSchemaMutations,
         saveSchemaMutations,
+        resetSchemaMutations,
       },
     },
   }

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -23,9 +23,6 @@ import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { roles } from "@budibase/backend-core"
-import * as schemaUtils from "../../../utilities/schema"
-
-jest.mock("../../../utilities/schema")
 
 describe.each([
   ["internal", undefined],
@@ -318,15 +315,13 @@ describe.each([
       })
 
       it("required fields cannot be marked as readonly", async () => {
-        const isRequiredSpy = jest.spyOn(schemaUtils, "isRequired")
-        isRequiredSpy.mockReturnValueOnce(true)
-
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
               name: {
                 name: "name",
                 type: FieldType.STRING,
+                constraints: { presence: true },
               },
               description: {
                 name: "description",
@@ -341,6 +336,7 @@ describe.each([
           tableId: table._id!,
           schema: {
             name: {
+              visible: true,
               readonly: true,
             },
           },
@@ -350,7 +346,7 @@ describe.each([
           status: 400,
           body: {
             message:
-              'Field "name" cannot be readonly as it is a required field',
+              'You can\'t make field "name" readonly because it is a required field.',
             status: 400,
           },
         })
@@ -1348,4 +1344,123 @@ describe.each([
       })
     })
   })
+
+  describe("updating table schema", () => {
+    describe("existing columns changed to required", () => {
+      beforeEach(async () => {
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              id: {
+                name: "id",
+                type: FieldType.AUTO,
+                autocolumn: true,
+              },
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+      })
+
+      it("allows updating when no views constrains the field", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: { visible: true },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: { allowEmpty: false } },
+              },
+            },
+          },
+          { status: 200 }
+        )
+      })
+
+      it("rejects if field is readonly in any view", async () => {
+        mocks.licenses.useViewReadonlyColumns()
+
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: {
+              visible: true,
+              readonly: true,
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+
+      it("rejects if field is hidden in any view", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: { id: { visible: true } },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+    })
+  })
 })

packages/server/src/api/routes/utils/validators.ts

@@ -1,51 +1,88 @@
 import { auth, permissions } from "@budibase/backend-core"
 import { DataSourceOperation } from "../../../constants"
-import { WebhookActionType } from "@budibase/types"
+import { Table, WebhookActionType } from "@budibase/types"
-import Joi from "joi"
+import Joi, { CustomValidator } from "joi"
 import { ValidSnippetNameRegex } from "@budibase/shared-core"
+import { isRequired } from "../../../utilities/schema"
+import sdk from "../../../sdk"
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
 const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
+const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
+  if (table.views && Object.entries(table.views).length) {
+    const requiredFields = Object.entries(table.schema)
+      .filter(([_, v]) => isRequired(v.constraints))
+      .map(([key]) => key)
+    if (requiredFields.length) {
+      for (const view of Object.values(table.views)) {
+        if (!sdk.views.isV2(view)) {
+          continue
+        }
+
+        const editableViewFields = Object.entries(view.schema || {})
+          .filter(([_, f]) => f.visible && !f.readonly)
+          .map(([key]) => key)
+        const missingField = requiredFields.find(
+          f => !editableViewFields.includes(f)
+        )
+        if (missingField) {
+          return helpers.message({
+            custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
+          })
+        }
+      }
+    }
+  }
+  return table
+}
+
 export function tableValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    type: OPTIONAL_STRING.valid("table", "internal", "external"),
+      type: OPTIONAL_STRING.valid("table", "internal", "external"),
-    primaryDisplay: OPTIONAL_STRING,
+      primaryDisplay: OPTIONAL_STRING,
-    schema: Joi.object().required(),
+      schema: Joi.object().required(),
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    views: Joi.object(),
+      views: Joi.object(),
-    rows: Joi.array(),
+      rows: Joi.array(),
-  }).unknown(true))
+    })
+      .custom(validateViewSchemas)
+      .unknown(true),
+    { errorPrefix: "" }
+  )
 }
 
 export function nameValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: OPTIONAL_STRING,
+      name: OPTIONAL_STRING,
-  }))
+    })
+  )
 }
 
 export function datasourceValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: Joi.string(),
+      _id: Joi.string(),
-    _rev: Joi.string(),
+      _rev: Joi.string(),
-    type: OPTIONAL_STRING.allow("datasource_plus"),
+      type: OPTIONAL_STRING.allow("datasource_plus"),
-    relationships: Joi.array().items(Joi.object({
+      relationships: Joi.array().items(
-      from: Joi.string().required(),
+        Joi.object({
-      to: Joi.string().required(),
+          from: Joi.string().required(),
-      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+          to: Joi.string().required(),
-    })),
+          cardinality: Joi.valid("1:N", "1:1", "N:N").required(),
-  }).unknown(true))
+        })
+      ),
+    }).unknown(true)
+  )
 }
 
 function filterObject() {
-  // prettier-ignore
   return Joi.object({
     string: Joi.object().optional(),
     fuzzy: Joi.object().optional(),
@@ -62,17 +99,20 @@ function filterObject() {
 }
 
 export function internalSearchValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    tableId: OPTIONAL_STRING,
+      tableId: OPTIONAL_STRING,
-    query: filterObject(),
+      query: filterObject(),
-    limit: OPTIONAL_NUMBER,
+      limit: OPTIONAL_NUMBER,
-    sort: OPTIONAL_STRING,
+      sort: OPTIONAL_STRING,
-    sortOrder: OPTIONAL_STRING,
+      sortOrder: OPTIONAL_STRING,
-    sortType: OPTIONAL_STRING,
+      sortType: OPTIONAL_STRING,
-    paginate: Joi.boolean(),
+      paginate: Joi.boolean(),
-    bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
+      bookmark: Joi.alternatives()
-  }))
+        .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
+        .optional(),
+    })
+  )
 }
 
 export function externalSearchValidator() {
@@ -94,92 +134,110 @@ export function externalSearchValidator() {
 }
 
 export function datasourceQueryValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    endpoint: Joi.object({
+      endpoint: Joi.object({
-      datasourceId: Joi.string().required(),
+        datasourceId: Joi.string().required(),
-      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+        operation: Joi.string()
-      entityId: Joi.string().required(),
+          .required()
-    }).required(),
+          .valid(...Object.values(DataSourceOperation)),
-    resource: Joi.object({
+        entityId: Joi.string().required(),
-      fields: Joi.array().items(Joi.string()).optional(),
+      }).required(),
-    }).optional(),
+      resource: Joi.object({
-    body: Joi.object().optional(),
+        fields: Joi.array().items(Joi.string()).optional(),
-    sort: Joi.object().optional(),
+      }).optional(),
-    filters: filterObject().optional(),
+      body: Joi.object().optional(),
-    paginate: Joi.object({
+      sort: Joi.object().optional(),
-      page: Joi.string().alphanum().optional(),
+      filters: filterObject().optional(),
-      limit: Joi.number().optional(),
+      paginate: Joi.object({
-    }).optional(),
+        page: Joi.string().alphanum().optional(),
-  }))
+        limit: Joi.number().optional(),
+      }).optional(),
+    })
+  )
 }
 
 export function webhookValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    live: Joi.bool(),
+      live: Joi.bool(),
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    bodySchema: Joi.object().optional(),
+      bodySchema: Joi.object().optional(),
-    action: Joi.object({
+      action: Joi.object({
-      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
+        type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
-      target: Joi.string().required(),
+        target: Joi.string().required(),
-    }).required(),
+      }).required(),
-  }).unknown(true))
+    }).unknown(true)
+  )
 }
 
 export function roleValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.body(Joi.object({
+  return auth.joiValidator.body(
-    _id: OPTIONAL_STRING,
+    Joi.object({
-    _rev: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    name: Joi.string().regex(/^[a-zA-Z0-9_]*$/).required(),
+      _rev: OPTIONAL_STRING,
-    // this is the base permission ID (for now a built in)
+      name: Joi.string()
-    permissionId: Joi.string().valid(...Object.values(permissions.BuiltinPermissionID)).required(),
+        .regex(/^[a-zA-Z0-9_]*$/)
-    permissions: Joi.object()
+        .required(),
-      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+      // this is the base permission ID (for now a built in)
-      .optional(),
+      permissionId: Joi.string()
-    inherits: OPTIONAL_STRING,
+        .valid(...Object.values(permissions.BuiltinPermissionID))
-  }).unknown(true))
+        .required(),
+      permissions: Joi.object()
+        .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+        .optional(),
+      inherits: OPTIONAL_STRING,
+    }).unknown(true)
+  )
 }
 
 export function permissionValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.params(Joi.object({
+  return auth.joiValidator.params(
-    level: Joi.string().valid(...permLevelArray).required(),
+    Joi.object({
-    resourceId: Joi.string(),
+      level: Joi.string()
-    roleId: Joi.string(),
+        .valid(...permLevelArray)
-  }).unknown(true))
+        .required(),
+      resourceId: Joi.string(),
+      roleId: Joi.string(),
+    }).unknown(true)
+  )
 }
 
 export function screenValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    showNavigation: OPTIONAL_BOOLEAN,
+      showNavigation: OPTIONAL_BOOLEAN,
-    width: OPTIONAL_STRING,
+      width: OPTIONAL_STRING,
-    routing: Joi.object({
+      routing: Joi.object({
-      route: Joi.string().required(),
+        route: Joi.string().required(),
-      roleId: Joi.string().required().allow(""),
+        roleId: Joi.string().required().allow(""),
-      homeScreen: OPTIONAL_BOOLEAN,
+        homeScreen: OPTIONAL_BOOLEAN,
-    }).required().unknown(true),
+      })
-    props: Joi.object({
+        .required()
-      _id: Joi.string().required(),
+        .unknown(true),
-      _component: Joi.string().required(),
+      props: Joi.object({
-      _children: Joi.array().required(),
+        _id: Joi.string().required(),
-      _styles: Joi.object().required(),
+        _component: Joi.string().required(),
-      type: OPTIONAL_STRING,
+        _children: Joi.array().required(),
-      table: OPTIONAL_STRING,
+        _styles: Joi.object().required(),
-      layoutId: OPTIONAL_STRING,
+        type: OPTIONAL_STRING,
-    }).required().unknown(true),
+        table: OPTIONAL_STRING,
-  }).unknown(true))
+        layoutId: OPTIONAL_STRING,
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 function generateStepSchema(allowStepTypes: string[]) {
-  // prettier-ignore
   return Joi.object({
     stepId: Joi.string().required(),
     id: Joi.string().required(),
@@ -189,33 +247,39 @@ function generateStepSchema(allowStepTypes: string[]) {
     icon: Joi.string().required(),
     params: Joi.object(),
     args: Joi.object(),
-    type: Joi.string().required().valid(...allowStepTypes),
+    type: Joi.string()
+      .required()
+      .valid(...allowStepTypes),
   }).unknown(true)
 }
 
 export function automationValidator(existing = false) {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _id: existing ? Joi.string().required() : OPTIONAL_STRING,
-    _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    type: Joi.string().valid("automation").required(),
+      type: Joi.string().valid("automation").required(),
-    definition: Joi.object({
+      definition: Joi.object({
-      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
+        steps: Joi.array()
-      trigger: generateStepSchema(["TRIGGER"]).allow(null),
+          .required()
-    }).required().unknown(true),
+          .items(generateStepSchema(["ACTION", "LOGIC"])),
-  }).unknown(true))
+        trigger: generateStepSchema(["TRIGGER"]).allow(null),
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 export function applicationValidator(opts = { isCreate: true }) {
-  // prettier-ignore
   const base: any = {
     _id: OPTIONAL_STRING,
     _rev: OPTIONAL_STRING,
     url: OPTIONAL_STRING,
     template: Joi.object({
       templateString: OPTIONAL_STRING,
-    })
+    }),
   }
 
   const appNameValidator = Joi.string()

packages/server/src/sdk/app/views/index.ts

@@ -39,9 +39,7 @@ async function guardViewSchema(
   tableId: string,
   viewSchema?: Record<string, ViewUIFieldMetadata>
 ) {
-  if (!viewSchema || !Object.keys(viewSchema).length) {
+  viewSchema ??= {}
-    return
-  }
   const table = await sdk.tables.getTable(tableId)
 
   for (const field of Object.keys(viewSchema)) {
@@ -54,17 +52,13 @@ async function guardViewSchema(
     }
 
     if (viewSchema[field].readonly) {
-      if (!(await features.isViewReadonlyColumnsEnabled())) {
+      if (
+        !(await features.isViewReadonlyColumnsEnabled()) &&
+        !(tableSchemaField as ViewUIFieldMetadata).readonly
+      ) {
         throw new HTTPError(`Readonly fields are not enabled`, 400)
       }
 
-      if (isRequired(tableSchemaField.constraints)) {
-        throw new HTTPError(
-          `Field "${field}" cannot be readonly as it is a required field`,
-          400
-        )
-      }
-
       if (!viewSchema[field].visible) {
         throw new HTTPError(
           `Field "${field}" must be visible if you want to make it readonly`,
@@ -73,6 +67,20 @@ async function guardViewSchema(
       }
     }
   }
+
+  for (const field of Object.values(table.schema)) {
+    if (!isRequired(field.constraints)) {
+      continue
+    }
+
+    const viewSchemaField = viewSchema[field.name]
+    if (viewSchemaField?.readonly) {
+      throw new HTTPError(
+        `You can't make field "${field.name}" readonly because it is a required field.`,
+        400
+      )
+    }
+  }
 }
 
 export async function create(