Merge branch 'master' of github.com:Budibase/budibase into BUDI-7656/add-migration

charts/budibase/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: couchdb
   repository: https://apache.github.io/couchdb-helm
-  version: 4.3.0
+  version: 4.5.6
-digest: sha256:94449a7f195b186f5af33ec5aa66d58b36bede240fae710f021ca87837b30606
+digest: sha256:405f098633e632d6f4e140175f156ed4f02918b0d89193f1b66c9cbea211d6c9
-generated: "2023-11-20T17:43:02.777596Z"
+generated: "2024-06-05T14:41:05.979052+01:00"

charts/budibase/Chart.yaml

@@ -17,6 +17,6 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 4.5.3
+    version: 4.5.6
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled

charts/budibase/README.md

@@ -112,7 +112,9 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | awsAlbIngress.enabled | bool | `false` | Whether to create an ALB Ingress resource pointing to the Budibase proxy. Requires the AWS ALB Ingress Controller. |
 | couchdb.clusterSize | int | `1` | The number of replicas to run in the CouchDB cluster. We set this to 1 by default to make things simpler, but you can set it to 3 if you need a high-availability CouchDB cluster. |
 | couchdb.couchdbConfig.couchdb.uuid | string | `"budibase-couchdb"` | Unique identifier for this CouchDB server instance. You shouldn't need to change this. |
+| couchdb.extraPorts[0] | object | `{"containerPort":4984,"name":"sqs"}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. |
 | couchdb.image | object | `{}` | We use a custom CouchDB image for running Budibase and we don't support using any other CouchDB image. You shouldn't change this, and if you do we can't guarantee that Budibase will work. |
+| couchdb.service.extraPorts[0] | object | `{"name":"sqs","port":4984,"protocol":"TCP","targetPort":4984}` | Extra ports to expose on the CouchDB service. We expose the SQS port by default, but you can add more ports here if you need to. |
 | globals.apiEncryptionKey | string | `""` | Used for encrypting API keys and environment variables when stored in the database. You don't need to set this if `createSecrets` is true. |
 | globals.appVersion | string | `""` | The version of Budibase to deploy. Defaults to what's specified by {{ .Chart.AppVersion }}. Ends up being used as the image version tag for the apps, proxy, and worker images. |
 | globals.automationMaxIterations | string | `"200"` | The maximum number of iterations allows for an automation loop step. You can read more about looping here: <https://docs.budibase.com/docs/looping>. |
@@ -135,6 +137,8 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | globals.smtp.password | string | `""` | The password to use when authenticating with your SMTP server. |
 | globals.smtp.port | string | `"587"` | The port of your SMTP server. |
 | globals.smtp.user | string | `""` | The username to use when authenticating with your SMTP server. |
+| globals.sqs.enabled | bool | `false` | Whether to use the CouchDB "structured query service" or not. This is disabled by default for now, but will become the default in a future release. |
+| globals.tempBucketName | string | `""` |  |
 | globals.tenantFeatureFlags | string | `"*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR"` | Sets what feature flags are enabled and for which tenants. Should not ordinarily need to be changed. |
 | imagePullSecrets | list | `[]` | Passed to all pods created by this chart. Should not ordinarily need to be changed. |
 | ingress.className | string | `""` | What ingress class to use. |
@@ -152,6 +156,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.apps.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the apps service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the apps pods. |
 | services.apps.extraContainers | list | `[]` | Additional containers to be added to the apps pod. |
 | services.apps.extraEnv | list | `[]` | Extra environment variables to set for apps pods. Takes a list of name=value pairs. |
+| services.apps.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.apps.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main apps container. |
 | services.apps.extraVolumes | list | `[]` | Additional volumes to the apps pod. |
 | services.apps.httpLogging | int | `1` | Whether or not to log HTTP requests to the apps service. |
@@ -168,6 +173,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.automationWorkers.enabled | bool | `true` | Whether or not to enable the automation worker service. If you disable this, automations will be processed by the apps service. |
 | services.automationWorkers.extraContainers | list | `[]` | Additional containers to be added to the automationWorkers pod. |
 | services.automationWorkers.extraEnv | list | `[]` | Extra environment variables to set for automation worker pods. Takes a list of name=value pairs. |
+| services.automationWorkers.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.automationWorkers.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main automationWorkers container. |
 | services.automationWorkers.extraVolumes | list | `[]` | Additional volumes to the automationWorkers pod. |
 | services.automationWorkers.livenessProbe | object | HTTP health checks. | Liveness probe configuration for automation worker pods. You shouldn't need to change this, but if you want to you can find more information here: <https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/> |
@@ -195,7 +201,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.objectStore.region | string | `""` | AWS_REGION if using S3 |
 | services.objectStore.resources | object | `{}` | The resources to use for Minio pods. See <https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/> for more information on how to set these. |
 | services.objectStore.secretKey | string | `""` | AWS_SECRET_ACCESS_KEY if using S3 |
-| services.objectStore.storage | string | `"100Mi"` | How much storage to give Minio in its PersistentVolumeClaim. |
+| services.objectStore.storage | string | `"2Gi"` | How much storage to give Minio in its PersistentVolumeClaim. |
 | services.objectStore.storageClass | string | `""` | If defined, storageClassName: <storageClass> If set to "-", storageClassName: "", which disables dynamic provisioning If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. |
 | services.objectStore.url | string | `"http://minio-service:9000"` | URL to use for object storage. Only change this if you're using an external object store, such as S3. Remember to set `minio: false` if you do this. |
 | services.proxy.autoscaling.enabled | bool | `false` | Whether to enable horizontal pod autoscaling for the proxy service. |
@@ -227,6 +233,7 @@ $ helm install --create-namespace --namespace budibase budibase . -f values.yaml
 | services.worker.autoscaling.targetCPUUtilizationPercentage | int | `80` | Target CPU utilization percentage for the worker service. Note that for autoscaling to work, you will need to have metrics-server configured, and resources set for the worker pods. |
 | services.worker.extraContainers | list | `[]` | Additional containers to be added to the worker pod. |
 | services.worker.extraEnv | list | `[]` | Extra environment variables to set for worker pods. Takes a list of name=value pairs. |
+| services.worker.extraEnvFromSecret | list | `[]` | Name of the K8s Secret in the same namespace which contains the extra environment variables. This can be used to avoid storing sensitive information in the values.yaml file. |
 | services.worker.extraVolumeMounts | list | `[]` | Additional volumeMounts to the main worker container. |
 | services.worker.extraVolumes | list | `[]` | Additional volumes to the worker pod. |
 | services.worker.httpLogging | int | `1` | Whether or not to log HTTP requests to the worker service. |

charts/budibase/templates/app-service-deployment.yaml

@@ -42,6 +42,14 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        {{ if .Values.globals.sqs.enabled }}
+        - name: COUCH_DB_SQL_URL
+        {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url }}
+        {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+        {{ end }}
+        {{ end }}
         {{ if .Values.services.couchdb.enabled }}
         - name: COUCH_DB_USER
           valueFrom:
@@ -198,6 +206,10 @@ spec:
         - name: APP_FEATURES
           value: "api"
         {{- end }}
+        {{- if .Values.globals.sqs.enabled }}
+        - name: SQS_SEARCH_ENABLE
+          value: "true"
+        {{- end }}
         {{- range .Values.services.apps.extraEnv }}
         - name: {{ .name }}
           value: {{ .value | quote }}

charts/budibase/templates/worker-service-deployment.yaml

@@ -56,6 +56,14 @@ spec:
           {{ else }}
           value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
           {{ end }}
+        {{ if .Values.globals.sqs.enabled }}
+        - name: COUCH_DB_SQL_URL
+        {{ if .Values.globals.sqs.url }}
+          value: {{ .Values.globals.sqs.url }}
+        {{ else }}
+          value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
+        {{ end }}
+        {{ end }}
         - name: API_ENCRYPTION_KEY
           value: {{ .Values.globals.apiEncryptionKey | quote }}
         - name: HTTP_LOGGING
@@ -184,6 +192,10 @@ spec:
         - name: NODE_TLS_REJECT_UNAUTHORIZED
           value: {{ .Values.services.tlsRejectUnauthorized }}
         {{ end }}
+        {{- if .Values.globals.sqs.enabled }}
+        - name: SQS_SEARCH_ENABLE
+          value: "true"
+        {{- end }}
         {{- range .Values.services.worker.extraEnv }}
         - name: {{ .name }}
           value: {{ .value | quote }}

charts/budibase/values.yaml

@@ -138,6 +138,15 @@ globals:
     # -- The password to use when authenticating with your SMTP server.
     password: ""
 
+  sqs:
+    # -- Whether to use the CouchDB "structured query service" or not. This is disabled by
+    # default for now, but will become the default in a future release.
+    enabled: false
+    # @ignore
+    url: ""
+    # @ignore
+    port: "4984"
+
 services:
   # -- The DNS suffix to use for service discovery. You only need to change this
   # if you've configured your cluster to use a different DNS suffix.
@@ -636,6 +645,21 @@ couchdb:
     # @ignore
     pullPolicy: Always
 
+  extraPorts:
+    # -- Extra ports to expose on the CouchDB service. We expose the SQS port
+    # by default, but you can add more ports here if you need to.
+    - name: sqs
+      containerPort: 4984
+
+  service:
+    extraPorts:
+      # -- Extra ports to expose on the CouchDB service. We expose the SQS port
+      # by default, but you can add more ports here if you need to.
+      - name: sqs
+        port: 4984
+        targetPort: 4984
+        protocol: TCP
+
   # @ignore
   # This should remain false. We ship Clouseau ourselves as part of the
   # budibase/couchdb image, and it's not possible to disable it because it's a

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.27.6",
+  "version": "2.28.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/middleware/joi-validator.ts

@@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types"
 
 function validate(
   schema: Joi.ObjectSchema | Joi.ArraySchema,
-  property: string
+  property: string,
+  opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` }
 ) {
   // Return a Koa middleware function
   return (ctx: Ctx, next: any) => {
@@ -29,16 +30,26 @@ function validate(
 
     const { error } = schema.validate(params)
     if (error) {
-      ctx.throw(400, `Invalid ${property} - ${error.message}`)
+      let message = error.message
+      if (opts.errorPrefix) {
+        message = `Invalid ${property} - ${message}`
+      }
+      ctx.throw(400, message)
     }
     return next()
   }
 }
 
-export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function body(
-  return validate(schema, "body")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "body", opts)
 }
 
-export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function params(
-  return validate(schema, "params")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "params", opts)
 }

packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte

@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin } from "stores/portal"
+  import { admin, licensing } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@@ -28,6 +28,7 @@
     showAvatars={false}
     on:updatedatasource={handleGridViewUpdate}
     isCloud={$admin.cloud}
+    allowViewReadonlyColumns={$licensing.isViewReadonlyColumnsEnabled}
   >
     <svelte:fragment slot="filter">
       <GridFilterButton />

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ShowNotification.svelte

@@ -25,6 +25,8 @@
     },
   ]
 
+  const MAX_DURATION = 120000 // Maximum duration in milliseconds (2 minutes)
+
   onMount(() => {
     if (!parameters.type) {
       parameters.type = "success"
@@ -33,6 +35,14 @@
       parameters.autoDismiss = true
     }
   })
+
+  function handleDurationChange(event) {
+    let newDuration = event.detail
+    if (newDuration > MAX_DURATION) {
+      newDuration = MAX_DURATION
+    }
+    parameters.duration = newDuration
+  }
 </script>
 
 <div class="root">
@@ -47,6 +57,16 @@
   />
   <Label />
   <Checkbox text="Auto dismiss" bind:value={parameters.autoDismiss} />
+  {#if parameters.autoDismiss}
+    <Label>Duration (ms)</Label>
+    <DrawerBindableInput
+      title="Duration"
+      {bindings}
+      value={parameters.duration}
+      placeholder="3000"
+      on:change={handleDurationChange}
+    />
+  {/if}
 </div>
 
 <style>

packages/builder/src/dataBinding.js

@@ -728,7 +728,7 @@ const getRoleBindings = () => {
   return (get(rolesStore) || []).map(role => {
     return {
       type: "context",
-      runtimeBinding: `trim "${role._id}"`,
+      runtimeBinding: `'${role._id}'`,
       readableBinding: `Role.${role.name}`,
       category: "Role",
       icon: "UserGroup",

packages/builder/src/stores/portal/licensing.js

@@ -138,6 +138,11 @@ export const createLicensingStore = () => {
       const isViewPermissionsEnabled = license.features.includes(
         Constants.Features.VIEW_PERMISSIONS
       )
+
+      const isViewReadonlyColumnsEnabled = license.features.includes(
+        Constants.Features.VIEW_READONLY_COLUMNS
+      )
+
       store.update(state => {
         return {
           ...state,
@@ -157,6 +162,7 @@ export const createLicensingStore = () => {
           triggerAutomationRunEnabled,
           isViewPermissionsEnabled,
           perAppBuildersEnabled,
+          isViewReadonlyColumnsEnabled,
         }
       })
     },

packages/client/src/components/app/forms/InnerForm.svelte

@@ -206,7 +206,7 @@
           error: initialError,
           disabled:
             disabled || fieldDisabled || (isAutoColumn && !editAutoColumns),
-          readonly: readonly || fieldReadOnly,
+          readonly: readonly || fieldReadOnly || schema?.[field]?.readonly,
           defaultValue,
           validator,
           lastUpdate: Date.now(),

packages/client/src/stores/notification.js

@@ -1,7 +1,7 @@
 import { writable, get } from "svelte/store"
 import { routeStore } from "./routes"
 
-const NOTIFICATION_TIMEOUT = 3000
+const DEFAULT_NOTIFICATION_TIMEOUT = 3000
 
 const createNotificationStore = () => {
   let block = false
@@ -18,13 +18,13 @@ const createNotificationStore = () => {
     type = "info",
     icon,
     autoDismiss = true,
+    duration,
     count = 1
   ) => {
     if (block) {
       return
     }
 
-    // If peeking, pass notifications back to parent window
     if (get(routeStore).queryParams?.peek) {
       window.parent.postMessage({
         type: "notification",
@@ -32,11 +32,13 @@ const createNotificationStore = () => {
           message,
           type,
           icon,
+          duration,
           autoDismiss,
         },
       })
       return
     }
+
     const _id = id()
     store.update(state => {
       const duplicateError = state.find(err => err.message === message)
@@ -60,7 +62,7 @@ const createNotificationStore = () => {
     if (autoDismiss) {
       setTimeout(() => {
         dismiss(_id)
-      }, NOTIFICATION_TIMEOUT)
+      }, duration || DEFAULT_NOTIFICATION_TIMEOUT)
     }
   }
 
@@ -74,14 +76,14 @@ const createNotificationStore = () => {
     subscribe: store.subscribe,
     actions: {
       send,
-      info: (msg, autoDismiss) =>
+      info: (msg, autoDismiss, duration) =>
-        send(msg, "info", "Info", autoDismiss ?? true),
+        send(msg, "info", "Info", autoDismiss ?? true, duration),
-      success: (msg, autoDismiss) =>
+      success: (msg, autoDismiss, duration) =>
-        send(msg, "success", "CheckmarkCircle", autoDismiss ?? true),
+        send(msg, "success", "CheckmarkCircle", autoDismiss ?? true, duration),
-      warning: (msg, autoDismiss) =>
+      warning: (msg, autoDismiss, duration) =>
-        send(msg, "warning", "Alert", autoDismiss ?? true),
+        send(msg, "warning", "Alert", autoDismiss ?? true, duration),
-      error: (msg, autoDismiss) =>
+      error: (msg, autoDismiss, duration) =>
-        send(msg, "error", "Alert", autoDismiss ?? false),
+        send(msg, "error", "Alert", autoDismiss ?? false, duration),
       blockNotifications,
       dismiss,
     },

packages/client/src/utils/buttonActions.js

@@ -416,11 +416,11 @@ const continueIfHandler = action => {
 }
 
 const showNotificationHandler = action => {
-  const { message, type, autoDismiss } = action.parameters
+  const { message, type, autoDismiss, duration } = action.parameters
   if (!message || !type) {
     return
   }
-  notificationStore.actions[type]?.(message, autoDismiss)
+  notificationStore.actions[type]?.(message, autoDismiss, duration)
 }
 
 const promptUserHandler = () => {}

packages/frontend-core/src/components/grid/cells/DataCell.svelte

@@ -33,7 +33,8 @@
     column.schema.autocolumn ||
     column.schema.disabled ||
     column.schema.type === "formula" ||
-    (!$config.canEditRows && !row._isNewRow)
+    (!$config.canEditRows && !row._isNewRow) ||
+    column.schema.readonly
 
   // Register this cell API if the row is focused
   $: {

packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte

@@ -1,49 +1,98 @@
 <script>
   import { getContext } from "svelte"
-  import { ActionButton, Popover, Icon } from "@budibase/bbui"
+  import { ActionButton, Popover, Icon, notifications } from "@budibase/bbui"
   import { getColumnIcon } from "../lib/utils"
   import ToggleActionButtonGroup from "./ToggleActionButtonGroup.svelte"
+  import { helpers } from "@budibase/shared-core"
+
+  export let allowViewReadonlyColumns = false
 
   const { columns, datasource, stickyColumn, dispatch } = getContext("grid")
 
   let open = false
   let anchor
 
-  $: anyHidden = $columns.some(col => !col.visible)
+  $: allColumns = $stickyColumn ? [$stickyColumn, ...$columns] : $columns
-  $: text = getText($columns)
+
+  $: restrictedColumns = allColumns.filter(col => !col.visible || col.readonly)
+  $: anyRestricted = restrictedColumns.length
+  $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
 
   const toggleColumn = async (column, permission) => {
     const visible = permission !== PERMISSION_OPTIONS.HIDDEN
+    const readonly = permission === PERMISSION_OPTIONS.READONLY
 
-    datasource.actions.addSchemaMutation(column.name, { visible })
+    await datasource.actions.addSchemaMutation(column.name, {
-    await datasource.actions.saveSchemaMutations()
+      visible,
+      readonly,
+    })
+    try {
+      await datasource.actions.saveSchemaMutations()
+    } catch (e) {
+      notifications.error(e.message)
+    } finally {
+      await datasource.actions.resetSchemaMutations()
+      await datasource.actions.refreshDefinition()
+    }
     dispatch(visible ? "show-column" : "hide-column")
   }
 
-  const getText = columns => {
-    const hidden = columns.filter(col => !col.visible).length
-    return hidden ? `Columns (${hidden} restricted)` : "Columns"
-  }
-
   const PERMISSION_OPTIONS = {
     WRITABLE: "writable",
+    READONLY: "readonly",
     HIDDEN: "hidden",
   }
 
-  const options = [
+  $: displayColumns = allColumns.map(c => {
-    { icon: "Edit", value: PERMISSION_OPTIONS.WRITABLE, tooltip: "Writable" },
+    const isRequired = helpers.schema.isRequired(c.schema.constraints)
-    {
+    const isDisplayColumn = $stickyColumn === c
+
+    const requiredTooltip = isRequired && "Required columns must be writable"
+
+    const editEnabled =
+      !isRequired ||
+      columnToPermissionOptions(c) !== PERMISSION_OPTIONS.WRITABLE
+    const options = [
+      {
+        icon: "Edit",
+        value: PERMISSION_OPTIONS.WRITABLE,
+        tooltip: (!editEnabled && requiredTooltip) || "Writable",
+        disabled: !editEnabled,
+      },
+    ]
+    if ($datasource.type === "viewV2") {
+      options.push({
+        icon: "Visibility",
+        value: PERMISSION_OPTIONS.READONLY,
+        tooltip: allowViewReadonlyColumns
+          ? requiredTooltip || "Read only"
+          : "Read only (premium feature)",
+        disabled: !allowViewReadonlyColumns || isRequired,
+      })
+    }
+
+    options.push({
       icon: "VisibilityOff",
       value: PERMISSION_OPTIONS.HIDDEN,
-      tooltip: "Hidden",
+      disabled: isDisplayColumn || isRequired,
-    },
+      tooltip:
-  ]
+        (isDisplayColumn && "Display column cannot be hidden") ||
+        requiredTooltip ||
+        "Hidden",
+    })
+
+    return { ...c, options }
+  })
 
   function columnToPermissionOptions(column) {
-    if (!column.visible) {
+    if (!column.schema.visible) {
       return PERMISSION_OPTIONS.HIDDEN
     }
 
+    if (column.schema.readonly) {
+      return PERMISSION_OPTIONS.READONLY
+    }
+
     return PERMISSION_OPTIONS.WRITABLE
   }
 </script>
@@ -54,7 +103,7 @@
     quiet
     size="M"
     on:click={() => (open = !open)}
-    selected={open || anyHidden}
+    selected={open || anyRestricted}
     disabled={!$columns.length}
   >
     {text}
@@ -64,19 +113,7 @@
 <Popover bind:open {anchor} align="left">
   <div class="content">
     <div class="columns">
-      {#if $stickyColumn}
+      {#each displayColumns as column}
-        <div class="column">
-          <Icon size="S" name={getColumnIcon($stickyColumn)} />
-          {$stickyColumn.label}
-        </div>
-
-        <ToggleActionButtonGroup
-          disabled
-          value={PERMISSION_OPTIONS.WRITABLE}
-          {options}
-        />
-      {/if}
-      {#each $columns as column}
         <div class="column">
           <Icon size="S" name={getColumnIcon(column)} />
           {column.label}
@@ -84,7 +121,7 @@
         <ToggleActionButtonGroup
           on:click={e => toggleColumn(column, e.detail)}
           value={columnToPermissionOptions(column)}
-          {options}
+          options={column.options}
         />
       {/each}
     </div>

packages/frontend-core/src/components/grid/controls/ToggleActionButtonGroup.svelte

@@ -7,7 +7,6 @@
 
   export let value
   export let options
-  export let disabled
 </script>
 
 <div class="permissionPicker">
@@ -15,7 +14,7 @@
     <AbsTooltip text={option.tooltip} type={TooltipType.Info}>
       <ActionButton
         on:click={() => dispatch("click", option.value)}
-        {disabled}
+        disabled={option.disabled}
         size="S"
         icon={option.icon}
         quiet

packages/frontend-core/src/components/grid/layout/Grid.svelte

@@ -57,6 +57,7 @@
   export let buttons = null
   export let darkMode
   export let isCloud = null
+  export let allowViewReadonlyColumns = false
 
   // Unique identifier for DOM nodes inside this instance
   const gridID = `grid-${Math.random().toString().slice(2)}`
@@ -153,7 +154,7 @@
       <div class="controls-left">
         <slot name="filter" />
         <SortButton />
-        <ColumnsSettingButton />
+        <ColumnsSettingButton {allowViewReadonlyColumns} />
         <SizeButton />
         <slot name="controls" />
       </div>

packages/frontend-core/src/components/grid/stores/columns.js

@@ -146,6 +146,7 @@ export const initialise = context => {
             schema: fieldSchema,
             width: fieldSchema.width || oldColumn?.width || DefaultColumnWidth,
             visible: fieldSchema.visible ?? true,
+            readonly: fieldSchema.readonly,
             order: fieldSchema.order ?? oldColumn?.order,
             primaryDisplay: field === primaryDisplay,
           }

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -204,6 +204,10 @@ export const createActions = context => {
       ...$definition,
       schema: newSchema,
     })
+    resetSchemaMutations()
+  }
+
+  const resetSchemaMutations = () => {
     schemaMutations.set({})
   }
 
@@ -253,6 +257,7 @@ export const createActions = context => {
         addSchemaMutation,
         addSchemaMutations,
         saveSchemaMutations,
+        resetSchemaMutations,
       },
     },
   }

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -22,10 +22,7 @@ import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { roles } from "@budibase/backend-core"
+import { db, roles } from "@budibase/backend-core"
-import * as schemaUtils from "../../../utilities/schema"
-
-jest.mock("../../../utilities/schema")
 
 describe.each([
   ["internal", undefined],
@@ -120,6 +117,9 @@ describe.each([
       const newView: CreateViewRequest = {
         name: generator.name(),
         tableId: table._id!,
+        schema: {
+          id: { visible: true },
+        },
       }
       const res = await config.api.viewV2.create(newView)
 
@@ -134,7 +134,7 @@ describe.each([
       const newView: Required<CreateViewRequest> = {
         name: generator.name(),
         tableId: table._id!,
-        primaryDisplay: generator.word(),
+        primaryDisplay: "id",
         query: [
           {
             operator: SearchFilterOperator.EQUAL,
@@ -148,6 +148,7 @@ describe.each([
           type: SortType.STRING,
         },
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -158,6 +159,7 @@ describe.each([
       expect(res).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -172,6 +174,11 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+            visible: true,
+          },
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -193,6 +200,7 @@ describe.each([
       expect(createdView).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             order: 1,
@@ -209,6 +217,12 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            autocolumn: true,
+            visible: true,
+          },
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -230,8 +244,9 @@ describe.each([
       const newView: CreateViewRequest = {
         name: generator.name(),
         tableId: table._id!,
-        primaryDisplay: generator.word(),
+        primaryDisplay: "id",
         schema: {
+          id: { visible: true },
           Price: { visible: true },
           Category: { visible: false },
         },
@@ -241,6 +256,7 @@ describe.each([
       expect(res).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -255,6 +271,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           nonExisting: {
             visible: true,
           },
@@ -293,6 +310,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: true,
               readonly: true,
@@ -306,6 +324,7 @@ describe.each([
 
         const res = await config.api.viewV2.create(newView)
         expect(res.schema).toEqual({
+          id: { visible: true },
           name: {
             visible: true,
             readonly: true,
@@ -318,15 +337,13 @@ describe.each([
       })
 
       it("required fields cannot be marked as readonly", async () => {
-        const isRequiredSpy = jest.spyOn(schemaUtils, "isRequired")
-        isRequiredSpy.mockReturnValueOnce(true)
-
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
               name: {
                 name: "name",
                 type: FieldType.STRING,
+                constraints: { presence: true },
               },
               description: {
                 name: "description",
@@ -340,7 +357,9 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
+              visible: true,
               readonly: true,
             },
           },
@@ -350,7 +369,7 @@ describe.each([
           status: 400,
           body: {
             message:
-              'Field "name" cannot be readonly as it is a required field',
+              'You can\'t make "name" readonly because it is a required field.',
             status: 400,
           },
         })
@@ -376,6 +395,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: false,
               readonly: true,
@@ -414,6 +434,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: true,
               readonly: true,
@@ -424,12 +445,84 @@ describe.each([
         await config.api.viewV2.create(newView, {
           status: 400,
           body: {
-            message: "Readonly fields are not enabled for your tenant",
+            message: "Readonly fields are not enabled",
             status: 400,
           },
         })
       })
     })
+
+    it("display fields must be visible", async () => {
+      const table = await config.api.table.save(
+        saveTableRequest({
+          schema: {
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+            },
+            description: {
+              name: "description",
+              type: FieldType.STRING,
+            },
+          },
+        })
+      )
+
+      const newView: CreateViewRequest = {
+        name: generator.name(),
+        tableId: table._id!,
+        primaryDisplay: "name",
+        schema: {
+          id: { visible: true },
+          name: {
+            visible: false,
+          },
+        },
+      }
+
+      await config.api.viewV2.create(newView, {
+        status: 400,
+        body: {
+          message: 'You can\'t hide "name" because it is the display column.',
+          status: 400,
+        },
+      })
+    })
+
+    it("display fields can be readonly", async () => {
+      mocks.licenses.useViewReadonlyColumns()
+      const table = await config.api.table.save(
+        saveTableRequest({
+          schema: {
+            name: {
+              name: "name",
+              type: FieldType.STRING,
+            },
+            description: {
+              name: "description",
+              type: FieldType.STRING,
+            },
+          },
+        })
+      )
+
+      const newView: CreateViewRequest = {
+        name: generator.name(),
+        tableId: table._id!,
+        primaryDisplay: "name",
+        schema: {
+          id: { visible: true },
+          name: {
+            visible: true,
+            readonly: true,
+          },
+        },
+      }
+
+      await config.api.viewV2.create(newView, {
+        status: 201,
+      })
+    })
   })
 
   describe("update", () => {
@@ -441,6 +534,9 @@ describe.each([
       view = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
     })
 
@@ -475,7 +571,7 @@ describe.each([
         id: view.id,
         tableId,
         name: view.name,
-        primaryDisplay: generator.word(),
+        primaryDisplay: "Price",
         query: [
           {
             operator: SearchFilterOperator.EQUAL,
@@ -489,6 +585,7 @@ describe.each([
           type: SortType.STRING,
         },
         schema: {
+          id: { visible: true },
           Category: {
             visible: false,
           },
@@ -506,7 +603,7 @@ describe.each([
           schema: {
             ...table.schema,
             id: expect.objectContaining({
-              visible: false,
+              visible: true,
             }),
             Category: expect.objectContaining({
               visible: false,
@@ -603,6 +700,9 @@ describe.each([
       const anotherView = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
       const result = await config
         .request!.put(`/api/v2/views/${anotherView.id}`)
@@ -621,6 +721,7 @@ describe.each([
       const updatedView = await config.api.viewV2.update({
         ...view,
         schema: {
+          ...view.schema,
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -640,6 +741,7 @@ describe.each([
       expect(updatedView).toEqual({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             order: 1,
@@ -656,6 +758,7 @@ describe.each([
         {
           ...view,
           schema: {
+            ...view.schema,
             Price: {
               name: "Price",
               type: FieldType.NUMBER,
@@ -679,6 +782,7 @@ describe.each([
       view = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: true,
@@ -690,7 +794,7 @@ describe.each([
       await config.api.viewV2.update(view, {
         status: 400,
         body: {
-          message: "Readonly fields are not enabled for your tenant",
+          message: "Readonly fields are not enabled",
         },
       })
     })
@@ -701,6 +805,7 @@ describe.each([
       view = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: true,
@@ -715,6 +820,7 @@ describe.each([
       const res = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: false,
@@ -725,6 +831,7 @@ describe.each([
         expect.objectContaining({
           ...view,
           schema: {
+            id: { visible: true },
             Price: {
               visible: true,
               readonly: false,
@@ -733,6 +840,53 @@ describe.each([
         })
       )
     })
+
+    isInternal &&
+      it("updating schema will only validate modified field", async () => {
+        let view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          schema: {
+            id: { visible: true },
+            Price: {
+              visible: true,
+            },
+            Category: { visible: true },
+          },
+        })
+
+        // Update the view to an invalid state
+        const tableToUpdate = await config.api.table.get(table._id!)
+        ;(tableToUpdate.views![view.name] as ViewV2).schema!.id.visible = false
+        await db.getDB(config.appId!).put(tableToUpdate)
+
+        view = await config.api.viewV2.get(view.id)
+        await config.api.viewV2.update({
+          ...view,
+          schema: {
+            ...view.schema,
+            Price: {
+              visible: false,
+            },
+          },
+        })
+
+        expect(await config.api.viewV2.get(view.id)).toEqual(
+          expect.objectContaining({
+            schema: {
+              id: expect.objectContaining({
+                visible: false,
+              }),
+              Price: expect.objectContaining({
+                visible: false,
+              }),
+              Category: expect.objectContaining({
+                visible: true,
+              }),
+            },
+          })
+        )
+      })
   })
 
   describe("delete", () => {
@@ -742,6 +896,9 @@ describe.each([
       view = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
     })
 
@@ -764,6 +921,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           Price: { visible: false },
           Category: { visible: true },
         },
@@ -786,6 +944,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           Price: { visible: true, readonly: true },
         },
       })
@@ -821,6 +980,7 @@ describe.each([
         tableId: table._id!,
         name: generator.guid(),
         schema: {
+          id: { visible: true },
           Country: {
             visible: true,
           },
@@ -855,6 +1015,7 @@ describe.each([
         tableId: table._id!,
         name: generator.guid(),
         schema: {
+          id: { visible: true },
           two: { visible: true },
         },
       })
@@ -880,6 +1041,7 @@ describe.each([
           tableId: table._id!,
           name: generator.guid(),
           schema: {
+            id: { visible: true },
             one: { visible: true, readonly: true },
             two: { visible: true },
           },
@@ -921,6 +1083,7 @@ describe.each([
           tableId: table._id!,
           name: generator.guid(),
           schema: {
+            id: { visible: true },
             one: { visible: true, readonly: true },
             two: { visible: true },
           },
@@ -988,6 +1151,7 @@ describe.each([
             rows.map(r => ({
               _viewId: view.id,
               tableId: table._id,
+              id: r.id,
               _id: r._id,
               _rev: r._rev,
               ...(isInternal
@@ -1028,6 +1192,7 @@ describe.each([
             },
           ],
           schema: {
+            id: { visible: true },
             two: { visible: true },
           },
         })
@@ -1039,6 +1204,7 @@ describe.each([
             {
               _viewId: view.id,
               tableId: table._id,
+              id: two.id,
               two: two.two,
               _id: two._id,
               _rev: two._rev,
@@ -1192,7 +1358,11 @@ describe.each([
 
       describe("sorting", () => {
         let table: Table
-        const viewSchema = { age: { visible: true }, name: { visible: true } }
+        const viewSchema = {
+          id: { visible: true },
+          age: { visible: true },
+          name: { visible: true },
+        }
 
         beforeAll(async () => {
           table = await config.api.table.save(
@@ -1348,4 +1518,123 @@ describe.each([
       })
     })
   })
+
+  describe("updating table schema", () => {
+    describe("existing columns changed to required", () => {
+      beforeEach(async () => {
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              id: {
+                name: "id",
+                type: FieldType.AUTO,
+                autocolumn: true,
+              },
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+      })
+
+      it("allows updating when no views constrains the field", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: { visible: true },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: { allowEmpty: false } },
+              },
+            },
+          },
+          { status: 200 }
+        )
+      })
+
+      it("rejects if field is readonly in any view", async () => {
+        mocks.licenses.useViewReadonlyColumns()
+
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: {
+              visible: true,
+              readonly: true,
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+
+      it("rejects if field is hidden in any view", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: { id: { visible: true } },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+    })
+  })
 })

packages/server/src/api/routes/utils/validators.ts

@@ -1,51 +1,89 @@
 import { auth, permissions } from "@budibase/backend-core"
 import { DataSourceOperation } from "../../../constants"
-import { WebhookActionType } from "@budibase/types"
+import { Table, WebhookActionType } from "@budibase/types"
-import Joi from "joi"
+import Joi, { CustomValidator } from "joi"
-import { ValidSnippetNameRegex } from "@budibase/shared-core"
+import { ValidSnippetNameRegex, helpers } from "@budibase/shared-core"
+import sdk from "../../../sdk"
+
+const { isRequired } = helpers.schema
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
 const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
+const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
+  if (table.views && Object.entries(table.views).length) {
+    const requiredFields = Object.entries(table.schema)
+      .filter(([_, v]) => isRequired(v.constraints))
+      .map(([key]) => key)
+    if (requiredFields.length) {
+      for (const view of Object.values(table.views)) {
+        if (!sdk.views.isV2(view)) {
+          continue
+        }
+
+        const editableViewFields = Object.entries(view.schema || {})
+          .filter(([_, f]) => f.visible && !f.readonly)
+          .map(([key]) => key)
+        const missingField = requiredFields.find(
+          f => !editableViewFields.includes(f)
+        )
+        if (missingField) {
+          return helpers.message({
+            custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
+          })
+        }
+      }
+    }
+  }
+  return table
+}
+
 export function tableValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    type: OPTIONAL_STRING.valid("table", "internal", "external"),
+      type: OPTIONAL_STRING.valid("table", "internal", "external"),
-    primaryDisplay: OPTIONAL_STRING,
+      primaryDisplay: OPTIONAL_STRING,
-    schema: Joi.object().required(),
+      schema: Joi.object().required(),
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    views: Joi.object(),
+      views: Joi.object(),
-    rows: Joi.array(),
+      rows: Joi.array(),
-  }).unknown(true))
+    })
+      .custom(validateViewSchemas)
+      .unknown(true),
+    { errorPrefix: "" }
+  )
 }
 
 export function nameValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: OPTIONAL_STRING,
+      name: OPTIONAL_STRING,
-  }))
+    })
+  )
 }
 
 export function datasourceValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: Joi.string(),
+      _id: Joi.string(),
-    _rev: Joi.string(),
+      _rev: Joi.string(),
-    type: OPTIONAL_STRING.allow("datasource_plus"),
+      type: OPTIONAL_STRING.allow("datasource_plus"),
-    relationships: Joi.array().items(Joi.object({
+      relationships: Joi.array().items(
-      from: Joi.string().required(),
+        Joi.object({
-      to: Joi.string().required(),
+          from: Joi.string().required(),
-      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+          to: Joi.string().required(),
-    })),
+          cardinality: Joi.valid("1:N", "1:1", "N:N").required(),
-  }).unknown(true))
+        })
+      ),
+    }).unknown(true)
+  )
 }
 
 function filterObject() {
-  // prettier-ignore
   return Joi.object({
     string: Joi.object().optional(),
     fuzzy: Joi.object().optional(),
@@ -62,17 +100,20 @@ function filterObject() {
 }
 
 export function internalSearchValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    tableId: OPTIONAL_STRING,
+      tableId: OPTIONAL_STRING,
-    query: filterObject(),
+      query: filterObject(),
-    limit: OPTIONAL_NUMBER,
+      limit: OPTIONAL_NUMBER,
-    sort: OPTIONAL_STRING,
+      sort: OPTIONAL_STRING,
-    sortOrder: OPTIONAL_STRING,
+      sortOrder: OPTIONAL_STRING,
-    sortType: OPTIONAL_STRING,
+      sortType: OPTIONAL_STRING,
-    paginate: Joi.boolean(),
+      paginate: Joi.boolean(),
-    bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
+      bookmark: Joi.alternatives()
-  }))
+        .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
+        .optional(),
+    })
+  )
 }
 
 export function externalSearchValidator() {
@@ -94,92 +135,110 @@ export function externalSearchValidator() {
 }
 
 export function datasourceQueryValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    endpoint: Joi.object({
+      endpoint: Joi.object({
-      datasourceId: Joi.string().required(),
+        datasourceId: Joi.string().required(),
-      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+        operation: Joi.string()
-      entityId: Joi.string().required(),
+          .required()
-    }).required(),
+          .valid(...Object.values(DataSourceOperation)),
-    resource: Joi.object({
+        entityId: Joi.string().required(),
-      fields: Joi.array().items(Joi.string()).optional(),
+      }).required(),
-    }).optional(),
+      resource: Joi.object({
-    body: Joi.object().optional(),
+        fields: Joi.array().items(Joi.string()).optional(),
-    sort: Joi.object().optional(),
+      }).optional(),
-    filters: filterObject().optional(),
+      body: Joi.object().optional(),
-    paginate: Joi.object({
+      sort: Joi.object().optional(),
-      page: Joi.string().alphanum().optional(),
+      filters: filterObject().optional(),
-      limit: Joi.number().optional(),
+      paginate: Joi.object({
-    }).optional(),
+        page: Joi.string().alphanum().optional(),
-  }))
+        limit: Joi.number().optional(),
+      }).optional(),
+    })
+  )
 }
 
 export function webhookValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    live: Joi.bool(),
+      live: Joi.bool(),
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    bodySchema: Joi.object().optional(),
+      bodySchema: Joi.object().optional(),
-    action: Joi.object({
+      action: Joi.object({
-      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
+        type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
-      target: Joi.string().required(),
+        target: Joi.string().required(),
-    }).required(),
+      }).required(),
-  }).unknown(true))
+    }).unknown(true)
+  )
 }
 
 export function roleValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.body(Joi.object({
+  return auth.joiValidator.body(
-    _id: OPTIONAL_STRING,
+    Joi.object({
-    _rev: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    name: Joi.string().regex(/^[a-zA-Z0-9_]*$/).required(),
+      _rev: OPTIONAL_STRING,
-    // this is the base permission ID (for now a built in)
+      name: Joi.string()
-    permissionId: Joi.string().valid(...Object.values(permissions.BuiltinPermissionID)).required(),
+        .regex(/^[a-zA-Z0-9_]*$/)
-    permissions: Joi.object()
+        .required(),
-      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+      // this is the base permission ID (for now a built in)
-      .optional(),
+      permissionId: Joi.string()
-    inherits: OPTIONAL_STRING,
+        .valid(...Object.values(permissions.BuiltinPermissionID))
-  }).unknown(true))
+        .required(),
+      permissions: Joi.object()
+        .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+        .optional(),
+      inherits: OPTIONAL_STRING,
+    }).unknown(true)
+  )
 }
 
 export function permissionValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.params(Joi.object({
+  return auth.joiValidator.params(
-    level: Joi.string().valid(...permLevelArray).required(),
+    Joi.object({
-    resourceId: Joi.string(),
+      level: Joi.string()
-    roleId: Joi.string(),
+        .valid(...permLevelArray)
-  }).unknown(true))
+        .required(),
+      resourceId: Joi.string(),
+      roleId: Joi.string(),
+    }).unknown(true)
+  )
 }
 
 export function screenValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    showNavigation: OPTIONAL_BOOLEAN,
+      showNavigation: OPTIONAL_BOOLEAN,
-    width: OPTIONAL_STRING,
+      width: OPTIONAL_STRING,
-    routing: Joi.object({
+      routing: Joi.object({
-      route: Joi.string().required(),
+        route: Joi.string().required(),
-      roleId: Joi.string().required().allow(""),
+        roleId: Joi.string().required().allow(""),
-      homeScreen: OPTIONAL_BOOLEAN,
+        homeScreen: OPTIONAL_BOOLEAN,
-    }).required().unknown(true),
+      })
-    props: Joi.object({
+        .required()
-      _id: Joi.string().required(),
+        .unknown(true),
-      _component: Joi.string().required(),
+      props: Joi.object({
-      _children: Joi.array().required(),
+        _id: Joi.string().required(),
-      _styles: Joi.object().required(),
+        _component: Joi.string().required(),
-      type: OPTIONAL_STRING,
+        _children: Joi.array().required(),
-      table: OPTIONAL_STRING,
+        _styles: Joi.object().required(),
-      layoutId: OPTIONAL_STRING,
+        type: OPTIONAL_STRING,
-    }).required().unknown(true),
+        table: OPTIONAL_STRING,
-  }).unknown(true))
+        layoutId: OPTIONAL_STRING,
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 function generateStepSchema(allowStepTypes: string[]) {
-  // prettier-ignore
   return Joi.object({
     stepId: Joi.string().required(),
     id: Joi.string().required(),
@@ -189,33 +248,39 @@ function generateStepSchema(allowStepTypes: string[]) {
     icon: Joi.string().required(),
     params: Joi.object(),
     args: Joi.object(),
-    type: Joi.string().required().valid(...allowStepTypes),
+    type: Joi.string()
+      .required()
+      .valid(...allowStepTypes),
   }).unknown(true)
 }
 
 export function automationValidator(existing = false) {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _id: existing ? Joi.string().required() : OPTIONAL_STRING,
-    _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    type: Joi.string().valid("automation").required(),
+      type: Joi.string().valid("automation").required(),
-    definition: Joi.object({
+      definition: Joi.object({
-      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
+        steps: Joi.array()
-      trigger: generateStepSchema(["TRIGGER"]).allow(null),
+          .required()
-    }).required().unknown(true),
+          .items(generateStepSchema(["ACTION", "LOGIC"])),
-  }).unknown(true))
+        trigger: generateStepSchema(["TRIGGER"]).allow(null),
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 export function applicationValidator(opts = { isCreate: true }) {
-  // prettier-ignore
   const base: any = {
     _id: OPTIONAL_STRING,
     _rev: OPTIONAL_STRING,
     url: OPTIONAL_STRING,
     template: Joi.object({
       templateString: OPTIONAL_STRING,
-    })
+    }),
   }
 
   const appNameValidator = Joi.string()

packages/server/src/sdk/app/views/index.ts

@@ -8,7 +8,8 @@ import {
 } from "@budibase/types"
 import { HTTPError, db as dbCore } from "@budibase/backend-core"
 import { features } from "@budibase/pro"
-import { cloneDeep } from "lodash"
+import { helpers } from "@budibase/shared-core"
+import { cloneDeep } from "lodash/fp"
 
 import * as utils from "../../../db/utils"
 import { isExternalTableID } from "../../../integrations/utils"
@@ -16,7 +17,6 @@ import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./internal"
 import * as external from "./external"
 import sdk from "../../../sdk"
-import { isRequired } from "../../../utilities/schema"
 
 function pickApi(tableId: any) {
   if (isExternalTableID(tableId)) {
@@ -37,11 +37,9 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
 
 async function guardViewSchema(
   tableId: string,
-  viewSchema?: Record<string, ViewUIFieldMetadata>
+  view: Omit<ViewV2, "id" | "version">
 ) {
-  if (!viewSchema || !Object.keys(viewSchema).length) {
+  const viewSchema = view.schema || {}
-    return
-  }
   const table = await sdk.tables.getTable(tableId)
 
   for (const field of Object.keys(viewSchema)) {
@@ -54,18 +52,11 @@ async function guardViewSchema(
     }
 
     if (viewSchema[field].readonly) {
-      if (!(await features.isViewReadonlyColumnsEnabled())) {
+      if (
-        throw new HTTPError(
+        !(await features.isViewReadonlyColumnsEnabled()) &&
-          `Readonly fields are not enabled for your tenant`,
+        !(tableSchemaField as ViewUIFieldMetadata).readonly
-          400
+      ) {
-        )
+        throw new HTTPError(`Readonly fields are not enabled`, 400)
-      }
-
-      if (isRequired(tableSchemaField.constraints)) {
-        throw new HTTPError(
-          `Field "${field}" cannot be readonly as it is a required field`,
-          400
-        )
       }
 
       if (!viewSchema[field].visible) {
@@ -76,19 +67,61 @@ async function guardViewSchema(
       }
     }
   }
+
+  const existingView =
+    table?.views && (table.views[view.name] as ViewV2 | undefined)
+
+  for (const field of Object.values(table.schema)) {
+    if (!helpers.schema.isRequired(field.constraints)) {
+      continue
+    }
+
+    const viewSchemaField = viewSchema[field.name]
+    const existingViewSchema =
+      existingView?.schema && existingView.schema[field.name]
+    if (!viewSchemaField && !existingViewSchema?.visible) {
+      // Supporting existing configs with required columns but hidden in views
+      continue
+    }
+
+    if (!viewSchemaField?.visible) {
+      throw new HTTPError(
+        `You can't hide "${field.name}" because it is a required field.`,
+        400
+      )
+    }
+
+    if (viewSchemaField.readonly) {
+      throw new HTTPError(
+        `You can't make "${field.name}" readonly because it is a required field.`,
+        400
+      )
+    }
+  }
+
+  if (view.primaryDisplay) {
+    const viewSchemaField = viewSchema[view.primaryDisplay]
+
+    if (!viewSchemaField?.visible) {
+      throw new HTTPError(
+        `You can't hide "${view.primaryDisplay}" because it is the display column.`,
+        400
+      )
+    }
+  }
 }
 
 export async function create(
   tableId: string,
   viewRequest: Omit<ViewV2, "id" | "version">
 ): Promise<ViewV2> {
-  await guardViewSchema(tableId, viewRequest.schema)
+  await guardViewSchema(tableId, viewRequest)
 
   return pickApi(tableId).create(tableId, viewRequest)
 }
 
 export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
-  await guardViewSchema(tableId, view.schema)
+  await guardViewSchema(tableId, view)
 
   return pickApi(tableId).update(tableId, view)
 }

packages/server/src/utilities/schema.ts

@@ -4,9 +4,8 @@ import {
   TableSchema,
   FieldSchema,
   Row,
-  FieldConstraints,
 } from "@budibase/types"
-import { ValidColumnNameRegex, utils } from "@budibase/shared-core"
+import { ValidColumnNameRegex, helpers, utils } from "@budibase/shared-core"
 import { db } from "@budibase/backend-core"
 import { parseCsvExport } from "../api/controllers/view/exporters"
 
@@ -41,15 +40,6 @@ export function isRows(rows: any): rows is Rows {
   return Array.isArray(rows) && rows.every(row => typeof row === "object")
 }
 
-export function isRequired(constraints: FieldConstraints | undefined) {
-  const isRequired =
-    !!constraints &&
-    ((typeof constraints.presence !== "boolean" &&
-      constraints.presence?.allowEmpty === false) ||
-      constraints.presence === true)
-  return isRequired
-}
-
 export function validate(rows: Rows, schema: TableSchema): ValidationResults {
   const results: ValidationResults = {
     schemaValidation: {},
@@ -109,7 +99,7 @@ export function validate(rows: Rows, schema: TableSchema): ValidationResults {
           columnData,
           columnType,
           columnSubtype,
-          isRequired(constraints)
+          helpers.schema.isRequired(constraints)
         )
       ) {
         results.schemaValidation[columnName] = false

packages/shared-core/src/helpers/schema.ts

@@ -1,5 +1,6 @@
 import {
   BBReferenceFieldSubType,
+  FieldConstraints,
   FieldSchema,
   FieldType,
 } from "@budibase/types"
@@ -16,3 +17,12 @@ export function isDeprecatedSingleUserColumn(
     schema.constraints?.type !== "array"
   return result
 }
+
+export function isRequired(constraints: FieldConstraints | undefined) {
+  const isRequired =
+    !!constraints &&
+    ((typeof constraints.presence !== "boolean" &&
+      constraints.presence?.allowEmpty === false) ||
+      constraints.presence === true)
+  return isRequired
+}

packages/string-templates/src/helpers/javascript.ts

@@ -33,7 +33,12 @@ const removeSquareBrackets = (value: string) => {
 // Our context getter function provided to JS code as $.
 // Extracts a value from context.
 const getContextValue = (path: string, context: any) => {
+  const literalStringRegex = /^(["'`]).*\1$/
   let data = context
+  // check if it's a literal string - just return path if its quoted
+  if (literalStringRegex.test(path)) {
+    return path.substring(1, path.length - 1)
+  }
   path.split(".").forEach(key => {
     if (data == null || typeof data !== "object") {
       return null

packages/string-templates/test/javascript.spec.ts

@@ -149,4 +149,11 @@ describe("Javascript", () => {
       expect(output).toMatch(UUID_REGEX)
     })
   })
+
+  describe("JS literal strings", () => {
+    it("should be able to handle a literal string that is quoted (like role IDs)", () => {
+      const output = processJS(`return $("'Custom'")`)
+      expect(output).toBe("Custom")
+    })
+  })
 })