Merge branch 'feat/readonly-columns' into BUDI-8282/dont-treat-display-column-as-required

.github/workflows/force-release.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   ensure-is-master-tag:
     name: Ensure is a master tag
-    runs-on: qa-arc-runner-set
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout monorepo
         uses: actions/checkout@v4

charts/budibase/Chart.yaml

@@ -17,6 +17,6 @@ version: 0.0.0
 appVersion: 0.0.0
 dependencies:
   - name: couchdb
-    version: 4.3.0
+    version: 4.5.3
     repository: https://apache.github.io/couchdb-helm
     condition: services.couchdb.enabled

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.27.5",
+  "version": "2.27.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/middleware/joi-validator.ts

@@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types"
 
 function validate(
   schema: Joi.ObjectSchema | Joi.ArraySchema,
-  property: string
+  property: string,
+  opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` }
 ) {
   // Return a Koa middleware function
   return (ctx: Ctx, next: any) => {
@@ -29,16 +30,26 @@ function validate(
 
     const { error } = schema.validate(params)
     if (error) {
-      ctx.throw(400, `Invalid ${property} - ${error.message}`)
+      let message = error.message
+      if (opts.errorPrefix) {
+        message = `Invalid ${property} - ${message}`
+      }
+      ctx.throw(400, message)
     }
     return next()
   }
 }
 
-export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function body(
-  return validate(schema, "body")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "body", opts)
 }
 
-export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
+export function params(
-  return validate(schema, "params")
+  schema: Joi.ObjectSchema | Joi.ArraySchema,
+  opts?: { errorPrefix: string }
+) {
+  return validate(schema, "params", opts)
 }

packages/backend-core/src/objectStore/objectStore.ts

@@ -14,6 +14,7 @@ import { v4 } from "uuid"
 import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
 import fsp from "fs/promises"
 import { HeadObjectOutput } from "aws-sdk/clients/s3"
+import { ReadableStream } from "stream/web"
 
 const streamPipeline = promisify(stream.pipeline)
 // use this as a temporary store of buckets that are being created
@@ -41,10 +42,7 @@ type UploadParams = BaseUploadParams & {
   path?: string | PathLike
 }
 
-export type StreamTypes =
+export type StreamTypes = ReadStream | NodeJS.ReadableStream
-  | ReadStream
-  | NodeJS.ReadableStream
-  | ReadableStream<Uint8Array>
 
 export type StreamUploadParams = BaseUploadParams & {
   stream?: StreamTypes
@@ -222,6 +220,9 @@ export async function streamUpload({
   extra,
   ttl,
 }: StreamUploadParams) {
+  if (!stream) {
+    throw new Error("Stream to upload is invalid/undefined")
+  }
   const extension = filename.split(".").pop()
   const objectStore = ObjectStore(bucketName)
   const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
@@ -251,14 +252,27 @@ export async function streamUpload({
       : CONTENT_TYPE_MAP.txt
   }
 
+  const bucket = sanitizeBucket(bucketName),
+    objKey = sanitizeKey(filename)
   const params = {
-    Bucket: sanitizeBucket(bucketName),
+    Bucket: bucket,
-    Key: sanitizeKey(filename),
+    Key: objKey,
     Body: stream,
     ContentType: contentType,
     ...extra,
   }
-  return objectStore.upload(params).promise()
+
+  const details = await objectStore.upload(params).promise()
+  const headDetails = await objectStore
+    .headObject({
+      Bucket: bucket,
+      Key: objKey,
+    })
+    .promise()
+  return {
+    ...details,
+    ContentLength: headDetails.ContentLength,
+  }
 }
 
 /**

packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte

@@ -22,11 +22,17 @@
     const visible = permission !== PERMISSION_OPTIONS.HIDDEN
     const readonly = permission === PERMISSION_OPTIONS.READONLY
 
-    datasource.actions.addSchemaMutation(column.name, { visible, readonly })
+    await datasource.actions.addSchemaMutation(column.name, {
+      visible,
+      readonly,
+    })
     try {
       await datasource.actions.saveSchemaMutations()
     } catch (e) {
       notifications.error(e.message)
+    } finally {
+      await datasource.actions.resetSchemaMutations()
+      await datasource.actions.refreshDefinition()
     }
     dispatch(visible ? "show-column" : "hide-column")
   }

packages/frontend-core/src/components/grid/stores/datasource.js

@@ -204,6 +204,10 @@ export const createActions = context => {
       ...$definition,
       schema: newSchema,
     })
+    resetSchemaMutations()
+  }
+
+  const resetSchemaMutations = () => {
     schemaMutations.set({})
   }
 
@@ -253,6 +257,7 @@ export const createActions = context => {
         addSchemaMutation,
         addSchemaMutations,
         saveSchemaMutations,
+        resetSchemaMutations,
       },
     },
   }

packages/server/package.json

@@ -68,7 +68,6 @@
     "aws-sdk": "2.1030.0",
     "bcrypt": "5.1.0",
     "bcryptjs": "2.4.3",
-    "bl": "^6.0.12",
     "bull": "4.10.1",
     "chokidar": "3.5.3",
     "content-disposition": "^0.5.4",
@@ -116,7 +115,8 @@
     "uuid": "^8.3.2",
     "validate.js": "0.13.1",
     "worker-farm": "1.7.0",
-    "xml2js": "0.5.0"
+    "xml2js": "0.5.0",
+    "tmp": "0.2.3"
   },
   "devDependencies": {
     "@babel/preset-env": "7.16.11",
@@ -137,6 +137,7 @@
     "@types/supertest": "2.0.14",
     "@types/tar": "6.1.5",
     "@types/uuid": "8.3.4",
+    "@types/tmp": "0.2.6",
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "jest": "29.7.0",

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -23,9 +23,6 @@ import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
 import { roles } from "@budibase/backend-core"
-import * as schemaUtils from "../../../utilities/schema"
-
-jest.mock("../../../utilities/schema")
 
 describe.each([
   ["internal", undefined],
@@ -120,6 +117,9 @@ describe.each([
       const newView: CreateViewRequest = {
         name: generator.name(),
         tableId: table._id!,
+        schema: {
+          id: { visible: true },
+        },
       }
       const res = await config.api.viewV2.create(newView)
 
@@ -148,6 +148,7 @@ describe.each([
           type: SortType.STRING,
         },
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -158,6 +159,7 @@ describe.each([
       expect(res).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -172,6 +174,11 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+            visible: true,
+          },
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -193,6 +200,7 @@ describe.each([
       expect(createdView).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             order: 1,
@@ -209,6 +217,12 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            autocolumn: true,
+            visible: true,
+          },
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -232,6 +246,7 @@ describe.each([
         tableId: table._id!,
         primaryDisplay: generator.word(),
         schema: {
+          id: { visible: true },
           Price: { visible: true },
           Category: { visible: false },
         },
@@ -241,6 +256,7 @@ describe.each([
       expect(res).toEqual({
         ...newView,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
           },
@@ -255,6 +271,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           nonExisting: {
             visible: true,
           },
@@ -293,6 +310,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: true,
               readonly: true,
@@ -306,6 +324,7 @@ describe.each([
 
         const res = await config.api.viewV2.create(newView)
         expect(res.schema).toEqual({
+          id: { visible: true },
           name: {
             visible: true,
             readonly: true,
@@ -318,15 +337,13 @@ describe.each([
       })
 
       it("required fields cannot be marked as readonly", async () => {
-        const isRequiredSpy = jest.spyOn(schemaUtils, "isRequired")
-        isRequiredSpy.mockReturnValueOnce(true)
-
         const table = await config.api.table.save(
           saveTableRequest({
             schema: {
               name: {
                 name: "name",
                 type: FieldType.STRING,
+                constraints: { presence: true },
               },
               description: {
                 name: "description",
@@ -340,7 +357,9 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
+              visible: true,
               readonly: true,
             },
           },
@@ -350,7 +369,7 @@ describe.each([
           status: 400,
           body: {
             message:
-              'Field "name" cannot be readonly as it is a required field',
+              'You can\'t make "name" readonly because it is a required field.',
             status: 400,
           },
         })
@@ -376,6 +395,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: false,
               readonly: true,
@@ -414,6 +434,7 @@ describe.each([
           name: generator.name(),
           tableId: table._id!,
           schema: {
+            id: { visible: true },
             name: {
               visible: true,
               readonly: true,
@@ -441,6 +462,9 @@ describe.each([
       view = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
     })
 
@@ -489,6 +513,7 @@ describe.each([
           type: SortType.STRING,
         },
         schema: {
+          id: { visible: true },
           Category: {
             visible: false,
           },
@@ -506,7 +531,7 @@ describe.each([
           schema: {
             ...table.schema,
             id: expect.objectContaining({
-              visible: false,
+              visible: true,
             }),
             Category: expect.objectContaining({
               visible: false,
@@ -603,6 +628,9 @@ describe.each([
       const anotherView = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
       const result = await config
         .request!.put(`/api/v2/views/${anotherView.id}`)
@@ -621,6 +649,7 @@ describe.each([
       const updatedView = await config.api.viewV2.update({
         ...view,
         schema: {
+          ...view.schema,
           Price: {
             name: "Price",
             type: FieldType.NUMBER,
@@ -640,6 +669,7 @@ describe.each([
       expect(updatedView).toEqual({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             order: 1,
@@ -656,6 +686,7 @@ describe.each([
         {
           ...view,
           schema: {
+            ...view.schema,
             Price: {
               name: "Price",
               type: FieldType.NUMBER,
@@ -679,6 +710,7 @@ describe.each([
       view = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: true,
@@ -701,6 +733,7 @@ describe.each([
       view = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: true,
@@ -715,6 +748,7 @@ describe.each([
       const res = await config.api.viewV2.update({
         ...view,
         schema: {
+          id: { visible: true },
           Price: {
             visible: true,
             readonly: false,
@@ -725,6 +759,7 @@ describe.each([
         expect.objectContaining({
           ...view,
           schema: {
+            id: { visible: true },
             Price: {
               visible: true,
               readonly: false,
@@ -742,6 +777,9 @@ describe.each([
       view = await config.api.viewV2.create({
         tableId: table._id!,
         name: generator.guid(),
+        schema: {
+          id: { visible: true },
+        },
       })
     })
 
@@ -764,6 +802,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           Price: { visible: false },
           Category: { visible: true },
         },
@@ -786,6 +825,7 @@ describe.each([
         name: generator.name(),
         tableId: table._id!,
         schema: {
+          id: { visible: true },
           Price: { visible: true, readonly: true },
         },
       })
@@ -821,6 +861,7 @@ describe.each([
         tableId: table._id!,
         name: generator.guid(),
         schema: {
+          id: { visible: true },
           Country: {
             visible: true,
           },
@@ -855,6 +896,7 @@ describe.each([
         tableId: table._id!,
         name: generator.guid(),
         schema: {
+          id: { visible: true },
           two: { visible: true },
         },
       })
@@ -880,6 +922,7 @@ describe.each([
           tableId: table._id!,
           name: generator.guid(),
           schema: {
+            id: { visible: true },
             one: { visible: true, readonly: true },
             two: { visible: true },
           },
@@ -921,6 +964,7 @@ describe.each([
           tableId: table._id!,
           name: generator.guid(),
           schema: {
+            id: { visible: true },
             one: { visible: true, readonly: true },
             two: { visible: true },
           },
@@ -988,6 +1032,7 @@ describe.each([
             rows.map(r => ({
               _viewId: view.id,
               tableId: table._id,
+              id: r.id,
               _id: r._id,
               _rev: r._rev,
               ...(isInternal
@@ -1028,6 +1073,7 @@ describe.each([
             },
           ],
           schema: {
+            id: { visible: true },
             two: { visible: true },
           },
         })
@@ -1039,6 +1085,7 @@ describe.each([
             {
               _viewId: view.id,
               tableId: table._id,
+              id: two.id,
               two: two.two,
               _id: two._id,
               _rev: two._rev,
@@ -1192,7 +1239,11 @@ describe.each([
 
       describe("sorting", () => {
         let table: Table
-        const viewSchema = { age: { visible: true }, name: { visible: true } }
+        const viewSchema = {
+          id: { visible: true },
+          age: { visible: true },
+          name: { visible: true },
+        }
 
         beforeAll(async () => {
           table = await config.api.table.save(
@@ -1348,4 +1399,123 @@ describe.each([
       })
     })
   })
+
+  describe("updating table schema", () => {
+    describe("existing columns changed to required", () => {
+      beforeEach(async () => {
+        table = await config.api.table.save(
+          saveTableRequest({
+            schema: {
+              id: {
+                name: "id",
+                type: FieldType.AUTO,
+                autocolumn: true,
+              },
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+              },
+            },
+          })
+        )
+      })
+
+      it("allows updating when no views constrains the field", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: { visible: true },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: { allowEmpty: false } },
+              },
+            },
+          },
+          { status: 200 }
+        )
+      })
+
+      it("rejects if field is readonly in any view", async () => {
+        mocks.licenses.useViewReadonlyColumns()
+
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: {
+            id: { visible: true },
+            name: {
+              visible: true,
+              readonly: true,
+            },
+          },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+
+      it("rejects if field is hidden in any view", async () => {
+        await config.api.viewV2.create({
+          name: "view a",
+          tableId: table._id!,
+          schema: { id: { visible: true } },
+        })
+
+        table = await config.api.table.get(table._id!)
+        await config.api.table.save(
+          {
+            ...table,
+            schema: {
+              ...table.schema,
+              name: {
+                name: "name",
+                type: FieldType.STRING,
+                constraints: { presence: true },
+              },
+            },
+          },
+          {
+            status: 400,
+            body: {
+              status: 400,
+              message:
+                'To make field "name" required, this field must be present and writable in views: view a.',
+            },
+          }
+        )
+      })
+    })
+  })
 })

packages/server/src/api/routes/utils/validators.ts

@@ -1,51 +1,88 @@
 import { auth, permissions } from "@budibase/backend-core"
 import { DataSourceOperation } from "../../../constants"
-import { WebhookActionType } from "@budibase/types"
+import { Table, WebhookActionType } from "@budibase/types"
-import Joi from "joi"
+import Joi, { CustomValidator } from "joi"
 import { ValidSnippetNameRegex } from "@budibase/shared-core"
+import { isRequired } from "../../../utilities/schema"
+import sdk from "../../../sdk"
 
 const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
 const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
 const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
 const APP_NAME_REGEX = /^[\w\s]+$/
 
+const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
+  if (table.views && Object.entries(table.views).length) {
+    const requiredFields = Object.entries(table.schema)
+      .filter(([_, v]) => isRequired(v.constraints))
+      .map(([key]) => key)
+    if (requiredFields.length) {
+      for (const view of Object.values(table.views)) {
+        if (!sdk.views.isV2(view)) {
+          continue
+        }
+
+        const editableViewFields = Object.entries(view.schema || {})
+          .filter(([_, f]) => f.visible && !f.readonly)
+          .map(([key]) => key)
+        const missingField = requiredFields.find(
+          f => !editableViewFields.includes(f)
+        )
+        if (missingField) {
+          return helpers.message({
+            custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
+          })
+        }
+      }
+    }
+  }
+  return table
+}
+
 export function tableValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    type: OPTIONAL_STRING.valid("table", "internal", "external"),
+      type: OPTIONAL_STRING.valid("table", "internal", "external"),
-    primaryDisplay: OPTIONAL_STRING,
+      primaryDisplay: OPTIONAL_STRING,
-    schema: Joi.object().required(),
+      schema: Joi.object().required(),
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    views: Joi.object(),
+      views: Joi.object(),
-    rows: Joi.array(),
+      rows: Joi.array(),
-  }).unknown(true))
+    })
+      .custom(validateViewSchemas)
+      .unknown(true),
+    { errorPrefix: "" }
+  )
 }
 
 export function nameValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: OPTIONAL_STRING,
+      name: OPTIONAL_STRING,
-  }))
+    })
+  )
 }
 
 export function datasourceValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: Joi.string(),
+      _id: Joi.string(),
-    _rev: Joi.string(),
+      _rev: Joi.string(),
-    type: OPTIONAL_STRING.allow("datasource_plus"),
+      type: OPTIONAL_STRING.allow("datasource_plus"),
-    relationships: Joi.array().items(Joi.object({
+      relationships: Joi.array().items(
-      from: Joi.string().required(),
+        Joi.object({
-      to: Joi.string().required(),
+          from: Joi.string().required(),
-      cardinality: Joi.valid("1:N", "1:1", "N:N").required()
+          to: Joi.string().required(),
-    })),
+          cardinality: Joi.valid("1:N", "1:1", "N:N").required(),
-  }).unknown(true))
+        })
+      ),
+    }).unknown(true)
+  )
 }
 
 function filterObject() {
-  // prettier-ignore
   return Joi.object({
     string: Joi.object().optional(),
     fuzzy: Joi.object().optional(),
@@ -62,17 +99,20 @@ function filterObject() {
 }
 
 export function internalSearchValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    tableId: OPTIONAL_STRING,
+      tableId: OPTIONAL_STRING,
-    query: filterObject(),
+      query: filterObject(),
-    limit: OPTIONAL_NUMBER,
+      limit: OPTIONAL_NUMBER,
-    sort: OPTIONAL_STRING,
+      sort: OPTIONAL_STRING,
-    sortOrder: OPTIONAL_STRING,
+      sortOrder: OPTIONAL_STRING,
-    sortType: OPTIONAL_STRING,
+      sortType: OPTIONAL_STRING,
-    paginate: Joi.boolean(),
+      paginate: Joi.boolean(),
-    bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
+      bookmark: Joi.alternatives()
-  }))
+        .try(OPTIONAL_STRING, OPTIONAL_NUMBER)
+        .optional(),
+    })
+  )
 }
 
 export function externalSearchValidator() {
@@ -94,92 +134,110 @@ export function externalSearchValidator() {
 }
 
 export function datasourceQueryValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    endpoint: Joi.object({
+      endpoint: Joi.object({
-      datasourceId: Joi.string().required(),
+        datasourceId: Joi.string().required(),
-      operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
+        operation: Joi.string()
-      entityId: Joi.string().required(),
+          .required()
-    }).required(),
+          .valid(...Object.values(DataSourceOperation)),
-    resource: Joi.object({
+        entityId: Joi.string().required(),
-      fields: Joi.array().items(Joi.string()).optional(),
+      }).required(),
-    }).optional(),
+      resource: Joi.object({
-    body: Joi.object().optional(),
+        fields: Joi.array().items(Joi.string()).optional(),
-    sort: Joi.object().optional(),
+      }).optional(),
-    filters: filterObject().optional(),
+      body: Joi.object().optional(),
-    paginate: Joi.object({
+      sort: Joi.object().optional(),
-      page: Joi.string().alphanum().optional(),
+      filters: filterObject().optional(),
-      limit: Joi.number().optional(),
+      paginate: Joi.object({
-    }).optional(),
+        page: Joi.string().alphanum().optional(),
-  }))
+        limit: Joi.number().optional(),
+      }).optional(),
+    })
+  )
 }
 
 export function webhookValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    live: Joi.bool(),
+      live: Joi.bool(),
-    _id: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    _rev: OPTIONAL_STRING,
+      _rev: OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    bodySchema: Joi.object().optional(),
+      bodySchema: Joi.object().optional(),
-    action: Joi.object({
+      action: Joi.object({
-      type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
+        type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
-      target: Joi.string().required(),
+        target: Joi.string().required(),
-    }).required(),
+      }).required(),
-  }).unknown(true))
+    }).unknown(true)
+  )
 }
 
 export function roleValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.body(Joi.object({
+  return auth.joiValidator.body(
-    _id: OPTIONAL_STRING,
+    Joi.object({
-    _rev: OPTIONAL_STRING,
+      _id: OPTIONAL_STRING,
-    name: Joi.string().regex(/^[a-zA-Z0-9_]*$/).required(),
+      _rev: OPTIONAL_STRING,
-    // this is the base permission ID (for now a built in)
+      name: Joi.string()
-    permissionId: Joi.string().valid(...Object.values(permissions.BuiltinPermissionID)).required(),
+        .regex(/^[a-zA-Z0-9_]*$/)
-    permissions: Joi.object()
+        .required(),
-      .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+      // this is the base permission ID (for now a built in)
-      .optional(),
+      permissionId: Joi.string()
-    inherits: OPTIONAL_STRING,
+        .valid(...Object.values(permissions.BuiltinPermissionID))
-  }).unknown(true))
+        .required(),
+      permissions: Joi.object()
+        .pattern(/.*/, [Joi.string().valid(...permLevelArray)])
+        .optional(),
+      inherits: OPTIONAL_STRING,
+    }).unknown(true)
+  )
 }
 
 export function permissionValidator() {
   const permLevelArray = Object.values(permissions.PermissionLevel)
-  // prettier-ignore
+
-  return auth.joiValidator.params(Joi.object({
+  return auth.joiValidator.params(
-    level: Joi.string().valid(...permLevelArray).required(),
+    Joi.object({
-    resourceId: Joi.string(),
+      level: Joi.string()
-    roleId: Joi.string(),
+        .valid(...permLevelArray)
-  }).unknown(true))
+        .required(),
+      resourceId: Joi.string(),
+      roleId: Joi.string(),
+    }).unknown(true)
+  )
 }
 
 export function screenValidator() {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    showNavigation: OPTIONAL_BOOLEAN,
+      showNavigation: OPTIONAL_BOOLEAN,
-    width: OPTIONAL_STRING,
+      width: OPTIONAL_STRING,
-    routing: Joi.object({
+      routing: Joi.object({
-      route: Joi.string().required(),
+        route: Joi.string().required(),
-      roleId: Joi.string().required().allow(""),
+        roleId: Joi.string().required().allow(""),
-      homeScreen: OPTIONAL_BOOLEAN,
+        homeScreen: OPTIONAL_BOOLEAN,
-    }).required().unknown(true),
+      })
-    props: Joi.object({
+        .required()
-      _id: Joi.string().required(),
+        .unknown(true),
-      _component: Joi.string().required(),
+      props: Joi.object({
-      _children: Joi.array().required(),
+        _id: Joi.string().required(),
-      _styles: Joi.object().required(),
+        _component: Joi.string().required(),
-      type: OPTIONAL_STRING,
+        _children: Joi.array().required(),
-      table: OPTIONAL_STRING,
+        _styles: Joi.object().required(),
-      layoutId: OPTIONAL_STRING,
+        type: OPTIONAL_STRING,
-    }).required().unknown(true),
+        table: OPTIONAL_STRING,
-  }).unknown(true))
+        layoutId: OPTIONAL_STRING,
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 function generateStepSchema(allowStepTypes: string[]) {
-  // prettier-ignore
   return Joi.object({
     stepId: Joi.string().required(),
     id: Joi.string().required(),
@@ -189,33 +247,39 @@ function generateStepSchema(allowStepTypes: string[]) {
     icon: Joi.string().required(),
     params: Joi.object(),
     args: Joi.object(),
-    type: Joi.string().required().valid(...allowStepTypes),
+    type: Joi.string()
+      .required()
+      .valid(...allowStepTypes),
   }).unknown(true)
 }
 
 export function automationValidator(existing = false) {
-  // prettier-ignore
+  return auth.joiValidator.body(
-  return auth.joiValidator.body(Joi.object({
+    Joi.object({
-    _id: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _id: existing ? Joi.string().required() : OPTIONAL_STRING,
-    _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
+      _rev: existing ? Joi.string().required() : OPTIONAL_STRING,
-    name: Joi.string().required(),
+      name: Joi.string().required(),
-    type: Joi.string().valid("automation").required(),
+      type: Joi.string().valid("automation").required(),
-    definition: Joi.object({
+      definition: Joi.object({
-      steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
+        steps: Joi.array()
-      trigger: generateStepSchema(["TRIGGER"]).allow(null),
+          .required()
-    }).required().unknown(true),
+          .items(generateStepSchema(["ACTION", "LOGIC"])),
-  }).unknown(true))
+        trigger: generateStepSchema(["TRIGGER"]).allow(null),
+      })
+        .required()
+        .unknown(true),
+    }).unknown(true)
+  )
 }
 
 export function applicationValidator(opts = { isCreate: true }) {
-  // prettier-ignore
   const base: any = {
     _id: OPTIONAL_STRING,
     _rev: OPTIONAL_STRING,
     url: OPTIONAL_STRING,
     template: Joi.object({
       templateString: OPTIONAL_STRING,
-    })
+    }),
   }
 
   const appNameValidator = Joi.string()

packages/server/src/integrations/rest.ts

@@ -149,13 +149,12 @@ class RestIntegration implements IntegrationBase {
       { downloadImages: this.config.downloadImages }
     )
     let contentLength = response.headers.get("content-length")
-    if (!contentLength && raw) {
+    let isSuccess = response.status >= 200 && response.status < 300
-      contentLength = Buffer.byteLength(raw, "utf8").toString()
-    }
     if (
-      contentDisposition.includes("filename") ||
+      (contentDisposition.includes("filename") ||
-      contentDisposition.includes("attachment") ||
+        contentDisposition.includes("attachment") ||
-      contentDisposition.includes("form-data")
+        contentDisposition.includes("form-data")) &&
+      isSuccess
     ) {
       filename =
         path.basename(parse(contentDisposition).parameters?.filename) || ""
@@ -168,6 +167,9 @@ class RestIntegration implements IntegrationBase {
         return handleFileResponse(response, filename, this.startTimeMs)
       } else {
         responseTxt = response.text ? await response.text() : ""
+        if (!contentLength && responseTxt) {
+          contentLength = Buffer.byteLength(responseTxt, "utf8").toString()
+        }
         const hasContent =
           (contentLength && parseInt(contentLength) > 0) ||
           responseTxt.length > 0

packages/server/src/integrations/tests/rest.spec.ts

@@ -657,6 +657,7 @@ describe("REST Integration", () => {
       mockReadable.push(null)
       ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
         Promise.resolve({
+          status: 200,
           headers: {
             raw: () => ({
               "content-type": [contentType],
@@ -700,6 +701,7 @@ describe("REST Integration", () => {
       mockReadable.push(null)
       ;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
         Promise.resolve({
+          status: 200,
           headers: {
             raw: () => ({
               "content-type": [contentType],

packages/server/src/integrations/utils/utils.ts

@@ -9,10 +9,12 @@ import { context, objectStore, sql } from "@budibase/backend-core"
 import { v4 } from "uuid"
 import { parseStringPromise as xmlParser } from "xml2js"
 import { formatBytes } from "../../utilities"
-import bl from "bl"
 import env from "../../environment"
 import { InvalidColumns } from "../../constants"
 import { helpers, utils } from "@budibase/shared-core"
+import { pipeline } from "stream/promises"
+import tmp from "tmp"
+import fs from "fs"
 
 type PrimitiveTypes =
   | FieldType.STRING
@@ -360,35 +362,44 @@ export async function handleFileResponse(
   const key = `${context.getProdAppId()}/${processedFileName}`
   const bucket = objectStore.ObjectStoreBuckets.TEMP
 
-  const stream = response.body.pipe(bl((error, data) => data))
+  // put the response stream to disk temporarily as a buffer
+  const tmpObj = tmp.fileSync()
+  try {
+    await pipeline(response.body, fs.createWriteStream(tmpObj.name))
+    if (response.body) {
+      const contentLength = response.headers.get("content-length")
+      if (contentLength) {
+        size = parseInt(contentLength, 10)
+      }
 
-  if (response.body) {
+      const details = await objectStore.streamUpload({
-    const contentLength = response.headers.get("content-length")
+        bucket,
-    if (contentLength) {
+        filename: key,
-      size = parseInt(contentLength, 10)
+        stream: fs.createReadStream(tmpObj.name),
+        ttl: 1,
+        type: response.headers["content-type"],
+      })
+      if (!size && details.ContentLength) {
+        size = details.ContentLength
+      }
     }
-
+    presignedUrl = objectStore.getPresignedUrl(bucket, key)
-    await objectStore.streamUpload({
+    return {
-      bucket,
+      data: {
-      filename: key,
+        size,
-      stream,
+        name: processedFileName,
-      ttl: 1,
+        url: presignedUrl,
-      type: response.headers["content-type"],
+        extension: fileExtension,
-    })
+        key: key,
-  }
+      },
-  presignedUrl = objectStore.getPresignedUrl(bucket, key)
+      info: {
-  return {
+        code: response.status,
-    data: {
+        size: formatBytes(size.toString()),
-      size,
+        time: `${Math.round(performance.now() - startTime)}ms`,
-      name: processedFileName,
+      },
-      url: presignedUrl,
+    }
-      extension: fileExtension,
+  } finally {
-      key: key,
+    // cleanup tmp
-    },
+    tmpObj.removeCallback()
-    info: {
-      code: response.status,
-      size: formatBytes(size.toString()),
-      time: `${Math.round(performance.now() - startTime)}ms`,
-    },
   }
 }

packages/server/src/sdk/app/views/index.ts

@@ -39,9 +39,7 @@ async function guardViewSchema(
   tableId: string,
   viewSchema?: Record<string, ViewUIFieldMetadata>
 ) {
-  if (!viewSchema || !Object.keys(viewSchema).length) {
+  viewSchema ??= {}
-    return
-  }
   const table = await sdk.tables.getTable(tableId)
 
   for (const field of Object.keys(viewSchema)) {
@@ -54,17 +52,13 @@ async function guardViewSchema(
     }
 
     if (viewSchema[field].readonly) {
-      if (!(await features.isViewReadonlyColumnsEnabled())) {
+      if (
+        !(await features.isViewReadonlyColumnsEnabled()) &&
+        !(tableSchemaField as ViewUIFieldMetadata).readonly
+      ) {
         throw new HTTPError(`Readonly fields are not enabled`, 400)
       }
 
-      if (helpers.schema.isRequired(tableSchemaField.constraints)) {
-        throw new HTTPError(
-          `Field "${field}" cannot be readonly as it is a required field`,
-          400
-        )
-      }
-
       if (!viewSchema[field].visible) {
         throw new HTTPError(
           `Field "${field}" must be visible if you want to make it readonly`,
@@ -73,6 +67,28 @@ async function guardViewSchema(
       }
     }
   }
+
+  for (const field of Object.values(table.schema)) {
+    if (!helpers.schema.isRequired(field.constraints)) {
+      continue
+    }
+
+    const viewSchemaField = viewSchema[field.name]
+
+    if (!viewSchemaField?.visible) {
+      throw new HTTPError(
+        `You can't hide "${field.name} because it is a required field."`,
+        400
+      )
+    }
+
+    if (viewSchemaField.readonly) {
+      throw new HTTPError(
+        `You can't make "${field.name}" readonly because it is a required field.`,
+        400
+      )
+    }
+  }
 }
 
 export async function create(

packages/server/src/threads/query.ts

@@ -196,12 +196,22 @@ class QueryRunner {
     return { rows, keys, info, extra, pagination }
   }
 
-  async runAnotherQuery(queryId: string, parameters: any) {
+  async runAnotherQuery(
+    queryId: string,
+    currentParameters: Record<string, any>
+  ) {
     const db = context.getAppDB()
     const query = await db.get<Query>(queryId)
     const datasource = await sdk.datasources.get(query.datasourceId, {
       enriched: true,
     })
+    // enrich parameters with dynamic queries defaults
+    const defaultParams = query.parameters || []
+    for (let param of defaultParams) {
+      if (!currentParameters[param.name]) {
+        currentParameters[param.name] = param.default
+      }
+    }
     return new QueryRunner(
       {
         schema: query.schema,
@@ -210,7 +220,7 @@ class QueryRunner {
         transformer: query.transformer,
         nullDefaultSupport: query.nullDefaultSupport,
         ctx: this.ctx,
-        parameters,
+        parameters: currentParameters,
         datasource,
         queryId,
       },

packages/types/src/documents/app/automation.ts

@@ -245,7 +245,7 @@ export type AutomationAttachment = {
 
 export type AutomationAttachmentContent = {
   filename: string
-  content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
+  content: ReadStream | NodeJS.ReadableStream
 }
 
 export type BucketedContent = AutomationAttachmentContent & {

yarn.lock

@@ -6348,6 +6348,11 @@
   dependencies:
     "@types/estree" "*"
 
+"@types/tmp@0.2.6":
+  version "0.2.6"
+  resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.6.tgz#d785ee90c52d7cc020e249c948c36f7b32d1e217"
+  integrity sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==
+
 "@types/tough-cookie@*", "@types/tough-cookie@^4.0.2":
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
@@ -7700,7 +7705,7 @@ bl@^4.0.3, bl@^4.1.0:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
-bl@^6.0.12, bl@^6.0.3:
+bl@^6.0.3:
   version "6.0.12"
   resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
   integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
@@ -16065,10 +16070,10 @@ mute-stream@~1.0.0:
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e"
   integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==
 
-mysql2@3.9.7:
+mysql2@3.9.8:
-  version "3.9.7"
+  version "3.9.8"
-  resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.7.tgz#843755daf65b5ef08afe545fe14b8fb62824741a"
+  resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.8.tgz#fe8a0f975f2c495ed76ca988ddc5505801dc49ce"
-  integrity sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==
+  integrity sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==
   dependencies:
     denque "^2.1.0"
     generate-function "^2.3.1"
@@ -21283,6 +21288,11 @@ tlhunter-sorted-set@^0.1.0:
   resolved "https://registry.yarnpkg.com/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz#1c3eae28c0fa4dff97e9501d2e3c204b86406f4b"
   integrity sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw==
 
+tmp@0.2.3:
+  version "0.2.3"
+  resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae"
+  integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==
+
 tmp@^0.0.33:
   version "0.0.33"
   resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"