MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/readonly-columns' into BUDI-8282/dont-treat-display-column-as-required

This commit is contained in:
Adria Navarro 2024-06-04 12:09:20 +02:00
parent 10f77c83b6 a562655361
commit ba9b5c3271
17 changed files with 527 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/force-release.yml vendored
View File

@ -9,7 +9,7 @@ on:
jobs:
ensure-is-master-tag:
name: Ensure is a master tag
runs-on: qa-arc-runner-set
runs-on: ubuntu-latest
steps:
- name: Checkout monorepo
uses: actions/checkout@v4

2
charts/budibase/Chart.yaml
View File

@ -17,6 +17,6 @@ version: 0.0.0
appVersion: 0.0.0
dependencies:
- name: couchdb
version: 4.3.0
version: 4.5.3
repository: https://apache.github.io/couchdb-helm
condition: services.couchdb.enabled

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.27.5",
"version": "2.27.6",
"npmClient": "yarn",
"packages": [
"packages/*",

23
packages/backend-core/src/middleware/joi-validator.ts
View File

@ -3,7 +3,8 @@ import { Ctx } from "@budibase/types"
function validate(
schema: Joi.ObjectSchema | Joi.ArraySchema,
property: string
property: string,
opts: { errorPrefix: string } = { errorPrefix: `Invalid ${property}` }
) {
// Return a Koa middleware function
return (ctx: Ctx, next: any) => {
@ -29,16 +30,26 @@ function validate(
const { error } = schema.validate(params)
if (error) {
ctx.throw(400, `Invalid ${property} - ${error.message}`)
let message = error.message
if (opts.errorPrefix) {
message = `Invalid ${property} - ${message}`
}
ctx.throw(400, message)
}
return next()
}
}
export function body(schema: Joi.ObjectSchema | Joi.ArraySchema) {
return validate(schema, "body")
export function body(
schema: Joi.ObjectSchema | Joi.ArraySchema,
opts?: { errorPrefix: string }
) {
return validate(schema, "body", opts)
}
export function params(schema: Joi.ObjectSchema | Joi.ArraySchema) {
return validate(schema, "params")
export function params(
schema: Joi.ObjectSchema | Joi.ArraySchema,
opts?: { errorPrefix: string }
) {
return validate(schema, "params", opts)
}

28
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -14,6 +14,7 @@ import { v4 } from "uuid"
import { APP_PREFIX, APP_DEV_PREFIX } from "../db"
import fsp from "fs/promises"
import { HeadObjectOutput } from "aws-sdk/clients/s3"
import { ReadableStream } from "stream/web"
const streamPipeline = promisify(stream.pipeline)
// use this as a temporary store of buckets that are being created
@ -41,10 +42,7 @@ type UploadParams = BaseUploadParams & {
path?: string | PathLike
}
export type StreamTypes =
| ReadStream
| NodeJS.ReadableStream
| ReadableStream<Uint8Array>
export type StreamTypes = ReadStream | NodeJS.ReadableStream
export type StreamUploadParams = BaseUploadParams & {
stream?: StreamTypes
@ -222,6 +220,9 @@ export async function streamUpload({
extra,
ttl,
}: StreamUploadParams) {
if (!stream) {
throw new Error("Stream to upload is invalid/undefined")
}
const extension = filename.split(".").pop()
const objectStore = ObjectStore(bucketName)
const bucketCreated = await createBucketIfNotExists(objectStore, bucketName)
@ -251,14 +252,27 @@ export async function streamUpload({
: CONTENT_TYPE_MAP.txt
}
const bucket = sanitizeBucket(bucketName),
objKey = sanitizeKey(filename)
const params = {
Bucket: sanitizeBucket(bucketName),
Key: sanitizeKey(filename),
Bucket: bucket,
Key: objKey,
Body: stream,
ContentType: contentType,
...extra,
}
return objectStore.upload(params).promise()
const details = await objectStore.upload(params).promise()
const headDetails = await objectStore
.headObject({
Bucket: bucket,
Key: objKey,
})
.promise()
return {
...details,
ContentLength: headDetails.ContentLength,
}
}
/**

8
packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
View File

@ -22,11 +22,17 @@
const visible = permission !== PERMISSION_OPTIONS.HIDDEN
const readonly = permission === PERMISSION_OPTIONS.READONLY
datasource.actions.addSchemaMutation(column.name, { visible, readonly })
await datasource.actions.addSchemaMutation(column.name, {
visible,
readonly,
})
try {
await datasource.actions.saveSchemaMutations()
} catch (e) {
notifications.error(e.message)
} finally {
await datasource.actions.resetSchemaMutations()
await datasource.actions.refreshDefinition()
}
dispatch(visible ? "show-column" : "hide-column")
}

5
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -204,6 +204,10 @@ export const createActions = context => {
...$definition,
schema: newSchema,
})
resetSchemaMutations()
}
const resetSchemaMutations = () => {
schemaMutations.set({})
}
@ -253,6 +257,7 @@ export const createActions = context => {
addSchemaMutation,
addSchemaMutations,
saveSchemaMutations,
resetSchemaMutations,
},
},
}

5
packages/server/package.json
View File

@ -68,7 +68,6 @@
"aws-sdk": "2.1030.0",
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bl": "^6.0.12",
"bull": "4.10.1",
"chokidar": "3.5.3",
"content-disposition": "^0.5.4",
@ -116,7 +115,8 @@
"uuid": "^8.3.2",
"validate.js": "0.13.1",
"worker-farm": "1.7.0",
"xml2js": "0.5.0"
"xml2js": "0.5.0",
"tmp": "0.2.3"
},
"devDependencies": {
"@babel/preset-env": "7.16.11",
@ -137,6 +137,7 @@
"@types/supertest": "2.0.14",
"@types/tar": "6.1.5",
"@types/uuid": "8.3.4",
"@types/tmp": "0.2.6",
"copyfiles": "2.4.1",
"docker-compose": "0.23.17",
"jest": "29.7.0",

188
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -23,9 +23,6 @@ import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import merge from "lodash/merge"
import { quotas } from "@budibase/pro"
import { roles } from "@budibase/backend-core"
import * as schemaUtils from "../../../utilities/schema"
jest.mock("../../../utilities/schema")
describe.each([
["internal", undefined],
@ -120,6 +117,9 @@ describe.each([
const newView: CreateViewRequest = {
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
},
}
const res = await config.api.viewV2.create(newView)
@ -148,6 +148,7 @@ describe.each([
type: SortType.STRING,
},
schema: {
id: { visible: true },
Price: {
visible: true,
},
@ -158,6 +159,7 @@ describe.each([
expect(res).toEqual({
...newView,
schema: {
id: { visible: true },
Price: {
visible: true,
},
@ -172,6 +174,11 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: {
name: "id",
type: FieldType.NUMBER,
visible: true,
},
Price: {
name: "Price",
type: FieldType.NUMBER,
@ -193,6 +200,7 @@ describe.each([
expect(createdView).toEqual({
...newView,
schema: {
id: { visible: true },
Price: {
visible: true,
order: 1,
@ -209,6 +217,12 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: {
name: "id",
type: FieldType.AUTO,
autocolumn: true,
visible: true,
},
Price: {
name: "Price",
type: FieldType.NUMBER,
@ -232,6 +246,7 @@ describe.each([
tableId: table._id!,
primaryDisplay: generator.word(),
schema: {
id: { visible: true },
Price: { visible: true },
Category: { visible: false },
},
@ -241,6 +256,7 @@ describe.each([
expect(res).toEqual({
...newView,
schema: {
id: { visible: true },
Price: {
visible: true,
},
@ -255,6 +271,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
nonExisting: {
visible: true,
},
@ -293,6 +310,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
name: {
visible: true,
readonly: true,
@ -306,6 +324,7 @@ describe.each([
const res = await config.api.viewV2.create(newView)
expect(res.schema).toEqual({
id: { visible: true },
name: {
visible: true,
readonly: true,
@ -318,15 +337,13 @@ describe.each([
})
it("required fields cannot be marked as readonly", async () => {
const isRequiredSpy = jest.spyOn(schemaUtils, "isRequired")
isRequiredSpy.mockReturnValueOnce(true)
const table = await config.api.table.save(
saveTableRequest({
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: { presence: true },
},
description: {
name: "description",
@ -340,7 +357,9 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
name: {
visible: true,
readonly: true,
},
},
@ -350,7 +369,7 @@ describe.each([
status: 400,
body: {
message:
'Field "name" cannot be readonly as it is a required field',
'You can\'t make "name" readonly because it is a required field.',
status: 400,
},
})
@ -376,6 +395,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
name: {
visible: false,
readonly: true,
@ -414,6 +434,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
name: {
visible: true,
readonly: true,
@ -441,6 +462,9 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
},
})
})
@ -489,6 +513,7 @@ describe.each([
type: SortType.STRING,
},
schema: {
id: { visible: true },
Category: {
visible: false,
},
@ -506,7 +531,7 @@ describe.each([
schema: {
...table.schema,
id: expect.objectContaining({
visible: false,
visible: true,
}),
Category: expect.objectContaining({
visible: false,
@ -603,6 +628,9 @@ describe.each([
const anotherView = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
},
})
const result = await config
.request!.put(`/api/v2/views/${anotherView.id}`)
@ -621,6 +649,7 @@ describe.each([
const updatedView = await config.api.viewV2.update({
...view,
schema: {
...view.schema,
Price: {
name: "Price",
type: FieldType.NUMBER,
@ -640,6 +669,7 @@ describe.each([
expect(updatedView).toEqual({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
order: 1,
@ -656,6 +686,7 @@ describe.each([
{
...view,
schema: {
...view.schema,
Price: {
name: "Price",
type: FieldType.NUMBER,
@ -679,6 +710,7 @@ describe.each([
view = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: true,
@ -701,6 +733,7 @@ describe.each([
view = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: true,
@ -715,6 +748,7 @@ describe.each([
const res = await config.api.viewV2.update({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: false,
@ -725,6 +759,7 @@ describe.each([
expect.objectContaining({
...view,
schema: {
id: { visible: true },
Price: {
visible: true,
readonly: false,
@ -742,6 +777,9 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
},
})
})
@ -764,6 +802,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
Price: { visible: false },
Category: { visible: true },
},
@ -786,6 +825,7 @@ describe.each([
name: generator.name(),
tableId: table._id!,
schema: {
id: { visible: true },
Price: { visible: true, readonly: true },
},
})
@ -821,6 +861,7 @@ describe.each([
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
Country: {
visible: true,
},
@ -855,6 +896,7 @@ describe.each([
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
two: { visible: true },
},
})
@ -880,6 +922,7 @@ describe.each([
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
one: { visible: true, readonly: true },
two: { visible: true },
},
@ -921,6 +964,7 @@ describe.each([
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
one: { visible: true, readonly: true },
two: { visible: true },
},
@ -988,6 +1032,7 @@ describe.each([
rows.map(r => ({
_viewId: view.id,
tableId: table._id,
id: r.id,
_id: r._id,
_rev: r._rev,
...(isInternal
@ -1028,6 +1073,7 @@ describe.each([
},
],
schema: {
id: { visible: true },
two: { visible: true },
},
})
@ -1039,6 +1085,7 @@ describe.each([
{
_viewId: view.id,
tableId: table._id,
id: two.id,
two: two.two,
_id: two._id,
_rev: two._rev,
@ -1192,7 +1239,11 @@ describe.each([
describe("sorting", () => {
let table: Table
const viewSchema = { age: { visible: true }, name: { visible: true } }
const viewSchema = {
id: { visible: true },
age: { visible: true },
name: { visible: true },
}
beforeAll(async () => {
table = await config.api.table.save(
@ -1348,4 +1399,123 @@ describe.each([
})
})
})
describe("updating table schema", () => {
describe("existing columns changed to required", () => {
beforeEach(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
id: {
name: "id",
type: FieldType.AUTO,
autocolumn: true,
},
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
})
it("allows updating when no views constrains the field", async () => {
await config.api.viewV2.create({
name: "view a",
tableId: table._id!,
schema: {
id: { visible: true },
name: { visible: true },
},
})
table = await config.api.table.get(table._id!)
await config.api.table.save(
{
...table,
schema: {
...table.schema,
name: {
name: "name",
type: FieldType.STRING,
constraints: { presence: { allowEmpty: false } },
},
},
},
{ status: 200 }
)
})
it("rejects if field is readonly in any view", async () => {
mocks.licenses.useViewReadonlyColumns()
await config.api.viewV2.create({
name: "view a",
tableId: table._id!,
schema: {
id: { visible: true },
name: {
visible: true,
readonly: true,
},
},
})
table = await config.api.table.get(table._id!)
await config.api.table.save(
{
...table,
schema: {
...table.schema,
name: {
name: "name",
type: FieldType.STRING,
constraints: { presence: true },
},
},
},
{
status: 400,
body: {
status: 400,
message:
'To make field "name" required, this field must be present and writable in views: view a.',
},
}
)
})
it("rejects if field is hidden in any view", async () => {
await config.api.viewV2.create({
name: "view a",
tableId: table._id!,
schema: { id: { visible: true } },
})
table = await config.api.table.get(table._id!)
await config.api.table.save(
{
...table,
schema: {
...table.schema,
name: {
name: "name",
type: FieldType.STRING,
constraints: { presence: true },
},
},
},
{
status: 400,
body: {
status: 400,
message:
'To make field "name" required, this field must be present and writable in views: view a.',
},
}
)
})
})
})
})

310
packages/server/src/api/routes/utils/validators.ts
View File

@ -1,51 +1,88 @@
import { auth, permissions } from "@budibase/backend-core"
import { DataSourceOperation } from "../../../constants"
import { WebhookActionType } from "@budibase/types"
import Joi from "joi"
import { Table, WebhookActionType } from "@budibase/types"
import Joi, { CustomValidator } from "joi"
import { ValidSnippetNameRegex } from "@budibase/shared-core"
import { isRequired } from "../../../utilities/schema"
import sdk from "../../../sdk"
const OPTIONAL_STRING = Joi.string().optional().allow(null).allow("")
const OPTIONAL_NUMBER = Joi.number().optional().allow(null)
const OPTIONAL_BOOLEAN = Joi.boolean().optional().allow(null)
const APP_NAME_REGEX = /^[\w\s]+$/
const validateViewSchemas: CustomValidator<Table> = (table, helpers) => {
if (table.views && Object.entries(table.views).length) {
const requiredFields = Object.entries(table.schema)
.filter(([_, v]) => isRequired(v.constraints))
.map(([key]) => key)
if (requiredFields.length) {
for (const view of Object.values(table.views)) {
if (!sdk.views.isV2(view)) {
continue
}
const editableViewFields = Object.entries(view.schema || {})
.filter(([_, f]) => f.visible && !f.readonly)
.map(([key]) => key)
const missingField = requiredFields.find(
f => !editableViewFields.includes(f)
)
if (missingField) {
return helpers.message({
custom: `To make field "${missingField}" required, this field must be present and writable in views: ${view.name}.`,
})
}
}
}
}
return table
}
export function tableValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
type: OPTIONAL_STRING.valid("table", "internal", "external"),
primaryDisplay: OPTIONAL_STRING,
schema: Joi.object().required(),
name: Joi.string().required(),
views: Joi.object(),
rows: Joi.array(),
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
type: OPTIONAL_STRING.valid("table", "internal", "external"),
primaryDisplay: OPTIONAL_STRING,
schema: Joi.object().required(),
name: Joi.string().required(),
views: Joi.object(),
rows: Joi.array(),
})
.custom(validateViewSchemas)
.unknown(true),
{ errorPrefix: "" }
)
}
export function nameValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
name: OPTIONAL_STRING,
}))
return auth.joiValidator.body(
Joi.object({
name: OPTIONAL_STRING,
})
)
}
export function datasourceValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
_id: Joi.string(),
_rev: Joi.string(),
type: OPTIONAL_STRING.allow("datasource_plus"),
relationships: Joi.array().items(Joi.object({
from: Joi.string().required(),
to: Joi.string().required(),
cardinality: Joi.valid("1:N", "1:1", "N:N").required()
})),
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
_id: Joi.string(),
_rev: Joi.string(),
type: OPTIONAL_STRING.allow("datasource_plus"),
relationships: Joi.array().items(
Joi.object({
from: Joi.string().required(),
to: Joi.string().required(),
cardinality: Joi.valid("1:N", "1:1", "N:N").required(),
})
),
}).unknown(true)
)
}
function filterObject() {
// prettier-ignore
return Joi.object({
string: Joi.object().optional(),
fuzzy: Joi.object().optional(),
@ -62,17 +99,20 @@ function filterObject() {
}
export function internalSearchValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
tableId: OPTIONAL_STRING,
query: filterObject(),
limit: OPTIONAL_NUMBER,
sort: OPTIONAL_STRING,
sortOrder: OPTIONAL_STRING,
sortType: OPTIONAL_STRING,
paginate: Joi.boolean(),
bookmark: Joi.alternatives().try(OPTIONAL_STRING, OPTIONAL_NUMBER).optional(),
}))
return auth.joiValidator.body(
Joi.object({
tableId: OPTIONAL_STRING,
query: filterObject(),
limit: OPTIONAL_NUMBER,
sort: OPTIONAL_STRING,
sortOrder: OPTIONAL_STRING,
sortType: OPTIONAL_STRING,
paginate: Joi.boolean(),
bookmark: Joi.alternatives()
.try(OPTIONAL_STRING, OPTIONAL_NUMBER)
.optional(),
})
)
}
export function externalSearchValidator() {
@ -94,92 +134,110 @@ export function externalSearchValidator() {
}
export function datasourceQueryValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
endpoint: Joi.object({
datasourceId: Joi.string().required(),
operation: Joi.string().required().valid(...Object.values(DataSourceOperation)),
entityId: Joi.string().required(),
}).required(),
resource: Joi.object({
fields: Joi.array().items(Joi.string()).optional(),
}).optional(),
body: Joi.object().optional(),
sort: Joi.object().optional(),
filters: filterObject().optional(),
paginate: Joi.object({
page: Joi.string().alphanum().optional(),
limit: Joi.number().optional(),
}).optional(),
}))
return auth.joiValidator.body(
Joi.object({
endpoint: Joi.object({
datasourceId: Joi.string().required(),
operation: Joi.string()
.required()
.valid(...Object.values(DataSourceOperation)),
entityId: Joi.string().required(),
}).required(),
resource: Joi.object({
fields: Joi.array().items(Joi.string()).optional(),
}).optional(),
body: Joi.object().optional(),
sort: Joi.object().optional(),
filters: filterObject().optional(),
paginate: Joi.object({
page: Joi.string().alphanum().optional(),
limit: Joi.number().optional(),
}).optional(),
})
)
}
export function webhookValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
live: Joi.bool(),
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
name: Joi.string().required(),
bodySchema: Joi.object().optional(),
action: Joi.object({
type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
target: Joi.string().required(),
}).required(),
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
live: Joi.bool(),
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
name: Joi.string().required(),
bodySchema: Joi.object().optional(),
action: Joi.object({
type: Joi.string().required().valid(WebhookActionType.AUTOMATION),
target: Joi.string().required(),
}).required(),
}).unknown(true)
)
}
export function roleValidator() {
const permLevelArray = Object.values(permissions.PermissionLevel)
// prettier-ignore
return auth.joiValidator.body(Joi.object({
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
name: Joi.string().regex(/^[a-zA-Z0-9_]*$/).required(),
// this is the base permission ID (for now a built in)
permissionId: Joi.string().valid(...Object.values(permissions.BuiltinPermissionID)).required(),
permissions: Joi.object()
.pattern(/.*/, [Joi.string().valid(...permLevelArray)])
.optional(),
inherits: OPTIONAL_STRING,
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
name: Joi.string()
.regex(/^[a-zA-Z0-9_]*$/)
.required(),
// this is the base permission ID (for now a built in)
permissionId: Joi.string()
.valid(...Object.values(permissions.BuiltinPermissionID))
.required(),
permissions: Joi.object()
.pattern(/.*/, [Joi.string().valid(...permLevelArray)])
.optional(),
inherits: OPTIONAL_STRING,
}).unknown(true)
)
}
export function permissionValidator() {
const permLevelArray = Object.values(permissions.PermissionLevel)
// prettier-ignore
return auth.joiValidator.params(Joi.object({
level: Joi.string().valid(...permLevelArray).required(),
resourceId: Joi.string(),
roleId: Joi.string(),
}).unknown(true))
return auth.joiValidator.params(
Joi.object({
level: Joi.string()
.valid(...permLevelArray)
.required(),
resourceId: Joi.string(),
roleId: Joi.string(),
}).unknown(true)
)
}
export function screenValidator() {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
name: Joi.string().required(),
showNavigation: OPTIONAL_BOOLEAN,
width: OPTIONAL_STRING,
routing: Joi.object({
route: Joi.string().required(),
roleId: Joi.string().required().allow(""),
homeScreen: OPTIONAL_BOOLEAN,
}).required().unknown(true),
props: Joi.object({
_id: Joi.string().required(),
_component: Joi.string().required(),
_children: Joi.array().required(),
_styles: Joi.object().required(),
type: OPTIONAL_STRING,
table: OPTIONAL_STRING,
layoutId: OPTIONAL_STRING,
}).required().unknown(true),
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
name: Joi.string().required(),
showNavigation: OPTIONAL_BOOLEAN,
width: OPTIONAL_STRING,
routing: Joi.object({
route: Joi.string().required(),
roleId: Joi.string().required().allow(""),
homeScreen: OPTIONAL_BOOLEAN,
})
.required()
.unknown(true),
props: Joi.object({
_id: Joi.string().required(),
_component: Joi.string().required(),
_children: Joi.array().required(),
_styles: Joi.object().required(),
type: OPTIONAL_STRING,
table: OPTIONAL_STRING,
layoutId: OPTIONAL_STRING,
})
.required()
.unknown(true),
}).unknown(true)
)
}
function generateStepSchema(allowStepTypes: string[]) {
// prettier-ignore
return Joi.object({
stepId: Joi.string().required(),
id: Joi.string().required(),
@ -189,33 +247,39 @@ function generateStepSchema(allowStepTypes: string[]) {
icon: Joi.string().required(),
params: Joi.object(),
args: Joi.object(),
type: Joi.string().required().valid(...allowStepTypes),
type: Joi.string()
.required()
.valid(...allowStepTypes),
}).unknown(true)
}
export function automationValidator(existing = false) {
// prettier-ignore
return auth.joiValidator.body(Joi.object({
_id: existing ? Joi.string().required() : OPTIONAL_STRING,
_rev: existing ? Joi.string().required() : OPTIONAL_STRING,
name: Joi.string().required(),
type: Joi.string().valid("automation").required(),
definition: Joi.object({
steps: Joi.array().required().items(generateStepSchema(["ACTION", "LOGIC"])),
trigger: generateStepSchema(["TRIGGER"]).allow(null),
}).required().unknown(true),
}).unknown(true))
return auth.joiValidator.body(
Joi.object({
_id: existing ? Joi.string().required() : OPTIONAL_STRING,
_rev: existing ? Joi.string().required() : OPTIONAL_STRING,
name: Joi.string().required(),
type: Joi.string().valid("automation").required(),
definition: Joi.object({
steps: Joi.array()
.required()
.items(generateStepSchema(["ACTION", "LOGIC"])),
trigger: generateStepSchema(["TRIGGER"]).allow(null),
})
.required()
.unknown(true),
}).unknown(true)
)
}
export function applicationValidator(opts = { isCreate: true }) {
// prettier-ignore
const base: any = {
_id: OPTIONAL_STRING,
_rev: OPTIONAL_STRING,
url: OPTIONAL_STRING,
template: Joi.object({
templateString: OPTIONAL_STRING,
})
}),
}
const appNameValidator = Joi.string()

14
packages/server/src/integrations/rest.ts
View File

@ -149,13 +149,12 @@ class RestIntegration implements IntegrationBase {
{ downloadImages: this.config.downloadImages }
)
let contentLength = response.headers.get("content-length")
if (!contentLength && raw) {
contentLength = Buffer.byteLength(raw, "utf8").toString()
}
let isSuccess = response.status >= 200 && response.status < 300
if (
contentDisposition.includes("filename") ||
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")
(contentDisposition.includes("filename") ||
contentDisposition.includes("attachment") ||
contentDisposition.includes("form-data")) &&
isSuccess
) {
filename =
path.basename(parse(contentDisposition).parameters?.filename) || ""
@ -168,6 +167,9 @@ class RestIntegration implements IntegrationBase {
return handleFileResponse(response, filename, this.startTimeMs)
} else {
responseTxt = response.text ? await response.text() : ""
if (!contentLength && responseTxt) {
contentLength = Buffer.byteLength(responseTxt, "utf8").toString()
}
const hasContent =
(contentLength && parseInt(contentLength) > 0) ||
responseTxt.length > 0

2
packages/server/src/integrations/tests/rest.spec.ts
View File

@ -657,6 +657,7 @@ describe("REST Integration", () => {
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: {
raw: () => ({
"content-type": [contentType],
@ -700,6 +701,7 @@ describe("REST Integration", () => {
mockReadable.push(null)
;(fetch as unknown as jest.Mock).mockImplementationOnce(() =>
Promise.resolve({
status: 200,
headers: {
raw: () => ({
"content-type": [contentType],

69
packages/server/src/integrations/utils/utils.ts
View File

@ -9,10 +9,12 @@ import { context, objectStore, sql } from "@budibase/backend-core"
import { v4 } from "uuid"
import { parseStringPromise as xmlParser } from "xml2js"
import { formatBytes } from "../../utilities"
import bl from "bl"
import env from "../../environment"
import { InvalidColumns } from "../../constants"
import { helpers, utils } from "@budibase/shared-core"
import { pipeline } from "stream/promises"
import tmp from "tmp"
import fs from "fs"
type PrimitiveTypes =
| FieldType.STRING
@ -360,35 +362,44 @@ export async function handleFileResponse(
const key = `${context.getProdAppId()}/${processedFileName}`
const bucket = objectStore.ObjectStoreBuckets.TEMP
const stream = response.body.pipe(bl((error, data) => data))
// put the response stream to disk temporarily as a buffer
const tmpObj = tmp.fileSync()
try {
await pipeline(response.body, fs.createWriteStream(tmpObj.name))
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
}
if (response.body) {
const contentLength = response.headers.get("content-length")
if (contentLength) {
size = parseInt(contentLength, 10)
const details = await objectStore.streamUpload({
bucket,
filename: key,
stream: fs.createReadStream(tmpObj.name),
ttl: 1,
type: response.headers["content-type"],
})
if (!size && details.ContentLength) {
size = details.ContentLength
}
}
await objectStore.streamUpload({
bucket,
filename: key,
stream,
ttl: 1,
type: response.headers["content-type"],
})
}
presignedUrl = objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
presignedUrl = objectStore.getPresignedUrl(bucket, key)
return {
data: {
size,
name: processedFileName,
url: presignedUrl,
extension: fileExtension,
key: key,
},
info: {
code: response.status,
size: formatBytes(size.toString()),
time: `${Math.round(performance.now() - startTime)}ms`,
},
}
} finally {
// cleanup tmp
tmpObj.removeCallback()
}
}

38
packages/server/src/sdk/app/views/index.ts
View File

@ -39,9 +39,7 @@ async function guardViewSchema(
tableId: string,
viewSchema?: Record<string, ViewUIFieldMetadata>
) {
if (!viewSchema || !Object.keys(viewSchema).length) {
return
}
viewSchema ??= {}
const table = await sdk.tables.getTable(tableId)
for (const field of Object.keys(viewSchema)) {
@ -54,17 +52,13 @@ async function guardViewSchema(
}
if (viewSchema[field].readonly) {
if (!(await features.isViewReadonlyColumnsEnabled())) {
if (
!(await features.isViewReadonlyColumnsEnabled()) &&
!(tableSchemaField as ViewUIFieldMetadata).readonly
) {
throw new HTTPError(`Readonly fields are not enabled`, 400)
}
if (helpers.schema.isRequired(tableSchemaField.constraints)) {
throw new HTTPError(
`Field "${field}" cannot be readonly as it is a required field`,
400
)
}
if (!viewSchema[field].visible) {
throw new HTTPError(
`Field "${field}" must be visible if you want to make it readonly`,
@ -73,6 +67,28 @@ async function guardViewSchema(
}
}
}
for (const field of Object.values(table.schema)) {
if (!helpers.schema.isRequired(field.constraints)) {
continue
}
const viewSchemaField = viewSchema[field.name]
if (!viewSchemaField?.visible) {
throw new HTTPError(
`You can't hide "${field.name} because it is a required field."`,
400
)
}
if (viewSchemaField.readonly) {
throw new HTTPError(
`You can't make "${field.name}" readonly because it is a required field.`,
400
)
}
}
}
export async function create(

14
packages/server/src/threads/query.ts
View File

@ -196,12 +196,22 @@ class QueryRunner {
return { rows, keys, info, extra, pagination }
}
async runAnotherQuery(queryId: string, parameters: any) {
async runAnotherQuery(
queryId: string,
currentParameters: Record<string, any>
) {
const db = context.getAppDB()
const query = await db.get<Query>(queryId)
const datasource = await sdk.datasources.get(query.datasourceId, {
enriched: true,
})
// enrich parameters with dynamic queries defaults
const defaultParams = query.parameters || []
for (let param of defaultParams) {
if (!currentParameters[param.name]) {
currentParameters[param.name] = param.default
}
}
return new QueryRunner(
{
schema: query.schema,
@ -210,7 +220,7 @@ class QueryRunner {
transformer: query.transformer,
nullDefaultSupport: query.nullDefaultSupport,
ctx: this.ctx,
parameters,
parameters: currentParameters,
datasource,
queryId,
},

2
packages/types/src/documents/app/automation.ts
View File

@ -245,7 +245,7 @@ export type AutomationAttachment = {
export type AutomationAttachmentContent = {
filename: string
content: ReadStream | NodeJS.ReadableStream | ReadableStream<Uint8Array>
content: ReadStream | NodeJS.ReadableStream
}
export type BucketedContent = AutomationAttachmentContent & {

20
yarn.lock
View File

@ -6348,6 +6348,11 @@
dependencies:
"@types/estree" "*"
"@types/tmp@0.2.6":
version "0.2.6"
resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.6.tgz#d785ee90c52d7cc020e249c948c36f7b32d1e217"
integrity sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==
"@types/tough-cookie@*", "@types/tough-cookie@^4.0.2":
version "4.0.2"
resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397"
@ -7700,7 +7705,7 @@ bl@^4.0.3, bl@^4.1.0:
inherits "^2.0.4"
readable-stream "^3.4.0"
bl@^6.0.12, bl@^6.0.3:
bl@^6.0.3:
version "6.0.12"
resolved "https://registry.yarnpkg.com/bl/-/bl-6.0.12.tgz#77c35b96e13aeff028496c798b75389ddee9c7f8"
integrity sha512-EnEYHilP93oaOa2MnmNEjAcovPS3JlQZOyzGXi3EyEpPhm9qWvdDp7BmAVEVusGzp8LlwQK56Av+OkDoRjzE0w==
@ -16065,10 +16070,10 @@ mute-stream@~1.0.0:
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-1.0.0.tgz#e31bd9fe62f0aed23520aa4324ea6671531e013e"
integrity sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==
mysql2@3.9.7:
version "3.9.7"
resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.7.tgz#843755daf65b5ef08afe545fe14b8fb62824741a"
integrity sha512-KnJT8vYRcNAZv73uf9zpXqNbvBG7DJrs+1nACsjZP1HMJ1TgXEy8wnNilXAn/5i57JizXKtrUtwDB7HxT9DDpw==
mysql2@3.9.8:
version "3.9.8"
resolved "https://registry.yarnpkg.com/mysql2/-/mysql2-3.9.8.tgz#fe8a0f975f2c495ed76ca988ddc5505801dc49ce"
integrity sha512-+5JKNjPuks1FNMoy9TYpl77f+5frbTklz7eb3XDwbpsERRLEeXiW2PDEkakYF50UuKU2qwfGnyXpKYvukv8mGA==
dependencies:
denque "^2.1.0"
generate-function "^2.3.1"
@ -21283,6 +21288,11 @@ tlhunter-sorted-set@^0.1.0:
resolved "https://registry.yarnpkg.com/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz#1c3eae28c0fa4dff97e9501d2e3c204b86406f4b"
integrity sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw==
tmp@0.2.3:
version "0.2.3"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.3.tgz#eb783cc22bc1e8bebd0671476d46ea4eb32a79ae"
integrity sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==
tmp@^0.0.33:
version "0.0.33"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"