From 6bbdf0e4744e2a42283eeeb36d2c4cd0e9d0f2bc Mon Sep 17 00:00:00 2001 From: Dean Date: Thu, 18 Apr 2024 17:04:26 +0100 Subject: [PATCH 01/37] Bindings support for views and table row searches --- .../buttons/TableFilterButton.svelte | 71 ++++++++++++------- .../controls/FilterEditor/FilterDrawer.svelte | 1 + .../controls/FilterEditor/FilterUsers.svelte | 21 +++--- .../server/src/api/controllers/row/index.ts | 15 +++- .../src/api/controllers/row/utils/utils.ts | 41 ++++++++++- .../server/src/api/controllers/row/views.ts | 14 +++- 6 files changed, 122 insertions(+), 41 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index 91456da655..26b6624160 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -1,7 +1,9 @@ - + {text} - - dispatch("change", tempValue)} - > -
- (tempValue = e.detail)} - /> -
- - - + + + (tempValue = e.detail)} + {bindings} + /> + diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index 7f1ee8010d..74c081cd5b 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -304,6 +304,7 @@ OperatorOptions.ContainsAny.value, ].includes(filter.operator)} disabled={filter.noValue} + type={filter.valueType} /> {:else} diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte index 88383ba170..4613b8c40f 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte @@ -1,7 +1,6 @@ - option.email} - getOptionValue={option => option._id} - {disabled} -/> +
+ option.email} + getOptionValue={option => option._id} + {disabled} + /> +
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index c3d1f2cb47..7f99105ea4 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -2,7 +2,7 @@ import stream from "stream" import archiver from "archiver" import { quotas } from "@budibase/pro" -import { objectStore } from "@budibase/backend-core" +import { objectStore, context } from "@budibase/backend-core" import * as internal from "./internal" import * as external from "./external" import { isExternalTableID } from "../../../integrations/utils" @@ -198,8 +198,21 @@ export async function destroy(ctx: UserCtx) { export async function search(ctx: Ctx) { const tableId = utils.getTableId(ctx) + // Current user context for bindable search + const { _id, _rev, firstName, lastName, email, status, roleId } = ctx.user + + await context.ensureSnippetContext() + + const enrichedQuery = await utils.enrichSearchContext( + { ...ctx.request.body.query }, + { + user: { _id, _rev, firstName, lastName, email, status, roleId }, + } + ) + const searchParams: RowSearchParams = { ...ctx.request.body, + query: enrichedQuery, tableId, } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index f387a468cf..503f139783 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -7,6 +7,8 @@ import { FieldType, RelationshipsJson, Row, + SearchRowRequest, + SearchRowResponse, Table, UserCtx, } from "@budibase/types" @@ -22,7 +24,7 @@ import { getInternalRowId, } from "./basic" import sdk from "../../../../sdk" - +import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" validateJs.extend(validateJs.validators.datetime, { @@ -187,3 +189,40 @@ export async function sqlOutputProcessing( export function isUserMetadataTable(tableId: string) { return tableId === InternalTables.USER_METADATA } + +export async function enrichSearchContext( + fields: Record, + inputs = {}, + helpers = true +): Promise> { + const enrichedQuery: Record = {} + if (!fields || !inputs) { + return enrichedQuery + } + const parameters = { ...inputs } + // enrich the fields with dynamic parameters + for (let key of Object.keys(fields)) { + if (fields[key] == null) { + continue + } + if (typeof fields[key] === "object") { + // enrich nested fields object + enrichedQuery[key] = await enrichSearchContext( + fields[key], + parameters, + helpers + ) + } else if (typeof fields[key] === "string") { + // enrich string value as normal + enrichedQuery[key] = processStringSync(fields[key], parameters, { + noEscaping: true, + noHelpers: !helpers, + escapeNewlines: true, + }) + } else { + enrichedQuery[key] = fields[key] + } + } + + return enrichedQuery +} diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 2644446d82..18953ebe88 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -9,7 +9,8 @@ import { } from "@budibase/types" import { dataFilters } from "@budibase/shared-core" import sdk from "../../../sdk" -import { db } from "@budibase/backend-core" +import { db, context } from "@budibase/backend-core" +import { enrichSearchContext, userSearchFromContext } from "./utils" export async function searchView( ctx: UserCtx @@ -56,10 +57,19 @@ export async function searchView( }) } + // Current user search context. + const { _id, _rev, firstName, lastName, email, status, roleId } = ctx.user + + await context.ensureSnippetContext() + + const enrichedQuery = await enrichSearchContext(query, { + user: { _id, _rev, firstName, lastName, email, status, roleId }, + }) + const searchOptions: RequiredKeys & RequiredKeys> = { tableId: view.tableId, - query, + query: enrichedQuery, fields: viewFields, ...getSortOptions(body, view), limit: body.limit, From bdf15b21b1f62cea09a3c03d5e0b8c7fc9260d66 Mon Sep 17 00:00:00 2001 From: Dean Date: Fri, 19 Apr 2024 11:49:20 +0100 Subject: [PATCH 02/37] Fixes for filter drawer padding --- .../buttons/TableFilterButton.svelte | 19 ++++++++++--------- .../src/components/FilterBuilder.svelte | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte index 0fc3fd505e..140cac1533 100644 --- a/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte +++ b/packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte @@ -1,6 +1,6 @@ {#if schemaHasOptions(schema) && schema.type !== "array"} @@ -77,6 +100,35 @@ on:change={e => onChange(e, field)} useLabel={false} /> +{:else if schema.type === FieldType.ATTACHMENTS || schema.type === FieldType.ATTACHMENT_SINGLE} +
+ + onChange( + { + detail: + schema.type === FieldType.ATTACHMENT_SINGLE + ? e.detail.length > 0 + ? { url: e.detail[0].name, filename: e.detail[0].value } + : {} + : e.detail.map(({ name, value }) => ({ + url: name, + filename: value, + })), + }, + field + )} + object={handleAttachmentParams(value[field])} + allowJS + {bindings} + keyBindings + customButtonText={"Add attachment"} + keyPlaceholder={"URL"} + valuePlaceholder={"Filename"} + actionButtonDisabled={schema.type === FieldType.ATTACHMENT_SINGLE && + Object.keys(value[field]).length >= 1} + /> +
{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)} {/if} + + diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index 8ce9dda209..fb448cca8d 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -4,6 +4,7 @@ readableToRuntimeBinding, runtimeToReadableBinding, } from "dataBinding" + import { FieldType } from "@budibase/types" import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte" import { createEventDispatcher, setContext } from "svelte" @@ -102,6 +103,8 @@ longform: value => !isJSBinding(value), json: value => !isJSBinding(value), boolean: isValidBoolean, + attachment: false, + attachment_single: false, } const isValid = value => { @@ -116,7 +119,16 @@ if (type === "json" && !isJSBinding(value)) { return "json-slot-icon" } - if (!["string", "number", "bigint", "barcodeqr"].includes(type)) { + if ( + ![ + "string", + "number", + "bigint", + "barcodeqr", + "attachment", + "attachment_single", + ].includes(type) + ) { return "slot-icon" } return "" @@ -157,7 +169,7 @@ {updateOnChange} /> {/if} - {#if !disabled && type !== "formula"} + {#if !disabled && type !== "formula" && !disabled && type !== FieldType.ATTACHMENTS && !disabled && type !== FieldType.ATTACHMENT_SINGLE}
{ diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 5ed18a970a..6f69e71ccb 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -37,6 +37,7 @@ export let customButtonText = null export let keyBindings = false export let allowJS = false + export let actionButtonDisabled = false export let compare = (option, value) => option === value let fields = Object.entries(object || {}).map(([name, value]) => ({ @@ -189,7 +190,14 @@ {/if} {#if !readOnly && !noAddButton}
- + {#if customButtonText} {customButtonText} {:else} diff --git a/packages/server/src/automations/automationUtils.ts b/packages/server/src/automations/automationUtils.ts index 6730e04494..c94c166be1 100644 --- a/packages/server/src/automations/automationUtils.ts +++ b/packages/server/src/automations/automationUtils.ts @@ -4,8 +4,11 @@ import { encodeJSBinding, } from "@budibase/string-templates" import sdk from "../sdk" -import { Row } from "@budibase/types" +import { AutomationAttachment, FieldType, Row } from "@budibase/types" import { LoopInput, LoopStepType } from "../definitions/automations" +import { objectStore, context } from "@budibase/backend-core" +import * as uuid from "uuid" +import path from "path" /** * When values are input to the system generally they will be of type string as this is required for template strings. @@ -96,6 +99,98 @@ export function getError(err: any) { return typeof err !== "string" ? err.toString() : err } +export async function sendAutomationAttachmentsToStorage( + tableId: string, + row: Row +): Promise { + const table = await sdk.tables.getTable(tableId) + const attachmentRows: Record< + string, + AutomationAttachment[] | AutomationAttachment + > = {} + + for (const [prop, value] of Object.entries(row)) { + const schema = table.schema[prop] + if ( + schema?.type === FieldType.ATTACHMENTS || + schema?.type === FieldType.ATTACHMENT_SINGLE + ) { + attachmentRows[prop] = value + } + } + for (const [prop, attachments] of Object.entries(attachmentRows)) { + if (Array.isArray(attachments)) { + if (attachments.length) { + row[prop] = await Promise.all( + attachments.map(attachment => generateAttachmentRow(attachment)) + ) + } + } else if (Object.keys(row[prop]).length > 0) { + row[prop] = await generateAttachmentRow(attachments) + } + } + + return row +} + +async function generateAttachmentRow(attachment: AutomationAttachment) { + const prodAppId = context.getProdAppId() + + async function uploadToS3( + extension: string, + content: objectStore.StreamTypes + ) { + const fileName = `${uuid.v4()}${extension}` + const s3Key = `${prodAppId}/attachments/${fileName}` + + await objectStore.streamUpload({ + bucket: objectStore.ObjectStoreBuckets.APPS, + stream: content, + filename: s3Key, + }) + + return s3Key + } + + async function getSize(s3Key: string) { + return ( + await objectStore.getObjectMetadata( + objectStore.ObjectStoreBuckets.APPS, + s3Key + ) + ).ContentLength + } + + try { + const { filename } = attachment + const extension = path.extname(filename) + const attachmentResult = await objectStore.processAutomationAttachment( + attachment + ) + + let s3Key = "" + if ( + "path" in attachmentResult && + attachmentResult.path.startsWith(`${prodAppId}/attachments/`) + ) { + s3Key = attachmentResult.path + } else { + s3Key = await uploadToS3(extension, attachmentResult.content) + } + + const size = await getSize(s3Key) + + return { + size, + name: filename, + extension, + key: s3Key, + } + } catch (error) { + console.error("Failed to process attachment:", error) + throw error + } +} export function substituteLoopStep(hbsString: string, substitute: string) { let checkForJS = isJSBinding(hbsString) let substitutedHbsString = "" diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts index d11baab5c6..5b5084b465 100644 --- a/packages/server/src/automations/steps/createRow.ts +++ b/packages/server/src/automations/steps/createRow.ts @@ -1,5 +1,9 @@ import { save } from "../../api/controllers/row" -import { cleanUpRow, getError } from "../automationUtils" +import { + cleanUpRow, + getError, + sendAutomationAttachmentsToStorage, +} from "../automationUtils" import { buildCtx } from "./utils" import { AutomationActionStepId, @@ -89,6 +93,10 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { try { inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row) + inputs.row = await sendAutomationAttachmentsToStorage( + inputs.row.tableId, + inputs.row + ) await save(ctx) return { row: inputs.row, diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index fea3e981f3..348c5e8373 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -108,7 +108,15 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { try { if (tableId) { - inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row) + inputs.row = await automationUtils.cleanUpRow( + inputs.row.tableId, + inputs.row + ) + + inputs.row = await automationUtils.sendAutomationAttachmentsToStorage( + inputs.row.tableId, + inputs.row + ) } await rowController.patch(ctx) return { diff --git a/packages/server/src/automations/tests/createRow.spec.ts b/packages/server/src/automations/tests/createRow.spec.ts index 0098be39a5..e78236c5ac 100644 --- a/packages/server/src/automations/tests/createRow.spec.ts +++ b/packages/server/src/automations/tests/createRow.spec.ts @@ -1,5 +1,18 @@ import * as setup from "./utilities" +import { basicTableWithAttachmentField } from "../../tests/utilities/structures" +import { objectStore } from "@budibase/backend-core" +async function uploadTestFile(filename: string) { + let bucket = "testbucket" + await objectStore.upload({ + bucket, + filename, + body: Buffer.from("test data"), + }) + let presignedUrl = await objectStore.getPresignedUrl(bucket, filename, 60000) + + return presignedUrl +} describe("test the create row action", () => { let table: any let row: any @@ -43,4 +56,76 @@ describe("test the create row action", () => { const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {}) expect(res.success).toEqual(false) }) + + it("should check that an attachment field is sent to storage and parsed", async () => { + let attachmentTable = await config.createTable( + basicTableWithAttachmentField() + ) + + let attachmentRow: any = { + tableId: attachmentTable._id, + } + + let filename = "test1.txt" + let presignedUrl = await uploadTestFile(filename) + let attachmentObject = [ + { + url: presignedUrl, + filename, + }, + ] + + attachmentRow.file_attachment = attachmentObject + const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { + row: attachmentRow, + }) + + expect(res.success).toEqual(true) + expect(res.row.file_attachment[0]).toHaveProperty("key") + let s3Key = res.row.file_attachment[0].key + + const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS) + + const objectData = await client + .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key }) + .promise() + + expect(objectData).toBeDefined() + expect(objectData.ContentLength).toBeGreaterThan(0) + }) + + it("should check that an single attachment field is sent to storage and parsed", async () => { + let attachmentTable = await config.createTable( + basicTableWithAttachmentField() + ) + + let attachmentRow: any = { + tableId: attachmentTable._id, + } + + let filename = "test2.txt" + let presignedUrl = await uploadTestFile(filename) + let attachmentObject = { + url: presignedUrl, + filename, + } + + attachmentRow.single_file_attachment = attachmentObject + const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, { + row: attachmentRow, + }) + + expect(res.success).toEqual(true) + expect(res.row.single_file_attachment).toHaveProperty("key") + let s3Key = res.row.single_file_attachment.key + + const client = objectStore.ObjectStore(objectStore.ObjectStoreBuckets.APPS) + + const objectData = await client + .headObject({ Bucket: objectStore.ObjectStoreBuckets.APPS, Key: s3Key }) + .promise() + + expect(objectData).toBeDefined() + expect(objectData.ContentLength).toBeGreaterThan(0) + }) }) diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 0613c3ade8..f5a12c2cbf 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -139,13 +139,13 @@ class RestIntegration implements IntegrationBase { const contentType = response.headers.get("content-type") || "" const contentDisposition = response.headers.get("content-disposition") || "" if ( + contentDisposition.includes("filename") || contentDisposition.includes("attachment") || contentDisposition.includes("form-data") ) { filename = path.basename(parse(contentDisposition).parameters?.filename) || "" } - try { if (filename) { return handleFileResponse(response, filename, this.startTimeMs) diff --git a/packages/server/src/tests/utilities/structures.ts b/packages/server/src/tests/utilities/structures.ts index 2a32489c30..77a6431335 100644 --- a/packages/server/src/tests/utilities/structures.ts +++ b/packages/server/src/tests/utilities/structures.ts @@ -78,6 +78,32 @@ export function basicTable( ) } +export function basicTableWithAttachmentField( + datasource?: Datasource, + ...extra: Partial[] +): Table { + return tableForDatasource( + datasource, + { + name: "TestTable", + schema: { + file_attachment: { + type: FieldType.ATTACHMENTS, + name: "description", + constraints: { + type: "array", + }, + }, + single_file_attachment: { + type: FieldType.ATTACHMENT_SINGLE, + name: "description", + }, + }, + }, + ...extra + ) +} + export function basicView(tableId: string) { return { tableId, diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index efa1ff1bd8..8cb03de948 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -234,7 +234,7 @@ export async function outputProcessing( } } else if (column.type === FieldType.ATTACHMENT_SINGLE) { for (let row of enriched) { - if (!row[property]) { + if (!row[property] || Object.keys(row[property]).length === 0) { continue } diff --git a/packages/types/src/documents/app/automation.ts b/packages/types/src/documents/app/automation.ts index c3847a2c04..481a051e1c 100644 --- a/packages/types/src/documents/app/automation.ts +++ b/packages/types/src/documents/app/automation.ts @@ -1,6 +1,7 @@ import { Document } from "../document" import { EventEmitter } from "events" import { User } from "../global" +import { ReadStream } from "fs" export enum AutomationIOType { OBJECT = "object", @@ -235,3 +236,18 @@ export interface AutomationMetadata extends Document { errorCount?: number automationChainCount?: number } + +export type AutomationAttachment = { + url: string + filename: string +} + +export type AutomationAttachmentContent = { + filename: string + content: ReadStream | NodeJS.ReadableStream | ReadableStream +} + +export type BucketedContent = AutomationAttachmentContent & { + bucket: string + path: string +} diff --git a/packages/worker/src/utilities/email.ts b/packages/worker/src/utilities/email.ts index db9a635356..bf686f647c 100644 --- a/packages/worker/src/utilities/email.ts +++ b/packages/worker/src/utilities/email.ts @@ -6,8 +6,7 @@ import { processString } from "@budibase/string-templates" import { User, SendEmailOpts, SMTPInnerConfig } from "@budibase/types" import { configs, cache, objectStore } from "@budibase/backend-core" import ical from "ical-generator" -import fetch from "node-fetch" -import path from "path" +import _ from "lodash" const nodemailer = require("nodemailer") @@ -165,39 +164,12 @@ export async function sendEmail( }), } if (opts?.attachments) { - const attachments = await Promise.all( - opts.attachments?.map(async attachment => { - const isFullyFormedUrl = - attachment.url.startsWith("http://") || - attachment.url.startsWith("https://") - if (isFullyFormedUrl) { - const response = await fetch(attachment.url) - if (!response.ok) { - throw new Error(`unexpected response ${response.statusText}`) - } - const fallbackFilename = path.basename( - new URL(attachment.url).pathname - ) - return { - filename: attachment.filename || fallbackFilename, - content: response?.body, - } - } else { - const url = attachment.url - const result = objectStore.extractBucketAndPath(url) - if (result === null) { - throw new Error("Invalid signed URL") - } - const { bucket, path } = result - const readStream = await objectStore.getReadStream(bucket, path) - const fallbackFilename = path.split("/").pop() || "" - return { - filename: attachment.filename || fallbackFilename, - content: readStream, - } - } - }) + let attachments = await Promise.all( + opts.attachments?.map(objectStore.processAutomationAttachment) ) + attachments = attachments.map(attachment => { + return _.omit(attachment, "path") + }) message = { ...message, attachments } } From 381c33cfb54a4875dfcf789763b38c224bc802bb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 May 2024 15:10:05 +0100 Subject: [PATCH 27/37] Adding support for buffers in a few places - this helps with BYTE type columns in SQL. --- packages/server/src/api/controllers/row/utils/basic.ts | 5 ++++- packages/server/src/integrations/utils/utils.ts | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 6255e13c1c..0c2c786e18 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -73,12 +73,15 @@ export function basicProcessing({ // filter the row down to what is actually the row (not joined) for (let field of Object.values(table.schema)) { const fieldName = field.name - const value = extractFieldValue({ + let value = extractFieldValue({ row, tableName: table.name, fieldName, isLinked, }) + if (value instanceof Buffer) { + value = value.toString() + } // all responses include "select col as table.col" so that overlaps are handled if (value != null) { thisRow[fieldName] = value diff --git a/packages/server/src/integrations/utils/utils.ts b/packages/server/src/integrations/utils/utils.ts index aac3f5f74a..e2fba11a4e 100644 --- a/packages/server/src/integrations/utils/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -192,6 +192,11 @@ export function generateRowIdField(keyProps: any[] = []) { if (!Array.isArray(keyProps)) { keyProps = [keyProps] } + for (let index in keyProps) { + if (keyProps[index] instanceof Buffer) { + keyProps[index] = keyProps[index].toString() + } + } // this conserves order and types // we have to swap the double quotes to single quotes for use in HBS statements // when using the literal helper the double quotes can break things From e928ff2ea25d4020e3fc25690754d2c3cdc898fb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 9 May 2024 15:26:53 +0100 Subject: [PATCH 28/37] Adding test case to confirm it works. --- .../src/integration-test/postgres.spec.ts | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index cf80f2359f..ec4cb90a86 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -1200,4 +1200,38 @@ describe("postgres integrations", () => { expect(Object.keys(schema).sort()).toEqual(["id", "val1"]) }) }) + + describe("check custom column types", () => { + beforeAll(async () => { + await rawQuery( + rawDatasource, + `CREATE TABLE binaryTable ( + id BYTEA PRIMARY KEY, + column1 TEXT, + column2 INT + ); + ` + ) + }) + + it("should handle binary columns", async () => { + const response = await makeRequest( + "post", + `/api/datasources/${datasource._id}/schema` + ) + expect(response.body).toBeDefined() + expect(response.body.datasource.entities).toBeDefined() + const table = response.body.datasource.entities["binarytable"] + expect(table).toBeDefined() + expect(table.schema.id.externalType).toBe("bytea") + const row = await config.api.row.save(table._id, { + id: "1111", + column1: "hello", + column2: 222, + }) + expect(row._id).toBeDefined() + const decoded = decodeURIComponent(row._id!).replace(/'/g, '"') + expect(JSON.parse(decoded)[0]).toBe("1111") + }) + }) }) From bfc63bd4e26a0498c382ffd4eb3f3a4f0b7fc1cf Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 9 May 2024 16:26:08 +0100 Subject: [PATCH 29/37] Remove the last internal.spec.ts file. --- .../src/api/routes/tests/search.spec.ts | 2 + .../app/rows/search/tests/internal.spec.ts | 117 ------------------ 2 files changed, 2 insertions(+), 117 deletions(-) delete mode 100644 packages/server/src/sdk/app/rows/search/tests/internal.spec.ts diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 7ea3f063de..d036da646e 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -402,6 +402,7 @@ describe.each([ }, ]) }) + // TODO(samwho): fix for SQS !isSqs && it("should match the session user id in a multi user field", async () => { @@ -419,6 +420,7 @@ describe.each([ ]) }) + // TODO(samwho): fix for SQS !isSqs && it("should not match the session user id in a multi user field", async () => { await expectQuery({ diff --git a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts deleted file mode 100644 index 1c5f396737..0000000000 --- a/packages/server/src/sdk/app/rows/search/tests/internal.spec.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { - FieldType, - Row, - Table, - RowSearchParams, - INTERNAL_TABLE_SOURCE_ID, - TableSourceType, -} from "@budibase/types" -import TestConfiguration from "../../../../../tests/utilities/TestConfiguration" -import { search } from "../internal" -import { - expectAnyInternalColsAttributes, - generator, -} from "@budibase/backend-core/tests" - -describe("internal", () => { - const config = new TestConfiguration() - - const tableData: Table = { - name: generator.word(), - type: "table", - sourceId: INTERNAL_TABLE_SOURCE_ID, - sourceType: TableSourceType.INTERNAL, - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - type: FieldType.STRING, - }, - }, - surname: { - name: "surname", - type: FieldType.STRING, - constraints: { - type: FieldType.STRING, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, - constraints: { - type: FieldType.NUMBER, - }, - }, - address: { - name: "address", - type: FieldType.STRING, - constraints: { - type: FieldType.STRING, - }, - }, - }, - } - - beforeAll(async () => { - await config.init() - }) - - describe("search", () => { - const rows: Row[] = [] - beforeAll(async () => { - await config.createTable(tableData) - for (let i = 0; i < 10; i++) { - rows.push( - await config.createRow({ - name: generator.first(), - surname: generator.last(), - age: generator.age(), - address: generator.address(), - }) - ) - } - }) - - it("default search returns all the data", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! - - const searchParams: RowSearchParams = { - tableId, - query: {}, - } - const result = await search(searchParams, config.table!) - - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining(rows.map(r => expect.objectContaining(r))) - ) - }) - }) - - it("querying by fields will always return data attribute columns", async () => { - await config.doInContext(config.appId, async () => { - const tableId = config.table!._id! - - const searchParams: RowSearchParams = { - tableId, - query: {}, - fields: ["name", "age"], - } - const result = await search(searchParams, config.table!) - - expect(result.rows).toHaveLength(10) - expect(result.rows).toEqual( - expect.arrayContaining( - rows.map(r => ({ - ...expectAnyInternalColsAttributes, - name: r.name, - age: r.age, - })) - ) - ) - }) - }) - }) -}) From 76449782b5ba0cff104d9b899e841c6ce2e82326 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:27:49 +0100 Subject: [PATCH 30/37] Fixes an issue with fetch information being passed up from DatabaseImpl, making sure errors are fully sanitised. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d220d0a8ac..d54e23217b 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -3,11 +3,11 @@ import { AllDocsResponse, AnyDocument, Database, - DatabaseOpts, - DatabaseQueryOpts, - DatabasePutOpts, DatabaseCreateIndexOpts, DatabaseDeleteIndexOpts, + DatabaseOpts, + DatabasePutOpts, + DatabaseQueryOpts, Document, isDocument, RowResponse, @@ -17,7 +17,7 @@ import { import { getCouchInfo } from "./connections" import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" -import { WriteStream, ReadStream } from "fs" +import { ReadStream, WriteStream } from "fs" import { newid } from "../../docIds/newid" import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" @@ -38,6 +38,34 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise +class CouchDBError extends Error { + status: number + statusCode: number + reason: string + name: string + errid: string | undefined + description: string | undefined + + constructor( + message: string, + info: { + status: number + name: string + errid: string + description: string + reason: string + } + ) { + super(message) + this.status = info.status + this.statusCode = info.status + this.reason = info.reason + this.name = info.name + this.errid = info.errid + this.description = info.description + } +} + export function DatabaseWithConnection( dbName: string, connection: string, @@ -119,7 +147,7 @@ export class DatabaseImpl implements Database { } catch (err: any) { // Handling race conditions if (err.statusCode !== 412) { - throw err + throw new CouchDBError(err.message, err) } } } @@ -138,10 +166,15 @@ export class DatabaseImpl implements Database { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() return await this.performCall(call) - } else if (err.statusCode) { - err.status = err.statusCode } - throw err + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, { + status: err.status || err.statusCode, + name: err.name, + errid: err.errid, + description: err.description, + reason: err.reason, + }) } } @@ -281,16 +314,9 @@ export class DatabaseImpl implements Database { } async destroy() { - try { - return await this.nano().db.destroy(this.name) - } catch (err: any) { - // didn't exist, don't worry - if (err.statusCode === 404) { - return - } else { - throw { ...err, status: err.statusCode } - } - } + return this.performCall(async () => { + return () => this.nano().db.destroy(this.name) + }) } async compact() { From 1aa89c61b61bdf5cd95ca2c2482512526a0b99e0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:32:57 +0100 Subject: [PATCH 31/37] One small change to keep 404 functionality on destroy DB. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index d54e23217b..ca8a22b54e 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -314,9 +314,16 @@ export class DatabaseImpl implements Database { } async destroy() { - return this.performCall(async () => { - return () => this.nano().db.destroy(this.name) - }) + try { + return await this.nano().db.destroy(this.name) + } catch (err: any) { + // didn't exist, don't worry + if (err.statusCode === 404) { + return + } else { + throw new CouchDBError(err.message, err) + } + } } async compact() { From de2d0e6b89e7943bc2226b837e0c51a0af54f4d6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:51:57 +0100 Subject: [PATCH 32/37] Adding error field. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ca8a22b54e..c520f4d81f 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,8 +43,9 @@ class CouchDBError extends Error { statusCode: number reason: string name: string - errid: string | undefined - description: string | undefined + errid: string + error: string + description: string constructor( message: string, @@ -54,6 +55,7 @@ class CouchDBError extends Error { errid: string description: string reason: string + error: string } ) { super(message) @@ -63,6 +65,7 @@ class CouchDBError extends Error { this.name = info.name this.errid = info.errid this.description = info.description + this.error = info.error } } @@ -168,13 +171,7 @@ export class DatabaseImpl implements Database { return await this.performCall(call) } // stripping the error down the props which are safe/useful, drop everything else - throw new CouchDBError(`CouchDB error: ${err.message}`, { - status: err.status || err.statusCode, - name: err.name, - errid: err.errid, - description: err.description, - reason: err.reason, - }) + throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } From 10608f9bb71edfaf9b567f656111466b241181bd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 11:59:11 +0100 Subject: [PATCH 33/37] Final final fix. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c520f4d81f..ef351f7d4d 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -50,7 +50,8 @@ class CouchDBError extends Error { constructor( message: string, info: { - status: number + status: number | undefined + statusCode: number | undefined name: string errid: string description: string @@ -59,8 +60,9 @@ class CouchDBError extends Error { } ) { super(message) - this.status = info.status - this.statusCode = info.status + const statusCode = info.status || info.statusCode || 500 + this.status = statusCode + this.statusCode = statusCode this.reason = info.reason this.name = info.name this.errid = info.errid From 78be5c16e4727828ab8d13df6fd6c7c4af7d0444 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 10 May 2024 12:03:24 +0100 Subject: [PATCH 34/37] Updating pro reference. --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 479879246a..ff397e5454 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 479879246aac5dd3073cc695945c62c41fae5b0e +Subproject commit ff397e5454ad3361b25efdf14746c36dcbd3f409 From a62279a5f162083128f78fec57772d121ddeaa5b Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:10:28 +0000 Subject: [PATCH 35/37] Bump version to 2.24.3 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 9c5a6c6bab..7daf0b039b 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.2", + "version": "2.24.3", "npmClient": "yarn", "packages": [ "packages/*", From 58538cc2019e2e5e98f0a0f0ac5a4520fd9a3763 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Fri, 10 May 2024 11:13:00 +0000 Subject: [PATCH 36/37] Bump version to 2.25.0 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 7daf0b039b..16dc73aa30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.24.3", + "version": "2.25.0", "npmClient": "yarn", "packages": [ "packages/*", From efaedbccde8adc17fb389abbd2989b26ad0257ad Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Fri, 10 May 2024 13:18:30 +0100 Subject: [PATCH 37/37] Allow Fancy Input validation to be triggered onBlur (#13658) * Add free_trial to deploy camunda script * Allow for more validation customisation on fancy input --- packages/bbui/src/FancyForm/FancyInput.svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/bbui/src/FancyForm/FancyInput.svelte b/packages/bbui/src/FancyForm/FancyInput.svelte index 0c58b9b045..f665fa5724 100644 --- a/packages/bbui/src/FancyForm/FancyInput.svelte +++ b/packages/bbui/src/FancyForm/FancyInput.svelte @@ -11,6 +11,7 @@ export let error = null export let validate = null export let suffix = null + export let validateOn = "change" const dispatch = createEventDispatcher() @@ -24,7 +25,16 @@ const newValue = e.target.value dispatch("change", newValue) value = newValue - if (validate) { + if (validate && (error || validateOn === "change")) { + error = validate(newValue) + } + } + + const onBlur = e => { + focused = false + const newValue = e.target.value + dispatch("blur", newValue) + if (validate && validateOn === "blur") { error = validate(newValue) } } @@ -61,7 +71,7 @@ type={type || "text"} on:input={onChange} on:focus={() => (focused = true)} - on:blur={() => (focused = false)} + on:blur={onBlur} class:placeholder bind:this={ref} />