From 1717fb7930ad68bfc8f0cc13dd04354cfc848d58 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Dec 2023 11:30:02 +0000 Subject: [PATCH 1/7] Instrument CouchDB client with DD APM traces. --- packages/backend-core/package.json | 1 + .../backend-core/src/db/couch/DatabaseImpl.ts | 6 +- packages/backend-core/src/db/db.ts | 3 +- .../backend-core/src/db/instrumentation.ts | 159 ++++++++++++++++++ 4 files changed, 167 insertions(+), 2 deletions(-) create mode 100644 packages/backend-core/src/db/instrumentation.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 6edeca3674..36b8de0f56 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -32,6 +32,7 @@ "bcryptjs": "2.4.3", "bull": "4.10.1", "correlation-id": "4.0.0", + "dd-trace": "3.13.2", "dotenv": "16.0.1", "ioredis": "5.3.2", "joi": "17.6.0", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 8588a7157a..c7c4f98e98 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -17,6 +17,7 @@ import { directCouchUrlCall } from "./utils" import { getPouchDB } from "./pouchDB" import { WriteStream, ReadStream } from "fs" import { newid } from "../../docIds/newid" +import { DDInstrumentedDatabase } from "../instrumentation" function buildNano(couchInfo: { url: string; cookie: string }) { return Nano({ @@ -38,7 +39,10 @@ export function DatabaseWithConnection( if (!connection) { throw new Error("Must provide connection details") } - return new DatabaseImpl(dbName, opts, connection) + return new DDInstrumentedDatabase( + new DatabaseImpl(dbName, opts, connection), + "couchdb" + ) } export class DatabaseImpl implements Database { diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 3e69d49f0e..3d0f522139 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,8 +1,9 @@ import { directCouchQuery, DatabaseImpl } from "./couch" import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" +import { DDInstrumentedDatabase } from "./instrumentation" export function getDB(dbName: string, opts?: DatabaseOpts): Database { - return new DatabaseImpl(dbName, opts) + return new DDInstrumentedDatabase(new DatabaseImpl(dbName, opts), "couchdb") } // we have to use a callback for this so that we can close diff --git a/packages/backend-core/src/db/instrumentation.ts b/packages/backend-core/src/db/instrumentation.ts new file mode 100644 index 0000000000..6d34478952 --- /dev/null +++ b/packages/backend-core/src/db/instrumentation.ts @@ -0,0 +1,159 @@ +import { + DocumentScope, + DocumentDestroyResponse, + DocumentInsertResponse, + DocumentBulkResponse, + OkResponse, +} from "@budibase/nano" +import { + AllDocsResponse, + AnyDocument, + Database, + DatabaseDumpOpts, + DatabasePutOpts, + DatabaseQueryOpts, + Document, +} from "@budibase/types" +import tracer from "dd-trace" +import { Writable } from "stream" + +export class DDInstrumentedDatabase implements Database { + constructor( + private readonly db: Database, + private readonly resource: string + ) {} + + get name(): string { + return this.db.name + } + + exists(): Promise { + return tracer.trace("exists", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.exists() + }) + } + + checkSetup(): Promise> { + return tracer.trace("checkSetup", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.checkSetup() + }) + } + + get(id?: string | undefined): Promise { + return tracer.trace("get", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.get(id) + }) + } + + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean | undefined } | undefined + ): Promise { + return tracer.trace("getMultiple", { resource: this.resource }, span => { + span?.addTags({ + db_name: this.name, + num_docs: ids.length, + allow_missing: opts?.allowMissing, + }) + return this.db.getMultiple(ids, opts) + }) + } + + remove( + id: string | Document, + rev?: string | undefined + ): Promise { + return tracer.trace("remove", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name, doc_id: id }) + return this.db.remove(id, rev) + }) + } + + put( + document: AnyDocument, + opts?: DatabasePutOpts | undefined + ): Promise { + return tracer.trace("put", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name, doc_id: document._id }) + return this.db.put(document, opts) + }) + } + + bulkDocs(documents: AnyDocument[]): Promise { + return tracer.trace("bulkDocs", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name, num_docs: documents.length }) + return this.db.bulkDocs(documents) + }) + } + + allDocs( + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("allDocs", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.allDocs(params) + }) + } + + query( + viewName: string, + params: DatabaseQueryOpts + ): Promise> { + return tracer.trace("query", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name, view_name: viewName }) + return this.db.query(viewName, params) + }) + } + + destroy(): Promise { + return tracer.trace("destroy", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.destroy() + }) + } + + compact(): Promise { + return tracer.trace("compact", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.compact() + }) + } + + dump(stream: Writable, opts?: DatabaseDumpOpts | undefined): Promise { + return tracer.trace("dump", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.dump(stream, opts) + }) + } + + load(...args: any[]): Promise { + return tracer.trace("load", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.load(...args) + }) + } + + createIndex(...args: any[]): Promise { + return tracer.trace("createIndex", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.createIndex(...args) + }) + } + + deleteIndex(...args: any[]): Promise { + return tracer.trace("deleteIndex", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.deleteIndex(...args) + }) + } + + getIndexes(...args: any[]): Promise { + return tracer.trace("getIndexes", { resource: this.resource }, span => { + span?.addTags({ db_name: this.name }) + return this.db.getIndexes(...args) + }) + } +} From e6cfe20432df23b0f3dda85117a3f4e4bc16b377 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 18 Dec 2023 17:41:15 +0000 Subject: [PATCH 2/7] Fixing build issue. --- scripts/build.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build.js b/scripts/build.js index b0ec8433bf..c76217bb8b 100755 --- a/scripts/build.js +++ b/scripts/build.js @@ -59,6 +59,7 @@ function runBuild(entry, outfile) { "pouchdb", "bcrypt", "bcryptjs", + "graphql/*", ], } From 78d039c949b22cf587e1d8b140b0b3dc6809617a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 18 Dec 2023 17:43:50 +0000 Subject: [PATCH 3/7] Update backend-core to dd-trace 4.20.0 --- packages/backend-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 36b8de0f56..a0dd320dff 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -32,7 +32,7 @@ "bcryptjs": "2.4.3", "bull": "4.10.1", "correlation-id": "4.0.0", - "dd-trace": "3.13.2", + "dd-trace": "4.20.0", "dotenv": "16.0.1", "ioredis": "5.3.2", "joi": "17.6.0", From d681d5298bb3da65c41425676eb0556f0b7621d6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 18 Dec 2023 18:33:04 +0000 Subject: [PATCH 4/7] Type fixes. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 16 +++++++++++----- packages/server/src/integrations/couchdb.ts | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index c7c4f98e98..35b11fbd59 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -31,7 +31,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) { }) } -export function DatabaseWithConnection( +export function DatabaseWithConnectionNoDD( dbName: string, connection: string, opts?: DatabaseOpts @@ -39,10 +39,16 @@ export function DatabaseWithConnection( if (!connection) { throw new Error("Must provide connection details") } - return new DDInstrumentedDatabase( - new DatabaseImpl(dbName, opts, connection), - "couchdb" - ) + return new DatabaseImpl(dbName, opts, connection) +} + +export function DatabaseWithConnection( + dbName: string, + connection: string, + opts?: DatabaseOpts +) { + const db = DatabaseWithConnectionNoDD(dbName, connection, opts) + return new DDInstrumentedDatabase(db, "couchdb") } export class DatabaseImpl implements Database { diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index b55468fd93..4d78ace0ec 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -8,6 +8,7 @@ import { QueryType, } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" +import { DatabaseWithConnectionNoDD } from "@budibase/backend-core/src/db" interface CouchDBConfig { url: string @@ -69,7 +70,7 @@ class CouchDBIntegration implements IntegrationBase { private readonly client: dbCore.DatabaseImpl constructor(config: CouchDBConfig) { - this.client = dbCore.DatabaseWithConnection(config.database, config.url) + this.client = dbCore.DatabaseWithConnectionNoDD(config.database, config.url) } async testConnection() { From 93da29611d29494212fe62a288652205761cd943 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 19 Dec 2023 10:11:51 +0000 Subject: [PATCH 5/7] Fix type error. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 13 +------------ packages/server/src/integrations/couchdb.ts | 6 +++--- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 35b11fbd59..45aefc36f7 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -31,23 +31,12 @@ function buildNano(couchInfo: { url: string; cookie: string }) { }) } -export function DatabaseWithConnectionNoDD( - dbName: string, - connection: string, - opts?: DatabaseOpts -) { - if (!connection) { - throw new Error("Must provide connection details") - } - return new DatabaseImpl(dbName, opts, connection) -} - export function DatabaseWithConnection( dbName: string, connection: string, opts?: DatabaseOpts ) { - const db = DatabaseWithConnectionNoDD(dbName, connection, opts) + const db = new DatabaseImpl(dbName, opts, connection) return new DDInstrumentedDatabase(db, "couchdb") } diff --git a/packages/server/src/integrations/couchdb.ts b/packages/server/src/integrations/couchdb.ts index 4d78ace0ec..079f646b60 100644 --- a/packages/server/src/integrations/couchdb.ts +++ b/packages/server/src/integrations/couchdb.ts @@ -1,5 +1,6 @@ import { ConnectionInfo, + Database, DatasourceFeature, DatasourceFieldType, Document, @@ -8,7 +9,6 @@ import { QueryType, } from "@budibase/types" import { db as dbCore } from "@budibase/backend-core" -import { DatabaseWithConnectionNoDD } from "@budibase/backend-core/src/db" interface CouchDBConfig { url: string @@ -67,10 +67,10 @@ const SCHEMA: Integration = { } class CouchDBIntegration implements IntegrationBase { - private readonly client: dbCore.DatabaseImpl + private readonly client: Database constructor(config: CouchDBConfig) { - this.client = dbCore.DatabaseWithConnectionNoDD(config.database, config.url) + this.client = dbCore.DatabaseWithConnection(config.database, config.url) } async testConnection() { From 3697ff3efcbced61dedbf701032787c0c05c48c4 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 19 Dec 2023 10:26:28 +0000 Subject: [PATCH 6/7] Support Barcode, BigInt and User column types in automations (#12610) * Support barcode and bigint in automations * Support users in LinkedRowSelector * Fix clear relationships if empty * Make sure clearRelationships is initialised to false * Revert yarn lock * Refactor * Refactor --- .../automation/SetupPanel/RowSelectorTypes.svelte | 10 +++++++++- .../src/components/common/LinkedRowSelector.svelte | 8 +++++--- .../common/bindings/DrawerBindableSlot.svelte | 2 +- packages/server/src/automations/steps/updateRow.ts | 6 ++++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte index 3fd2708186..851c5b39c9 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte @@ -69,7 +69,15 @@ on:change={e => onChange(e, field)} useLabel={false} /> -{:else if schema.type === "string" || schema.type === "number"} +{:else if schema.type === "bb_reference"} + onChange(e, field)} + useLabel={false} + /> +{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)} row?._id || row ) - $: label = capitalise(schema.name) - $: linkedTableId = schema.tableId + $: label = label || capitalise(schema.name) + $: linkedTableId = linkedTableId || schema.tableId $: linkedTable = $tables.list.find(table => table._id === linkedTableId) $: fetchRows(linkedTableId) @@ -57,7 +59,7 @@ {:else} row._id} diff --git a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte index bfef0acac1..6689ddb650 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableSlot.svelte @@ -113,7 +113,7 @@ if (type === "json" && !isJSBinding(value)) { return "json-slot-icon" } - if (type !== "string" && type !== "number") { + if (!["string", "number", "bigint", "barcodeqr"].includes(type)) { return "slot-icon" } return "" diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 5ed4e19f61..fea3e981f3 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -84,9 +84,11 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { // clear any undefined, null or empty string properties so that they aren't updated for (let propKey of Object.keys(inputs.row)) { + const clearRelationships = + inputs.meta?.fields?.[propKey]?.clearRelationships if ( - (inputs.row[propKey] == null || inputs.row[propKey] === "") && - !inputs.meta?.fields?.[propKey]?.clearRelationships + (inputs.row[propKey] == null || inputs.row[propKey]?.length === 0) && + !clearRelationships ) { delete inputs.row[propKey] } From 3f115972514df8b5d802711f6b583181c509b087 Mon Sep 17 00:00:00 2001 From: melohagan <101575380+melohagan@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:02:39 +0000 Subject: [PATCH 7/7] Support both presence constraint variants (#12617) --- .../settings/controls/ValidationEditor/ValidationDrawer.svelte | 3 ++- packages/client/src/components/app/forms/validation.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte b/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte index 0bff2ccce6..cd50f526b5 100644 --- a/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/ValidationEditor/ValidationDrawer.svelte @@ -164,7 +164,8 @@ // Required constraint if ( field === dataSourceSchema?.table?.primaryDisplay || - constraints.presence?.allowEmpty === false + constraints.presence?.allowEmpty === false || + constraints.presence === true ) { rules.push({ constraint: "required", diff --git a/packages/client/src/components/app/forms/validation.js b/packages/client/src/components/app/forms/validation.js index 9224572c10..3b3a5d6e1d 100644 --- a/packages/client/src/components/app/forms/validation.js +++ b/packages/client/src/components/app/forms/validation.js @@ -23,7 +23,8 @@ export const createValidatorFromConstraints = ( // Required constraint if ( field === table?.primaryDisplay || - schemaConstraints.presence?.allowEmpty === false + schemaConstraints.presence?.allowEmpty === false || + schemaConstraints.presence === true ) { rules.push({ type: schemaConstraints.type == "array" ? "array" : "string",