From eb748db3bedd0fa51d9aa17155da3dfc5f5fdaaa Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 10:45:27 +0200 Subject: [PATCH 001/105] Add user type column selector --- .../backend/DataTable/modals/CreateEditColumn.svelte | 6 ++++++ packages/builder/src/constants/backend/index.js | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 44c37813d6..bfe7783e4f 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -20,6 +20,7 @@ import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, + DEV_FIELDS, RelationshipType, ALLOWABLE_STRING_OPTIONS, ALLOWABLE_NUMBER_OPTIONS, @@ -34,6 +35,7 @@ import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" + import { admin } from "stores/portal" const AUTO_TYPE = "auto" const FORMULA_TYPE = FIELDS.FORMULA.type @@ -70,6 +72,10 @@ fieldName: $tables.selected.name, } + if ($admin.isDev) { + fieldDefinitions = { ...fieldDefinitions, ...cloneDeep(DEV_FIELDS) } + } + $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ed0549eeca..45cd2d5ed4 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -122,6 +122,15 @@ export const FIELDS = { }, } +export const DEV_FIELDS = { + USER: { + name: "User", + type: "internal", + subtype: "user", + icon: "User", + }, +} + export const AUTO_COLUMN_SUB_TYPES = { AUTO_ID: "autoID", CREATED_BY: "createdBy", From 85a9b9ab21d17abb801004e092b8bc6ccc8519a2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 15:18:54 +0200 Subject: [PATCH 002/105] Typings --- .../backend/DataTable/modals/CreateEditColumn.svelte | 2 +- packages/server/src/api/controllers/table/utils.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index bfe7783e4f..c4342f9b0b 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -66,7 +66,7 @@ let jsonSchemaModal let allowedTypes = [] let editableColumn = { - type: "string", + type: fieldDefinitions.STRING.type, constraints: fieldDefinitions.STRING.constraints, // Initial value for column name in other table for linked records fieldName: $tables.selected.name, diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 0e5b784c66..e77c2b7647 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -422,13 +422,11 @@ export function hasTypeChanged(table: Table, oldTable: Table | undefined) { if (!oldTable) { return false } - let key: any - let field: any - for ([key, field] of Object.entries(oldTable.schema)) { - const oldType = field.type + for (let [key, field] of Object.entries(oldTable.schema)) { if (!table.schema[key]) { continue } + const oldType = field.type const newType = table.schema[key].type if (oldType !== newType && !areSwitchableTypes(oldType, newType)) { return true From 29b2e3b8f153ba7b1eb09ec8b3b6aa750ed1fcb3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 16:06:02 +0200 Subject: [PATCH 003/105] Handle internal types frontend --- .../DataTable/modals/CreateEditColumn.svelte | 38 ++++++++++++++++++- .../src/components/grid/lib/utils.js | 13 ++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index c4342f9b0b..f61e69943d 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -72,8 +72,28 @@ fieldName: $tables.selected.name, } + const typeMapping = {} + if ($admin.isDev) { - fieldDefinitions = { ...fieldDefinitions, ...cloneDeep(DEV_FIELDS) } + fieldDefinitions = { + ...fieldDefinitions, + ...Object.entries(DEV_FIELDS).reduce((p, [key, field]) => { + if (field.subtype) { + const composedType = `${field.type}_${field.subtype}` + p[key] = { + ...field, + type: composedType, + } + typeMapping[composedType] = { + type: field.type, + subtype: field.subtype, + } + } else { + p[key] = field + } + return p + }, {}), + } } $: if (primaryDisplay) { @@ -82,6 +102,7 @@ const initialiseField = (field, savingColumn) => { isCreating = !field + if (field && !savingColumn) { editableColumn = cloneDeep(field) originalName = editableColumn.name ? editableColumn.name + "" : null @@ -89,6 +110,14 @@ primaryDisplay = $tables.selected.primaryDisplay == null || $tables.selected.primaryDisplay === editableColumn.name + + const mapped = Object.entries(typeMapping).find( + ([_, v]) => v.type === field.type && v.subtype === field.subtype + ) + if (mapped) { + editableColumn.type = mapped[0] + delete editableColumn.subtype + } } else if (!savingColumn) { let highestNumber = 0 Object.keys(table.schema).forEach(columnName => { @@ -105,6 +134,7 @@ editableColumn.name = "Column 01" } } + allowedTypes = getAllowedTypes() } @@ -186,6 +216,10 @@ let saveColumn = cloneDeep(editableColumn) + if (typeMapping[saveColumn.type]) { + saveColumn = { ...saveColumn, ...typeMapping[saveColumn.type] } + } + if (saveColumn.type === AUTO_TYPE) { saveColumn = buildAutoColumn( $tables.selected.name, @@ -426,6 +460,8 @@ onMount(() => { mounted = true }) + + $: console.log({ type: editableColumn.type, allowedTypes, typeMapping }) diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js index d4298e6a1b..feff3da625 100644 --- a/packages/frontend-core/src/components/grid/lib/utils.js +++ b/packages/frontend-core/src/components/grid/lib/utils.js @@ -19,12 +19,21 @@ const TypeIconMap = { formula: "Calculator", json: "Brackets", bigint: "TagBold", + internal: { + user: "User", + }, } export const getColumnIcon = column => { if (column.schema.autocolumn) { return "MagicWand" } - const type = column.schema.type - return TypeIconMap[type] || "Text" + const { type, subtype } = column.schema + + const result = + typeof TypeIconMap[type] === "object" && subtype + ? TypeIconMap[type][subtype] + : TypeIconMap[type] + + return result || "Text" } From 1b1819b4a5d9d8d22acb1670ae43976d361d70b5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 16:16:47 +0200 Subject: [PATCH 004/105] Clean --- .../DataTable/modals/CreateEditColumn.svelte | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index f61e69943d..d9f95b493c 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -74,28 +74,6 @@ const typeMapping = {} - if ($admin.isDev) { - fieldDefinitions = { - ...fieldDefinitions, - ...Object.entries(DEV_FIELDS).reduce((p, [key, field]) => { - if (field.subtype) { - const composedType = `${field.type}_${field.subtype}` - p[key] = { - ...field, - type: composedType, - } - typeMapping[composedType] = { - type: field.type, - subtype: field.subtype, - } - } else { - p[key] = field - } - return p - }, {}), - } - } - $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } @@ -368,9 +346,35 @@ ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 ) { return ALLOWABLE_NUMBER_OPTIONS - } else if (!external) { + } + + let devFieldDefinitions = {} + if ($admin.isDev) { + devFieldDefinitions = Object.entries(DEV_FIELDS).reduce( + (p, [key, field]) => { + if (field.subtype) { + const composedType = `${field.type}_${field.subtype}` + p[key] = { + ...field, + type: composedType, + } + typeMapping[composedType] = { + type: field.type, + subtype: field.subtype, + } + } else { + p[key] = field + } + return p + }, + {} + ) + } + + if (!external) { return [ ...Object.values(fieldDefinitions), + ...Object.values(devFieldDefinitions), { name: "Auto Column", type: AUTO_TYPE }, ] } else { From 3c4e550c7d716fa2ba9e5d35d3eccd3b33baf3e1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 16:43:34 +0200 Subject: [PATCH 005/105] Clean --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index d9f95b493c..3619787a38 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -464,8 +464,6 @@ onMount(() => { mounted = true }) - - $: console.log({ type: editableColumn.type, allowedTypes, typeMapping }) From 78c5216ee8bc901484fb2da66c5ccebd0a60cc2c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 16:49:41 +0200 Subject: [PATCH 006/105] Merge dev fields to fields --- .../DataTable/modals/CreateEditColumn.svelte | 50 +++++++++---------- .../builder/src/constants/backend/index.js | 3 -- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 3619787a38..eb66d4d995 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -20,7 +20,6 @@ import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, - DEV_FIELDS, RelationshipType, ALLOWABLE_STRING_OPTIONS, ALLOWABLE_NUMBER_OPTIONS, @@ -73,6 +72,30 @@ } const typeMapping = {} + if (!$admin.isDev) { + delete fieldDefinitions.USER + } + + // Handling fields with subtypes + fieldDefinitions = Object.entries(fieldDefinitions).reduce( + (p, [key, field]) => { + if (field.subtype) { + const composedType = `${field.type}_${field.subtype}` + p[key] = { + ...field, + type: composedType, + } + typeMapping[composedType] = { + type: field.type, + subtype: field.subtype, + } + } else { + p[key] = field + } + return p + }, + {} + ) $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } @@ -348,33 +371,9 @@ return ALLOWABLE_NUMBER_OPTIONS } - let devFieldDefinitions = {} - if ($admin.isDev) { - devFieldDefinitions = Object.entries(DEV_FIELDS).reduce( - (p, [key, field]) => { - if (field.subtype) { - const composedType = `${field.type}_${field.subtype}` - p[key] = { - ...field, - type: composedType, - } - typeMapping[composedType] = { - type: field.type, - subtype: field.subtype, - } - } else { - p[key] = field - } - return p - }, - {} - ) - } - if (!external) { return [ ...Object.values(fieldDefinitions), - ...Object.values(devFieldDefinitions), { name: "Auto Column", type: AUTO_TYPE }, ] } else { @@ -393,6 +392,7 @@ if (!external || table.sql) { fields = [...fields, FIELDS.LINK, FIELDS.ARRAY] } + // fields = [...fields, ...Object.values(devFieldDefinitions)] return fields } } diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index 45cd2d5ed4..ccbc925672 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -120,9 +120,6 @@ export const FIELDS = { presence: false, }, }, -} - -export const DEV_FIELDS = { USER: { name: "User", type: "internal", From 6f78825592c50528b02f496dddc99d0cefaa161b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 16:51:15 +0200 Subject: [PATCH 007/105] Clean --- .../components/backend/DataTable/modals/CreateEditColumn.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index eb66d4d995..7fa016af22 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -392,7 +392,6 @@ if (!external || table.sql) { fields = [...fields, FIELDS.LINK, FIELDS.ARRAY] } - // fields = [...fields, ...Object.values(devFieldDefinitions)] return fields } } From 3ff4f9702b6fe4d4ac0c4a1a495984dd9685e284 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 17:46:50 +0200 Subject: [PATCH 008/105] Rename user type --- packages/builder/src/constants/backend/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ccbc925672..fbcab7fd8b 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -122,7 +122,7 @@ export const FIELDS = { }, USER: { name: "User", - type: "internal", + type: "bb_reference", subtype: "user", icon: "User", }, From 321bd8eec4b096e7e33c03ec4fc8c90035bbb5ba Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 17:47:45 +0200 Subject: [PATCH 009/105] Add type --- packages/types/src/documents/app/row.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index c09dc79b95..6001695b13 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -16,6 +16,7 @@ export enum FieldType { INTERNAL = "internal", BARCODEQR = "barcodeqr", BIGINT = "bigint", + BB_REFERENCE = "bb_reference", } export interface RowAttachment { From 2c715f5144f09bf120e1cc5bec7717cd539124d3 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 17:49:48 +0200 Subject: [PATCH 010/105] Renamings --- .../DataTable/modals/CreateEditColumn.svelte | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 7fa016af22..dad5660c8e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -35,6 +35,7 @@ import JSONSchemaModal from "./JSONSchemaModal.svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" import { admin } from "stores/portal" + import { FieldType } from "@budibase/types" const AUTO_TYPE = "auto" const FORMULA_TYPE = FIELDS.FORMULA.type @@ -71,7 +72,7 @@ fieldName: $tables.selected.name, } - const typeMapping = {} + const bbRefTypeMapping = {} if (!$admin.isDev) { delete fieldDefinitions.USER } @@ -79,13 +80,13 @@ // Handling fields with subtypes fieldDefinitions = Object.entries(fieldDefinitions).reduce( (p, [key, field]) => { - if (field.subtype) { + if (field.type === FieldType.BB_REFERENCE) { const composedType = `${field.type}_${field.subtype}` p[key] = { ...field, type: composedType, } - typeMapping[composedType] = { + bbRefTypeMapping[composedType] = { type: field.type, subtype: field.subtype, } @@ -112,7 +113,7 @@ $tables.selected.primaryDisplay == null || $tables.selected.primaryDisplay === editableColumn.name - const mapped = Object.entries(typeMapping).find( + const mapped = Object.entries(bbRefTypeMapping).find( ([_, v]) => v.type === field.type && v.subtype === field.subtype ) if (mapped) { @@ -217,8 +218,11 @@ let saveColumn = cloneDeep(editableColumn) - if (typeMapping[saveColumn.type]) { - saveColumn = { ...saveColumn, ...typeMapping[saveColumn.type] } + if (bbRefTypeMapping[saveColumn.type]) { + saveColumn = { + ...saveColumn, + ...bbRefTypeMapping[saveColumn.type], + } } if (saveColumn.type === AUTO_TYPE) { From 8564509ca91c8bdd26736f4b557a77700388312d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 14 Sep 2023 17:51:03 +0200 Subject: [PATCH 011/105] Allow bb_refs to externals --- .../backend/DataTable/modals/CreateEditColumn.svelte | 3 +++ packages/server/src/integrations/base/sqlTable.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index dad5660c8e..62cd7e413e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -396,6 +396,9 @@ if (!external || table.sql) { fields = [...fields, FIELDS.LINK, FIELDS.ARRAY] } + if (fieldDefinitions.USER) { + fields.push(fieldDefinitions.USER) + } return fields } } diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 47b09dadee..4383167f4a 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -41,6 +41,7 @@ function generateSchema( case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: case FieldTypes.BARCODEQR: + case FieldTypes.BB_REFERENCE: schema.text(key) break case FieldTypes.NUMBER: From 1cbfeafe39a395acbed19207ce708777e4e2f6c5 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 09:56:28 +0200 Subject: [PATCH 012/105] Remove context refs in row processor --- .../src/api/controllers/row/external.ts | 2 +- .../src/api/controllers/row/internal.ts | 4 ++-- .../server/src/api/controllers/table/utils.ts | 2 +- .../src/utilities/rowProcessor/index.ts | 20 +++++++++---------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 3881d1dd08..fa494d7b9c 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -93,7 +93,7 @@ export async function save(ctx: UserCtx) { const table = await sdk.tables.getTable(tableId) const { table: updatedTable, row } = inputProcessing( - ctx.user, + ctx.user?._id, cloneDeep(table), inputs ) diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index b80bd339e6..1441fe92a1 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -59,7 +59,7 @@ export async function patch(ctx: UserCtx) { const tableClone = cloneDeep(dbTable) // this returns the table and row incase they have been updated - let { table, row } = inputProcessing(ctx.user, tableClone, combinedRow) + let { table, row } = inputProcessing(ctx.user?._id, tableClone, combinedRow) const validateResult = await sdk.rows.utils.validate({ row, table, @@ -106,7 +106,7 @@ export async function save(ctx: UserCtx) { // need to copy the table so it can be differenced on way out const tableClone = cloneDeep(dbTable) - let { table, row } = inputProcessing(ctx.user, tableClone, inputs) + let { table, row } = inputProcessing(ctx.user?._id, tableClone, inputs) const validateResult = await sdk.rows.utils.validate({ row, diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index e77c2b7647..3a5b22b702 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -113,7 +113,7 @@ export function importToRows( // We use a reference to table here and update it after input processing, // so that we can auto increment auto IDs in imported data properly - const processed = inputProcessing(user, table, row, { + const processed = inputProcessing(user?._id, table, row, { noAutoRelationships: true, }) row = processed.row diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 8e95a15dca..232ba94808 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -5,8 +5,8 @@ import { ObjectStoreBuckets } from "../../constants" import { context, db as dbCore, objectStore } from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { Row, RowAttachment, Table, ContextUser } from "@budibase/types" -const { cloneDeep } = require("lodash/fp") +import { Row, RowAttachment, Table } from "@budibase/types" +import { cloneDeep } from "lodash/fp" export * from "./utils" type AutoColumnProcessingOpts = { @@ -48,12 +48,12 @@ function getRemovedAttachmentKeys( * for automatic ID purposes. */ export function processAutoColumn( - user: ContextUser | null, + userId: string | null | undefined, table: Table, row: Row, opts?: AutoColumnProcessingOpts ) { - let noUser = !user || !user.userId + let noUser = !userId let isUserTable = table._id === InternalTables.USER_METADATA let now = new Date().toISOString() // if a row doesn't have a revision then it doesn't exist yet @@ -70,8 +70,8 @@ export function processAutoColumn( } switch (schema.subtype) { case AutoFieldSubTypes.CREATED_BY: - if (creating && shouldUpdateUserFields && user) { - row[key] = [user.userId] + if (creating && shouldUpdateUserFields && userId) { + row[key] = [userId] } break case AutoFieldSubTypes.CREATED_AT: @@ -80,8 +80,8 @@ export function processAutoColumn( } break case AutoFieldSubTypes.UPDATED_BY: - if (shouldUpdateUserFields && user) { - row[key] = [user.userId] + if (shouldUpdateUserFields && userId) { + row[key] = [userId] } break case AutoFieldSubTypes.UPDATED_AT: @@ -131,7 +131,7 @@ export function coerce(row: any, type: string) { * @returns {object} the row which has been prepared to be written to the DB. */ export function inputProcessing( - user: ContextUser | null, + userId: string | null | undefined, table: Table, row: Row, opts?: AutoColumnProcessingOpts @@ -174,7 +174,7 @@ export function inputProcessing( } // handle auto columns - this returns an object like {table, row} - return processAutoColumn(user, table, clonedRow, opts) + return processAutoColumn(userId, table, clonedRow, opts) } /** From edd8879d0421b4b63cdf15c7a5ea61cbc712b116 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 10:31:54 +0200 Subject: [PATCH 013/105] inputProcessing async --- packages/server/src/api/controllers/application.ts | 2 +- .../server/src/api/controllers/row/external.ts | 2 +- .../server/src/api/controllers/row/internal.ts | 8 ++++++-- .../src/api/controllers/table/tests/utils.spec.ts | 4 ++-- packages/server/src/api/controllers/table/utils.ts | 6 +++--- .../src/db/defaultData/datasource_bb_default.ts | 14 +++++++------- .../server/src/utilities/rowProcessor/index.ts | 2 +- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 012aa7c66d..71bd034b9f 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -168,7 +168,7 @@ export const addSampleData = async (ctx: UserCtx) => { // Check if default datasource exists before creating it await sdk.datasources.get(DEFAULT_BB_DATASOURCE_ID) } catch (err: any) { - const defaultDbDocs = buildDefaultDocs() + const defaultDbDocs = await buildDefaultDocs() // add in the default db data docs - tables, datasource, rows and links await db.bulkDocs([...defaultDbDocs]) diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index fa494d7b9c..9de4f00ccb 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -92,7 +92,7 @@ export async function save(ctx: UserCtx) { } const table = await sdk.tables.getTable(tableId) - const { table: updatedTable, row } = inputProcessing( + const { table: updatedTable, row } = await inputProcessing( ctx.user?._id, cloneDeep(table), inputs diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 1441fe92a1..f33df4536f 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -59,7 +59,11 @@ export async function patch(ctx: UserCtx) { const tableClone = cloneDeep(dbTable) // this returns the table and row incase they have been updated - let { table, row } = inputProcessing(ctx.user?._id, tableClone, combinedRow) + let { table, row } = await inputProcessing( + ctx.user?._id, + tableClone, + combinedRow + ) const validateResult = await sdk.rows.utils.validate({ row, table, @@ -106,7 +110,7 @@ export async function save(ctx: UserCtx) { // need to copy the table so it can be differenced on way out const tableClone = cloneDeep(dbTable) - let { table, row } = inputProcessing(ctx.user?._id, tableClone, inputs) + let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs) const validateResult = await sdk.rows.utils.validate({ row, diff --git a/packages/server/src/api/controllers/table/tests/utils.spec.ts b/packages/server/src/api/controllers/table/tests/utils.spec.ts index 957cf51ecd..931a1cd90f 100644 --- a/packages/server/src/api/controllers/table/tests/utils.spec.ts +++ b/packages/server/src/api/controllers/table/tests/utils.spec.ts @@ -42,7 +42,7 @@ describe("utils", () => { const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }] - const result = importToRows(data, table, config.user) + const result = await importToRows(data, table, config.user) expect(result).toEqual([ expect.objectContaining({ autoId: 1, @@ -89,7 +89,7 @@ describe("utils", () => { const data = [{ name: "Alice" }, { name: "Bob" }, { name: "Claire" }] - const result = importToRows(data, table) + const result = await importToRows(data, table) expect(result).toHaveLength(3) }) }) diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 3a5b22b702..e0d564db2a 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -99,7 +99,7 @@ export function makeSureTableUpToDate(table: any, tableToSave: any) { return tableToSave } -export function importToRows( +export async function importToRows( data: any[], table: Table, user: ContextUser | null = null @@ -113,7 +113,7 @@ export function importToRows( // We use a reference to table here and update it after input processing, // so that we can auto increment auto IDs in imported data properly - const processed = inputProcessing(user?._id, table, row, { + const processed = await inputProcessing(user?._id, table, row, { noAutoRelationships: true, }) row = processed.row @@ -158,7 +158,7 @@ export async function handleDataImport( const db = context.getAppDB() const data = parse(rows, schema) - let finalData: any = importToRows(data, table, user) + let finalData: any = await importToRows(data, table, user) //Set IDs of finalData to match existing row if an update is expected if (identifierFields.length > 0) { diff --git a/packages/server/src/db/defaultData/datasource_bb_default.ts b/packages/server/src/db/defaultData/datasource_bb_default.ts index d01f598ce4..a4821667ff 100644 --- a/packages/server/src/db/defaultData/datasource_bb_default.ts +++ b/packages/server/src/db/defaultData/datasource_bb_default.ts @@ -34,9 +34,9 @@ function syncLastIds(table: Table, rowCount: number) { }) } -function tableImport(table: Table, data: Row[]) { +async function tableImport(table: Table, data: Row[]) { const cloneTable = cloneDeep(table) - const rowDocs = importToRows(data, cloneTable) + const rowDocs = await importToRows(data, cloneTable) syncLastIds(cloneTable, rowDocs.length) return { rows: rowDocs, table: cloneTable } } @@ -601,20 +601,20 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = { }, } -export function buildDefaultDocs() { - const inventoryData = tableImport( +export async function buildDefaultDocs() { + const inventoryData = await tableImport( DEFAULT_INVENTORY_TABLE_SCHEMA, inventoryImport ) - const employeeData = tableImport( + const employeeData = await tableImport( DEFAULT_EMPLOYEE_TABLE_SCHEMA, employeeImport ) - const jobData = tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport) + const jobData = await tableImport(DEFAULT_JOBS_TABLE_SCHEMA, jobsImport) - const expensesData = tableImport( + const expensesData = await tableImport( DEFAULT_EXPENSES_TABLE_SCHEMA, expensesImport ) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 232ba94808..92afd0dcb1 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -130,7 +130,7 @@ export function coerce(row: any, type: string) { * @param {object} opts some input processing options (like disabling auto-column relationships). * @returns {object} the row which has been prepared to be written to the DB. */ -export function inputProcessing( +export async function inputProcessing( userId: string | null | undefined, table: Table, row: Row, From 0e3f316eb236d4f5642fa37c72f7882a4114ec6c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 10:33:36 +0200 Subject: [PATCH 014/105] Process bb ref on input processing --- .../rowProcessor/bbReferenceProcessor.ts | 6 ++ .../src/utilities/rowProcessor/index.ts | 10 ++- .../tests/inputProcessing.spec.ts | 67 +++++++++++++++++++ packages/types/src/documents/app/row.ts | 10 +++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts create mode 100644 packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts new file mode 100644 index 0000000000..ae31810856 --- /dev/null +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -0,0 +1,6 @@ +import { FieldSubtype } from "@budibase/types" + +export async function processOutputBBReferences( + value: string, + subtype: FieldSubtype +) {} diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 92afd0dcb1..702c6c1d7d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -5,8 +5,9 @@ import { ObjectStoreBuckets } from "../../constants" import { context, db as dbCore, objectStore } from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { Row, RowAttachment, Table } from "@budibase/types" +import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" +import { processOutputBBReferences } from "./bbReferenceProcessor" export * from "./utils" type AutoColumnProcessingOpts = { @@ -166,6 +167,13 @@ export async function inputProcessing( }) } } + + if (field.type === FieldTypes.BB_REFERENCE) { + clonedRow[key] = await processOutputBBReferences( + value, + field.subtype as FieldSubtype + ) + } } if (!clonedRow._id || !clonedRow._rev) { diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts new file mode 100644 index 0000000000..d04221c1d0 --- /dev/null +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -0,0 +1,67 @@ +import TestConfiguration from "../../../tests/utilities/TestConfiguration" +import { inputProcessing } from ".." +import { generator, structures } from "@budibase/backend-core/tests" +import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types" +import * as bbReferenceProcessor from "../bbReferenceProcessor" + +const config = new TestConfiguration() + +jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ + processOutputBBReferences: jest.fn(), +})) + +describe("rowProcessor - inputProcessing", () => { + beforeAll(async () => { + await config.init() + }) + + it("populate user BB references", async () => { + const userId = generator.guid() + + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + name: "user", + constraints: { + presence: true, + type: "string", + }, + }, + }, + } + + const newRow = { + name: "Jack", + user: "123", + } + + const user = structures.users.user() + + ;( + bbReferenceProcessor.processOutputBBReferences as jest.Mock + ).mockResolvedValue(user) + + const { row } = await inputProcessing(userId, table, newRow) + + expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1) + expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith( + "123", + "user" + ) + + expect(row).toEqual({ ...newRow, user }) + }) +}) diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts index 6001695b13..708cbc3b9e 100644 --- a/packages/types/src/documents/app/row.ts +++ b/packages/types/src/documents/app/row.ts @@ -34,3 +34,13 @@ export interface Row extends Document { _viewId?: string [key: string]: any } + +export enum FieldSubtype { + USER = "user", +} + +export const FieldTypeSubtypes = { + BB_REFERENCE: { + USER: FieldSubtype.USER, + }, +} From e36fd75bc1e0b54396f3d2444d70e234dfbb4fd2 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 10:41:39 +0200 Subject: [PATCH 015/105] Add tests --- .../tests/inputProcessing.spec.ts | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index d04221c1d0..10e4a95cf9 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -15,7 +15,11 @@ describe("rowProcessor - inputProcessing", () => { await config.init() }) - it("populate user BB references", async () => { + beforeEach(() => { + jest.resetAllMocks() + }) + + it("processes BB references if on the schema and it's populated", async () => { const userId = generator.guid() const table: Table = { @@ -64,4 +68,83 @@ describe("rowProcessor - inputProcessing", () => { expect(row).toEqual({ ...newRow, user }) }) + + it("it does not processe BB references if on the schema but it is not populated", async () => { + const userId = generator.guid() + + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, + }, + }, + } + + const newRow = { + name: "Jack", + } + + const { row } = await inputProcessing(userId, table, newRow) + + expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + expect(row).toEqual({ ...newRow, user: undefined }) + }) + + it("it does not processe BB references if not in the schema", async () => { + const userId = generator.guid() + + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.NUMBER, + name: "user", + constraints: { + presence: true, + type: "string", + }, + }, + }, + } + + const newRow = { + name: "Jack", + user: "123", + } + + const { row } = await inputProcessing(userId, table, newRow) + + expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + expect(row).toEqual({ + name: "Jack", + user: 123, + }) + }) }) From 8a9474c2acae0dfe88fd98c8b07adefd6c07990e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 10:44:50 +0200 Subject: [PATCH 016/105] Clean test --- .../utilities/rowProcessor/tests/inputProcessing.spec.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 10e4a95cf9..a13dbbb769 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -1,20 +1,13 @@ -import TestConfiguration from "../../../tests/utilities/TestConfiguration" import { inputProcessing } from ".." import { generator, structures } from "@budibase/backend-core/tests" import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types" import * as bbReferenceProcessor from "../bbReferenceProcessor" -const config = new TestConfiguration() - jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ processOutputBBReferences: jest.fn(), })) describe("rowProcessor - inputProcessing", () => { - beforeAll(async () => { - await config.init() - }) - beforeEach(() => { jest.resetAllMocks() }) From 4c4c2e118b5cea6e1cd240fe5b41b44343173d46 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 10:54:43 +0200 Subject: [PATCH 017/105] Renames --- packages/server/src/utilities/rowProcessor/index.ts | 4 ++-- .../rowProcessor/tests/inputProcessing.spec.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 702c6c1d7d..a69bebd0a2 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -7,7 +7,7 @@ import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" -import { processOutputBBReferences } from "./bbReferenceProcessor" +import { processInputBBReferences } from "./bbReferenceProcessor" export * from "./utils" type AutoColumnProcessingOpts = { @@ -169,7 +169,7 @@ export async function inputProcessing( } if (field.type === FieldTypes.BB_REFERENCE) { - clonedRow[key] = await processOutputBBReferences( + clonedRow[key] = await processInputBBReferences( value, field.subtype as FieldSubtype ) diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index a13dbbb769..e291b62663 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -4,7 +4,7 @@ import { FieldType, FieldTypeSubtypes, Table } from "@budibase/types" import * as bbReferenceProcessor from "../bbReferenceProcessor" jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ - processOutputBBReferences: jest.fn(), + processInputBBReferences: jest.fn(), })) describe("rowProcessor - inputProcessing", () => { @@ -48,13 +48,13 @@ describe("rowProcessor - inputProcessing", () => { const user = structures.users.user() ;( - bbReferenceProcessor.processOutputBBReferences as jest.Mock + bbReferenceProcessor.processInputBBReferences as jest.Mock ).mockResolvedValue(user) const { row } = await inputProcessing(userId, table, newRow) - expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1) - expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith( + expect(bbReferenceProcessor.processInputBBReferences).toBeCalledTimes(1) + expect(bbReferenceProcessor.processInputBBReferences).toBeCalledWith( "123", "user" ) @@ -96,7 +96,7 @@ describe("rowProcessor - inputProcessing", () => { const { row } = await inputProcessing(userId, table, newRow) - expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled() expect(row).toEqual({ ...newRow, user: undefined }) }) @@ -134,7 +134,7 @@ describe("rowProcessor - inputProcessing", () => { const { row } = await inputProcessing(userId, table, newRow) - expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled() expect(row).toEqual({ name: "Jack", user: 123, From 1ff3f5db808f9bff417072421a49f5549a538ec1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 11:21:10 +0200 Subject: [PATCH 018/105] Handle ids --- .../tests/core/utilities/structures/shared.ts | 4 +- .../rowProcessor/bbReferenceProcessor.ts | 25 +++++++- .../tests/bbReferenceProcessor.spec.ts | 62 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) create mode 100644 packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts diff --git a/packages/backend-core/tests/core/utilities/structures/shared.ts b/packages/backend-core/tests/core/utilities/structures/shared.ts index de0e19486c..6e6292e3f4 100644 --- a/packages/backend-core/tests/core/utilities/structures/shared.ts +++ b/packages/backend-core/tests/core/utilities/structures/shared.ts @@ -1,12 +1,13 @@ import { User } from "@budibase/types" import { generator } from "./generator" import { uuid } from "./common" +import { tenant } from "." export const newEmail = () => { return `${uuid()}@test.com` } -export const user = (userProps?: any): User => { +export const user = (userProps?: Partial): User => { return { email: newEmail(), password: "test", @@ -14,6 +15,7 @@ export const user = (userProps?: any): User => { firstName: generator.first(), lastName: generator.last(), pictureUrl: "http://test.com", + tenantId: tenant.id(), ...userProps, } } diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index ae31810856..2d296cea3f 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -1,6 +1,27 @@ +import { cache } from "@budibase/backend-core" +import { utils } from "@budibase/shared-core" import { FieldSubtype } from "@budibase/types" -export async function processOutputBBReferences( +export async function processInputBBReferences( value: string, subtype: FieldSubtype -) {} +) { + const result = [] + const ids = value.split(",").map((id: string) => id.trim()) + + switch (subtype) { + case FieldSubtype.USER: + for (const id of ids) { + result.push(await cache.user.getUser(id)) + } + break + default: + utils.unreachable(subtype) + } + + if (result.length > 1) { + return result + } + + return result[0] +} diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts new file mode 100644 index 0000000000..95716b0467 --- /dev/null +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -0,0 +1,62 @@ +import * as backendCore from "@budibase/backend-core" +import { FieldSubtype, User } from "@budibase/types" +import { processInputBBReferences } from "../bbReferenceProcessor" +import { generator, structures } from "@budibase/backend-core/tests" + +jest.mock("@budibase/backend-core", (): typeof backendCore => { + const actual = jest.requireActual("@budibase/backend-core") + return { + ...actual, + cache: { + ...actual.cache, + user: { + getUser: jest.fn(), + invalidateUser: jest.fn(), + }, + }, + } +}) + +describe("bbReferenceProcessor", () => { + const mockedCacheGetUser = backendCore.cache.user.getUser as jest.Mock + + beforeEach(() => { + jest.resetAllMocks() + }) + + describe("processInputBBReferences", () => { + describe("subtype user", () => { + it("fetches by user id", async () => { + const input = generator.guid() + + const userFromCache = structures.users.user() + mockedCacheGetUser.mockResolvedValueOnce(userFromCache) + + const result = await processInputBBReferences(input, FieldSubtype.USER) + + expect(result).toEqual(userFromCache) + expect(mockedCacheGetUser).toBeCalledTimes(1) + expect(mockedCacheGetUser).toBeCalledWith(input) + }) + + it("fetches by user id when send as csv", async () => { + const users: Record = {} + for (let i = 0; i < 5; i++) { + const userId = generator.guid() + const user = structures.users.user({ _id: userId, userId }) + mockedCacheGetUser.mockResolvedValueOnce(user) + users[userId] = user + } + + const input = Object.keys(users).join(" , ") + const result = await processInputBBReferences(input, FieldSubtype.USER) + + expect(result).toEqual(Object.values(users)) + expect(mockedCacheGetUser).toBeCalledTimes(5) + Object.keys(users).forEach(userId => { + expect(mockedCacheGetUser).toBeCalledWith(userId) + }) + }) + }) + }) +}) From 0d3f9dac8c4119330976811026157cc70e33527d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 12:07:25 +0200 Subject: [PATCH 019/105] Validate and return csv --- .../rowProcessor/bbReferenceProcessor.ts | 33 ++++++++------ .../src/utilities/rowProcessor/errors.ts | 7 +++ .../tests/bbReferenceProcessor.spec.ts | 45 +++++++++++++++---- 3 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 packages/server/src/utilities/rowProcessor/errors.ts diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 2d296cea3f..7e78eb9693 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -1,27 +1,34 @@ import { cache } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" -import { FieldSubtype } from "@budibase/types" +import { Document, FieldSubtype } from "@budibase/types" +import { InvalidBBRefError } from "./errors" export async function processInputBBReferences( - value: string, + value: string | string[] | { _id: string } | { _id: string }[], subtype: FieldSubtype -) { - const result = [] - const ids = value.split(",").map((id: string) => id.trim()) +): Promise { + const result: string[] = [] switch (subtype) { case FieldSubtype.USER: - for (const id of ids) { - result.push(await cache.user.getUser(id)) + if (Array.isArray(value)) { + result.push(...value.map(x => (typeof x === "string" ? x : x._id))) + } else if (typeof value !== "string") { + result.push(value._id) + } else { + result.push(...value.split(",").map((id: string) => id.trim())) + } + + for (const id of result) { + const user = await cache.user.getUser(id) + if (!user) { + throw new InvalidBBRefError(id, FieldSubtype.USER) + } } break default: - utils.unreachable(subtype) + throw utils.unreachable(subtype) } - if (result.length > 1) { - return result - } - - return result[0] + return result.join(",") } diff --git a/packages/server/src/utilities/rowProcessor/errors.ts b/packages/server/src/utilities/rowProcessor/errors.ts new file mode 100644 index 0000000000..279a528b5f --- /dev/null +++ b/packages/server/src/utilities/rowProcessor/errors.ts @@ -0,0 +1,7 @@ +import { FieldSubtype } from "@budibase/types" + +export class InvalidBBRefError extends Error { + constructor(id: string, subtype: FieldSubtype) { + super(`Id "${id}" is not valid for the subtype "${subtype}"`) + } +} diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index 95716b0467..7e4312b398 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -2,6 +2,7 @@ import * as backendCore from "@budibase/backend-core" import { FieldSubtype, User } from "@budibase/types" import { processInputBBReferences } from "../bbReferenceProcessor" import { generator, structures } from "@budibase/backend-core/tests" +import { InvalidBBRefError } from "../errors" jest.mock("@budibase/backend-core", (): typeof backendCore => { const actual = jest.requireActual("@budibase/backend-core") @@ -26,7 +27,7 @@ describe("bbReferenceProcessor", () => { describe("processInputBBReferences", () => { describe("subtype user", () => { - it("fetches by user id", async () => { + it("validate valid string id", async () => { const input = generator.guid() const userFromCache = structures.users.user() @@ -34,29 +35,57 @@ describe("bbReferenceProcessor", () => { const result = await processInputBBReferences(input, FieldSubtype.USER) - expect(result).toEqual(userFromCache) + expect(result).toEqual(input) expect(mockedCacheGetUser).toBeCalledTimes(1) expect(mockedCacheGetUser).toBeCalledWith(input) }) - it("fetches by user id when send as csv", async () => { - const users: Record = {} + it("throws an error given an invalid id", async () => { + const input = generator.guid() + + await expect( + processInputBBReferences(input, FieldSubtype.USER) + ).rejects.toThrowError(new InvalidBBRefError(input, FieldSubtype.USER)) + }) + + it("validates valid user ids as csv", async () => { + const userIds: string[] = [] for (let i = 0; i < 5; i++) { const userId = generator.guid() const user = structures.users.user({ _id: userId, userId }) mockedCacheGetUser.mockResolvedValueOnce(user) - users[userId] = user + userIds.push(userId) } - const input = Object.keys(users).join(" , ") + const input = userIds.join(" , ") const result = await processInputBBReferences(input, FieldSubtype.USER) - expect(result).toEqual(Object.values(users)) + expect(result).toEqual(userIds.join(",")) expect(mockedCacheGetUser).toBeCalledTimes(5) - Object.keys(users).forEach(userId => { + userIds.forEach(userId => { expect(mockedCacheGetUser).toBeCalledWith(userId) }) }) + + it("throws an error given an invalid id in a csv", async () => { + const userId1 = generator.guid() + const userId2 = generator.guid() + const userId3 = generator.guid() + mockedCacheGetUser.mockResolvedValueOnce( + structures.users.user({ _id: userId1 }) + ) + mockedCacheGetUser.mockResolvedValueOnce( + structures.users.user({ _id: userId2 }) + ) + + const input = [userId1, userId2, userId3].join(" , ") + + await expect( + processInputBBReferences(input, FieldSubtype.USER) + ).rejects.toThrowError( + new InvalidBBRefError(userId3, FieldSubtype.USER) + ) + }) }) }) }) From d62e9a475f45b0939460a6f05aa1523f5842cfc8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 12:10:59 +0200 Subject: [PATCH 020/105] Validate objects --- .../tests/bbReferenceProcessor.spec.ts | 57 +++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index 7e4312b398..efc9805ef3 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -28,24 +28,24 @@ describe("bbReferenceProcessor", () => { describe("processInputBBReferences", () => { describe("subtype user", () => { it("validate valid string id", async () => { - const input = generator.guid() + const userId = generator.guid() const userFromCache = structures.users.user() mockedCacheGetUser.mockResolvedValueOnce(userFromCache) - const result = await processInputBBReferences(input, FieldSubtype.USER) + const result = await processInputBBReferences(userId, FieldSubtype.USER) - expect(result).toEqual(input) + expect(result).toEqual(userId) expect(mockedCacheGetUser).toBeCalledTimes(1) - expect(mockedCacheGetUser).toBeCalledWith(input) + expect(mockedCacheGetUser).toBeCalledWith(userId) }) it("throws an error given an invalid id", async () => { - const input = generator.guid() + const userId = generator.guid() await expect( - processInputBBReferences(input, FieldSubtype.USER) - ).rejects.toThrowError(new InvalidBBRefError(input, FieldSubtype.USER)) + processInputBBReferences(userId, FieldSubtype.USER) + ).rejects.toThrowError(new InvalidBBRefError(userId, FieldSubtype.USER)) }) it("validates valid user ids as csv", async () => { @@ -57,8 +57,11 @@ describe("bbReferenceProcessor", () => { userIds.push(userId) } - const input = userIds.join(" , ") - const result = await processInputBBReferences(input, FieldSubtype.USER) + const userIdCsv = userIds.join(" , ") + const result = await processInputBBReferences( + userIdCsv, + FieldSubtype.USER + ) expect(result).toEqual(userIds.join(",")) expect(mockedCacheGetUser).toBeCalledTimes(5) @@ -78,14 +81,46 @@ describe("bbReferenceProcessor", () => { structures.users.user({ _id: userId2 }) ) - const input = [userId1, userId2, userId3].join(" , ") + const userIdCsv = [userId1, userId2, userId3].join(" , ") await expect( - processInputBBReferences(input, FieldSubtype.USER) + processInputBBReferences(userIdCsv, FieldSubtype.USER) ).rejects.toThrowError( new InvalidBBRefError(userId3, FieldSubtype.USER) ) }) + + it("validate valid user object", async () => { + const userId = generator.guid() + + const userFromCache = structures.users.user() + mockedCacheGetUser.mockResolvedValueOnce(userFromCache) + + const result = await processInputBBReferences( + { _id: userId }, + FieldSubtype.USER + ) + + expect(result).toEqual(userId) + expect(mockedCacheGetUser).toBeCalledTimes(1) + expect(mockedCacheGetUser).toBeCalledWith(userId) + }) + + it("validate valid user object array", async () => { + const users = Array.from({ length: 3 }, () => ({ + _id: generator.guid(), + })) + + mockedCacheGetUser.mockResolvedValue(structures.users.user()) + + const result = await processInputBBReferences(users, FieldSubtype.USER) + + expect(result).toEqual(users.map(x => x._id).join(",")) + expect(mockedCacheGetUser).toBeCalledTimes(3) + for (const user of users) { + expect(mockedCacheGetUser).toBeCalledWith(user._id) + } + }) }) }) }) From 679f56f32fd2edc0a889b6069085f0b765ac137a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 12:29:57 +0200 Subject: [PATCH 021/105] Error handling --- .../utilities/rowProcessor/bbReferenceProcessor.ts | 13 ++++++++++--- .../rowProcessor/tests/bbReferenceProcessor.spec.ts | 5 +++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 7e78eb9693..a9a4e5fddf 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -20,9 +20,16 @@ export async function processInputBBReferences( } for (const id of result) { - const user = await cache.user.getUser(id) - if (!user) { - throw new InvalidBBRefError(id, FieldSubtype.USER) + try { + const user = await cache.user.getUser(id) + if (!user) { + throw new InvalidBBRefError(id, FieldSubtype.USER) + } + } catch (err: any) { + if (err != null && err.status === 404 && err.error === "not_found") { + throw new InvalidBBRefError(id, FieldSubtype.USER) + } + throw err } } break diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index efc9805ef3..748bc7efb6 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -43,6 +43,11 @@ describe("bbReferenceProcessor", () => { it("throws an error given an invalid id", async () => { const userId = generator.guid() + mockedCacheGetUser.mockRejectedValue({ + status: 404, + error: "not_found", + }) + await expect( processInputBBReferences(userId, FieldSubtype.USER) ).rejects.toThrowError(new InvalidBBRefError(userId, FieldSubtype.USER)) From 9510c37b7a344490a0113390d27fefdce471b3bc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 13:04:45 +0200 Subject: [PATCH 022/105] Handle bb ref only if there is a value --- .../src/utilities/rowProcessor/index.ts | 2 +- .../tests/inputProcessing.spec.ts | 46 ++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index a69bebd0a2..e30eecf829 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -168,7 +168,7 @@ export async function inputProcessing( } } - if (field.type === FieldTypes.BB_REFERENCE) { + if (field.type === FieldTypes.BB_REFERENCE && value) { clonedRow[key] = await processInputBBReferences( value, field.subtype as FieldSubtype diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index e291b62663..65630da2b9 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -62,7 +62,7 @@ describe("rowProcessor - inputProcessing", () => { expect(row).toEqual({ ...newRow, user }) }) - it("it does not processe BB references if on the schema but it is not populated", async () => { + it("it does not process BB references if on the schema but it is not populated", async () => { const userId = generator.guid() const table: Table = { @@ -100,7 +100,49 @@ describe("rowProcessor - inputProcessing", () => { expect(row).toEqual({ ...newRow, user: undefined }) }) - it("it does not processe BB references if not in the schema", async () => { + it.each([undefined, null, ""])( + "it does not process BB references the field is $%", + async userValue => { + const userId = generator.guid() + + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, + }, + }, + } + + const newRow = { + name: "Jack", + user: userValue, + } + + const { row } = await inputProcessing(userId, table, newRow) + + expect(bbReferenceProcessor.processInputBBReferences).not.toBeCalled() + expect(row).toEqual(newRow) + } + ) + + it("it does not process BB references if not in the schema", async () => { const userId = generator.guid() const table: Table = { From bebe342b33803fb68333e7db430d75f92b4de3eb Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 13:31:22 +0200 Subject: [PATCH 023/105] Handle output processing --- .../rowProcessor/bbReferenceProcessor.ts | 30 +++- .../tests/inputProcessing.spec.ts | 1 + .../tests/outputProcessing.spec.ts | 142 ++++++++++++++++++ 3 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index a9a4e5fddf..10b81f9788 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -1,6 +1,6 @@ import { cache } from "@budibase/backend-core" import { utils } from "@budibase/shared-core" -import { Document, FieldSubtype } from "@budibase/types" +import { FieldSubtype } from "@budibase/types" import { InvalidBBRefError } from "./errors" export async function processInputBBReferences( @@ -39,3 +39,31 @@ export async function processInputBBReferences( return result.join(",") } + +export async function processOutputBBReferences( + value: string, + subtype: FieldSubtype +) { + const result = [] + + switch (subtype) { + case FieldSubtype.USER: + for (const id of value.split(",")) { + try { + const user = await cache.user.getUser(id) + if (user) { + result.push(user) + } + } catch {} + } + break + default: + throw utils.unreachable(subtype) + } + + if (result.length > 1) { + return result + } + + return result[0] +} diff --git a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts index 65630da2b9..18d5128986 100644 --- a/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts @@ -5,6 +5,7 @@ import * as bbReferenceProcessor from "../bbReferenceProcessor" jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ processInputBBReferences: jest.fn(), + processOutputBBReferences: jest.fn(), })) describe("rowProcessor - inputProcessing", () => { diff --git a/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts new file mode 100644 index 0000000000..1b780bed54 --- /dev/null +++ b/packages/server/src/utilities/rowProcessor/tests/outputProcessing.spec.ts @@ -0,0 +1,142 @@ +import { + FieldSubtype, + FieldType, + FieldTypeSubtypes, + Table, +} from "@budibase/types" +import { outputProcessing } from ".." +import { generator, structures } from "@budibase/backend-core/tests" +import * as bbReferenceProcessor from "../bbReferenceProcessor" + +jest.mock("../bbReferenceProcessor", (): typeof bbReferenceProcessor => ({ + processInputBBReferences: jest.fn(), + processOutputBBReferences: jest.fn(), +})) + +describe("rowProcessor - outputProcessing", () => { + beforeEach(() => { + jest.resetAllMocks() + }) + + const processOutputBBReferencesMock = + bbReferenceProcessor.processOutputBBReferences as jest.Mock + + it("fetches bb user references given a populated field", async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, + }, + }, + } + + const row = { + name: "Jack", + user: "123", + } + + const user = structures.users.user() + processOutputBBReferencesMock.mockResolvedValue(user) + + const result = await outputProcessing(table, row, { squash: false }) + + expect(result).toEqual({ name: "Jack", user }) + + expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledTimes(1) + expect(bbReferenceProcessor.processOutputBBReferences).toBeCalledWith( + "123", + FieldSubtype.USER + ) + }) + + it("does not fetch bb references when fields are empty", async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.BB_REFERENCE, + subtype: FieldTypeSubtypes.BB_REFERENCE.USER, + name: "user", + constraints: { + presence: false, + type: "string", + }, + }, + }, + } + + const row = { + name: "Jack", + } + + const result = await outputProcessing(table, row, { squash: false }) + + expect(result).toEqual({ name: "Jack" }) + + expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + }) + + it("does not fetch bb references when not in the schema", async () => { + const table: Table = { + _id: generator.guid(), + name: "TestTable", + type: "table", + schema: { + name: { + type: FieldType.STRING, + name: "name", + constraints: { + presence: true, + type: "string", + }, + }, + user: { + type: FieldType.NUMBER, + name: "user", + constraints: { + presence: false, + type: "string", + }, + }, + }, + } + + const row = { + name: "Jack", + user: "123", + } + + const result = await outputProcessing(table, row, { squash: false }) + + expect(result).toEqual({ name: "Jack", user: "123" }) + + expect(bbReferenceProcessor.processOutputBBReferences).not.toBeCalled() + }) +}) From 385989eca4fe6d914a877b3a031947e452762692 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 13:31:56 +0200 Subject: [PATCH 024/105] Process output --- .../src/utilities/rowProcessor/index.ts | 15 ++++++- .../tests/bbReferenceProcessor.spec.ts | 45 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index e30eecf829..8aea0a08aa 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -7,7 +7,10 @@ import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" -import { processInputBBReferences } from "./bbReferenceProcessor" +import { + processInputBBReferences, + processOutputBBReferences, +} from "./bbReferenceProcessor" export * from "./utils" type AutoColumnProcessingOpts = { @@ -224,6 +227,16 @@ export async function outputProcessing( attachment.url = objectStore.getAppFileUrl(attachment.key) }) } + } else if (column.type == FieldTypes.BB_REFERENCE) { + for (let row of enriched) { + if (!row[property]) { + continue + } + row[property] = await processOutputBBReferences( + row[property], + column.subtype as FieldSubtype + ) + } } } if (opts.squash) { diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index 748bc7efb6..abba713fba 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -1,6 +1,9 @@ import * as backendCore from "@budibase/backend-core" import { FieldSubtype, User } from "@budibase/types" -import { processInputBBReferences } from "../bbReferenceProcessor" +import { + processInputBBReferences, + processOutputBBReferences, +} from "../bbReferenceProcessor" import { generator, structures } from "@budibase/backend-core/tests" import { InvalidBBRefError } from "../errors" @@ -128,4 +131,44 @@ describe("bbReferenceProcessor", () => { }) }) }) + + describe("processOutputBBReferences", () => { + describe("subtype user", () => { + it("fetches user given a valid string id", async () => { + const userId = generator.guid() + + const userFromCache = structures.users.user() + mockedCacheGetUser.mockResolvedValueOnce(userFromCache) + + const result = await processOutputBBReferences( + userId, + FieldSubtype.USER + ) + + expect(result).toEqual(userFromCache) + expect(mockedCacheGetUser).toBeCalledTimes(1) + expect(mockedCacheGetUser).toBeCalledWith(userId) + }) + + it("fetches user given a valid string id csv", async () => { + const userId1 = generator.guid() + const userId2 = generator.guid() + + const userFromCache1 = structures.users.user({ _id: userId1 }) + const userFromCache2 = structures.users.user({ _id: userId2 }) + mockedCacheGetUser.mockResolvedValueOnce(userFromCache1) + mockedCacheGetUser.mockResolvedValueOnce(userFromCache2) + + const result = await processOutputBBReferences( + [userId1, userId2].join(","), + FieldSubtype.USER + ) + + expect(result).toEqual([userFromCache1, userFromCache2]) + expect(mockedCacheGetUser).toBeCalledTimes(2) + expect(mockedCacheGetUser).toBeCalledWith(userId1) + expect(mockedCacheGetUser).toBeCalledWith(userId2) + }) + }) + }) }) From 29eaeacc4c05b8ef4b7cc379805be7949c73d1d7 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 13:47:08 +0200 Subject: [PATCH 025/105] Fix deletions --- .../server/src/utilities/rowProcessor/bbReferenceProcessor.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 10b81f9788..48a26796a8 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -44,6 +44,10 @@ export async function processOutputBBReferences( value: string, subtype: FieldSubtype ) { + if (typeof value !== "string") { + return value + } + const result = [] switch (subtype) { From 60d94e76cfb3ccbcbdb3796d41e4dcbe5070fde0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 13:49:24 +0200 Subject: [PATCH 026/105] Lint --- .../server/src/utilities/rowProcessor/bbReferenceProcessor.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 48a26796a8..d580ebacae 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -58,7 +58,9 @@ export async function processOutputBBReferences( if (user) { result.push(user) } - } catch {} + } catch { + // If user cannot be found, we just strip it + } } break default: From 7f6ef551c9ac11c7ecd3ad99946f518cfb88be1d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 17:34:59 +0200 Subject: [PATCH 027/105] Return always arrays for bb processor fields --- .../src/utilities/rowProcessor/bbReferenceProcessor.ts | 6 +----- .../rowProcessor/tests/bbReferenceProcessor.spec.ts | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index d580ebacae..23b3a8deee 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -67,9 +67,5 @@ export async function processOutputBBReferences( throw utils.unreachable(subtype) } - if (result.length > 1) { - return result - } - - return result[0] + return result } diff --git a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts index abba713fba..8b82ee24c7 100644 --- a/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts +++ b/packages/server/src/utilities/rowProcessor/tests/bbReferenceProcessor.spec.ts @@ -145,7 +145,7 @@ describe("bbReferenceProcessor", () => { FieldSubtype.USER ) - expect(result).toEqual(userFromCache) + expect(result).toEqual([userFromCache]) expect(mockedCacheGetUser).toBeCalledTimes(1) expect(mockedCacheGetUser).toBeCalledWith(userId) }) From ead2a2795e19a74a4c6df959da3c0305b4072430 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 19:06:06 +0200 Subject: [PATCH 028/105] Fix deletion --- .../server/src/utilities/rowProcessor/bbReferenceProcessor.ts | 2 +- packages/server/src/utilities/rowProcessor/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts index 23b3a8deee..89b74d4e09 100644 --- a/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts +++ b/packages/server/src/utilities/rowProcessor/bbReferenceProcessor.ts @@ -52,7 +52,7 @@ export async function processOutputBBReferences( switch (subtype) { case FieldSubtype.USER: - for (const id of value.split(",")) { + for (const id of value.split(",").filter(x => !!x)) { try { const user = await cache.user.getUser(id) if (user) { diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 8aea0a08aa..7098eefa47 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -229,7 +229,7 @@ export async function outputProcessing( } } else if (column.type == FieldTypes.BB_REFERENCE) { for (let row of enriched) { - if (!row[property]) { + if (!row[property] == null) { continue } row[property] = await processOutputBBReferences( From 256ab3a296a49a718019dcb16919e9c6ace60f6b Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 18 Sep 2023 12:22:18 +0200 Subject: [PATCH 029/105] Fix test --- packages/server/src/utilities/rowProcessor/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 7098eefa47..8aea0a08aa 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -229,7 +229,7 @@ export async function outputProcessing( } } else if (column.type == FieldTypes.BB_REFERENCE) { for (let row of enriched) { - if (!row[property] == null) { + if (!row[property]) { continue } row[property] = await processOutputBBReferences( From 43bbf8d093e762e218067c9eca242a8bcf428b2a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 14:15:15 +0200 Subject: [PATCH 030/105] Display icon in column --- packages/frontend-core/src/components/grid/lib/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js index feff3da625..9383f69f66 100644 --- a/packages/frontend-core/src/components/grid/lib/utils.js +++ b/packages/frontend-core/src/components/grid/lib/utils.js @@ -19,7 +19,7 @@ const TypeIconMap = { formula: "Calculator", json: "Brackets", bigint: "TagBold", - internal: { + bb_reference: { user: "User", }, } From ae237d3714e92c26c525e4ac8c29f1a16cbaac40 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 17:05:38 +0200 Subject: [PATCH 031/105] Reference cell --- .../components/grid/cells/BBReferenceCell.svelte | 16 ++++++++++++++++ .../src/components/grid/lib/renderers.js | 2 ++ 2 files changed, 18 insertions(+) create mode 100644 packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte new file mode 100644 index 0000000000..a7f65a337c --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -0,0 +1,16 @@ + + + diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js index 104984ca4e..f5d4cfe297 100644 --- a/packages/frontend-core/src/components/grid/lib/renderers.js +++ b/packages/frontend-core/src/components/grid/lib/renderers.js @@ -9,6 +9,7 @@ import BooleanCell from "../cells/BooleanCell.svelte" import FormulaCell from "../cells/FormulaCell.svelte" import JSONCell from "../cells/JSONCell.svelte" import AttachmentCell from "../cells/AttachmentCell.svelte" +import BBReferenceCell from "../cells/BBReferenceCell.svelte" const TypeComponentMap = { text: TextCell, @@ -23,6 +24,7 @@ const TypeComponentMap = { link: RelationshipCell, formula: FormulaCell, json: JSONCell, + bb_reference: BBReferenceCell, } export const getCellRenderer = column => { return TypeComponentMap[column?.schema?.type] || TextCell From 45a474f7ea836bb3231c0deeafe4b8f435f1e64c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 17:28:18 +0200 Subject: [PATCH 032/105] Fix refs --- .../src/components/grid/cells/BBReferenceCell.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte index a7f65a337c..1f41f42a08 100644 --- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -1,10 +1,10 @@ - + diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte index ac7869bc4b..74610ee431 100644 --- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte @@ -30,6 +30,8 @@ export let invertX = false export let invertY = false export let contentLines = 1 + export let searchFunction = API.searchTable + export let primaryDisplay const { API, dispatch } = getContext("grid") const color = getColor(0) @@ -38,7 +40,6 @@ let searchResults let searchString let lastSearchString - let primaryDisplay let candidateIndex let lastSearchId let searching = false @@ -96,7 +97,7 @@ lastSearchId = Math.random() searching = true const thisSearchId = lastSearchId - const results = await API.searchTable({ + const results = await searchFunction({ paginate: false, tableId: schema.tableId, limit: 20, From e06d3296b043c7cde9391efe02b34f3248d0ca11 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 19:06:06 +0200 Subject: [PATCH 034/105] Fix deletion --- .../src/components/grid/cells/BBReferenceCell.svelte | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte index a075aaf8ee..4f87250b4d 100644 --- a/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/BBReferenceCell.svelte @@ -32,5 +32,4 @@ {schema} {searchFunction} primaryDisplay={"email"} - relationshipType={"many-to-one"} /> From 98a46fa15558580bba04d7e7d5d4268ecf486190 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 15 Sep 2023 19:11:45 +0200 Subject: [PATCH 035/105] Display --- .../src/components/grid/cells/RelationshipCell.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte index 74610ee431..93356e2e7e 100644 --- a/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte +++ b/packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte @@ -260,14 +260,16 @@ on:wheel={e => (focused ? e.stopPropagation() : null)} > {#each value || [] as relationship} - {#if relationship.primaryDisplay} + {#if relationship[primaryDisplay] || relationship.primaryDisplay}
showRelationship(relationship._id) : null} > - {readable(relationship.primaryDisplay)} + {readable( + relationship[primaryDisplay] || relationship.primaryDisplay + )} {#if editable} Date: Mon, 18 Sep 2023 10:10:52 +0200 Subject: [PATCH 036/105] Handle single/multiple users on creation and display --- .../DataTable/modals/CreateEditColumn.svelte | 102 ++++++++++++------ 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 62cd7e413e..a00448c918 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -35,7 +35,7 @@ import JSONSchemaModal from "./JSONSchemaModal.svelte" import { ValidColumnNameRegex } from "@budibase/shared-core" import { admin } from "stores/portal" - import { FieldType } from "@budibase/types" + import { FieldSubtype, FieldType } from "@budibase/types" const AUTO_TYPE = "auto" const FORMULA_TYPE = FIELDS.FORMULA.type @@ -44,6 +44,11 @@ const NUMBER_TYPE = FIELDS.NUMBER.type const JSON_TYPE = FIELDS.JSON.type const DATE_TYPE = FIELDS.DATETIME.type + const BB_REFERENCE_TYPE = FieldType.BB_REFERENCE + const BB_USER_REFERENCE_TYPE = composeType( + FieldType.BB_REFERENCE, + FieldSubtype.USER + ) const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] @@ -77,11 +82,15 @@ delete fieldDefinitions.USER } + function composeType(fieldType, subtype) { + return `${fieldType}_${subtype}` + } + // Handling fields with subtypes fieldDefinitions = Object.entries(fieldDefinitions).reduce( (p, [key, field]) => { - if (field.type === FieldType.BB_REFERENCE) { - const composedType = `${field.type}_${field.subtype}` + if (field.type === BB_REFERENCE_TYPE) { + const composedType = composeType(field.type, field.subtype) p[key] = { ...field, type: composedType, @@ -98,6 +107,8 @@ {} ) + $: isBBReference = !!bbRefTypeMapping[editableColumn.type] + $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } @@ -303,9 +314,10 @@ // Default relationships many to many if (editableColumn.type === LINK_TYPE) { editableColumn.relationshipType = RelationshipType.MANY_TO_MANY - } - if (editableColumn.type === FORMULA_TYPE) { + } else if (editableColumn.type === FORMULA_TYPE) { editableColumn.formulaType = "dynamic" + } else if (editableColumn.type === BB_USER_REFERENCE_TYPE) { + editableColumn.relationshipType = RelationshipType.ONE_TO_MANY } } @@ -334,32 +346,52 @@ } function getRelationshipOptions(field) { - if (!field || !field.tableId) { - return null + switch (field.type) { + case FieldType.LINK: + if (!field || !field.tableId) { + return null + } + const linkTable = tableOptions?.find( + table => table._id === field.tableId + ) + if (!linkTable) { + return null + } + const thisName = truncate(table.name, { length: 14 }), + linkName = truncate(linkTable.name, { length: 14 }) + return [ + { + name: `Many ${thisName} rows → many ${linkName} rows`, + alt: `Many ${table.name} rows → many ${linkTable.name} rows`, + value: RelationshipType.MANY_TO_MANY, + }, + { + name: `One ${linkName} row → many ${thisName} rows`, + alt: `One ${linkTable.name} rows → many ${table.name} rows`, + value: RelationshipType.ONE_TO_MANY, + }, + { + name: `One ${thisName} row → many ${linkName} rows`, + alt: `One ${table.name} rows → many ${linkTable.name} rows`, + value: RelationshipType.MANY_TO_ONE, + }, + ] + case BB_USER_REFERENCE_TYPE: + return [ + { + name: `Single user`, + alt: `Single user`, + value: RelationshipType.ONE_TO_MANY, + }, + { + name: `Multiple users`, + alt: `Multiple users`, + value: RelationshipType.MANY_TO_ONE, + }, + ] + default: + return null } - const linkTable = tableOptions?.find(table => table._id === field.tableId) - if (!linkTable) { - return null - } - const thisName = truncate(table.name, { length: 14 }), - linkName = truncate(linkTable.name, { length: 14 }) - return [ - { - name: `Many ${thisName} rows → many ${linkName} rows`, - alt: `Many ${table.name} rows → many ${linkTable.name} rows`, - value: RelationshipType.MANY_TO_MANY, - }, - { - name: `One ${linkName} row → many ${thisName} rows`, - alt: `One ${linkTable.name} rows → many ${table.name} rows`, - value: RelationshipType.ONE_TO_MANY, - }, - { - name: `One ${thisName} row → many ${linkName} rows`, - alt: `One ${table.name} rows → many ${linkTable.name} rows`, - value: RelationshipType.MANY_TO_ONE, - }, - ] } function getAllowedTypes() { @@ -666,6 +698,16 @@ + {:else if isBBReference} + option.name} + getOptionValue={option => option.value} + getOptionTitle={option => option.alt} + /> {/if} {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} populateExtraQuery(extraQueryFields)} - bind:value={extraQueryFields[key]} - /> - {/if} + + {#if type === "string"} + populateExtraQuery(extraQueryFields)} + bind:value={extraQueryFields[key]} + /> + {/if} - {#if type === "list"} - populateExtraQuery(extraQueryFields)} + bind:value={extraQueryFields[key]} + options={config[key].data[query.queryVerb]} + getOptionLabel={current => current} + /> + {/if} {/each} - - diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 5d35498cfe..096d5c0f71 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -34,6 +34,7 @@ export let bindings = [] export let bindingDrawerLeft export let allowHelpers = true + export let customButtonText = null let fields = Object.entries(object || {}).map(([name, value]) => ({ name, @@ -158,9 +159,13 @@ {/if} {#if !readOnly && !noAddButton}
- Add{name ? ` ${lowercase(name)}` : ""} + + {#if customButtonText} + {customButtonText} + {:else} + {`Add${name ? ` ${lowercase(name)}` : ""}`} + {/if} +
{/if} diff --git a/packages/builder/src/components/integration/QueryViewer.svelte b/packages/builder/src/components/integration/QueryViewer.svelte index 4683bc6335..6e7ee52268 100644 --- a/packages/builder/src/components/integration/QueryViewer.svelte +++ b/packages/builder/src/components/integration/QueryViewer.svelte @@ -1,364 +1,443 @@ - { - navigateTo = null - }} -> - { - await saveQuery() - override = true - resumeNavigation() - }} - onCancel={async () => { - override = true - resumeNavigation() - }} - > - Leaving this section will mean losing and changes to your query - - - -
- - Query {integrationInfo?.friendlyName} - - Config -
-
- - { - let newValue = e.target.value || "" - if (newValue.match(ValidQueryNameRegex)) { - query.name = newValue.trim() - nameError = null - } else { - nameError = "Invalid query name" - } - }} - error={nameError} - /> -
- {#if queryConfig} -
- - { + let newValue = e.target.value || "" + if (newValue.match(ValidQueryNameRegex)) { + newQuery.name = newValue.trim() + nameError = null + } else { + nameError = "Invalid query name" + } + }} + error={nameError} + /> + {#if integration.query} + +