diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index 5a0ac4f283..7b411e801c 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -188,4 +188,17 @@ describe("utils", () => { expectResult(false) }) }) + + describe("hasCircularStructure", () => { + it("should detect a circular structure", () => { + const a: any = { b: "b" } + const b = { a } + a.b = b + expect(utils.hasCircularStructure(b)).toBe(true) + }) + + it("should allow none circular structures", () => { + expect(utils.hasCircularStructure({ a: "b" })).toBe(false) + }) + }) }) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b92471a7a4..1c1ca8473b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -237,3 +237,17 @@ export function timeout(timeMs: number) { export function isAudited(event: Event) { return !!AuditedEventFriendlyName[event] } + +export function hasCircularStructure(json: any) { + if (typeof json !== "object") { + return false + } + try { + JSON.stringify(json) + } catch (err) { + if (err instanceof Error && err?.message.includes("circular structure")) { + return true + } + } + return false +} diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 66439d3411..2cf3da3dda 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) { } ctx.body = Object.values(deployments.history).reverse() } catch (err) { - console.error(err) ctx.body = [] } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c3c5468840..7c98fecb9b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -23,7 +23,6 @@ import { breakRowIdField, convertRowId, generateRowIdField, - getPrimaryDisplay, isRowId, isSQL, } from "../../../integrations/utils" @@ -237,7 +236,7 @@ function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" - return processFormulas(table, thisRow) + return thisRow } function fixArrayTypes(row: Row, table: Table) { @@ -392,7 +391,7 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } - squashRelationshipColumns( + processRelationshipFields( table: Table, row: Row, relationships: RelationshipsJson[] @@ -402,7 +401,6 @@ export class ExternalRequest { if (!linkedTable || !row[relationship.column]) { continue } - const display = linkedTable.primaryDisplay for (let key of Object.keys(row[relationship.column])) { let relatedRow: Row = row[relationship.column][key] // add this row as context for the relationship @@ -411,15 +409,10 @@ export class ExternalRequest { relatedRow[col.name] = [row] } } + // process additional types + relatedRow = processDates(table, relatedRow) relatedRow = processFormulas(linkedTable, relatedRow) - let relatedDisplay - if (display) { - relatedDisplay = getPrimaryDisplay(relatedRow[display]) - } - row[relationship.column][key] = { - primaryDisplay: relatedDisplay || "Invalid display column", - _id: relatedRow._id, - } + row[relationship.column][key] = relatedRow } } return row @@ -521,14 +514,14 @@ export class ExternalRequest { ) } - // Process some additional data types - let finalRowArray = Object.values(finalRows) - finalRowArray = processDates(table, finalRowArray) - finalRowArray = processFormulas(table, finalRowArray) as Row[] - - return finalRowArray.map((row: Row) => - this.squashRelationshipColumns(table, row, relationships) + // make sure all related rows are correct + let finalRowArray = Object.values(finalRows).map(row => + this.processRelationshipFields(table, row, relationships) ) + + // process some additional types + finalRowArray = processDates(table, finalRowArray) + return finalRowArray } /** @@ -663,7 +656,7 @@ export class ExternalRequest { linkPrimary, linkSecondary, }: { - row: { [key: string]: any } + row: Row linkPrimary: string linkSecondary?: string }) { diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 0515b6b97e..287b2ae6aa 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx) { relationships: true, }) const enrichedRow = await outputProcessing(table, row, { + squash: true, preserveLinks: true, }) return { @@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) { }) return { ...response, - row: await outputProcessing(table, row, { preserveLinks: true }), + row: await outputProcessing(table, row, { + preserveLinks: true, + squash: true, + }), } } else { return response @@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise { const table = await sdk.tables.getTable(tableId) // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case return await outputProcessing(table, row, { - squash: false, + squash: true, preserveLinks: true, }) } @@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { tables, filters: { oneOf: { @@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { }, includeSqlRelationships: IncludeRelationship.INCLUDE, }) + row[fieldName] = await outputProcessing(linkedTable, relatedRows, { + squash: true, + preserveLinks: true, + }) } return row } diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 87d551bc4b..8d52b6a05c 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) { for (let row of rows) { // find the enriched row, if found process the formulas const enrichedRow = enrichedRows.find( - (enriched: any) => enriched._id === row._id + (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { const processed = processFormulas(table, cloneDeep(row), { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // values have changed, need to add to bulk docs to update if (!isEqual(processed, row)) { @@ -139,7 +139,7 @@ export async function finaliseRow( // use enriched row to generate formulas for saving, specifically only use as context row = processFormulas(table, row, { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will @@ -163,7 +163,9 @@ export async function finaliseRow( const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev - enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + enrichedRow = processFormulas(table, enrichedRow, { + dynamic: false, + }) // this updates the related formulas in other rows based on the relations to this row if (updateFormula) { await updateRelatedFormula(table, enrichedRow) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index c426d59f4e..060f6e46c1 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -10,6 +10,7 @@ import { FieldSchema, FieldType, FieldTypeSubtypes, + FormulaTypes, INTERNAL_TABLE_SOURCE_ID, MonthlyQuotaName, PermissionLevel, @@ -2000,4 +2001,52 @@ describe.each([ }) }) }) + + describe("Formula fields", () => { + let relationshipTable: Table, tableId: string, relatedRow: Row + + beforeAll(async () => { + const otherTableId = config.table!._id! + const cfg = generateTableConfig() + relationshipTable = await config.createLinkedTable( + RelationshipType.ONE_TO_MANY, + ["links"], + { + ...cfg, + // needs to be a short name + name: "b", + schema: { + ...cfg.schema, + formula: { + name: "formula", + type: FieldType.FORMULA, + formula: "{{ links.0.name }}", + formulaType: FormulaTypes.DYNAMIC, + }, + }, + } + ) + + tableId = relationshipTable._id! + + relatedRow = await config.api.row.save(otherTableId, { + name: generator.word(), + description: generator.paragraph(), + }) + await config.api.row.save(tableId, { + name: generator.word(), + description: generator.paragraph(), + tableId, + links: [relatedRow._id], + }) + }) + + it("should be able to search for rows containing formulas", async () => { + const { rows } = await config.api.row.search(tableId) + expect(rows.length).toBe(1) + expect(rows[0].links.length).toBe(1) + const row = rows[0] + expect(row.formula).toBe(relatedRow.name) + }) + }) }) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 8cccf1b96a..7324fa1d94 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -2,7 +2,6 @@ import LinkController from "./LinkController" import { IncludeDocs, getLinkDocuments, - createLinkView, getUniqueByProp, getRelatedTableForField, getLinkedTableIDs, diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index db9a0dc7d5..5942e7e5a1 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -8,6 +8,7 @@ import { LinkDocumentValue, Table, } from "@budibase/types" +import sdk from "../../sdk" export { createLinkView } from "../views/staticViews" @@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] { } export async function getLinkedTable(id: string, tables: Table[]) { - const db = context.getAppDB() let linkedTable = tables.find(table => table._id === id) if (linkedTable) { return linkedTable } - linkedTable = await db.get(id) + linkedTable = await sdk.tables.getTable(id) if (linkedTable) { tables.push(linkedTable) } diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 60416853b3..8dc49a9489 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -923,7 +923,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) @@ -932,7 +931,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 974900ba6d..2fc6caeb39 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -80,7 +80,10 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - rows = await outputProcessing(table, rows, { preserveLinks: true }) + rows = await outputProcessing(table, rows, { + preserveLinks: true, + squash: true, + }) // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } @@ -185,6 +188,7 @@ export async function fetch(tableId: string): Promise { const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response, { preserveLinks: true, + squash: true, }) } diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts index 20b1d6f9ee..3d4cf6c82c 100644 --- a/packages/server/src/tests/utilities/api/row.ts +++ b/packages/server/src/tests/utilities/api/row.ts @@ -6,6 +6,7 @@ import { ExportRowsRequest, BulkImportRequest, BulkImportResponse, + SearchRowResponse, } from "@budibase/types" import TestConfiguration from "../TestConfiguration" import { TestAPI } from "./base" @@ -154,7 +155,7 @@ export class RowAPI extends TestAPI { search = async ( sourceId: string, { expectStatus } = { expectStatus: 200 } - ): Promise => { + ): Promise => { const request = this.request .post(`/api/${sourceId}/search`) .set(this.config.defaultHeaders()) diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 098962c646..0e53422a4f 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows" import { FieldTypes, AutoFieldSubTypes } from "../../constants" import { processFormulas, fixAutoColumnSubType } from "./utils" import { ObjectStoreBuckets } from "../../constants" -import { context, db as dbCore, objectStore } from "@budibase/backend-core" +import { + context, + db as dbCore, + objectStore, + utils, +} from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { - AutoColumnFieldMetadata, - FieldSubtype, - Row, - RowAttachment, - Table, -} from "@budibase/types" +import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { processInputBBReferences, @@ -233,6 +232,11 @@ export async function outputProcessing( }) : safeRows + // make sure squash is enabled if needed + if (!opts.squash && utils.hasCircularStructure(rows)) { + opts.squash = true + } + // process complex types: attachements, bb references... for (let [property, column] of Object.entries(table.schema)) { if (column.type === FieldTypes.ATTACHMENT) { @@ -258,7 +262,7 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = processFormulas(table, enriched, { dynamic: true }) as Row[] + enriched = processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { enriched = (await linkRows.squashLinksToPrimaryDisplay( diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 48697af6a9..9eb725dd7c 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -12,6 +12,11 @@ import { Table, } from "@budibase/types" +interface FormulaOpts { + dynamic?: boolean + contextRows?: Row[] +} + /** * If the subtype has been lost for any reason this works out what * subtype the auto column should be. @@ -40,52 +45,50 @@ export function fixAutoColumnSubType( /** * Looks through the rows provided and finds formulas - which it then processes. */ -export function processFormulas( +export function processFormulas( table: Table, - rows: Row[] | Row, - { dynamic, contextRows }: any = { dynamic: true } -) { - const single = !Array.isArray(rows) - let rowArray: Row[] - if (single) { - rowArray = [rows] - contextRows = contextRows ? [contextRows] : contextRows - } else { - rowArray = rows - } - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + inputRows: T, + { dynamic, contextRows }: FormulaOpts = { dynamic: true } +): T { + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rowArray.length; i++) { - let row = rowArray[i] - let context = contextRows ? contextRows[i] : row - rowArray[i] = { - ...row, - [column]: processStringSync(schema.formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } } } - } - return single ? rowArray[0] : rowArray + return Array.isArray(inputRows) ? rows : rows[0] } /** * Processes any date columns and ensures that those without the ignoreTimezones * flag set are parsed as UTC rather than local time. */ -export function processDates(table: Table, rows: Row[]) { - let datesWithTZ = [] +export function processDates( + table: Table, + inputRows: T +): T { + let rows = Array.isArray(inputRows) ? inputRows : [inputRows] + let datesWithTZ: string[] = [] for (let [column, schema] of Object.entries(table.schema)) { if (schema.type !== FieldTypes.DATETIME) { continue @@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) { } } } - return rows + + return Array.isArray(inputRows) ? rows : rows[0] }