From 635af0f76aec1c8da2139701ecd3ab99f606a38e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Nov 2023 19:03:37 +0000 Subject: [PATCH 01/20] Getting external rows to be more consistent with the internal API - the formulas should be processed using the outputProcessing as they were being processed, but was causing an overwrite. The problem was that the external system internally processed the formulas/relationships, then squashed the relationships. Once it got to the external API, it would go through normal output processing, which would run over the squashed rows, causing an inconsistent behaviour. --- .../src/api/controllers/deploy/index.ts | 1 - .../api/controllers/row/ExternalRequest.ts | 33 ++++---- .../src/api/controllers/row/external.ts | 14 +++- .../src/api/controllers/row/staticFormula.ts | 14 ++-- packages/server/src/db/linkedRows/index.ts | 1 - .../server/src/db/linkedRows/linkUtils.ts | 4 +- .../src/sdk/app/rows/search/external.ts | 5 +- .../src/utilities/rowProcessor/index.ts | 2 +- .../src/utilities/rowProcessor/utils.ts | 76 ++++++++++--------- 9 files changed, 79 insertions(+), 71 deletions(-) diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index 66439d3411..2cf3da3dda 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) { } ctx.body = Object.values(deployments.history).reverse() } catch (err) { - console.error(err) ctx.body = [] } } diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index c3c5468840..7c98fecb9b 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -23,7 +23,6 @@ import { breakRowIdField, convertRowId, generateRowIdField, - getPrimaryDisplay, isRowId, isSQL, } from "../../../integrations/utils" @@ -237,7 +236,7 @@ function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" - return processFormulas(table, thisRow) + return thisRow } function fixArrayTypes(row: Row, table: Table) { @@ -392,7 +391,7 @@ export class ExternalRequest { return { row: newRow, manyRelationships } } - squashRelationshipColumns( + processRelationshipFields( table: Table, row: Row, relationships: RelationshipsJson[] @@ -402,7 +401,6 @@ export class ExternalRequest { if (!linkedTable || !row[relationship.column]) { continue } - const display = linkedTable.primaryDisplay for (let key of Object.keys(row[relationship.column])) { let relatedRow: Row = row[relationship.column][key] // add this row as context for the relationship @@ -411,15 +409,10 @@ export class ExternalRequest { relatedRow[col.name] = [row] } } + // process additional types + relatedRow = processDates(table, relatedRow) relatedRow = processFormulas(linkedTable, relatedRow) - let relatedDisplay - if (display) { - relatedDisplay = getPrimaryDisplay(relatedRow[display]) - } - row[relationship.column][key] = { - primaryDisplay: relatedDisplay || "Invalid display column", - _id: relatedRow._id, - } + row[relationship.column][key] = relatedRow } } return row @@ -521,14 +514,14 @@ export class ExternalRequest { ) } - // Process some additional data types - let finalRowArray = Object.values(finalRows) - finalRowArray = processDates(table, finalRowArray) - finalRowArray = processFormulas(table, finalRowArray) as Row[] - - return finalRowArray.map((row: Row) => - this.squashRelationshipColumns(table, row, relationships) + // make sure all related rows are correct + let finalRowArray = Object.values(finalRows).map(row => + this.processRelationshipFields(table, row, relationships) ) + + // process some additional types + finalRowArray = processDates(table, finalRowArray) + return finalRowArray } /** @@ -663,7 +656,7 @@ export class ExternalRequest { linkPrimary, linkSecondary, }: { - row: { [key: string]: any } + row: Row linkPrimary: string linkSecondary?: string }) { diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 0515b6b97e..287b2ae6aa 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx) { relationships: true, }) const enrichedRow = await outputProcessing(table, row, { + squash: true, preserveLinks: true, }) return { @@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) { }) return { ...response, - row: await outputProcessing(table, row, { preserveLinks: true }), + row: await outputProcessing(table, row, { + preserveLinks: true, + squash: true, + }), } } else { return response @@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise { const table = await sdk.tables.getTable(tableId) // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case return await outputProcessing(table, row, { - squash: false, + squash: true, preserveLinks: true, }) } @@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) const primaryLink = linkedTable.primary?.[0] as string - row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, { + const relatedRows = await handleRequest(Operation.READ, linkedTableId!, { tables, filters: { oneOf: { @@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { }, includeSqlRelationships: IncludeRelationship.INCLUDE, }) + row[fieldName] = await outputProcessing(linkedTable, relatedRows, { + squash: true, + preserveLinks: true, + }) } return row } diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 6f426c6fa0..87cbdad23f 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) { const db = context.getAppDB() // start by getting the raw rows (which will be written back to DB after update) let rows = ( - await db.allDocs( + await db.allDocs( getRowParams(table._id, null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc as Row) // now enrich the rows, note the clone so that we have the base state of the // rows so that we don't write any of the enriched information back let enrichedRows = await outputProcessing(table, cloneDeep(rows), { @@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) { for (let row of rows) { // find the enriched row, if found process the formulas const enrichedRow = enrichedRows.find( - (enriched: any) => enriched._id === row._id + (enriched: Row) => enriched._id === row._id ) if (enrichedRow) { const processed = processFormulas(table, cloneDeep(row), { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // values have changed, need to add to bulk docs to update if (!isEqual(processed, row)) { @@ -139,7 +139,7 @@ export async function finaliseRow( // use enriched row to generate formulas for saving, specifically only use as context row = processFormulas(table, row, { dynamic: false, - contextRows: enrichedRow, + contextRows: [enrichedRow], }) // don't worry about rev, tables handle rev/lastID updates // if another row has been written since processing this will @@ -163,7 +163,9 @@ export async function finaliseRow( const response = await db.put(row) // for response, calculate the formulas for the enriched row enrichedRow._rev = response.rev - enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false }) + enrichedRow = processFormulas(table, enrichedRow, { + dynamic: false, + }) // this updates the related formulas in other rows based on the relations to this row if (updateFormula) { await updateRelatedFormula(table, enrichedRow) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7a7a06551e..0dba773ad7 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -2,7 +2,6 @@ import LinkController from "./LinkController" import { IncludeDocs, getLinkDocuments, - createLinkView, getUniqueByProp, getRelatedTableForField, getLinkedTableIDs, diff --git a/packages/server/src/db/linkedRows/linkUtils.ts b/packages/server/src/db/linkedRows/linkUtils.ts index db9a0dc7d5..5942e7e5a1 100644 --- a/packages/server/src/db/linkedRows/linkUtils.ts +++ b/packages/server/src/db/linkedRows/linkUtils.ts @@ -8,6 +8,7 @@ import { LinkDocumentValue, Table, } from "@budibase/types" +import sdk from "../../sdk" export { createLinkView } from "../views/staticViews" @@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] { } export async function getLinkedTable(id: string, tables: Table[]) { - const db = context.getAppDB() let linkedTable = tables.find(table => table._id === id) if (linkedTable) { return linkedTable } - linkedTable = await db.get(id) + linkedTable = await sdk.tables.getTable(id) if (linkedTable) { tables.push(linkedTable) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 981ae1bf8d..71c12f0a79 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -80,7 +80,10 @@ export async function search(options: SearchParams) { rows = rows.map((r: any) => pick(r, fields)) } - rows = await outputProcessing(table, rows, { preserveLinks: true }) + rows = await outputProcessing(table, rows, { + preserveLinks: true, + squash: true, + }) // need wrapper object for bookmarks etc when paginating return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 098962c646..6070d40f7d 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -258,7 +258,7 @@ export async function outputProcessing( } // process formulas after the complex types had been processed - enriched = processFormulas(table, enriched, { dynamic: true }) as Row[] + enriched = processFormulas(table, enriched, { dynamic: true }) if (opts.squash) { enriched = (await linkRows.squashLinksToPrimaryDisplay( diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 48697af6a9..8d4af29a6b 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -12,6 +12,11 @@ import { Table, } from "@budibase/types" +interface FormulaOpts { + dynamic?: boolean + contextRows?: Row[] +} + /** * If the subtype has been lost for any reason this works out what * subtype the auto column should be. @@ -40,52 +45,50 @@ export function fixAutoColumnSubType( /** * Looks through the rows provided and finds formulas - which it then processes. */ -export function processFormulas( +export function processFormulas( table: Table, - rows: Row[] | Row, - { dynamic, contextRows }: any = { dynamic: true } -) { - const single = !Array.isArray(rows) - let rowArray: Row[] - if (single) { - rowArray = [rows] - contextRows = contextRows ? [contextRows] : contextRows - } else { - rowArray = rows - } - for (let [column, schema] of Object.entries(table.schema)) { - if (schema.type !== FieldTypes.FORMULA) { - continue - } + inputRows: T, + { dynamic, contextRows }: FormulaOpts = { dynamic: true } +): Promise { + const rows = Array.isArray(inputRows) ? inputRows : [inputRows] + if (rows) + for (let [column, schema] of Object.entries(table.schema)) { + if (schema.type !== FieldTypes.FORMULA) { + continue + } - const isStatic = schema.formulaType === FormulaTypes.STATIC + const isStatic = schema.formulaType === FormulaTypes.STATIC - if ( - schema.formula == null || - (dynamic && isStatic) || - (!dynamic && !isStatic) - ) { - continue - } - // iterate through rows and process formula - for (let i = 0; i < rowArray.length; i++) { - let row = rowArray[i] - let context = contextRows ? contextRows[i] : row - rowArray[i] = { - ...row, - [column]: processStringSync(schema.formula, context), + if ( + schema.formula == null || + (dynamic && isStatic) || + (!dynamic && !isStatic) + ) { + continue + } + // iterate through rows and process formula + for (let i = 0; i < rows.length; i++) { + let row = rows[i] + let context = contextRows ? contextRows[i] : row + rows[i] = { + ...row, + [column]: processStringSync(schema.formula, context), + } } } - } - return single ? rowArray[0] : rowArray + return Array.isArray(inputRows) ? rows : rows[0] } /** * Processes any date columns and ensures that those without the ignoreTimezones * flag set are parsed as UTC rather than local time. */ -export function processDates(table: Table, rows: Row[]) { - let datesWithTZ = [] +export function processDates( + table: Table, + inputRows: T +): T { + let rows = Array.isArray(inputRows) ? inputRows : [inputRows] + let datesWithTZ: string[] = [] for (let [column, schema] of Object.entries(table.schema)) { if (schema.type !== FieldTypes.DATETIME) { continue @@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) { } } } - return rows + + return Array.isArray(inputRows) ? rows : rows[0] } From 49b00ee7eeb23717b5f075064b021e0aa74e6429 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 11:07:25 +0000 Subject: [PATCH 02/20] Fixing build issue. --- packages/server/src/utilities/rowProcessor/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts index 8d4af29a6b..9eb725dd7c 100644 --- a/packages/server/src/utilities/rowProcessor/utils.ts +++ b/packages/server/src/utilities/rowProcessor/utils.ts @@ -49,7 +49,7 @@ export function processFormulas( table: Table, inputRows: T, { dynamic, contextRows }: FormulaOpts = { dynamic: true } -): Promise { +): T { const rows = Array.isArray(inputRows) ? inputRows : [inputRows] if (rows) for (let [column, schema] of Object.entries(table.schema)) { From fdae7ab913ec22e7ce22ff01e3356c13e1d53d05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:37:45 +0000 Subject: [PATCH 03/20] Fixing issues discovered by tests. --- packages/server/src/integration-test/postgres.spec.ts | 2 -- packages/server/src/sdk/app/rows/search/external.ts | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts index 60416853b3..8dc49a9489 100644 --- a/packages/server/src/integration-test/postgres.spec.ts +++ b/packages/server/src/integration-test/postgres.spec.ts @@ -923,7 +923,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) @@ -932,7 +931,6 @@ describe("postgres integrations", () => { [m2mFieldName]: [ { _id: row._id, - primaryDisplay: "Invalid display column", }, ], }) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 71c12f0a79..1edd6b30f2 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -186,6 +186,7 @@ export async function fetch(tableId: string): Promise { const table = await sdk.tables.getTable(tableId) return await outputProcessing(table, response, { preserveLinks: true, + squash: true, }) } From 312415ca7d1e99f71d3442f0b9674b1f73d8b349 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:47:21 +0000 Subject: [PATCH 04/20] Enforcing squash for rows which contain circular structures. --- packages/backend-core/src/utils/utils.ts | 14 +++++++++++++ .../src/utilities/rowProcessor/index.ts | 20 +++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/utils/utils.ts b/packages/backend-core/src/utils/utils.ts index b92471a7a4..1c1ca8473b 100644 --- a/packages/backend-core/src/utils/utils.ts +++ b/packages/backend-core/src/utils/utils.ts @@ -237,3 +237,17 @@ export function timeout(timeMs: number) { export function isAudited(event: Event) { return !!AuditedEventFriendlyName[event] } + +export function hasCircularStructure(json: any) { + if (typeof json !== "object") { + return false + } + try { + JSON.stringify(json) + } catch (err) { + if (err instanceof Error && err?.message.includes("circular structure")) { + return true + } + } + return false +} diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 6070d40f7d..0e53422a4f 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows" import { FieldTypes, AutoFieldSubTypes } from "../../constants" import { processFormulas, fixAutoColumnSubType } from "./utils" import { ObjectStoreBuckets } from "../../constants" -import { context, db as dbCore, objectStore } from "@budibase/backend-core" +import { + context, + db as dbCore, + objectStore, + utils, +} from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" -import { - AutoColumnFieldMetadata, - FieldSubtype, - Row, - RowAttachment, - Table, -} from "@budibase/types" +import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types" import { cloneDeep } from "lodash/fp" import { processInputBBReferences, @@ -233,6 +232,11 @@ export async function outputProcessing( }) : safeRows + // make sure squash is enabled if needed + if (!opts.squash && utils.hasCircularStructure(rows)) { + opts.squash = true + } + // process complex types: attachements, bb references... for (let [property, column] of Object.entries(table.schema)) { if (column.type === FieldTypes.ATTACHMENT) { From 8d35453f015b53d4f9a159ea6a494bfb227cd702 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 7 Nov 2023 13:56:42 +0000 Subject: [PATCH 05/20] Adding test case for circular detection. --- packages/backend-core/src/utils/tests/utils.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/backend-core/src/utils/tests/utils.spec.ts b/packages/backend-core/src/utils/tests/utils.spec.ts index 5a0ac4f283..7b411e801c 100644 --- a/packages/backend-core/src/utils/tests/utils.spec.ts +++ b/packages/backend-core/src/utils/tests/utils.spec.ts @@ -188,4 +188,17 @@ describe("utils", () => { expectResult(false) }) }) + + describe("hasCircularStructure", () => { + it("should detect a circular structure", () => { + const a: any = { b: "b" } + const b = { a } + a.b = b + expect(utils.hasCircularStructure(b)).toBe(true) + }) + + it("should allow none circular structures", () => { + expect(utils.hasCircularStructure({ a: "b" })).toBe(false) + }) + }) }) From fdfda100c1f019067bda247924d452eb4344c1a2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 7 Nov 2023 18:14:52 +0000 Subject: [PATCH 06/20] Ensure that the DB always returns Documents. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 6 ++- packages/backend-core/src/db/views.ts | 27 ++++++----- packages/backend-core/src/users/db.ts | 10 ++-- packages/backend-core/src/users/users.ts | 2 +- packages/server/src/api/controllers/layout.ts | 4 +- .../server/src/api/controllers/permission.ts | 6 +-- packages/server/src/api/controllers/role.ts | 12 +++-- .../src/api/controllers/row/internal.ts | 9 ++-- .../src/api/controllers/row/staticFormula.ts | 4 +- .../server/src/api/controllers/view/utils.ts | 4 +- packages/server/src/automations/triggers.ts | 10 ++-- packages/server/src/db/linkedRows/index.ts | 23 +++++---- .../server/src/sdk/app/applications/import.ts | 2 +- .../server/src/sdk/app/applications/sync.ts | 10 ++-- .../src/sdk/app/datasources/datasources.ts | 6 +-- packages/server/src/sdk/app/tables/getters.ts | 4 +- packages/server/src/sdk/users/utils.ts | 30 +++++++----- packages/server/src/utilities/global.ts | 48 +++++++++---------- .../server/src/utilities/routing/index.ts | 4 +- packages/types/src/documents/app/user.ts | 10 ++-- packages/types/src/documents/pouch.ts | 8 ++-- packages/types/src/sdk/db.ts | 6 ++- .../worker/src/constants/templates/index.ts | 6 +-- packages/worker/src/utilities/email.ts | 11 +---- 24 files changed, 140 insertions(+), 122 deletions(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 29ca4123f5..330b15e680 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -175,12 +175,14 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: DatabaseQueryOpts): Promise> { + async allDocs( + params: DatabaseQueryOpts + ): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query( + async query( viewName: string, params: DatabaseQueryOpts ): Promise> { diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0980ad217..fbcdbde5fc 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -7,7 +7,12 @@ import { } from "../constants" import { getGlobalDB } from "../context" import { doWithDB } from "./" -import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types" +import { + AllDocsResponse, + Database, + DatabaseQueryOpts, + Document, +} from "@budibase/types" import env from "../environment" const DESIGN_DB = "_design/database" @@ -109,7 +114,7 @@ export interface QueryViewOptions { arrayResponse?: boolean } -export async function queryViewRaw( +export async function queryViewRaw( viewName: ViewName, params: DatabaseQueryOpts, db: Database, @@ -137,18 +142,16 @@ export async function queryViewRaw( } } -export const queryView = async ( +export const queryView = async ( viewName: ViewName, params: DatabaseQueryOpts, db: Database, createFunc: any, opts?: QueryViewOptions -): Promise => { +): Promise => { const response = await queryViewRaw(viewName, params, db, createFunc, opts) const rows = response.rows - const docs = rows.map((row: any) => - params.include_docs ? row.doc : row.value - ) + const docs = rows.map(row => (params.include_docs ? row.doc! : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -198,11 +201,11 @@ export const createPlatformUserView = async () => { await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE) } -export const queryPlatformView = async ( +export const queryPlatformView = async ( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions -): Promise => { +): Promise => { const CreateFuncByName: any = { [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView, [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, @@ -220,7 +223,7 @@ const CreateFuncByName: any = { [ViewName.USER_BY_APP]: createUserAppView, } -export const queryGlobalView = async ( +export const queryGlobalView = async ( viewName: ViewName, params: DatabaseQueryOpts, db?: Database, @@ -231,10 +234,10 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db!, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } -export async function queryGlobalViewRaw( +export async function queryGlobalViewRaw( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index c071064713..59f698d99c 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -413,15 +413,13 @@ export class UserDB { } // Get users and delete - const allDocsResponse: AllDocsResponse = await db.allDocs({ + const allDocsResponse = await db.allDocs({ include_docs: true, keys: userIds, }) - const usersToDelete: User[] = allDocsResponse.rows.map( - (user: RowResponse) => { - return user.doc - } - ) + const usersToDelete = allDocsResponse.rows.map(user => { + return user.doc! + }) // Delete from DB const toDelete = usersToDelete.map(user => ({ diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 6dc8750b62..9f4a41f6df 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async ( include_docs: true, }) params.startkey = opts && opts.startkey ? opts.startkey : params.startkey - let response = await queryGlobalView(ViewName.USER_BY_APP, params) + let response = await queryGlobalView(ViewName.USER_BY_APP, params) if (!response) { response = [] diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts index c00252d643..2a359592c7 100644 --- a/packages/server/src/api/controllers/layout.ts +++ b/packages/server/src/api/controllers/layout.ts @@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) { layoutRev = ctx.params.layoutRev const layoutsUsedByScreens = ( - await db.allDocs( + await db.allDocs( getScreenParams(null, { include_docs: true, }) ) - ).rows.map(element => element.doc.layoutId) + ).rows.map(element => element.doc!.layoutId) if (layoutsUsedByScreens.includes(layoutId)) { ctx.throw(400, "Cannot delete a layout that's being used by a screen") } diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index a9cd686674..e2bd6c40e5 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db: Database) { - const body = await db.allDocs( + const body = await db.allDocs( getRoleParams(null, { include_docs: true, }) ) - return body.rows.map(row => row.doc) + return body.rows.map(row => row.doc!) } async function updatePermissionOnRole( @@ -79,7 +79,7 @@ async function updatePermissionOnRole( ) { rolePermissions[resourceId] = typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId]] + ? [rolePermissions[resourceId] as unknown as string] : [] } // handle the removal/updating the role which has this permission first diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 3697bbe925..ae6b89e6d4 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -6,7 +6,13 @@ import { Header, } from "@budibase/backend-core" import { getUserMetadataParams, InternalTables } from "../../db/utils" -import { Database, Role, UserCtx, UserRoles } from "@budibase/types" +import { + Database, + Role, + UserCtx, + UserMetadata, + UserRoles, +} from "@budibase/types" import { sdk as sharedSdk } from "@budibase/shared-core" import sdk from "../../sdk" @@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) { const role = await db.get(roleId) // first check no users actively attached to role const users = ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(undefined, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const usersWithRole = users.filter(user => user.roleId === roleId) if (usersWithRole.length !== 0) { ctx.throw(400, "Cannot delete role when it is in use.") diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 80a69cf92b..d53345a239 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -233,17 +233,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const tableId = utils.getTableId(ctx) const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs - let response = await Promise.all([ + const [table, row, links] = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - const table = response[0] as Table - const row = response[1] as Row - const linkVals = response[2] as LinkDocumentValue[] + const linkVals = links as LinkDocumentValue[] + // look up the actual rows based on the ids const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc) + let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc!) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 6f426c6fa0..87d551bc4b 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) { const db = context.getAppDB() // start by getting the raw rows (which will be written back to DB after update) let rows = ( - await db.allDocs( + await db.allDocs( getRowParams(table._id, null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) // now enrich the rows, note the clone so that we have the base state of the // rows so that we don't write any of the enriched information back let enrichedRows = await outputProcessing(table, cloneDeep(rows), { diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts index 189c4ede51..9ffa091a8c 100644 --- a/packages/server/src/api/controllers/view/utils.ts +++ b/packages/server/src/api/controllers/view/utils.ts @@ -53,12 +53,12 @@ export async function getViews() { } } else { const views = ( - await db.allDocs( + await db.allDocs( getMemoryViewParams({ include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) for (let viewDoc of views) { response.push({ name: viewDoc.name, diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 922bc10343..f0eca759f5 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -20,10 +20,10 @@ const JOB_OPTS = { async function getAllAutomations() { const db = context.getAppDB() - let automations = await db.allDocs( + let automations = await db.allDocs( getAutomationParams(null, { include_docs: true }) ) - return automations.rows.map(row => row.doc) + return automations.rows.map(row => row.doc!) } async function queueRelevantRowAutomations( @@ -45,19 +45,19 @@ async function queueRelevantRowAutomations( for (let automation of automations) { let automationDef = automation.definition - let automationTrigger = automationDef ? automationDef.trigger : {} + let automationTrigger = automationDef?.trigger // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) if ( !env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId) && - !(await checkTestFlag(automation._id)) + !(await checkTestFlag(automation._id!)) ) { continue } if ( - automationTrigger.inputs && + automationTrigger?.inputs && automationTrigger.inputs.tableId === event.row.tableId ) { await automationQueue.add({ automation, event }, JOB_OPTS) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7a7a06551e..8cccf1b96a 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -14,7 +14,14 @@ import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types" +import { + Table, + Row, + LinkDocumentValue, + FieldType, + LinkDocument, + ContextUser, +} from "@budibase/types" import sdk from "../../sdk" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -73,18 +80,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc + let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( + row => row.doc! ) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) - .filter(row => row != null) + .filter(row => row != null) as Row[] // need to handle users as specific cases let [users, other] = partition(linked, linkRow => - linkRow._id.startsWith(USER_METDATA_PREFIX) + linkRow._id!.startsWith(USER_METDATA_PREFIX) ) - users = await getGlobalUsersFromMetadata(users) + users = await getGlobalUsersFromMetadata(users as ContextUser[]) return [...other, ...users] } @@ -176,7 +183,7 @@ export async function attachFullLinkedDocs( // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = [] + let linked: Row[] = [] if (linksWithoutFromRow.length > 0) { linked = await getFullLinkedDocs(linksWithoutFromRow) } @@ -189,7 +196,7 @@ export async function attachFullLinkedDocs( if (opts?.fromRow && opts?.fromRow?._id === link.id) { linkedRow = opts.fromRow! } else { - linkedRow = linked.find(row => row._id === link.id) + linkedRow = linked.find(row => row._id === link.id)! } if (linkedRow) { const linkedTableId = diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts index 158e4772b2..c3415bdb36 100644 --- a/packages/server/src/sdk/app/applications/import.ts +++ b/packages/server/src/sdk/app/applications/import.ts @@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) { // map the responses to the document itself let documents: Document[] = [] for (let response of await Promise.all(docPromises)) { - documents = documents.concat(response.rows.map(row => row.doc)) + documents = documents.concat(response.rows.map(row => row.doc!)) } // remove the _rev, stops it being written documents.forEach(doc => { diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 6e1e6747e1..4813f6dd21 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core" import { User, ContextUser, UserGroup } from "@budibase/types" import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" -import { getGlobalUsers, processUser } from "../../../utilities/global" +import { + getGlobalUsers, + getRawGlobalUsers, + processUser, +} from "../../../utilities/global" import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } @@ -77,9 +81,7 @@ async function syncUsersToApp( export async function syncUsersToAllApps(userIds: string[]) { // list of users, if one has been deleted it will be undefined in array - const users = (await getGlobalUsers(userIds, { - noProcessing: true, - })) as User[] + const users = (await getRawGlobalUsers(userIds)) as User[] const groups = await proSdk.groups.fetch() const finalUsers: (User | DeletedUser)[] = [] for (let userId of userIds) { diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index fb5d04b03e..51cceeab94 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -51,12 +51,12 @@ export async function fetch(opts?: { // Get external datasources const datasources = ( - await db.allDocs( + await db.allDocs( getDatasourceParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([ bbInternalDb, @@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise { }) ) - return externalDatasources.rows.map(r => r.doc) + return externalDatasources.rows.map(r => r.doc!) } diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index c0d5fe8da8..a7074f95b2 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, }) @@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs( + const internalTableDocs = await db.allDocs
( getMultiIDParams(internalTableIds) ) tables = tables.concat(internalTableDocs.rows.map(row => row.doc!)) diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 03ddc954e9..9632ac29d8 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -7,12 +7,17 @@ import { InternalTables, } from "../../db/utils" import isEqual from "lodash/isEqual" -import { ContextUser, UserMetadata, User, Database } from "@budibase/types" +import { + ContextUser, + UserMetadata, + Database, + ContextUserMetadata, +} from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, metadata: UserMetadata | UserMetadata[] -) { +): ContextUserMetadata | null { const metadataId = generateUserMetadataID(user._id!) const found = Array.isArray(metadata) ? metadata.find(doc => doc._id === metadataId) @@ -51,33 +56,33 @@ export function combineMetadataAndUser( return null } -export async function rawUserMetadata(db?: Database) { +export async function rawUserMetadata(db?: Database): Promise { if (!db) { db = context.getAppDB() } return ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } -export async function fetchMetadata() { +export async function fetchMetadata(): Promise { const global = await getGlobalUsers() const metadata = await rawUserMetadata() - const users = [] + const users: ContextUserMetadata[] = [] for (let user of global) { // find the metadata that matches up to the global ID - const info = metadata.find(meta => meta._id.includes(user._id)) + const info = metadata.find(meta => meta._id!.includes(user._id!)) // remove these props, not for the correct DB users.push({ ...user, ...info, tableId: InternalTables.USER_METADATA, // make sure the ID is always a local ID, not a global one - _id: generateUserMetadataID(user._id), + _id: generateUserMetadataID(user._id!), }) } return users @@ -90,9 +95,10 @@ export async function syncGlobalUsers() { if (!(await db.exists())) { continue } - const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)]) - const users = resp[0] as User[] - const metadata = resp[1] as UserMetadata[] + const [users, metadata] = await Promise.all([ + getGlobalUsers(), + rawUserMetadata(db), + ]) const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 5aa201990c..cdc2d84513 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -71,69 +71,67 @@ export async function processUser( return user } -export async function getCachedSelf(ctx: UserCtx, appId: string) { +export async function getCachedSelf( + ctx: UserCtx, + appId: string +): Promise { // this has to be tenant aware, can't depend on the context to find it out // running some middlewares before the tenancy causes context to break const user = await cache.user.getUser(ctx.user?._id!) return processUser(user, { appId }) } -export async function getRawGlobalUser(userId: string) { +export async function getRawGlobalUser(userId: string): Promise { const db = tenancy.getGlobalDB() return db.get(getGlobalIDFromUserMetadataID(userId)) } -export async function getGlobalUser(userId: string) { +export async function getGlobalUser(userId: string): Promise { const appId = context.getAppId() let user = await getRawGlobalUser(userId) return processUser(user, { appId }) } -export async function getGlobalUsers( - userIds?: string[], - opts?: { noProcessing?: boolean } -) { - const appId = context.getAppId() +export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() - let globalUsers + let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc + globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( + row => row.doc! ) } else { globalUsers = ( - await db.allDocs( + await db.allDocs( dbCore.getGlobalUserParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } - globalUsers = globalUsers + return globalUsers .filter(user => user != null) .map(user => { delete user.password delete user.forceResetPassword return user }) +} - if (opts?.noProcessing || !appId) { - return globalUsers - } else { - // pass in the groups, meaning we don't actually need to retrieve them for - // each user individually - const allGroups = await groups.fetch() - return Promise.all( - globalUsers.map(user => processUser(user, { groups: allGroups })) - ) - } +export async function getGlobalUsers( + userIds?: string[] +): Promise { + const users = await getRawGlobalUsers(userIds) + const allGroups = await groups.fetch() + return Promise.all( + users.map(user => processUser(user, { groups: allGroups })) + ) } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( - globalUser => globalUser && user._id?.includes(globalUser._id) + globalUser => globalUser && user._id?.includes(globalUser._id!) ) return { ...globalUser, diff --git a/packages/server/src/utilities/routing/index.ts b/packages/server/src/utilities/routing/index.ts index de966a946b..82d45743ce 100644 --- a/packages/server/src/utilities/routing/index.ts +++ b/packages/server/src/utilities/routing/index.ts @@ -1,9 +1,9 @@ import { createRoutingView } from "../../db/views/staticViews" import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils" import { context } from "@budibase/backend-core" -import { ScreenRouting } from "@budibase/types" +import { ScreenRouting, Document } from "@budibase/types" -type ScreenRoutesView = { +interface ScreenRoutesView extends Document { id: string routing: ScreenRouting } diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts index 4defd4a414..207997245e 100644 --- a/packages/types/src/documents/app/user.ts +++ b/packages/types/src/documents/app/user.ts @@ -1,6 +1,6 @@ -import { Document } from "../document" +import { User } from "../global" +import { Row } from "./row" +import { ContextUser } from "../../sdk" -export interface UserMetadata extends Document { - roleId: string - email?: string -} +export type UserMetadata = User & Row +export type ContextUserMetadata = ContextUser & Row diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index d484f4700d..11efc502be 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -1,17 +1,19 @@ +import { Document } from "../" + export interface RowValue { rev: string deleted: boolean } -export interface RowResponse { +export interface RowResponse { id: string key: string error: string value: T | RowValue - doc?: T | any + doc?: T } -export interface AllDocsResponse { +export interface AllDocsResponse { offset: number total_rows: number rows: RowResponse[] diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 05f72f5524..12c53a9561 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -101,8 +101,10 @@ export interface Database { opts?: DatabasePutOpts ): Promise bulkDocs(documents: AnyDocument[]): Promise - allDocs(params: DatabaseQueryOpts): Promise> - query( + allDocs( + params: DatabaseQueryOpts + ): Promise> + query( viewName: string, params: DatabaseQueryOpts ): Promise> diff --git a/packages/worker/src/constants/templates/index.ts b/packages/worker/src/constants/templates/index.ts index 1feac62040..6dd3f556a6 100644 --- a/packages/worker/src/constants/templates/index.ts +++ b/packages/worker/src/constants/templates/index.ts @@ -56,12 +56,12 @@ export async function getTemplates({ id, }: { ownerId?: string; type?: string; id?: string } = {}) { const db = tenancy.getGlobalDB() - const response = await db.allDocs( + const response = await db.allDocs