From 6d8921978b7bc5f5c596d8666b49c0b198bfc930 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:30:39 +0100 Subject: [PATCH 1/8] Quick temporary fix for issue - previously the invalid state was never reset, which can cause apps to go missing completely, updating this so it will fix it self after a short while. --- .../backend-core/src/cache/appMetadata.ts | 6 ++--- .../backend-core/src/db/couch/DatabaseImpl.ts | 25 +++++++++++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index d442511fb8..144836029f 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -11,6 +11,7 @@ export interface DeletedApp { } const EXPIRY_SECONDS = 3600 +const INVALID_EXPIRY_SECONDS = 60 /** * The default populate app metadata function @@ -48,9 +49,8 @@ export async function getAppMetadata(appId: string): Promise { // app DB left around, but no metadata, it is invalid if (err && err.status === 404) { metadata = { state: AppState.INVALID } - // don't expire the reference to an invalid app, it'll only be - // updated if a metadata doc actually gets stored (app is remade/reverted) - expiry = undefined + // expire invalid apps regularly, in-case it was only briefly invalid + expiry = INVALID_EXPIRY_SECONDS } else { throw err } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index aa4656bf64..180cd22efa 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -43,6 +43,9 @@ function buildNano(couchInfo: { url: string; cookie: string }) { } type DBCall = () => Promise +type DBCallback = ( + db: Nano.DocumentScope +) => Promise> | DBCall class CouchDBError extends Error implements DBError { status: number @@ -171,8 +174,8 @@ export class DatabaseImpl implements Database { } // this function fetches the DB and handles if DB creation is needed - private async performCall( - call: (db: Nano.DocumentScope) => Promise> | DBCall + private async performCallWithDBCreation( + call: DBCallback ): Promise { const db = this.getDb() const fnc = await call(db) @@ -181,13 +184,24 @@ export class DatabaseImpl implements Database { } catch (err: any) { if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) { await this.checkAndCreateDb() - return await this.performCall(call) + return await this.performCallWithDBCreation(call) } // stripping the error down the props which are safe/useful, drop everything else throw new CouchDBError(`CouchDB error: ${err.message}`, err) } } + private async performCall(call: DBCallback): Promise { + const db = this.getDb() + const fnc = await call(db) + try { + return await fnc() + } catch (err: any) { + // stripping the error down the props which are safe/useful, drop everything else + throw new CouchDBError(`CouchDB error: ${err.message}`, err) + } + } + async get(id?: string): Promise { return this.performCall(db => { if (!id) { @@ -227,6 +241,7 @@ export class DatabaseImpl implements Database { } async remove(idOrDoc: string | Document, rev?: string) { + // not a read call - but don't create a DB to delete a document return this.performCall(db => { let _id: string let _rev: string @@ -286,7 +301,7 @@ export class DatabaseImpl implements Database { if (!document._id) { throw new Error("Cannot store document without _id field.") } - return this.performCall(async db => { + return this.performCallWithDBCreation(async db => { if (!document.createdAt) { document.createdAt = new Date().toISOString() } @@ -309,7 +324,7 @@ export class DatabaseImpl implements Database { async bulkDocs(documents: AnyDocument[]) { const now = new Date().toISOString() - return this.performCall(db => { + return this.performCallWithDBCreation(db => { return () => db.bulk({ docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })), From ad60f8a8119103478cc46f992e8edee3182b813d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:47:07 +0100 Subject: [PATCH 2/8] All docs returns no docs if 404. --- .../backend-core/src/db/couch/DatabaseImpl.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 180cd22efa..650f5ca8b6 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -336,7 +336,21 @@ export class DatabaseImpl implements Database { params: DatabaseQueryOpts ): Promise> { return this.performCall(db => { - return () => db.list(params) + return async () => { + try { + return (await db.list(params)) as AllDocsResponse + } catch (err: any) { + if (err.status === 404) { + return { + offset: 0, + total_rows: 0, + rows: [], + } + } else { + throw err + } + } + } }) } From a5cb1b39dcc7605c962c590fea2d8d8989f3d62d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Sep 2024 15:51:41 +0100 Subject: [PATCH 3/8] All docs no error fix. --- packages/backend-core/src/db/couch/DatabaseImpl.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 650f5ca8b6..274c1b9e93 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -340,7 +340,7 @@ export class DatabaseImpl implements Database { try { return (await db.list(params)) as AllDocsResponse } catch (err: any) { - if (err.status === 404) { + if (err.reason === DATABASE_NOT_FOUND) { return { offset: 0, total_rows: 0, From d7d8284caf06b512b915c58c1e5d52cd56ebf6bc Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:42:25 +0200 Subject: [PATCH 4/8] Add test --- .../src/api/routes/tests/viewV2.spec.ts | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c4a39ae8a9..c34f4fb3ac 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -22,6 +22,7 @@ import { TableSchema, ViewFieldMetadata, RenameColumn, + FeatureFlag, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2213,6 +2214,100 @@ describe.each([ }) ) }) + + describe("foreign relationship columns", () => { + const createMainTable = async ( + links: { + name: string + tableId: string + fk: string + }[] + ) => { + const table = await config.api.table.save( + saveTableRequest({ + schema: { title: { name: "title", type: FieldType.STRING } }, + }) + ) + await config.api.table.save({ + ...table, + schema: { + ...table.schema, + ...links.reduce((acc, c) => { + acc[c.name] = { + name: c.name, + relationshipType: RelationshipType.ONE_TO_MANY, + type: FieldType.LINK, + tableId: c.tableId, + fieldName: c.fk, + constraints: { type: "array" }, + } + return acc + }, {}), + }, + }) + return table + } + const createAuxTable = (schema: TableSchema) => + config.api.table.save( + saveTableRequest({ + primaryDisplay: "name", + schema: { + ...schema, + name: { name: "name", type: FieldType.STRING }, + }, + }) + ) + + it("returns squashed fields respecting the view config", async () => { + const auxTable = await createAuxTable({ + age: { name: "age", type: FieldType.NUMBER }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + age: generator.age(), + }) + const row = await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: false, readonly: false }, + age: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + age: auxRow.age, + }, + ], + }), + ]) + }) + }) }) describe("permissions", () => { From 53b4634cffb48d254c7d4e1b5a91f4a61f017880 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:51:05 +0200 Subject: [PATCH 5/8] Add enrichment tests --- .../src/api/routes/tests/viewV2.spec.ts | 66 ++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index c34f4fb3ac..a2af95bee3 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -23,6 +23,7 @@ import { ViewFieldMetadata, RenameColumn, FeatureFlag, + BBReferenceFieldSubType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -2270,7 +2271,7 @@ describe.each([ name: generator.name(), age: generator.age(), }) - const row = await config.api.row.save(table._id!, { + await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], }) @@ -2307,6 +2308,69 @@ describe.each([ }), ]) }) + + it("enriches squashed fields", async () => { + const auxTable = await createAuxTable({ + user: { + name: "user", + type: FieldType.BB_REFERENCE_SINGLE, + subtype: BBReferenceFieldSubType.USER, + constraints: { presence: true }, + }, + }) + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) + + const user = config.getUser() + const auxRow = await config.api.row.save(auxTable._id!, { + name: generator.name(), + user: user._id, + }) + await config.api.row.save(table._id!, { + title: generator.word(), + aux: [auxRow], + }) + + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + title: { visible: true }, + aux: { + visible: true, + columns: { + name: { visible: true, readonly: true }, + user: { visible: true, readonly: true }, + }, + }, + }, + }) + + const response = await withCoreEnv( + { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + () => config.api.viewV2.search(view.id) + ) + + expect(response.rows).toEqual([ + expect.objectContaining({ + aux: [ + { + _id: auxRow._id, + primaryDisplay: auxRow.name, + name: auxRow.name, + user: { + _id: user._id, + email: user.email, + firstName: user.firstName, + lastName: user.lastName, + primaryDisplay: user.email, + }, + }, + ], + }), + ]) + }) }) }) From 7a7ce3dc629ed365a67cbdbae4d812d80ffc7440 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 24 Sep 2024 17:56:07 +0200 Subject: [PATCH 6/8] Fix --- packages/server/src/db/linkedRows/index.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 2c8d1f77ac..c2b043785f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -10,7 +10,7 @@ import flatten from "lodash/flatten" import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" -import { processFormulas } from "../../utilities/rowProcessor" +import { outputProcessing, processFormulas } from "../../utilities/rowProcessor" import { context, features } from "@budibase/backend-core" import { ContextUser, @@ -275,7 +275,7 @@ export async function squashLinks( // will populate this as we find them const linkedTables = [table] const isArray = Array.isArray(enriched) - const enrichedArray = !isArray ? [enriched] : enriched + const enrichedArray = !isArray ? [enriched as Row] : (enriched as Row[]) for (const row of enrichedArray) { // this only fetches the table if its not already in array const rowTable = await getLinkedTable(row.tableId!, linkedTables) @@ -292,6 +292,9 @@ export async function squashLinks( obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable) if (viewSchema[column]?.columns) { + const enrichedLink = await outputProcessing(linkedTable, link, { + squash: false, + }) const squashFields = Object.entries(viewSchema[column].columns) .filter(([columnName, viewColumnConfig]) => { const tableColumn = linkedTable.schema[columnName] @@ -312,7 +315,7 @@ export async function squashLinks( .map(([columnName]) => columnName) for (const relField of squashFields) { - obj[relField] = link[relField] + obj[relField] = enrichedLink[relField] } } @@ -321,5 +324,5 @@ export async function squashLinks( row[column] = newLinks } } - return isArray ? enrichedArray : enrichedArray[0] + return (isArray ? enrichedArray : enrichedArray[0]) as T } From 9ecb64a99229abc626012be597af55f54eba2e0a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 10:32:58 +0200 Subject: [PATCH 7/8] Use sqs flag correctly on test --- packages/server/src/api/routes/tests/viewV2.spec.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a2af95bee3..a47d6dd828 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -2291,8 +2291,17 @@ describe.each([ }, }) + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (isLucene) { + flags.push("*:!SQS") + } else if (isSqs) { + flags.push("*:SQS") + } + const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, + { + TENANT_FEATURE_FLAGS: flags.join(","), + }, () => config.api.viewV2.search(view.id) ) From 7072244f31a165a0b3af80bae55922b725e6d13a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Wed, 25 Sep 2024 12:44:30 +0200 Subject: [PATCH 8/8] Fix --- .../src/api/routes/tests/viewV2.spec.ts | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index a47d6dd828..e38e4c2ed5 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -34,6 +34,7 @@ import { roles, withEnv as withCoreEnv, setEnv as setCoreEnv, + env, } from "@budibase/backend-core" import sdk from "../../../sdk" @@ -696,22 +697,23 @@ describe.each([ ) }) - it("cannot update views v1", async () => { - const viewV1 = await config.api.legacyView.save({ - tableId: table._id!, - name: generator.guid(), - filters: [], - schema: {}, - }) + isInternal && + it("cannot update views v1", async () => { + const viewV1 = await config.api.legacyView.save({ + tableId: table._id!, + name: generator.guid(), + filters: [], + schema: {}, + }) - await config.api.viewV2.update(viewV1 as unknown as ViewV2, { - status: 400, - body: { - message: "Only views V2 can be updated", + await config.api.viewV2.update(viewV1 as unknown as ViewV2, { status: 400, - }, + body: { + message: "Only views V2 can be updated", + status: 400, + }, + }) }) - }) it("cannot update the a view with unmatching ids between url and body", async () => { const anotherView = await config.api.viewV2.create({ @@ -2217,6 +2219,21 @@ describe.each([ }) describe("foreign relationship columns", () => { + let envCleanup: () => void + beforeAll(() => { + const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] + if (env.TENANT_FEATURE_FLAGS) { + flags.push(...env.TENANT_FEATURE_FLAGS.split(",")) + } + envCleanup = setCoreEnv({ + TENANT_FEATURE_FLAGS: flags.join(","), + }) + }) + + afterAll(() => { + envCleanup?.() + }) + const createMainTable = async ( links: { name: string @@ -2263,14 +2280,14 @@ describe.each([ const auxTable = await createAuxTable({ age: { name: "age", type: FieldType.NUMBER }, }) - const table = await createMainTable([ - { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, - ]) - const auxRow = await config.api.row.save(auxTable._id!, { name: generator.name(), age: generator.age(), }) + + const table = await createMainTable([ + { name: "aux", tableId: auxTable._id!, fk: "fk_aux" }, + ]) await config.api.row.save(table._id!, { title: generator.word(), aux: [auxRow], @@ -2291,20 +2308,7 @@ describe.each([ }, }) - const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`] - if (isLucene) { - flags.push("*:!SQS") - } else if (isSqs) { - flags.push("*:SQS") - } - - const response = await withCoreEnv( - { - TENANT_FEATURE_FLAGS: flags.join(","), - }, - () => config.api.viewV2.search(view.id) - ) - + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({ aux: [ @@ -2356,10 +2360,7 @@ describe.each([ }, }) - const response = await withCoreEnv( - { TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}` }, - () => config.api.viewV2.search(view.id) - ) + const response = await config.api.viewV2.search(view.id) expect(response.rows).toEqual([ expect.objectContaining({