Merge master.

packages/backend-core/src/cache/appMetadata.ts

@@ -11,6 +11,7 @@ export interface DeletedApp {
 }
 
 const EXPIRY_SECONDS = 3600
+const INVALID_EXPIRY_SECONDS = 60
 
 /**
  * The default populate app metadata function
@@ -48,9 +49,8 @@ export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
       // app DB left around, but no metadata, it is invalid
       if (err && err.status === 404) {
         metadata = { state: AppState.INVALID }
-        // don't expire the reference to an invalid app, it'll only be
-        // updated if a metadata doc actually gets stored (app is remade/reverted)
-        expiry = undefined
+        // expire invalid apps regularly, in-case it was only briefly invalid
+        expiry = INVALID_EXPIRY_SECONDS
       } else {
         throw err
       }

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -43,6 +43,9 @@ function buildNano(couchInfo: { url: string; cookie: string }) {
 }
 
 type DBCall<T> = () => Promise<T>
+type DBCallback<T> = (
+  db: Nano.DocumentScope<any>
+) => Promise<DBCall<T>> | DBCall<T>
 
 class CouchDBError extends Error implements DBError {
   status: number
@@ -171,8 +174,8 @@ export class DatabaseImpl implements Database {
   }
 
   // this function fetches the DB and handles if DB creation is needed
-  private async performCall<T>(
-    call: (db: Nano.DocumentScope<any>) => Promise<DBCall<T>> | DBCall<T>
+  private async performCallWithDBCreation<T>(
+    call: DBCallback<T>
   ): Promise<any> {
     const db = this.getDb()
     const fnc = await call(db)
@@ -181,13 +184,24 @@ export class DatabaseImpl implements Database {
     } catch (err: any) {
       if (err.statusCode === 404 && err.reason === DATABASE_NOT_FOUND) {
         await this.checkAndCreateDb()
-        return await this.performCall(call)
+        return await this.performCallWithDBCreation(call)
       }
       // stripping the error down the props which are safe/useful, drop everything else
       throw new CouchDBError(`CouchDB error: ${err.message}`, err)
     }
   }
 
+  private async performCall<T>(call: DBCallback<T>): Promise<any> {
+    const db = this.getDb()
+    const fnc = await call(db)
+    try {
+      return await fnc()
+    } catch (err: any) {
+      // stripping the error down the props which are safe/useful, drop everything else
+      throw new CouchDBError(`CouchDB error: ${err.message}`, err)
+    }
+  }
+
   async get<T extends Document>(id?: string): Promise<T> {
     return this.performCall(db => {
       if (!id) {
@@ -227,6 +241,7 @@ export class DatabaseImpl implements Database {
   }
 
   async remove(idOrDoc: string | Document, rev?: string) {
+    // not a read call - but don't create a DB to delete a document
     return this.performCall(db => {
       let _id: string
       let _rev: string
@@ -286,7 +301,7 @@ export class DatabaseImpl implements Database {
     if (!document._id) {
       throw new Error("Cannot store document without _id field.")
     }
-    return this.performCall(async db => {
+    return this.performCallWithDBCreation(async db => {
       if (!document.createdAt) {
         document.createdAt = new Date().toISOString()
       }
@@ -309,7 +324,7 @@ export class DatabaseImpl implements Database {
 
   async bulkDocs(documents: AnyDocument[]) {
     const now = new Date().toISOString()
-    return this.performCall(db => {
+    return this.performCallWithDBCreation(db => {
       return () =>
         db.bulk({
           docs: documents.map(d => ({ createdAt: now, ...d, updatedAt: now })),
@@ -321,7 +336,21 @@ export class DatabaseImpl implements Database {
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {
     return this.performCall(db => {
-      return () => db.list(params)
+      return async () => {
+        try {
+          return (await db.list(params)) as AllDocsResponse<T>
+        } catch (err: any) {
+          if (err.reason === DATABASE_NOT_FOUND) {
+            return {
+              offset: 0,
+              total_rows: 0,
+              rows: [],
+            }
+          } else {
+            throw err
+          }
+        }
+      }
     })
   }
 

packages/server/src/api/routes/tests/search.spec.ts

@@ -45,14 +45,14 @@ import { generateRowIdField } from "../../../integrations/utils"
 import { cloneDeep } from "lodash/fp"
 
 describe.each([
-  ["in-memory", undefined],
-  ["lucene", undefined],
+  // ["in-memory", undefined],
+  // ["lucene", undefined],
   ["sqs", undefined],
-  [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
-  [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
-  [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
-  [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-  [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
+  // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
+  // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
+  // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
+  // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
+  // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)],
 ])("search (%s)", (name, dsProvider) => {
   const isSqs = name === "sqs"
   const isLucene = name === "lucene"
@@ -2757,7 +2757,7 @@ describe.each([
         })
       })
 
-      it("can filter by the row ID with limit 1", async () => {
+      it.only("can filter by the row ID with limit 1", async () => {
         await expectSearch({
           query: {
             equal: { _id: row._id },

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -23,6 +23,8 @@ import {
   TableSchema,
   RenameColumn,
   ViewUIFieldMetadata,
+  FeatureFlag,
+  BBReferenceFieldSubType,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -33,6 +35,7 @@ import {
   roles,
   withEnv as withCoreEnv,
   setEnv as setCoreEnv,
+  env,
 } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 
@@ -695,22 +698,23 @@ describe.each([
         )
       })
 
-      it("cannot update views v1", async () => {
-        const viewV1 = await config.api.legacyView.save({
-          tableId: table._id!,
-          name: generator.guid(),
-          filters: [],
-          schema: {},
-        })
+      isInternal &&
+        it("cannot update views v1", async () => {
+          const viewV1 = await config.api.legacyView.save({
+            tableId: table._id!,
+            name: generator.guid(),
+            filters: [],
+            schema: {},
+          })
 
-        await config.api.viewV2.update(viewV1 as unknown as ViewV2, {
-          status: 400,
-          body: {
-            message: "Only views V2 can be updated",
+          await config.api.viewV2.update(viewV1 as unknown as ViewV2, {
             status: 400,
-          },
+            body: {
+              message: "Only views V2 can be updated",
+              status: 400,
+            },
+          })
         })
-      })
 
       it("cannot update the a view with unmatching ids between url and body", async () => {
         const anotherView = await config.api.viewV2.create({
@@ -2215,6 +2219,171 @@ describe.each([
         )
       })
 
+      describe("foreign relationship columns", () => {
+        let envCleanup: () => void
+        beforeAll(() => {
+          const flags = [`*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`]
+          if (env.TENANT_FEATURE_FLAGS) {
+            flags.push(...env.TENANT_FEATURE_FLAGS.split(","))
+          }
+          envCleanup = setCoreEnv({
+            TENANT_FEATURE_FLAGS: flags.join(","),
+          })
+        })
+
+        afterAll(() => {
+          envCleanup?.()
+        })
+
+        const createMainTable = async (
+          links: {
+            name: string
+            tableId: string
+            fk: string
+          }[]
+        ) => {
+          const table = await config.api.table.save(
+            saveTableRequest({
+              schema: { title: { name: "title", type: FieldType.STRING } },
+            })
+          )
+          await config.api.table.save({
+            ...table,
+            schema: {
+              ...table.schema,
+              ...links.reduce<TableSchema>((acc, c) => {
+                acc[c.name] = {
+                  name: c.name,
+                  relationshipType: RelationshipType.ONE_TO_MANY,
+                  type: FieldType.LINK,
+                  tableId: c.tableId,
+                  fieldName: c.fk,
+                  constraints: { type: "array" },
+                }
+                return acc
+              }, {}),
+            },
+          })
+          return table
+        }
+        const createAuxTable = (schema: TableSchema) =>
+          config.api.table.save(
+            saveTableRequest({
+              primaryDisplay: "name",
+              schema: {
+                ...schema,
+                name: { name: "name", type: FieldType.STRING },
+              },
+            })
+          )
+
+        it("returns squashed fields respecting the view config", async () => {
+          const auxTable = await createAuxTable({
+            age: { name: "age", type: FieldType.NUMBER },
+          })
+          const auxRow = await config.api.row.save(auxTable._id!, {
+            name: generator.name(),
+            age: generator.age(),
+          })
+
+          const table = await createMainTable([
+            { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
+          ])
+          await config.api.row.save(table._id!, {
+            title: generator.word(),
+            aux: [auxRow],
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              title: { visible: true },
+              aux: {
+                visible: true,
+                columns: {
+                  name: { visible: false, readonly: false },
+                  age: { visible: true, readonly: true },
+                },
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id)
+          expect(response.rows).toEqual([
+            expect.objectContaining({
+              aux: [
+                {
+                  _id: auxRow._id,
+                  primaryDisplay: auxRow.name,
+                  age: auxRow.age,
+                },
+              ],
+            }),
+          ])
+        })
+
+        it("enriches squashed fields", async () => {
+          const auxTable = await createAuxTable({
+            user: {
+              name: "user",
+              type: FieldType.BB_REFERENCE_SINGLE,
+              subtype: BBReferenceFieldSubType.USER,
+              constraints: { presence: true },
+            },
+          })
+          const table = await createMainTable([
+            { name: "aux", tableId: auxTable._id!, fk: "fk_aux" },
+          ])
+
+          const user = config.getUser()
+          const auxRow = await config.api.row.save(auxTable._id!, {
+            name: generator.name(),
+            user: user._id,
+          })
+          await config.api.row.save(table._id!, {
+            title: generator.word(),
+            aux: [auxRow],
+          })
+
+          const view = await config.api.viewV2.create({
+            tableId: table._id!,
+            name: generator.guid(),
+            schema: {
+              title: { visible: true },
+              aux: {
+                visible: true,
+                columns: {
+                  name: { visible: true, readonly: true },
+                  user: { visible: true, readonly: true },
+                },
+              },
+            },
+          })
+
+          const response = await config.api.viewV2.search(view.id)
+
+          expect(response.rows).toEqual([
+            expect.objectContaining({
+              aux: [
+                {
+                  _id: auxRow._id,
+                  primaryDisplay: auxRow.name,
+                  name: auxRow.name,
+                  user: {
+                    _id: user._id,
+                    email: user.email,
+                    firstName: user.firstName,
+                    lastName: user.lastName,
+                    primaryDisplay: user.email,
+                  },
+                },
+              ],
+            }),
+          ])
+        })
+      })
+
       describe("calculations", () => {
         let table: Table
         let rows: Row[]
@@ -2245,7 +2414,7 @@ describe.each([
           )
         })
 
-        it.only("should be able to search by calculations", async () => {
+        it("should be able to search by calculations", async () => {
           const view = await config.api.viewV2.create({
             tableId: table._id!,
             name: generator.guid(),

packages/server/src/db/linkedRows/index.ts

@@ -10,7 +10,7 @@ import flatten from "lodash/flatten"
 import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
-import { processFormulas } from "../../utilities/rowProcessor"
+import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
 import { context, features } from "@budibase/backend-core"
 import {
   ContextUser,
@@ -284,7 +284,7 @@ export async function squashLinks<T = Row[] | Row>(
   // will populate this as we find them
   const linkedTables = [table]
   const isArray = Array.isArray(enriched)
-  const enrichedArray = !isArray ? [enriched] : enriched
+  const enrichedArray = !isArray ? [enriched as Row] : (enriched as Row[])
   for (const row of enrichedArray) {
     // this only fetches the table if its not already in array
     const rowTable = await getLinkedTable(row.tableId!, linkedTables)
@@ -301,6 +301,9 @@ export async function squashLinks<T = Row[] | Row>(
         obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
 
         if (viewSchema[column]?.columns) {
+          const enrichedLink = await outputProcessing(linkedTable, link, {
+            squash: false,
+          })
           const squashFields = Object.entries(viewSchema[column].columns)
             .filter(([columnName, viewColumnConfig]) => {
               const tableColumn = linkedTable.schema[columnName]
@@ -321,7 +324,7 @@ export async function squashLinks<T = Row[] | Row>(
             .map(([columnName]) => columnName)
 
           for (const relField of squashFields) {
-            obj[relField] = link[relField]
+            obj[relField] = enrichedLink[relField]
           }
         }
 
@@ -330,5 +333,5 @@ export async function squashLinks<T = Row[] | Row>(
       row[column] = newLinks
     }
   }
-  return isArray ? enrichedArray : enrichedArray[0]
+  return (isArray ? enrichedArray : enrichedArray[0]) as T
 }