Updating DesignDocument and View typing - making it more clear how view and lucene indexing works.

packages/backend-core/src/constants/db.ts

@@ -28,7 +28,7 @@ export enum ViewName {
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
 
-export const DeprecatedViews = {
+export const DeprecatedViews: Record<string, string[]> = {
   [ViewName.USER_BY_EMAIL]: [
     // removed due to inaccuracy in view doc filter logic
     "by_email",

packages/backend-core/src/db/views.ts

@@ -12,12 +12,14 @@ import {
   Database,
   DatabaseQueryOpts,
   Document,
+  DesignDocument,
+  DBView,
 } from "@budibase/types"
 import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
-function DesignDoc() {
+function DesignDoc(): DesignDocument {
   return {
     _id: DESIGN_DB,
     // view collation information, read before writing any complex views:
@@ -26,20 +28,14 @@ function DesignDoc() {
   }
 }
 
-interface DesignDocument {
-  views: any
-}
-
 async function removeDeprecated(db: Database, viewName: ViewName) {
-  // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
   }
   try {
     const designDoc = await db.get<DesignDocument>(DESIGN_DB)
-    // @ts-ignore
     for (let deprecatedNames of DeprecatedViews[viewName]) {
-      delete designDoc.views[deprecatedNames]
+      delete designDoc.views?.[deprecatedNames]
     }
     await db.put(designDoc)
   } catch (err) {
@@ -48,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
 }
 
 export async function createView(
-  db: any,
+  db: Database,
   viewJs: string,
   viewName: string
 ): Promise<void> {
   let designDoc
   try {
-    designDoc = (await db.get(DESIGN_DB)) as DesignDocument
+    designDoc = await db.get<DesignDocument>(DESIGN_DB)
   } catch (err) {
     // no design doc, make one
     designDoc = DesignDoc()
   }
-  const view = {
+  const view: DBView = {
     map: viewJs,
   }
   designDoc.views = {

packages/server/src/api/controllers/table/utils.ts

@@ -333,29 +333,33 @@ export async function checkForViewUpdates(
   columnRename?: RenameColumn
 ) {
   const views = await getViews()
-  const tableViews = views.filter(view => view.meta.tableId === table._id)
+  const tableViews = views.filter(view => view.meta?.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
   for (let view of tableViews) {
     let needsUpdated = false
+    const viewMetadata = view.meta as any
+    if (!viewMetadata) {
+      continue
+    }
 
     // First check for renames, otherwise check for deletions
     if (columnRename) {
       // Update calculation field if required
-      if (view.meta.field === columnRename.old) {
+      if (viewMetadata.field === columnRename.old) {
-        view.meta.field = columnRename.updated
+        viewMetadata.field = columnRename.updated
         needsUpdated = true
       }
 
       // Update group by field if required
-      if (view.meta.groupBy === columnRename.old) {
+      if (viewMetadata.groupBy === columnRename.old) {
-        view.meta.groupBy = columnRename.updated
+        viewMetadata.groupBy = columnRename.updated
         needsUpdated = true
       }
 
       // Update filters if required
-      if (view.meta.filters) {
+      if (viewMetadata.filters) {
-        view.meta.filters.forEach((filter: any) => {
+        viewMetadata.filters.forEach((filter: any) => {
           if (filter.key === columnRename.old) {
             filter.key = columnRename.updated
             needsUpdated = true
@@ -365,26 +369,26 @@ export async function checkForViewUpdates(
     } else if (deletedColumns) {
       deletedColumns.forEach((column: string) => {
         // Remove calculation statement if required
-        if (view.meta.field === column) {
+        if (viewMetadata.field === column) {
-          delete view.meta.field
+          delete viewMetadata.field
-          delete view.meta.calculation
+          delete viewMetadata.calculation
-          delete view.meta.groupBy
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove group by field if required
-        if (view.meta.groupBy === column) {
+        if (viewMetadata.groupBy === column) {
-          delete view.meta.groupBy
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove filters referencing deleted field if required
-        if (view.meta.filters && view.meta.filters.length) {
+        if (viewMetadata.filters && viewMetadata.filters.length) {
-          const initialLength = view.meta.filters.length
+          const initialLength = viewMetadata.filters.length
-          view.meta.filters = view.meta.filters.filter((filter: any) => {
+          viewMetadata.filters = viewMetadata.filters.filter((filter: any) => {
             return filter.key !== column
           })
-          if (initialLength !== view.meta.filters.length) {
+          if (initialLength !== viewMetadata.filters.length) {
             needsUpdated = true
           }
         }
@@ -397,15 +401,16 @@ export async function checkForViewUpdates(
         (field: any) => field.name == view.groupBy
       )
       const newViewTemplate = viewTemplate(
-        view.meta,
+        viewMetadata,
         groupByField?.type === FieldTypes.ARRAY
       )
-      await saveView(null, view.name, newViewTemplate)
+      const viewName = view.name!
-      if (!newViewTemplate.meta.schema) {
+      await saveView(null, viewName, newViewTemplate)
-        newViewTemplate.meta.schema = table.schema
+      if (!newViewTemplate.meta?.schema) {
+        newViewTemplate.meta!.schema = table.schema
       }
-      if (table.views?.[view.name]) {
+      if (table.views?.[viewName]) {
-        table.views[view.name] = newViewTemplate.meta as View
+        table.views[viewName] = newViewTemplate.meta as View
       }
     }
   }

packages/server/src/api/controllers/view/utils.ts

@@ -8,13 +8,13 @@ import {
 import env from "../../../environment"
 import { context } from "@budibase/backend-core"
 import viewBuilder from "./viewBuilder"
-import { Database } from "@budibase/types"
+import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
 
 export async function getView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    return designDoc.views[viewName]
+    return designDoc.views?.[viewName]
   } else {
     // This is a table view, don't read the view from the DB
     if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) {
@@ -22,7 +22,7 @@ export async function getView(viewName: string) {
     }
 
     try {
-      const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
+      const viewDoc = await db.get<InMemoryView>(generateMemoryViewID(viewName))
       return viewDoc.view
     } catch (err: any) {
       // Return null when PouchDB doesn't found the view
@@ -35,25 +35,28 @@ export async function getView(viewName: string) {
   }
 }
 
-export async function getViews() {
+export async function getViews(): Promise<DBView[]> {
   const db = context.getAppDB()
-  const response = []
+  const response: DBView[] = []
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    for (let name of Object.keys(designDoc.views)) {
+    for (let name of Object.keys(designDoc.views || {})) {
       // Only return custom views, not built ins
       const viewNames = Object.values(ViewName) as string[]
       if (viewNames.indexOf(name) !== -1) {
         continue
       }
+      const view = designDoc.views?.[name]
+      if (view) {
         response.push({
           name,
-        ...designDoc.views[name],
+          ...view,
         })
       }
+    }
   } else {
     const views = (
-      await db.allDocs<any>(
+      await db.allDocs<InMemoryView>(
         getMemoryViewParams({
           include_docs: true,
         })
@@ -72,11 +75,11 @@ export async function getViews() {
 export async function saveView(
   originalName: string | null,
   viewName: string,
-  viewTemplate: any
+  viewTemplate: DBView
 ) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
     designDoc.views = {
       ...designDoc.views,
       [viewName]: viewTemplate,
@@ -89,17 +92,17 @@ export async function saveView(
   } else {
     const id = generateMemoryViewID(viewName)
     const originalId = originalName ? generateMemoryViewID(originalName) : null
-    const viewDoc: any = {
+    const viewDoc: InMemoryView = {
       _id: id,
       view: viewTemplate,
       name: viewName,
-      tableId: viewTemplate.meta.tableId,
+      tableId: viewTemplate.meta!.tableId,
     }
     try {
-      const old = await db.get<any>(id)
+      const old = await db.get<InMemoryView>(id)
       if (originalId) {
-        const originalDoc = await db.get<any>(originalId)
+        const originalDoc = await db.get<InMemoryView>(originalId)
-        await db.remove(originalDoc._id, originalDoc._rev)
+        await db.remove(originalDoc._id!, originalDoc._rev)
       }
       if (old && old._rev) {
         viewDoc._rev = old._rev
@@ -114,52 +117,65 @@ export async function saveView(
 export async function deleteView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    const view = designDoc.views[viewName]
+    const view = designDoc.views?.[viewName]
-    delete designDoc.views[viewName]
+    delete designDoc.views?.[viewName]
     await db.put(designDoc)
     return view
   } else {
     const id = generateMemoryViewID(viewName)
-    const viewDoc = await db.get<any>(id)
+    const viewDoc = await db.get<InMemoryView>(id)
-    await db.remove(viewDoc._id, viewDoc._rev)
+    await db.remove(viewDoc._id!, viewDoc._rev)
     return viewDoc.view
   }
 }
 
 export async function migrateToInMemoryView(db: Database, viewName: string) {
   // delete the view initially
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
+  const meta = designDoc.views?.[viewName].meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
   // run the view back through the view builder to update it
-  const view = viewBuilder(designDoc.views[viewName].meta)
+  const view = viewBuilder(meta)
-  delete designDoc.views[viewName]
+  delete designDoc.views?.[viewName]
   await db.put(designDoc)
-  await exports.saveView(db, null, viewName, view)
+  await saveView(null, viewName, view)
 }
 
 export async function migrateToDesignView(db: Database, viewName: string) {
-  let view = await db.get<any>(generateMemoryViewID(viewName))
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
-  designDoc.views[viewName] = viewBuilder(view.view.meta)
+  const meta = view.view.meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
+  if (!designDoc.views) {
+    designDoc.views = {}
+  }
+  designDoc.views[viewName] = viewBuilder(meta)
   await db.put(designDoc)
-  await db.remove(view._id, view._rev)
+  await db.remove(view._id!, view._rev)
 }
 
 export async function getFromDesignDoc(db: Database, viewName: string) {
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
-  let view = designDoc.views[viewName]
+  let view = designDoc.views?.[viewName]
   if (view == null) {
     throw { status: 404, message: "Unable to get view" }
   }
   return view
 }
 
-export async function getFromMemoryDoc(db: Database, viewName: string) {
+export async function getFromMemoryDoc(
-  let view = await db.get<any>(generateMemoryViewID(viewName))
+  db: Database,
+  viewName: string
+): Promise<DBView> {
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
   if (view) {
-    view = view.view
+    return view.view
   } else {
     throw { status: 404, message: "Unable to get view" }
   }
-  return view
 }

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -1,13 +1,4 @@
-import { ViewFilter } from "@budibase/types"
+import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
-
-type ViewTemplateOpts = {
-  field: string
-  tableId: string
-  groupBy: string
-  filters: ViewFilter[]
-  calculation: string
-  groupByMulti: boolean
-}
 
 const TOKEN_MAP: Record<string, string> = {
   EQUALS: "===",
@@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) {
 export default function (
   { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
   groupByMulti?: boolean
-) {
+): DBView {
   // first filter can't have a conjunction
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction

packages/server/src/api/controllers/view/views.ts

@@ -47,8 +47,11 @@ export async function save(ctx: Ctx) {
 
   // add views to table document
   if (!table.views) table.views = {}
-  if (!view.meta.schema) {
+  if (!view.meta?.schema) {
-    view.meta.schema = table.schema
+    view.meta = {
+      ...view.meta!,
+      schema: table.schema,
+    }
   }
   table.views[viewName] = { ...view.meta, name: viewName }
   if (originalName) {
@@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const viewName = decodeURIComponent(ctx.params.viewName)
   const view = await deleteView(viewName)
+  if (!view || !view.meta) {
+    ctx.throw(400, "Unable to delete view - no metadata/view not found.")
+  }
   const table = await sdk.tables.getTable(view.meta.tableId)
   delete table.views![viewName]
   await db.put(table)
-  await events.view.deleted(view)
+  await events.view.deleted(view as View)
 
   ctx.body = view
   builderSocket?.emitTableUpdate(ctx, table)
@@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) {
     )
   }
 
-  if (view) {
+  if (view && view.meta) {
     ctx.params.viewName = viewName
     // Fetch view rows
     ctx.query = {

packages/server/src/db/inMemoryView.ts

@@ -1,5 +1,5 @@
 import newid from "./newid"
-import { Row, View, Document } from "@budibase/types"
+import { Row, Document, DBView } from "@budibase/types"
 
 // bypass the main application db config
 // use in memory pouchdb directly
@@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core"
 const Pouch = dbCore.getPouch({ inMemory: true })
 
 export async function runView(
-  view: View,
+  view: DBView,
   calculation: string,
   group: boolean,
   data: Row[]

packages/server/src/sdk/app/rows/search/internal.ts

@@ -185,8 +185,8 @@ export async function fetchView(
       group: !!group,
     })
   } else {
-    const tableId = viewInfo.meta.tableId
+    const tableId = viewInfo.meta!.tableId
-    const data = await fetchRaw(tableId)
+    const data = await fetchRaw(tableId!)
     response = await inMemoryViews.runView(
       viewInfo,
       calculation as string,
@@ -200,7 +200,7 @@ export async function fetchView(
     response.rows = response.rows.map(row => row.doc)
     let table: Table
     try {
-      table = await sdk.tables.getTable(viewInfo.meta.tableId)
+      table = await sdk.tables.getTable(viewInfo.meta!.tableId)
     } catch (err) {
       throw new Error("Unable to retrieve view table.")
     }

packages/types/src/documents/app/view.ts

@@ -1,5 +1,24 @@
 import { SearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
+import { Document } from "../document"
+import { DBView } from "../../sdk"
+
+export type ViewTemplateOpts = {
+  field: string
+  tableId: string
+  groupBy: string
+  filters: ViewFilter[]
+  schema: any
+  calculation: string
+  groupByMulti?: boolean
+}
+
+export interface InMemoryView extends Document {
+  view: DBView
+  name: string
+  tableId: string
+  groupBy?: string
+}
 
 export interface View {
   name?: string
@@ -10,7 +29,7 @@ export interface View {
   calculation?: ViewCalculation
   map?: string
   reduce?: any
-  meta?: Record<string, any>
+  meta?: ViewTemplateOpts
 }
 
 export interface ViewV2 {

packages/types/src/sdk/db.ts

@@ -1,5 +1,5 @@
 import Nano from "@budibase/nano"
-import { AllDocsResponse, AnyDocument, Document } from "../"
+import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../"
 import { Writable } from "stream"
 
 export enum SearchIndex {
@@ -20,6 +20,37 @@ export enum SortOption {
   DESCENDING = "desc",
 }
 
+export type IndexAnalyzer = {
+  name: string
+  default?: string
+  fields?: Record<string, string>
+}
+
+export type DBView = {
+  name?: string
+  map: string
+  reduce?: string
+  meta?: ViewTemplateOpts
+  groupBy?: string
+}
+
+export interface DesignDocument extends Document {
+  // we use this static reference for all design documents
+  _id: "_design/database"
+  language?: string
+  // CouchDB views
+  views?: {
+    [viewName: string]: DBView
+  }
+  // Lucene indexes
+  indexes?: {
+    [indexName: string]: {
+      index: string
+      analyzer?: string | IndexAnalyzer
+    }
+  }
+}
+
 export type CouchFindOptions = {
   selector: PouchDB.Find.Selector
   fields?: string[]