Merge remote-tracking branch 'origin/master' into feature/toggle-all-formblock-fields

hosting/couchdb/build-target-paths.sh

@@ -2,8 +2,8 @@
 
 echo ${TARGETBUILD} > /buildtarget.txt
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    # Azure AppService uses /home for persisent data & SSH on port 2222
+    # Azure AppService uses /home for persistent data & SSH on port 2222
-    DATA_DIR=/home
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     mkdir -p $DATA_DIR/{search,minio,couch}
     mkdir -p $DATA_DIR/couch/{dbs,views}

hosting/scripts/build-target-paths.sh

@@ -2,8 +2,8 @@
 
 echo ${TARGETBUILD} > /buildtarget.txt
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    # Azure AppService uses /home for persisent data & SSH on port 2222
+    # Azure AppService uses /home for persistent data & SSH on port 2222
-    DATA_DIR=/home
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     mkdir -p $DATA_DIR/{search,minio,couch}
     mkdir -p $DATA_DIR/couch/{dbs,views}

hosting/single/runner.sh

@@ -22,7 +22,7 @@ declare -a DOCKER_VARS=("APP_PORT" "APPS_URL" "ARCHITECTURE" "BUDIBASE_ENVIRONME
 
 # Azure App Service customisations
 if [[ "${TARGETBUILD}" = "aas" ]]; then
-    DATA_DIR=/home
+    DATA_DIR="${DATA_DIR:-/home}"
     WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
     /etc/init.d/ssh start
 else

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.4",
+  "version": "2.13.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/constants/db.ts

@@ -28,7 +28,7 @@ export enum ViewName {
   APP_BACKUP_BY_TRIGGER = "by_trigger",
 }
 
-export const DeprecatedViews = {
+export const DeprecatedViews: Record<string, string[]> = {
   [ViewName.USER_BY_EMAIL]: [
     // removed due to inaccuracy in view doc filter logic
     "by_email",

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -175,12 +175,14 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.bulk({ docs: documents }))
   }
 
-  async allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>> {
+  async allDocs<T extends Document>(
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>> {
     const db = await this.checkSetup()
     return this.updateOutput(() => db.list(params))
   }
 
-  async query<T>(
+  async query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>> {

packages/backend-core/src/db/views.ts

@@ -7,12 +7,19 @@ import {
 } from "../constants"
 import { getGlobalDB } from "../context"
 import { doWithDB } from "./"
-import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types"
+import {
+  AllDocsResponse,
+  Database,
+  DatabaseQueryOpts,
+  Document,
+  DesignDocument,
+  DBView,
+} from "@budibase/types"
 import env from "../environment"
 
 const DESIGN_DB = "_design/database"
 
-function DesignDoc() {
+function DesignDoc(): DesignDocument {
   return {
     _id: DESIGN_DB,
     // view collation information, read before writing any complex views:
@@ -21,20 +28,14 @@ function DesignDoc() {
   }
 }
 
-interface DesignDocument {
-  views: any
-}
-
 async function removeDeprecated(db: Database, viewName: ViewName) {
-  // @ts-ignore
   if (!DeprecatedViews[viewName]) {
     return
   }
   try {
     const designDoc = await db.get<DesignDocument>(DESIGN_DB)
-    // @ts-ignore
     for (let deprecatedNames of DeprecatedViews[viewName]) {
-      delete designDoc.views[deprecatedNames]
+      delete designDoc.views?.[deprecatedNames]
     }
     await db.put(designDoc)
   } catch (err) {
@@ -43,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) {
 }
 
 export async function createView(
-  db: any,
+  db: Database,
   viewJs: string,
   viewName: string
 ): Promise<void> {
   let designDoc
   try {
-    designDoc = (await db.get(DESIGN_DB)) as DesignDocument
+    designDoc = await db.get<DesignDocument>(DESIGN_DB)
   } catch (err) {
     // no design doc, make one
     designDoc = DesignDoc()
   }
-  const view = {
+  const view: DBView = {
     map: viewJs,
   }
   designDoc.views = {
@@ -109,7 +110,7 @@ export interface QueryViewOptions {
   arrayResponse?: boolean
 }
 
-export async function queryViewRaw<T>(
+export async function queryViewRaw<T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db: Database,
@@ -137,18 +138,16 @@ export async function queryViewRaw<T>(
   }
 }
 
-export const queryView = async <T>(
+export const queryView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db: Database,
   createFunc: any,
   opts?: QueryViewOptions
-): Promise<T[] | T | undefined> => {
+): Promise<T[] | T> => {
   const response = await queryViewRaw<T>(viewName, params, db, createFunc, opts)
   const rows = response.rows
-  const docs = rows.map((row: any) =>
+  const docs = rows.map(row => (params.include_docs ? row.doc! : row.value))
-    params.include_docs ? row.doc : row.value
-  )
 
   // if arrayResponse has been requested, always return array regardless of length
   if (opts?.arrayResponse) {
@@ -198,11 +197,11 @@ export const createPlatformUserView = async () => {
   await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE)
 }
 
-export const queryPlatformView = async <T>(
+export const queryPlatformView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions
-): Promise<T[] | T | undefined> => {
+): Promise<T[] | T> => {
   const CreateFuncByName: any = {
     [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView,
     [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView,
@@ -220,7 +219,7 @@ const CreateFuncByName: any = {
   [ViewName.USER_BY_APP]: createUserAppView,
 }
 
-export const queryGlobalView = async <T>(
+export const queryGlobalView = async <T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   db?: Database,
@@ -231,10 +230,10 @@ export const queryGlobalView = async <T>(
     db = getGlobalDB()
   }
   const createFn = CreateFuncByName[viewName]
-  return queryView(viewName, params, db!, createFn, opts)
+  return queryView<T>(viewName, params, db!, createFn, opts)
 }
 
-export async function queryGlobalViewRaw<T>(
+export async function queryGlobalViewRaw<T extends Document>(
   viewName: ViewName,
   params: DatabaseQueryOpts,
   opts?: QueryViewOptions

packages/backend-core/src/users/db.ts

@@ -413,15 +413,13 @@ export class UserDB {
     }
 
     // Get users and delete
-    const allDocsResponse: AllDocsResponse<User> = await db.allDocs({
+    const allDocsResponse = await db.allDocs<User>({
       include_docs: true,
       keys: userIds,
     })
-    const usersToDelete: User[] = allDocsResponse.rows.map(
+    const usersToDelete = allDocsResponse.rows.map(user => {
-      (user: RowResponse<User>) => {
+      return user.doc!
-        return user.doc
+    })
-      }
-    )
 
     // Delete from DB
     const toDelete = usersToDelete.map(user => ({

packages/backend-core/src/users/users.ts

@@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async (
     include_docs: true,
   })
   params.startkey = opts && opts.startkey ? opts.startkey : params.startkey
-  let response = await queryGlobalView(ViewName.USER_BY_APP, params)
+  let response = await queryGlobalView<User>(ViewName.USER_BY_APP, params)
 
   if (!response) {
     response = []

packages/backend-core/src/utils/tests/utils.spec.ts

@@ -188,4 +188,17 @@ describe("utils", () => {
       expectResult(false)
     })
   })
+
+  describe("hasCircularStructure", () => {
+    it("should detect a circular structure", () => {
+      const a: any = { b: "b" }
+      const b = { a }
+      a.b = b
+      expect(utils.hasCircularStructure(b)).toBe(true)
+    })
+
+    it("should allow none circular structures", () => {
+      expect(utils.hasCircularStructure({ a: "b" })).toBe(false)
+    })
+  })
 })

packages/backend-core/src/utils/utils.ts

@@ -237,3 +237,17 @@ export function timeout(timeMs: number) {
 export function isAudited(event: Event) {
   return !!AuditedEventFriendlyName[event]
 }
+
+export function hasCircularStructure(json: any) {
+  if (typeof json !== "object") {
+    return false
+  }
+  try {
+    JSON.stringify(json)
+  } catch (err) {
+    if (err instanceof Error && err?.message.includes("circular structure")) {
+      return true
+    }
+  }
+  return false
+}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowChart.svelte

@@ -48,15 +48,14 @@
     <UndoRedoControl store={automationHistoryStore} />
   </div>
   <div class="controls">
-    <div class="buttons">
-      <Icon hoverable size="M" name="Play" />
     <div
       on:click={() => {
         testDataModal.show()
       }}
+      class="buttons"
     >
-        Run test
+      <Icon hoverable size="M" name="Play" />
-      </div>
+      <div>Run test</div>
     </div>
     <div class="buttons">
       <Icon

packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte

@@ -27,7 +27,7 @@
   $: if (value?.queryId == null) value = { queryId: "" }
 </script>
 
-<div class="schema-fields">
+<div class="schema-field">
   <Label>Query</Label>
   <div class="field-width">
     <Select
@@ -41,8 +41,8 @@
 </div>
 
 {#if parameters.length}
-  <div class="schema-fields">
   {#each parameters as field}
+    <div class="schema-field">
       <Label>{field.name}</Label>
       <div class="field-width">
         <DrawerBindableInput
@@ -56,8 +56,8 @@
           updateOnChange={false}
         />
       </div>
-    {/each}
     </div>
+  {/each}
 {/if}
 
 <style>
@@ -65,7 +65,7 @@
     width: 320px;
   }
 
-  .schema-fields {
+  .schema-field {
     display: flex;
     justify-content: space-between;
     align-items: center;
@@ -76,7 +76,7 @@
     margin-bottom: 10px;
   }
 
-  .schema-fields :global(label) {
+  .schema-field :global(label) {
     text-transform: capitalize;
   }
 </style>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -114,10 +114,10 @@
 </div>
 {#if schemaFields.length}
   {#each schemaFields as [field, schema]}
+    {#if !schema.autocolumn && schema.type !== "attachment"}
       <div class="schema-fields">
         <Label>{field}</Label>
         <div class="field-width">
-        {#if !schema.autocolumn && schema.type !== "attachment"}
           {#if isTestModal}
             <RowSelectorTypes
               {isTestModal}
@@ -151,7 +151,6 @@
               />
             </DrawerBindableSlot>
           {/if}
-        {/if}
 
           {#if isUpdateRow && schema.type === "link"}
             <div class="checkbox-field">
@@ -165,6 +164,7 @@
           {/if}
         </div>
       </div>
+    {/if}
   {/each}
 {/if}
 

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -67,6 +67,7 @@
     bind:linkedRows={value[field]}
     {schema}
     on:change={e => onChange(e, field)}
+    useLabel={false}
   />
 {:else if schema.type === "string" || schema.type === "number"}
   <svelte:component

packages/builder/src/components/common/LinkedRowSelector.svelte

@@ -7,7 +7,7 @@
 
   export let schema
   export let linkedRows = []
-
+  export let useLabel = true
   const dispatch = createEventDispatcher()
 
   let rows = []
@@ -51,7 +51,7 @@
       linkedIds = e.detail ? [e.detail] : []
       dispatch("change", linkedIds)
     }}
-    {label}
+    label={useLabel ? label : null}
     sort
   />
 {:else}

packages/server/src/api/controllers/auth.ts

@@ -26,7 +26,7 @@ export async function fetchSelf(ctx: UserCtx) {
   }
 
   const appId = context.getAppId()
-  let user: ContextUser = await getFullUser(ctx, userId)
+  let user: ContextUser = await getFullUser(userId)
   // this shouldn't be returned by the app self
   delete user.roles
   // forward the csrf token from the session

packages/server/src/api/controllers/deploy/index.ts

@@ -106,7 +106,6 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
-    console.error(err)
     ctx.body = []
   }
 }

packages/server/src/api/controllers/layout.ts

@@ -1,7 +1,7 @@
 import { EMPTY_LAYOUT } from "../../constants/layouts"
 import { generateLayoutID, getScreenParams } from "../../db/utils"
 import { events, context } from "@budibase/backend-core"
-import { BBContext } from "@budibase/types"
+import { BBContext, Layout } from "@budibase/types"
 
 export async function save(ctx: BBContext) {
   const db = context.getAppDB()
@@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) {
     layoutRev = ctx.params.layoutRev
 
   const layoutsUsedByScreens = (
-    await db.allDocs(
+    await db.allDocs<Layout>(
       getScreenParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(element => element.doc.layoutId)
+  ).rows.map(element => element.doc!.layoutId)
   if (layoutsUsedByScreens.includes(layoutId)) {
     ctx.throw(400, "Cannot delete a layout that's being used by a screen")
   }

packages/server/src/api/controllers/permission.ts

@@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS
 
 // utility function to stop this repetition - permissions always stored under roles
 async function getAllDBRoles(db: Database) {
-  const body = await db.allDocs(
+  const body = await db.allDocs<Role>(
     getRoleParams(null, {
       include_docs: true,
     })
   )
-  return body.rows.map(row => row.doc)
+  return body.rows.map(row => row.doc!)
 }
 
 async function updatePermissionOnRole(
@@ -79,7 +79,7 @@ async function updatePermissionOnRole(
     ) {
       rolePermissions[resourceId] =
         typeof rolePermissions[resourceId] === "string"
-          ? [rolePermissions[resourceId]]
+          ? [rolePermissions[resourceId] as unknown as string]
           : []
     }
     // handle the removal/updating the role which has this permission first

packages/server/src/api/controllers/role.ts

@@ -6,7 +6,13 @@ import {
   Header,
 } from "@budibase/backend-core"
 import { getUserMetadataParams, InternalTables } from "../../db/utils"
-import { Database, Role, UserCtx, UserRoles } from "@budibase/types"
+import {
+  Database,
+  Role,
+  UserCtx,
+  UserMetadata,
+  UserRoles,
+} from "@budibase/types"
 import { sdk as sharedSdk } from "@budibase/shared-core"
 import sdk from "../../sdk"
 
@@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) {
   const role = await db.get<Role>(roleId)
   // first check no users actively attached to role
   const users = (
-    await db.allDocs(
+    await db.allDocs<UserMetadata>(
       getUserMetadataParams(undefined, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
   const usersWithRole = users.filter(user => user.roleId === roleId)
   if (usersWithRole.length !== 0) {
     ctx.throw(400, "Cannot delete role when it is in use.")

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -23,7 +23,6 @@ import {
   breakRowIdField,
   convertRowId,
   generateRowIdField,
-  getPrimaryDisplay,
   isRowId,
   isSQL,
 } from "../../../integrations/utils"
@@ -237,7 +236,7 @@ function basicProcessing({
   thisRow._id = generateIdForRow(row, table, isLinked)
   thisRow.tableId = table._id
   thisRow._rev = "rev"
-  return processFormulas(table, thisRow)
+  return thisRow
 }
 
 function fixArrayTypes(row: Row, table: Table) {
@@ -392,7 +391,7 @@ export class ExternalRequest<T extends Operation> {
     return { row: newRow, manyRelationships }
   }
 
-  squashRelationshipColumns(
+  processRelationshipFields(
     table: Table,
     row: Row,
     relationships: RelationshipsJson[]
@@ -402,7 +401,6 @@ export class ExternalRequest<T extends Operation> {
       if (!linkedTable || !row[relationship.column]) {
         continue
       }
-      const display = linkedTable.primaryDisplay
       for (let key of Object.keys(row[relationship.column])) {
         let relatedRow: Row = row[relationship.column][key]
         // add this row as context for the relationship
@@ -411,15 +409,10 @@ export class ExternalRequest<T extends Operation> {
             relatedRow[col.name] = [row]
           }
         }
+        // process additional types
+        relatedRow = processDates(table, relatedRow)
         relatedRow = processFormulas(linkedTable, relatedRow)
-        let relatedDisplay
+        row[relationship.column][key] = relatedRow
-        if (display) {
-          relatedDisplay = getPrimaryDisplay(relatedRow[display])
-        }
-        row[relationship.column][key] = {
-          primaryDisplay: relatedDisplay || "Invalid display column",
-          _id: relatedRow._id,
-        }
       }
     }
     return row
@@ -521,14 +514,14 @@ export class ExternalRequest<T extends Operation> {
       )
     }
 
-    // Process some additional data types
+    // make sure all related rows are correct
-    let finalRowArray = Object.values(finalRows)
+    let finalRowArray = Object.values(finalRows).map(row =>
-    finalRowArray = processDates(table, finalRowArray)
+      this.processRelationshipFields(table, row, relationships)
-    finalRowArray = processFormulas(table, finalRowArray) as Row[]
-
-    return finalRowArray.map((row: Row) =>
-      this.squashRelationshipColumns(table, row, relationships)
     )
+
+    // process some additional types
+    finalRowArray = processDates(table, finalRowArray)
+    return finalRowArray
   }
 
   /**
@@ -663,7 +656,7 @@ export class ExternalRequest<T extends Operation> {
         linkPrimary,
         linkSecondary,
       }: {
-        row: { [key: string]: any }
+        row: Row
         linkPrimary: string
         linkSecondary?: string
       }) {

packages/server/src/api/controllers/row/external.ts

@@ -76,6 +76,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     relationships: true,
   })
   const enrichedRow = await outputProcessing(table, row, {
+    squash: true,
     preserveLinks: true,
   })
   return {
@@ -119,7 +120,10 @@ export async function save(ctx: UserCtx) {
     })
     return {
       ...response,
-      row: await outputProcessing(table, row, { preserveLinks: true }),
+      row: await outputProcessing(table, row, {
+        preserveLinks: true,
+        squash: true,
+      }),
     }
   } else {
     return response
@@ -140,7 +144,7 @@ export async function find(ctx: UserCtx): Promise<Row> {
   const table = await sdk.tables.getTable(tableId)
   // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
   return await outputProcessing(table, row, {
-    squash: false,
+    squash: true,
     preserveLinks: true,
   })
 }
@@ -207,7 +211,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
     const primaryLink = linkedTable.primary?.[0] as string
-    row[fieldName] = await handleRequest(Operation.READ, linkedTableId!, {
+    const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
       tables,
       filters: {
         oneOf: {
@@ -216,6 +220,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
       },
       includeSqlRelationships: IncludeRelationship.INCLUDE,
     })
+    row[fieldName] = await outputProcessing(linkedTable, relatedRows, {
+      squash: true,
+      preserveLinks: true,
+    })
   }
   return row
 }

packages/server/src/api/controllers/row/internal.ts

@@ -2,7 +2,6 @@ import * as linkRows from "../../../db/linkedRows"
 import {
   generateRowID,
   getMultiIDParams,
-  getTableIDFromRowID,
   InternalTables,
 } from "../../../db/utils"
 import * as userController from "../user"
@@ -89,7 +88,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
     ctx.request.body = row as any
-    await userController.updateMetadata(ctx)
+    await userController.updateMetadata(ctx as any)
     return { row: ctx.body as Row, table }
   }
 
@@ -233,17 +232,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const tableId = utils.getTableId(ctx)
   const rowId = ctx.params.rowId as string
   // need table to work out where links go in row, as well as the link docs
-  let response = await Promise.all([
+  const [table, row, links] = await Promise.all([
     sdk.tables.getTable(tableId),
     utils.findRow(ctx, tableId, rowId),
     linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
   ])
-  const table = response[0] as Table
+  const linkVals = links as LinkDocumentValue[]
-  const row = response[1] as Row
+
-  const linkVals = response[2] as LinkDocumentValue[]
   // look up the actual rows based on the ids
   const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-  let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc)
+  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)

packages/server/src/api/controllers/row/staticFormula.ts

@@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) {
   const db = context.getAppDB()
   // start by getting the raw rows (which will be written back to DB after update)
   let rows = (
-    await db.allDocs(
+    await db.allDocs<Row>(
       getRowParams(table._id, null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
   // now enrich the rows, note the clone so that we have the base state of the
   // rows so that we don't write any of the enriched information back
   let enrichedRows = await outputProcessing(table, cloneDeep(rows), {
@@ -101,12 +101,12 @@ export async function updateAllFormulasInTable(table: Table) {
   for (let row of rows) {
     // find the enriched row, if found process the formulas
     const enrichedRow = enrichedRows.find(
-      (enriched: any) => enriched._id === row._id
+      (enriched: Row) => enriched._id === row._id
     )
     if (enrichedRow) {
       const processed = processFormulas(table, cloneDeep(row), {
         dynamic: false,
-        contextRows: enrichedRow,
+        contextRows: [enrichedRow],
       })
       // values have changed, need to add to bulk docs to update
       if (!isEqual(processed, row)) {
@@ -139,7 +139,7 @@ export async function finaliseRow(
   // use enriched row to generate formulas for saving, specifically only use as context
   row = processFormulas(table, row, {
     dynamic: false,
-    contextRows: enrichedRow,
+    contextRows: [enrichedRow],
   })
   // don't worry about rev, tables handle rev/lastID updates
   // if another row has been written since processing this will
@@ -163,7 +163,9 @@ export async function finaliseRow(
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
-  enrichedRow = await processFormulas(table, enrichedRow, { dynamic: false })
+  enrichedRow = processFormulas(table, enrichedRow, {
+    dynamic: false,
+  })
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)

packages/server/src/api/controllers/table/utils.ts

@@ -333,29 +333,33 @@ export async function checkForViewUpdates(
   columnRename?: RenameColumn
 ) {
   const views = await getViews()
-  const tableViews = views.filter(view => view.meta.tableId === table._id)
+  const tableViews = views.filter(view => view.meta?.tableId === table._id)
 
   // Check each table view to see if impacted by this table action
   for (let view of tableViews) {
     let needsUpdated = false
+    const viewMetadata = view.meta as any
+    if (!viewMetadata) {
+      continue
+    }
 
     // First check for renames, otherwise check for deletions
     if (columnRename) {
       // Update calculation field if required
-      if (view.meta.field === columnRename.old) {
+      if (viewMetadata.field === columnRename.old) {
-        view.meta.field = columnRename.updated
+        viewMetadata.field = columnRename.updated
         needsUpdated = true
       }
 
       // Update group by field if required
-      if (view.meta.groupBy === columnRename.old) {
+      if (viewMetadata.groupBy === columnRename.old) {
-        view.meta.groupBy = columnRename.updated
+        viewMetadata.groupBy = columnRename.updated
         needsUpdated = true
       }
 
       // Update filters if required
-      if (view.meta.filters) {
+      if (viewMetadata.filters) {
-        view.meta.filters.forEach((filter: any) => {
+        viewMetadata.filters.forEach((filter: any) => {
           if (filter.key === columnRename.old) {
             filter.key = columnRename.updated
             needsUpdated = true
@@ -365,26 +369,26 @@ export async function checkForViewUpdates(
     } else if (deletedColumns) {
       deletedColumns.forEach((column: string) => {
         // Remove calculation statement if required
-        if (view.meta.field === column) {
+        if (viewMetadata.field === column) {
-          delete view.meta.field
+          delete viewMetadata.field
-          delete view.meta.calculation
+          delete viewMetadata.calculation
-          delete view.meta.groupBy
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove group by field if required
-        if (view.meta.groupBy === column) {
+        if (viewMetadata.groupBy === column) {
-          delete view.meta.groupBy
+          delete viewMetadata.groupBy
           needsUpdated = true
         }
 
         // Remove filters referencing deleted field if required
-        if (view.meta.filters && view.meta.filters.length) {
+        if (viewMetadata.filters && viewMetadata.filters.length) {
-          const initialLength = view.meta.filters.length
+          const initialLength = viewMetadata.filters.length
-          view.meta.filters = view.meta.filters.filter((filter: any) => {
+          viewMetadata.filters = viewMetadata.filters.filter((filter: any) => {
             return filter.key !== column
           })
-          if (initialLength !== view.meta.filters.length) {
+          if (initialLength !== viewMetadata.filters.length) {
             needsUpdated = true
           }
         }
@@ -397,15 +401,16 @@ export async function checkForViewUpdates(
         (field: any) => field.name == view.groupBy
       )
       const newViewTemplate = viewTemplate(
-        view.meta,
+        viewMetadata,
         groupByField?.type === FieldTypes.ARRAY
       )
-      await saveView(null, view.name, newViewTemplate)
+      const viewName = view.name!
-      if (!newViewTemplate.meta.schema) {
+      await saveView(null, viewName, newViewTemplate)
-        newViewTemplate.meta.schema = table.schema
+      if (!newViewTemplate.meta?.schema) {
+        newViewTemplate.meta!.schema = table.schema
       }
-      if (table.views?.[view.name]) {
+      if (table.views?.[viewName]) {
-        table.views[view.name] = newViewTemplate.meta as View
+        table.views[viewName] = newViewTemplate.meta as View
       }
     }
   }

packages/server/src/api/controllers/user.ts

@@ -1,14 +1,26 @@
 import { generateUserFlagID, InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
-import { Ctx, UserCtx } from "@budibase/types"
+import {
+  ContextUserMetadata,
+  Ctx,
+  FetchUserMetadataResponse,
+  FindUserMetadataResponse,
+  Flags,
+  SetFlagRequest,
+  UserCtx,
+  UserMetadata,
+} from "@budibase/types"
 import sdk from "../../sdk"
+import { DocumentInsertResponse } from "@budibase/nano"
 
-export async function fetchMetadata(ctx: Ctx) {
+export async function fetchMetadata(ctx: Ctx<void, FetchUserMetadataResponse>) {
   ctx.body = await sdk.users.fetchMetadata()
 }
 
-export async function updateSelfMetadata(ctx: UserCtx) {
+export async function updateSelfMetadata(
+  ctx: UserCtx<UserMetadata, DocumentInsertResponse>
+) {
   // overwrite the ID with current users
   ctx.request.body._id = ctx.user?._id
   // make sure no stale rev
@@ -18,19 +30,21 @@ export async function updateSelfMetadata(ctx: UserCtx) {
   await updateMetadata(ctx)
 }
 
-export async function updateMetadata(ctx: UserCtx) {
+export async function updateMetadata(
+  ctx: UserCtx<UserMetadata, DocumentInsertResponse>
+) {
   const db = context.getAppDB()
   const user = ctx.request.body
-  // this isn't applicable to the user
+  const metadata: ContextUserMetadata = {
-  delete user.roles
-  const metadata = {
     tableId: InternalTables.USER_METADATA,
     ...user,
   }
+  // this isn't applicable to the user
+  delete metadata.roles
   ctx.body = await db.put(metadata)
 }
 
-export async function destroyMetadata(ctx: UserCtx) {
+export async function destroyMetadata(ctx: UserCtx<void, { message: string }>) {
   const db = context.getAppDB()
   try {
     const dbUser = await sdk.users.get(ctx.params.id)
@@ -43,11 +57,15 @@ export async function destroyMetadata(ctx: UserCtx) {
   }
 }
 
-export async function findMetadata(ctx: UserCtx) {
+export async function findMetadata(
-  ctx.body = await getFullUser(ctx, ctx.params.id)
+  ctx: UserCtx<void, FindUserMetadataResponse>
+) {
+  ctx.body = await getFullUser(ctx.params.id)
 }
 
-export async function setFlag(ctx: UserCtx) {
+export async function setFlag(
+  ctx: UserCtx<SetFlagRequest, { message: string }>
+) {
   const userId = ctx.user?._id
   const { flag, value } = ctx.request.body
   if (!flag) {
@@ -55,9 +73,9 @@ export async function setFlag(ctx: UserCtx) {
   }
   const flagDocId = generateUserFlagID(userId!)
   const db = context.getAppDB()
-  let doc
+  let doc: Flags
   try {
-    doc = await db.get<any>(flagDocId)
+    doc = await db.get<Flags>(flagDocId)
   } catch (err) {
     doc = { _id: flagDocId }
   }
@@ -66,13 +84,13 @@ export async function setFlag(ctx: UserCtx) {
   ctx.body = { message: "Flag set successfully" }
 }
 
-export async function getFlags(ctx: UserCtx) {
+export async function getFlags(ctx: UserCtx<void, Flags>) {
   const userId = ctx.user?._id
   const docId = generateUserFlagID(userId!)
   const db = context.getAppDB()
-  let doc
+  let doc: Flags
   try {
-    doc = await db.get(docId)
+    doc = await db.get<Flags>(docId)
   } catch (err) {
     doc = { _id: docId }
   }

packages/server/src/api/controllers/view/utils.ts

@@ -8,13 +8,13 @@ import {
 import env from "../../../environment"
 import { context } from "@budibase/backend-core"
 import viewBuilder from "./viewBuilder"
-import { Database } from "@budibase/types"
+import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
 
 export async function getView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    return designDoc.views[viewName]
+    return designDoc.views?.[viewName]
   } else {
     // This is a table view, don't read the view from the DB
     if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) {
@@ -22,7 +22,7 @@ export async function getView(viewName: string) {
     }
 
     try {
-      const viewDoc = await db.get<any>(generateMemoryViewID(viewName))
+      const viewDoc = await db.get<InMemoryView>(generateMemoryViewID(viewName))
       return viewDoc.view
     } catch (err: any) {
       // Return null when PouchDB doesn't found the view
@@ -35,30 +35,33 @@ export async function getView(viewName: string) {
   }
 }
 
-export async function getViews() {
+export async function getViews(): Promise<DBView[]> {
   const db = context.getAppDB()
-  const response = []
+  const response: DBView[] = []
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    for (let name of Object.keys(designDoc.views)) {
+    for (let name of Object.keys(designDoc.views || {})) {
       // Only return custom views, not built ins
       const viewNames = Object.values(ViewName) as string[]
       if (viewNames.indexOf(name) !== -1) {
         continue
       }
+      const view = designDoc.views?.[name]
+      if (view) {
         response.push({
           name,
-        ...designDoc.views[name],
+          ...view,
         })
       }
+    }
   } else {
     const views = (
-      await db.allDocs(
+      await db.allDocs<InMemoryView>(
         getMemoryViewParams({
           include_docs: true,
         })
       )
-    ).rows.map(row => row.doc)
+    ).rows.map(row => row.doc!)
     for (let viewDoc of views) {
       response.push({
         name: viewDoc.name,
@@ -72,11 +75,11 @@ export async function getViews() {
 export async function saveView(
   originalName: string | null,
   viewName: string,
-  viewTemplate: any
+  viewTemplate: DBView
 ) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
     designDoc.views = {
       ...designDoc.views,
       [viewName]: viewTemplate,
@@ -89,17 +92,17 @@ export async function saveView(
   } else {
     const id = generateMemoryViewID(viewName)
     const originalId = originalName ? generateMemoryViewID(originalName) : null
-    const viewDoc: any = {
+    const viewDoc: InMemoryView = {
       _id: id,
       view: viewTemplate,
       name: viewName,
-      tableId: viewTemplate.meta.tableId,
+      tableId: viewTemplate.meta!.tableId,
     }
     try {
-      const old = await db.get<any>(id)
+      const old = await db.get<InMemoryView>(id)
       if (originalId) {
-        const originalDoc = await db.get<any>(originalId)
+        const originalDoc = await db.get<InMemoryView>(originalId)
-        await db.remove(originalDoc._id, originalDoc._rev)
+        await db.remove(originalDoc._id!, originalDoc._rev)
       }
       if (old && old._rev) {
         viewDoc._rev = old._rev
@@ -114,52 +117,65 @@ export async function saveView(
 export async function deleteView(viewName: string) {
   const db = context.getAppDB()
   if (env.SELF_HOSTED) {
-    const designDoc = await db.get<any>("_design/database")
+    const designDoc = await db.get<DesignDocument>("_design/database")
-    const view = designDoc.views[viewName]
+    const view = designDoc.views?.[viewName]
-    delete designDoc.views[viewName]
+    delete designDoc.views?.[viewName]
     await db.put(designDoc)
     return view
   } else {
     const id = generateMemoryViewID(viewName)
-    const viewDoc = await db.get<any>(id)
+    const viewDoc = await db.get<InMemoryView>(id)
-    await db.remove(viewDoc._id, viewDoc._rev)
+    await db.remove(viewDoc._id!, viewDoc._rev)
     return viewDoc.view
   }
 }
 
 export async function migrateToInMemoryView(db: Database, viewName: string) {
   // delete the view initially
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
+  const meta = designDoc.views?.[viewName].meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
   // run the view back through the view builder to update it
-  const view = viewBuilder(designDoc.views[viewName].meta)
+  const view = viewBuilder(meta)
-  delete designDoc.views[viewName]
+  delete designDoc.views?.[viewName]
   await db.put(designDoc)
-  await exports.saveView(db, null, viewName, view)
+  await saveView(null, viewName, view)
 }
 
 export async function migrateToDesignView(db: Database, viewName: string) {
-  let view = await db.get<any>(generateMemoryViewID(viewName))
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
-  designDoc.views[viewName] = viewBuilder(view.view.meta)
+  const meta = view.view.meta
+  if (!meta) {
+    throw new Error("Unable to migrate view - no metadata")
+  }
+  if (!designDoc.views) {
+    designDoc.views = {}
+  }
+  designDoc.views[viewName] = viewBuilder(meta)
   await db.put(designDoc)
-  await db.remove(view._id, view._rev)
+  await db.remove(view._id!, view._rev)
 }
 
 export async function getFromDesignDoc(db: Database, viewName: string) {
-  const designDoc = await db.get<any>("_design/database")
+  const designDoc = await db.get<DesignDocument>("_design/database")
-  let view = designDoc.views[viewName]
+  let view = designDoc.views?.[viewName]
   if (view == null) {
     throw { status: 404, message: "Unable to get view" }
   }
   return view
 }
 
-export async function getFromMemoryDoc(db: Database, viewName: string) {
+export async function getFromMemoryDoc(
-  let view = await db.get<any>(generateMemoryViewID(viewName))
+  db: Database,
+  viewName: string
+): Promise<DBView> {
+  let view = await db.get<InMemoryView>(generateMemoryViewID(viewName))
   if (view) {
-    view = view.view
+    return view.view
   } else {
     throw { status: 404, message: "Unable to get view" }
   }
-  return view
 }

packages/server/src/api/controllers/view/viewBuilder.ts

@@ -1,13 +1,4 @@
-import { ViewFilter } from "@budibase/types"
+import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types"
-
-type ViewTemplateOpts = {
-  field: string
-  tableId: string
-  groupBy: string
-  filters: ViewFilter[]
-  calculation: string
-  groupByMulti: boolean
-}
 
 const TOKEN_MAP: Record<string, string> = {
   EQUALS: "===",
@@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) {
 export default function (
   { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts,
   groupByMulti?: boolean
-) {
+): DBView {
   // first filter can't have a conjunction
   if (filters && filters.length > 0 && filters[0].conjunction) {
     delete filters[0].conjunction

packages/server/src/api/controllers/view/views.ts

@@ -47,8 +47,11 @@ export async function save(ctx: Ctx) {
 
   // add views to table document
   if (!table.views) table.views = {}
-  if (!view.meta.schema) {
+  if (!view.meta?.schema) {
-    view.meta.schema = table.schema
+    view.meta = {
+      ...view.meta!,
+      schema: table.schema,
+    }
   }
   table.views[viewName] = { ...view.meta, name: viewName }
   if (originalName) {
@@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) {
   const db = context.getAppDB()
   const viewName = decodeURIComponent(ctx.params.viewName)
   const view = await deleteView(viewName)
+  if (!view || !view.meta) {
+    ctx.throw(400, "Unable to delete view - no metadata/view not found.")
+  }
   const table = await sdk.tables.getTable(view.meta.tableId)
   delete table.views![viewName]
   await db.put(table)
-  await events.view.deleted(view)
+  await events.view.deleted(view as View)
 
   ctx.body = view
   builderSocket?.emitTableUpdate(ctx, table)
@@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) {
     )
   }
 
-  if (view) {
+  if (view && view.meta) {
     ctx.params.viewName = viewName
     // Fetch view rows
     ctx.query = {

packages/server/src/api/routes/tests/row.spec.ts

@@ -10,6 +10,7 @@ import {
   FieldSchema,
   FieldType,
   FieldTypeSubtypes,
+  FormulaTypes,
   INTERNAL_TABLE_SOURCE_ID,
   MonthlyQuotaName,
   PermissionLevel,
@@ -32,6 +33,7 @@ import {
   structures,
 } from "@budibase/backend-core/tests"
 import _ from "lodash"
+import * as uuid from "uuid"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -68,7 +70,7 @@ describe.each([
 
   const generateTableConfig: () => SaveTableRequest = () => {
     return {
-      name: generator.word(),
+      name: uuid.v4(),
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
@@ -481,7 +483,7 @@ describe.each([
       })
 
       const createViewResponse = await config.createView({
-        name: generator.word(),
+        name: uuid.v4(),
         schema: {
           Country: {
             visible: true,
@@ -816,7 +818,8 @@ describe.each([
             RelationshipType.ONE_TO_MANY,
             ["link"],
             {
-              name: generator.word(),
+              // Making sure that the combined table name + column name is within postgres limits
+              name: uuid.v4().replace(/-/g, "").substring(0, 16),
               type: "table",
               primary: ["id"],
               primaryDisplay: "id",
@@ -949,7 +952,7 @@ describe.each([
   describe("view 2.0", () => {
     async function userTable(): Promise<Table> {
       return {
-        name: `users_${generator.word()}`,
+        name: `users_${uuid.v4()}`,
         sourceId: INTERNAL_TABLE_SOURCE_ID,
         sourceType: TableSourceType.INTERNAL,
         type: "table",
@@ -1133,7 +1136,7 @@ describe.each([
       const viewSchema = { age: { visible: true }, name: { visible: true } }
       async function userTable(): Promise<Table> {
         return {
-          name: `users_${generator.word()}`,
+          name: `users_${uuid.v4()}`,
           sourceId: INTERNAL_TABLE_SOURCE_ID,
           sourceType: TableSourceType.INTERNAL,
           type: "table",
@@ -1630,7 +1633,7 @@ describe.each([
       }),
       (tableId: string) =>
         config.api.row.save(tableId, {
-          name: generator.word(),
+          name: uuid.v4(),
           description: generator.paragraph(),
           tableId,
         }),
@@ -1998,4 +2001,52 @@ describe.each([
       })
     })
   })
+
+  describe("Formula fields", () => {
+    let relationshipTable: Table, tableId: string, relatedRow: Row
+
+    beforeAll(async () => {
+      const otherTableId = config.table!._id!
+      const cfg = generateTableConfig()
+      relationshipTable = await config.createLinkedTable(
+        RelationshipType.ONE_TO_MANY,
+        ["links"],
+        {
+          ...cfg,
+          // needs to be a short name
+          name: "b",
+          schema: {
+            ...cfg.schema,
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: "{{ links.0.name }}",
+              formulaType: FormulaTypes.DYNAMIC,
+            },
+          },
+        }
+      )
+
+      tableId = relationshipTable._id!
+
+      relatedRow = await config.api.row.save(otherTableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+      })
+      await config.api.row.save(tableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+        tableId,
+        links: [relatedRow._id],
+      })
+    })
+
+    it("should be able to search for rows containing formulas", async () => {
+      const { rows } = await config.api.row.search(tableId)
+      expect(rows.length).toBe(1)
+      expect(rows[0].links.length).toBe(1)
+      const row = rows[0]
+      expect(row.formula).toBe(relatedRow.name)
+    })
+  })
 })

packages/server/src/api/routes/tests/user.spec.js

@@ -1,208 +0,0 @@
-const { roles, utils } = require("@budibase/backend-core")
-const { checkPermissionsEndpoint } = require("./utilities/TestFunctions")
-const setup = require("./utilities")
-const { BUILTIN_ROLE_IDS } = roles
-
-jest.setTimeout(30000)
-
-jest.mock("../../../utilities/workerRequests", () => ({
-  getGlobalUsers: jest.fn(() => {
-    return {}
-  }),
-  getGlobalSelf: jest.fn(() => {
-    return {}
-  }),
-  deleteGlobalUser: jest.fn(),
-}))
-
-describe("/users", () => {
-  let request = setup.getRequest()
-  let config = setup.getConfig()
-
-  afterAll(setup.afterAll)
-
-  beforeAll(async () => {
-    await config.init()
-  })
-
-  describe("fetch", () => {
-    it("returns a list of users from an instance db", async () => {
-      await config.createUser({ id: "uuidx" })
-      await config.createUser({ id: "uuidy" })
-      const res = await request
-        .get(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      expect(res.body.length).toBe(3)
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidx`)).toBeDefined()
-      expect(res.body.find(u => u._id === `ro_ta_users_us_uuidy`)).toBeDefined()
-    })
-
-    it("should apply authorization to endpoint", async () => {
-      await config.createUser()
-      await checkPermissionsEndpoint({
-        config,
-        request,
-        method: "GET",
-        url: `/api/users/metadata`,
-        passRole: BUILTIN_ROLE_IDS.ADMIN,
-        failRole: BUILTIN_ROLE_IDS.PUBLIC,
-      })
-    })
-  })
-
-  describe("update", () => {
-    it("should be able to update the user", async () => {
-      const user = await config.createUser({ id: `us_update${utils.newid()}` })
-      user.roleId = BUILTIN_ROLE_IDS.BASIC
-      delete user._rev
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send(user)
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.ok).toEqual(true)
-    })
-
-    it("should be able to update the user multiple times", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      const res1 = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      const res = await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, _rev: res1.body.rev, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      expect(res.body.ok).toEqual(true)
-    })
-
-    it("should require the _rev field for multiple updates", async () => {
-      const user = await config.createUser()
-      delete user._rev
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.BASIC })
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      await request
-        .put(`/api/users/metadata`)
-        .set(config.defaultHeaders())
-        .send({ ...user, roleId: BUILTIN_ROLE_IDS.POWER })
-        .expect(409)
-        .expect("Content-Type", /json/)
-    })
-  })
-
-  describe("destroy", () => {
-    it("should be able to delete the user", async () => {
-      const user = await config.createUser()
-      const res = await request
-        .delete(`/api/users/metadata/${user._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toBeDefined()
-    })
-  })
-
-  describe("find", () => {
-    it("should be able to find the user", async () => {
-      const user = await config.createUser()
-      const res = await request
-        .get(`/api/users/metadata/${user._id}`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body._id).toEqual(user._id)
-      expect(res.body.roleId).toEqual(BUILTIN_ROLE_IDS.ADMIN)
-      expect(res.body.tableId).toBeDefined()
-    })
-  })
-
-  describe("setFlag", () => {
-    it("should throw an error if a flag is not provided", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test" })
-        .expect(400)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
-    })
-
-    it("should be able to set a flag on the user", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test", flag: "test" })
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Flag set successfully")
-    })
-  })
-
-  describe("getFlags", () => {
-    it("should get flags for a specific user", async () => {
-      let flagData = { value: "test", flag: "test" }
-      await config.createUser()
-      await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send(flagData)
-        .expect(200)
-        .expect("Content-Type", /json/)
-
-      const res = await request
-        .get(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body[flagData.value]).toEqual(flagData.flag)
-    })
-  })
-
-  describe("setFlag", () => {
-    it("should throw an error if a flag is not provided", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test" })
-        .expect(400)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual(
-        "Must supply a 'flag' field in request body."
-      )
-    })
-
-    it("should be able to set a flag on the user", async () => {
-      await config.createUser()
-      const res = await request
-        .post(`/api/users/flags`)
-        .set(config.defaultHeaders())
-        .send({ value: "test", flag: "test" })
-        .expect(200)
-        .expect("Content-Type", /json/)
-      expect(res.body.message).toEqual("Flag set successfully")
-    })
-  })
-})

packages/server/src/api/routes/tests/user.spec.ts

@@ -0,0 +1,144 @@
+import { roles, utils } from "@budibase/backend-core"
+import { checkPermissionsEndpoint } from "./utilities/TestFunctions"
+import * as setup from "./utilities"
+import { UserMetadata } from "@budibase/types"
+
+jest.setTimeout(30000)
+
+jest.mock("../../../utilities/workerRequests", () => ({
+  getGlobalUsers: jest.fn(() => {
+    return {}
+  }),
+  getGlobalSelf: jest.fn(() => {
+    return {}
+  }),
+  deleteGlobalUser: jest.fn(),
+}))
+
+describe("/users", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("fetch", () => {
+    it("returns a list of users from an instance db", async () => {
+      await config.createUser({ id: "uuidx" })
+      await config.createUser({ id: "uuidy" })
+
+      const res = await config.api.user.fetch()
+      expect(res.length).toBe(3)
+
+      const ids = res.map(u => u._id)
+      expect(ids).toContain(`ro_ta_users_us_uuidx`)
+      expect(ids).toContain(`ro_ta_users_us_uuidy`)
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      await config.createUser()
+      await checkPermissionsEndpoint({
+        config,
+        request,
+        method: "GET",
+        url: `/api/users/metadata`,
+        passRole: roles.BUILTIN_ROLE_IDS.ADMIN,
+        failRole: roles.BUILTIN_ROLE_IDS.PUBLIC,
+      })
+    })
+  })
+
+  describe("update", () => {
+    it("should be able to update the user", async () => {
+      const user: UserMetadata = await config.createUser({
+        id: `us_update${utils.newid()}`,
+      })
+      user.roleId = roles.BUILTIN_ROLE_IDS.BASIC
+      delete user._rev
+      const res = await config.api.user.update(user)
+      expect(res.ok).toEqual(true)
+    })
+
+    it("should be able to update the user multiple times", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      const res1 = await config.api.user.update({
+        ...user,
+        roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+      })
+      const res2 = await config.api.user.update({
+        ...user,
+        _rev: res1.rev,
+        roleId: roles.BUILTIN_ROLE_IDS.POWER,
+      })
+      expect(res2.ok).toEqual(true)
+    })
+
+    it("should require the _rev field for multiple updates", async () => {
+      const user = await config.createUser()
+      delete user._rev
+
+      await config.api.user.update({
+        ...user,
+        roleId: roles.BUILTIN_ROLE_IDS.BASIC,
+      })
+      await config.api.user.update(
+        { ...user, roleId: roles.BUILTIN_ROLE_IDS.POWER },
+        { expectStatus: 409 }
+      )
+    })
+  })
+
+  describe("destroy", () => {
+    it("should be able to delete the user", async () => {
+      const user = await config.createUser()
+      const res = await config.api.user.destroy(user._id!)
+      expect(res.message).toBeDefined()
+    })
+  })
+
+  describe("find", () => {
+    it("should be able to find the user", async () => {
+      const user = await config.createUser()
+      const res = await config.api.user.find(user._id!)
+      expect(res._id).toEqual(user._id)
+      expect(res.roleId).toEqual(roles.BUILTIN_ROLE_IDS.ADMIN)
+      expect(res.tableId).toBeDefined()
+    })
+  })
+
+  describe("setFlag", () => {
+    it("should throw an error if a flag is not provided", async () => {
+      await config.createUser()
+      const res = await request
+        .post(`/api/users/flags`)
+        .set(config.defaultHeaders())
+        .send({ value: "test" })
+        .expect(400)
+        .expect("Content-Type", /json/)
+      expect(res.body.message).toEqual(
+        "Must supply a 'flag' field in request body."
+      )
+    })
+
+    it("should be able to set a flag on the user", async () => {
+      await config.createUser()
+      const res = await config.api.user.setFlag("test", true)
+      expect(res.message).toEqual("Flag set successfully")
+    })
+  })
+
+  describe("getFlags", () => {
+    it("should get flags for a specific user", async () => {
+      await config.createUser()
+      await config.api.user.setFlag("test", "test")
+
+      const res = await config.api.user.getFlags()
+      expect(res.test).toEqual("test")
+    })
+  })
+})

packages/server/src/automations/triggers.ts

@@ -20,10 +20,10 @@ const JOB_OPTS = {
 
 async function getAllAutomations() {
   const db = context.getAppDB()
-  let automations = await db.allDocs(
+  let automations = await db.allDocs<Automation>(
     getAutomationParams(null, { include_docs: true })
   )
-  return automations.rows.map(row => row.doc)
+  return automations.rows.map(row => row.doc!)
 }
 
 async function queueRelevantRowAutomations(
@@ -45,19 +45,19 @@ async function queueRelevantRowAutomations(
 
     for (let automation of automations) {
       let automationDef = automation.definition
-      let automationTrigger = automationDef ? automationDef.trigger : {}
+      let automationTrigger = automationDef?.trigger
       // don't queue events which are for dev apps, only way to test automations is
       // running tests on them, in production the test flag will never
       // be checked due to lazy evaluation (first always false)
       if (
         !env.ALLOW_DEV_AUTOMATIONS &&
         isDevAppID(event.appId) &&
-        !(await checkTestFlag(automation._id))
+        !(await checkTestFlag(automation._id!))
       ) {
         continue
       }
       if (
-        automationTrigger.inputs &&
+        automationTrigger?.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
         await automationQueue.add({ automation, event }, JOB_OPTS)

packages/server/src/db/inMemoryView.ts

@@ -1,5 +1,5 @@
 import newid from "./newid"
-import { Row, View, Document } from "@budibase/types"
+import { Row, Document, DBView } from "@budibase/types"
 
 // bypass the main application db config
 // use in memory pouchdb directly
@@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core"
 const Pouch = dbCore.getPouch({ inMemory: true })
 
 export async function runView(
-  view: View,
+  view: DBView,
   calculation: string,
   group: boolean,
   data: Row[]

packages/server/src/db/linkedRows/index.ts

@@ -2,7 +2,6 @@ import LinkController from "./LinkController"
 import {
   IncludeDocs,
   getLinkDocuments,
-  createLinkView,
   getUniqueByProp,
   getRelatedTableForField,
   getLinkedTableIDs,
@@ -14,7 +13,14 @@ import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types"
+import {
+  Table,
+  Row,
+  LinkDocumentValue,
+  FieldType,
+  LinkDocument,
+  ContextUser,
+} from "@budibase/types"
 import sdk from "../../sdk"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
@@ -73,18 +79,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map(
+  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
-    row => row.doc
+    row => row.doc!
   )
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))
-    .filter(row => row != null)
+    .filter(row => row != null) as Row[]
   // need to handle users as specific cases
   let [users, other] = partition(linked, linkRow =>
-    linkRow._id.startsWith(USER_METDATA_PREFIX)
+    linkRow._id!.startsWith(USER_METDATA_PREFIX)
   )
-  users = await getGlobalUsersFromMetadata(users)
+  users = await getGlobalUsersFromMetadata(users as ContextUser[])
   return [...other, ...users]
 }
 
@@ -176,7 +182,7 @@ export async function attachFullLinkedDocs(
   // clear any existing links that could be dupe'd
   rows = clearRelationshipFields(table, rows)
   // now get the docs and combine into the rows
-  let linked = []
+  let linked: Row[] = []
   if (linksWithoutFromRow.length > 0) {
     linked = await getFullLinkedDocs(linksWithoutFromRow)
   }
@@ -189,7 +195,7 @@ export async function attachFullLinkedDocs(
       if (opts?.fromRow && opts?.fromRow?._id === link.id) {
         linkedRow = opts.fromRow!
       } else {
-        linkedRow = linked.find(row => row._id === link.id)
+        linkedRow = linked.find(row => row._id === link.id)!
       }
       if (linkedRow) {
         const linkedTableId =

packages/server/src/db/linkedRows/linkUtils.ts

@@ -8,6 +8,7 @@ import {
   LinkDocumentValue,
   Table,
 } from "@budibase/types"
+import sdk from "../../sdk"
 
 export { createLinkView } from "../views/staticViews"
 
@@ -110,12 +111,11 @@ export function getLinkedTableIDs(table: Table): string[] {
 }
 
 export async function getLinkedTable(id: string, tables: Table[]) {
-  const db = context.getAppDB()
   let linkedTable = tables.find(table => table._id === id)
   if (linkedTable) {
     return linkedTable
   }
-  linkedTable = await db.get(id)
+  linkedTable = await sdk.tables.getTable(id)
   if (linkedTable) {
     tables.push(linkedTable)
   }

packages/server/src/integration-test/postgres.spec.ts

@@ -923,7 +923,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })
@@ -932,7 +931,6 @@ describe("postgres integrations", () => {
             [m2mFieldName]: [
               {
                 _id: row._id,
-                primaryDisplay: "Invalid display column",
               },
             ],
           })

packages/server/src/sdk/app/applications/import.ts

@@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) {
   // map the responses to the document itself
   let documents: Document[] = []
   for (let response of await Promise.all(docPromises)) {
-    documents = documents.concat(response.rows.map(row => row.doc))
+    documents = documents.concat(response.rows.map(row => row.doc!))
   }
   // remove the _rev, stops it being written
   documents.forEach(doc => {

packages/server/src/sdk/app/applications/sync.ts

@@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core"
 import { User, ContextUser, UserGroup } from "@budibase/types"
 import { sdk as proSdk } from "@budibase/pro"
 import sdk from "../../"
-import { getGlobalUsers, processUser } from "../../../utilities/global"
+import {
+  getGlobalUsers,
+  getRawGlobalUsers,
+  processUser,
+} from "../../../utilities/global"
 import { generateUserMetadataID, InternalTables } from "../../../db/utils"
 
 type DeletedUser = { _id: string; deleted: boolean }
@@ -77,9 +81,7 @@ async function syncUsersToApp(
 
 export async function syncUsersToAllApps(userIds: string[]) {
   // list of users, if one has been deleted it will be undefined in array
-  const users = (await getGlobalUsers(userIds, {
+  const users = await getRawGlobalUsers(userIds)
-    noProcessing: true,
-  })) as User[]
   const groups = await proSdk.groups.fetch()
   const finalUsers: (User | DeletedUser)[] = []
   for (let userId of userIds) {

packages/server/src/sdk/app/datasources/datasources.ts

@@ -51,12 +51,12 @@ export async function fetch(opts?: {
 
   // Get external datasources
   const datasources = (
-    await db.allDocs(
+    await db.allDocs<Datasource>(
       getDatasourceParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
 
   const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
     bbInternalDb,
@@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
     })
   )
 
-  return externalDatasources.rows.map(r => r.doc)
+  return externalDatasources.rows.map(r => r.doc!)
 }

packages/server/src/sdk/app/rows/search/external.ts

@@ -80,7 +80,10 @@ export async function search(options: SearchParams) {
       rows = rows.map((r: any) => pick(r, fields))
     }
 
-    rows = await outputProcessing(table, rows, { preserveLinks: true })
+    rows = await outputProcessing(table, rows, {
+      preserveLinks: true,
+      squash: true,
+    })
 
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
@@ -185,6 +188,7 @@ export async function fetch(tableId: string): Promise<Row[]> {
   const table = await sdk.tables.getTable(tableId)
   return await outputProcessing<Row[]>(table, response, {
     preserveLinks: true,
+    squash: true,
   })
 }
 

packages/server/src/sdk/app/rows/search/internal.ts

@@ -185,8 +185,8 @@ export async function fetchView(
       group: !!group,
     })
   } else {
-    const tableId = viewInfo.meta.tableId
+    const tableId = viewInfo.meta!.tableId
-    const data = await fetchRaw(tableId)
+    const data = await fetchRaw(tableId!)
     response = await inMemoryViews.runView(
       viewInfo,
       calculation as string,
@@ -200,7 +200,7 @@ export async function fetchView(
     response.rows = response.rows.map(row => row.doc)
     let table: Table
     try {
-      table = await sdk.tables.getTable(viewInfo.meta.tableId)
+      table = await sdk.tables.getTable(viewInfo.meta!.tableId)
     } catch (err) {
       throw new Error("Unable to retrieve view table.")
     }

packages/server/src/sdk/app/tables/getters.ts

@@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise<Table[]> {
   if (!db) {
     db = context.getAppDB()
   }
-  const internalTables = await db.allDocs<Table[]>(
+  const internalTables = await db.allDocs<Table>(
     getTableParams(null, {
       include_docs: true,
     })
@@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table[]>(
+    const internalTableDocs = await db.allDocs<Table>(
       getMultiIDParams(internalTableIds)
     )
     tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))

packages/server/src/sdk/users/utils.ts

@@ -7,12 +7,17 @@ import {
   InternalTables,
 } from "../../db/utils"
 import isEqual from "lodash/isEqual"
-import { ContextUser, UserMetadata, User, Database } from "@budibase/types"
+import {
+  ContextUser,
+  UserMetadata,
+  Database,
+  ContextUserMetadata,
+} from "@budibase/types"
 
 export function combineMetadataAndUser(
   user: ContextUser,
   metadata: UserMetadata | UserMetadata[]
-) {
+): ContextUserMetadata | null {
   const metadataId = generateUserMetadataID(user._id!)
   const found = Array.isArray(metadata)
     ? metadata.find(doc => doc._id === metadataId)
@@ -51,33 +56,33 @@ export function combineMetadataAndUser(
   return null
 }
 
-export async function rawUserMetadata(db?: Database) {
+export async function rawUserMetadata(db?: Database): Promise<UserMetadata[]> {
   if (!db) {
     db = context.getAppDB()
   }
   return (
-    await db.allDocs(
+    await db.allDocs<UserMetadata>(
       getUserMetadataParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map(row => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
-export async function fetchMetadata() {
+export async function fetchMetadata(): Promise<ContextUserMetadata[]> {
   const global = await getGlobalUsers()
   const metadata = await rawUserMetadata()
-  const users = []
+  const users: ContextUserMetadata[] = []
   for (let user of global) {
     // find the metadata that matches up to the global ID
-    const info = metadata.find(meta => meta._id.includes(user._id))
+    const info = metadata.find(meta => meta._id!.includes(user._id!))
     // remove these props, not for the correct DB
     users.push({
       ...user,
       ...info,
       tableId: InternalTables.USER_METADATA,
       // make sure the ID is always a local ID, not a global one
-      _id: generateUserMetadataID(user._id),
+      _id: generateUserMetadataID(user._id!),
     })
   }
   return users
@@ -90,9 +95,10 @@ export async function syncGlobalUsers() {
     if (!(await db.exists())) {
       continue
     }
-    const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)])
+    const [users, metadata] = await Promise.all([
-    const users = resp[0] as User[]
+      getGlobalUsers(),
-    const metadata = resp[1] as UserMetadata[]
+      rawUserMetadata(db),
+    ])
     const toWrite = []
     for (let user of users) {
       const combined = combineMetadataAndUser(user, metadata)

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -264,7 +264,7 @@ class TestConfiguration {
     admin = false,
     email = this.defaultUserValues.email,
     roles,
-  }: any = {}) {
+  }: any = {}): Promise<User> {
     const db = tenancy.getTenantDB(this.getTenantId())
     let existing
     try {

packages/server/src/tests/utilities/api/index.ts

@@ -9,6 +9,7 @@ import { ScreenAPI } from "./screen"
 import { ApplicationAPI } from "./application"
 import { BackupAPI } from "./backup"
 import { AttachmentAPI } from "./attachment"
+import { UserAPI } from "./user"
 
 export default class API {
   table: TableAPI
@@ -21,6 +22,7 @@ export default class API {
   application: ApplicationAPI
   backup: BackupAPI
   attachment: AttachmentAPI
+  user: UserAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
@@ -33,5 +35,6 @@ export default class API {
     this.application = new ApplicationAPI(config)
     this.backup = new BackupAPI(config)
     this.attachment = new AttachmentAPI(config)
+    this.user = new UserAPI(config)
   }
 }

packages/server/src/tests/utilities/api/row.ts

@@ -6,6 +6,7 @@ import {
   ExportRowsRequest,
   BulkImportRequest,
   BulkImportResponse,
+  SearchRowResponse,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -154,7 +155,7 @@ export class RowAPI extends TestAPI {
   search = async (
     sourceId: string,
     { expectStatus } = { expectStatus: 200 }
-  ): Promise<Row[]> => {
+  ): Promise<SearchRowResponse> => {
     const request = this.request
       .post(`/api/${sourceId}/search`)
       .set(this.config.defaultHeaders())

packages/server/src/tests/utilities/api/user.ts

@@ -0,0 +1,157 @@
+import {
+  FetchUserMetadataResponse,
+  FindUserMetadataResponse,
+  Flags,
+  UserMetadata,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+import { DocumentInsertResponse } from "@budibase/nano"
+
+export class UserAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  fetch = async (
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<FetchUserMetadataResponse> => {
+    const res = await this.request
+      .get(`/api/users/metadata`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body
+  }
+
+  find = async (
+    id: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<FindUserMetadataResponse> => {
+    const res = await this.request
+      .get(`/api/users/metadata/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body
+  }
+
+  update = async (
+    user: UserMetadata,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<DocumentInsertResponse> => {
+    const res = await this.request
+      .put(`/api/users/metadata`)
+      .set(this.config.defaultHeaders())
+      .send(user)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as DocumentInsertResponse
+  }
+
+  updateSelf = async (
+    user: UserMetadata,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<DocumentInsertResponse> => {
+    const res = await this.request
+      .post(`/api/users/metadata/self`)
+      .set(this.config.defaultHeaders())
+      .send(user)
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as DocumentInsertResponse
+  }
+
+  destroy = async (
+    id: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<{ message: string }> => {
+    const res = await this.request
+      .delete(`/api/users/metadata/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as { message: string }
+  }
+
+  setFlag = async (
+    flag: string,
+    value: any,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<{ message: string }> => {
+    const res = await this.request
+      .post(`/api/users/flags`)
+      .set(this.config.defaultHeaders())
+      .send({ flag, value })
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as { message: string }
+  }
+
+  getFlags = async (
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Flags> => {
+    const res = await this.request
+      .get(`/api/users/flags`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    if (res.status !== expectStatus) {
+      throw new Error(
+        `Expected status ${expectStatus} but got ${
+          res.status
+        } with body ${JSON.stringify(res.body)}`
+      )
+    }
+
+    return res.body as Flags
+  }
+}

packages/server/src/utilities/global.ts

@@ -71,69 +71,67 @@ export async function processUser(
   return user
 }
 
-export async function getCachedSelf(ctx: UserCtx, appId: string) {
+export async function getCachedSelf(
+  ctx: UserCtx,
+  appId: string
+): Promise<ContextUser> {
   // this has to be tenant aware, can't depend on the context to find it out
   // running some middlewares before the tenancy causes context to break
   const user = await cache.user.getUser(ctx.user?._id!)
   return processUser(user, { appId })
 }
 
-export async function getRawGlobalUser(userId: string) {
+export async function getRawGlobalUser(userId: string): Promise<User> {
   const db = tenancy.getGlobalDB()
   return db.get<User>(getGlobalIDFromUserMetadataID(userId))
 }
 
-export async function getGlobalUser(userId: string) {
+export async function getGlobalUser(userId: string): Promise<ContextUser> {
   const appId = context.getAppId()
   let user = await getRawGlobalUser(userId)
   return processUser(user, { appId })
 }
 
-export async function getGlobalUsers(
+export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
-  userIds?: string[],
-  opts?: { noProcessing?: boolean }
-) {
-  const appId = context.getAppId()
   const db = tenancy.getGlobalDB()
-  let globalUsers
+  let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map(
+    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
-      row => row.doc
+      row => row.doc!
     )
   } else {
     globalUsers = (
-      await db.allDocs(
+      await db.allDocs<User>(
         dbCore.getGlobalUserParams(null, {
           include_docs: true,
         })
       )
-    ).rows.map(row => row.doc)
+    ).rows.map(row => row.doc!)
   }
-  globalUsers = globalUsers
+  return globalUsers
     .filter(user => user != null)
     .map(user => {
       delete user.password
       delete user.forceResetPassword
       return user
     })
+}
 
-  if (opts?.noProcessing || !appId) {
+export async function getGlobalUsers(
-    return globalUsers
+  userIds?: string[]
-  } else {
+): Promise<ContextUser[]> {
-    // pass in the groups, meaning we don't actually need to retrieve them for
+  const users = await getRawGlobalUsers(userIds)
-    // each user individually
   const allGroups = await groups.fetch()
   return Promise.all(
-      globalUsers.map(user => processUser(user, { groups: allGroups }))
+    users.map(user => processUser(user, { groups: allGroups }))
   )
-  }
 }
 
 export async function getGlobalUsersFromMetadata(users: ContextUser[]) {
   const globalUsers = await getGlobalUsers(users.map(user => user._id!))
   return users.map(user => {
     const globalUser = globalUsers.find(
-      globalUser => globalUser && user._id?.includes(globalUser._id)
+      globalUser => globalUser && user._id?.includes(globalUser._id!)
     )
     return {
       ...globalUser,

packages/server/src/utilities/routing/index.ts

@@ -1,9 +1,9 @@
 import { createRoutingView } from "../../db/views/staticViews"
 import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils"
 import { context } from "@budibase/backend-core"
-import { ScreenRouting } from "@budibase/types"
+import { ScreenRouting, Document } from "@budibase/types"
 
-type ScreenRoutesView = {
+interface ScreenRoutesView extends Document {
   id: string
   routing: ScreenRouting
 }

packages/server/src/utilities/rowProcessor/index.ts

@@ -2,16 +2,15 @@ import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
-import { context, db as dbCore, objectStore } from "@budibase/backend-core"
+import {
+  context,
+  db as dbCore,
+  objectStore,
+  utils,
+} from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import {
+import { FieldSubtype, Row, RowAttachment, Table } from "@budibase/types"
-  AutoColumnFieldMetadata,
-  FieldSubtype,
-  Row,
-  RowAttachment,
-  Table,
-} from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
   processInputBBReferences,
@@ -233,6 +232,11 @@ export async function outputProcessing<T extends Row[] | Row>(
       })
     : safeRows
 
+  // make sure squash is enabled if needed
+  if (!opts.squash && utils.hasCircularStructure(rows)) {
+    opts.squash = true
+  }
+
   // process complex types: attachements, bb references...
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
@@ -258,7 +262,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
+  enriched = processFormulas(table, enriched, { dynamic: true })
 
   if (opts.squash) {
     enriched = (await linkRows.squashLinksToPrimaryDisplay(

packages/server/src/utilities/rowProcessor/utils.ts

@@ -12,6 +12,11 @@ import {
   Table,
 } from "@budibase/types"
 
+interface FormulaOpts {
+  dynamic?: boolean
+  contextRows?: Row[]
+}
+
 /**
  * If the subtype has been lost for any reason this works out what
  * subtype the auto column should be.
@@ -40,19 +45,13 @@ export function fixAutoColumnSubType(
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */
-export function processFormulas(
+export function processFormulas<T extends Row | Row[]>(
   table: Table,
-  rows: Row[] | Row,
+  inputRows: T,
-  { dynamic, contextRows }: any = { dynamic: true }
+  { dynamic, contextRows }: FormulaOpts = { dynamic: true }
-) {
+): T {
-  const single = !Array.isArray(rows)
+  const rows = Array.isArray(inputRows) ? inputRows : [inputRows]
-  let rowArray: Row[]
+  if (rows)
-  if (single) {
-    rowArray = [rows]
-    contextRows = contextRows ? [contextRows] : contextRows
-  } else {
-    rowArray = rows
-  }
     for (let [column, schema] of Object.entries(table.schema)) {
       if (schema.type !== FieldTypes.FORMULA) {
         continue
@@ -68,24 +67,28 @@ export function processFormulas(
         continue
       }
       // iterate through rows and process formula
-    for (let i = 0; i < rowArray.length; i++) {
+      for (let i = 0; i < rows.length; i++) {
-      let row = rowArray[i]
+        let row = rows[i]
         let context = contextRows ? contextRows[i] : row
-      rowArray[i] = {
+        rows[i] = {
           ...row,
           [column]: processStringSync(schema.formula, context),
         }
       }
     }
-  return single ? rowArray[0] : rowArray
+  return Array.isArray(inputRows) ? rows : rows[0]
 }
 
 /**
  * Processes any date columns and ensures that those without the ignoreTimezones
  * flag set are parsed as UTC rather than local time.
  */
-export function processDates(table: Table, rows: Row[]) {
+export function processDates<T extends Row | Row[]>(
-  let datesWithTZ = []
+  table: Table,
+  inputRows: T
+): T {
+  let rows = Array.isArray(inputRows) ? inputRows : [inputRows]
+  let datesWithTZ: string[] = []
   for (let [column, schema] of Object.entries(table.schema)) {
     if (schema.type !== FieldTypes.DATETIME) {
       continue
@@ -102,5 +105,6 @@ export function processDates(table: Table, rows: Row[]) {
       }
     }
   }
-  return rows
+
+  return Array.isArray(inputRows) ? rows : rows[0]
 }

packages/server/src/utilities/users.ts

@@ -1,11 +1,13 @@
 import { InternalTables } from "../db/utils"
 import { getGlobalUser } from "./global"
 import { context, roles } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
+import { ContextUserMetadata, UserCtx, UserMetadata } from "@budibase/types"
 
-export async function getFullUser(ctx: UserCtx, userId: string) {
+export async function getFullUser(
+  userId: string
+): Promise<ContextUserMetadata> {
   const global = await getGlobalUser(userId)
-  let metadata: any = {}
+  let metadata: UserMetadata | undefined = undefined
 
   // always prefer the user metadata _id and _rev
   delete global._id
@@ -14,11 +16,11 @@ export async function getFullUser(ctx: UserCtx, userId: string) {
   try {
     // this will throw an error if the db doesn't exist, or there is no appId
     const db = context.getAppDB()
-    metadata = await db.get(userId)
+    metadata = await db.get<UserMetadata>(userId)
+    delete metadata.csrfToken
   } catch (err) {
     // it is fine if there is no user metadata yet
   }
-  delete metadata.csrfToken
   return {
     ...metadata,
     ...global,

packages/types/src/api/web/app/index.ts

@@ -6,3 +6,4 @@ export * from "./rows"
 export * from "./table"
 export * from "./permission"
 export * from "./attachment"
+export * from "./user"

packages/types/src/api/web/app/user.ts

@@ -0,0 +1,9 @@
+import { ContextUserMetadata } from "../../../"
+
+export type FetchUserMetadataResponse = ContextUserMetadata[]
+export type FindUserMetadataResponse = ContextUserMetadata
+
+export interface SetFlagRequest {
+  flag: string
+  value: any
+}

packages/types/src/documents/account/flag.ts

@@ -0,0 +1,5 @@
+import { Document } from "../../"
+
+export interface Flags extends Document {
+  [key: string]: any
+}

packages/types/src/documents/account/index.ts

@@ -1,2 +1,3 @@
 export * from "./account"
 export * from "./user"
+export * from "./flag"

packages/types/src/documents/app/layout.ts

@@ -2,4 +2,5 @@ import { Document } from "../document"
 
 export interface Layout extends Document {
   props: any
+  layoutId?: string
 }

packages/types/src/documents/app/user.ts

@@ -1,6 +1,6 @@
-import { Document } from "../document"
+import { User } from "../global"
+import { Row } from "./row"
+import { ContextUser } from "../../sdk"
 
-export interface UserMetadata extends Document {
+export type UserMetadata = User & Row
-  roleId: string
+export type ContextUserMetadata = ContextUser & Row
-  email?: string
-}

packages/types/src/documents/app/view.ts

@@ -1,5 +1,24 @@
 import { SearchFilter, SortOrder, SortType } from "../../api"
 import { UIFieldMetadata } from "./table"
+import { Document } from "../document"
+import { DBView } from "../../sdk"
+
+export type ViewTemplateOpts = {
+  field: string
+  tableId: string
+  groupBy: string
+  filters: ViewFilter[]
+  schema: any
+  calculation: string
+  groupByMulti?: boolean
+}
+
+export interface InMemoryView extends Document {
+  view: DBView
+  name: string
+  tableId: string
+  groupBy?: string
+}
 
 export interface View {
   name?: string
@@ -10,7 +29,7 @@ export interface View {
   calculation?: ViewCalculation
   map?: string
   reduce?: any
-  meta?: Record<string, any>
+  meta?: ViewTemplateOpts
 }
 
 export interface ViewV2 {

packages/types/src/documents/pouch.ts

@@ -1,17 +1,19 @@
+import { Document } from "../"
+
 export interface RowValue {
   rev: string
   deleted: boolean
 }
 
-export interface RowResponse<T> {
+export interface RowResponse<T extends Document> {
   id: string
   key: string
   error: string
   value: T | RowValue
-  doc?: T | any
+  doc?: T
 }
 
-export interface AllDocsResponse<T> {
+export interface AllDocsResponse<T extends Document> {
   offset: number
   total_rows: number
   rows: RowResponse<T>[]

packages/types/src/sdk/db.ts

@@ -1,5 +1,5 @@
 import Nano from "@budibase/nano"
-import { AllDocsResponse, AnyDocument, Document } from "../"
+import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../"
 import { Writable } from "stream"
 
 export enum SearchIndex {
@@ -20,6 +20,37 @@ export enum SortOption {
   DESCENDING = "desc",
 }
 
+export type IndexAnalyzer = {
+  name: string
+  default?: string
+  fields?: Record<string, string>
+}
+
+export type DBView = {
+  name?: string
+  map: string
+  reduce?: string
+  meta?: ViewTemplateOpts
+  groupBy?: string
+}
+
+export interface DesignDocument extends Document {
+  // we use this static reference for all design documents
+  _id: "_design/database"
+  language?: string
+  // CouchDB views
+  views?: {
+    [viewName: string]: DBView
+  }
+  // Lucene indexes
+  indexes?: {
+    [indexName: string]: {
+      index: string
+      analyzer?: string | IndexAnalyzer
+    }
+  }
+}
+
 export type CouchFindOptions = {
   selector: PouchDB.Find.Selector
   fields?: string[]
@@ -101,8 +132,10 @@ export interface Database {
     opts?: DatabasePutOpts
   ): Promise<Nano.DocumentInsertResponse>
   bulkDocs(documents: AnyDocument[]): Promise<Nano.DocumentBulkResponse[]>
-  allDocs<T>(params: DatabaseQueryOpts): Promise<AllDocsResponse<T>>
+  allDocs<T extends Document>(
-  query<T>(
+    params: DatabaseQueryOpts
+  ): Promise<AllDocsResponse<T>>
+  query<T extends Document>(
     viewName: string,
     params: DatabaseQueryOpts
   ): Promise<AllDocsResponse<T>>

packages/worker/src/constants/templates/index.ts

@@ -56,12 +56,12 @@ export async function getTemplates({
   id,
 }: { ownerId?: string; type?: string; id?: string } = {}) {
   const db = tenancy.getGlobalDB()
-  const response = await db.allDocs(
+  const response = await db.allDocs<Template>(
     dbCore.getTemplateParams(ownerId || GLOBAL_OWNER, id, {
       include_docs: true,
     })
   )
-  let templates = response.rows.map(row => row.doc)
+  let templates = response.rows.map(row => row.doc!)
   // should only be one template with ID
   if (id) {
     return templates[0]
@@ -73,6 +73,6 @@ export async function getTemplates({
 }
 
 export async function getTemplateByPurpose(type: string, purpose: string) {
-  const templates = await getTemplates({ type })
+  const templates = (await getTemplates({ type })) as Template[]
   return templates.find((template: Template) => template.purpose === purpose)
 }

packages/worker/src/utilities/email.ts

@@ -99,8 +99,6 @@ async function buildEmail(
   if (!base || !body || !core) {
     throw "Unable to build email, missing base components"
   }
-  base = base.contents
-  body = body.contents
 
   let name = user ? user.name : undefined
   if (user && !name && user.firstName) {
@@ -114,15 +112,10 @@ async function buildEmail(
     user: user || {},
   }
 
-  // Prepend the core template
-  const fullBody = core + body
-
-  body = await processString(fullBody, context)
-
   // this should now be the core email HTML
-  return processString(base, {
+  return processString(base.contents, {
     ...context,
-    body,
+    body: await processString(core + body?.contents, context),
   })
 }