Merge branch 'v3-ui' of github.com:Budibase/budibase into new-rbac-ui

.tool-versions

@@ -1,3 +1,3 @@
 nodejs 20.10.0
 python 3.10.0
-yarn 1.22.19
+yarn 1.22.22

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.32.8",
+  "version": "2.32.10",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit 558a32dfd1f55bd894804a503e7e1090937df88c
+Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0

packages/backend-core/src/context/mainContext.ts

@@ -10,7 +10,7 @@ import {
   StaticDatabases,
   DEFAULT_TENANT_ID,
 } from "../constants"
-import { Database, IdentityContext, Snippet, App } from "@budibase/types"
+import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types"
 import { ContextMap } from "./types"
 
 let TEST_APP_ID: string | null = null
@@ -394,3 +394,20 @@ export function setFeatureFlags(key: string, value: Record<string, any>) {
   context.featureFlagCache ??= {}
   context.featureFlagCache[key] = value
 }
+
+export function getTableForView(viewId: string): Table | undefined {
+  const context = getCurrentContext()
+  if (!context) {
+    return
+  }
+  return context.viewToTableCache?.[viewId]
+}
+
+export function setTableForView(viewId: string, table: Table) {
+  const context = getCurrentContext()
+  if (!context) {
+    return
+  }
+  context.viewToTableCache ??= {}
+  context.viewToTableCache[viewId] = table
+}

packages/backend-core/src/context/types.ts

@@ -1,4 +1,4 @@
-import { IdentityContext, Snippet, VM } from "@budibase/types"
+import { IdentityContext, Snippet, Table, VM } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 
@@ -21,4 +21,5 @@ export type ContextMap = {
   featureFlagCache?: {
     [key: string]: Record<string, any>
   }
+  viewToTableCache?: Record<string, Table>
 }

packages/backend-core/src/db/lucene.ts

@@ -612,7 +612,6 @@ async function runQuery<T>(
  *   limit {number} The number of results to fetch
  *   bookmark {string|null} Current bookmark in the recursive search
  *   rows {array|null} Current results in the recursive search
- * @returns {Promise<*[]|*>}
  */
 async function recursiveSearch<T>(
   dbName: string,

packages/backend-core/src/docIds/params.ts

@@ -6,7 +6,7 @@ import {
   ViewName,
 } from "../constants"
 import { getProdAppID } from "./conversions"
-import { DatabaseQueryOpts } from "@budibase/types"
+import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types"
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
@@ -66,9 +66,8 @@ export function getQueryIndex(viewName: ViewName) {
 
 /**
  * Check if a given ID is that of a table.
- * @returns {boolean}
  */
-export const isTableId = (id: string) => {
+export const isTableId = (id: string): boolean => {
   // this includes datasource plus tables
   return (
     !!id &&
@@ -77,13 +76,16 @@ export const isTableId = (id: string) => {
   )
 }
 
+export function isViewId(id: string): boolean {
+  return !!id && id.startsWith(`${VirtualDocumentType.VIEW}${SEPARATOR}`)
+}
+
 /**
  * Check if a given ID is that of a datasource or datasource plus.
- * @returns {boolean}
  */
-export const isDatasourceId = (id: string) => {
+export const isDatasourceId = (id: string): boolean => {
   // this covers both datasources and datasource plus
-  return id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
+  return !!id && id.startsWith(`${DocumentType.DATASOURCE}${SEPARATOR}`)
 }
 
 /**

packages/backend-core/src/sql/sql.ts

@@ -11,10 +11,12 @@ import {
 } from "./utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
+  Aggregation,
   AnySearchFilter,
   ArrayOperator,
   BasicOperator,
   BBReferenceFieldMetadata,
+  CalculationType,
   FieldSchema,
   FieldType,
   INTERNAL_TABLE_SOURCE_ID,
@@ -69,18 +71,6 @@ function prioritisedArraySort(toSort: string[], priorities: string[]) {
   })
 }
 
-function getTableName(table?: Table): string | undefined {
-  // SQS uses the table ID rather than the table name
-  if (
-    table?.sourceType === TableSourceType.INTERNAL ||
-    table?.sourceId === INTERNAL_TABLE_SOURCE_ID
-  ) {
-    return table?._id
-  } else {
-    return table?.name
-  }
-}
-
 function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
   if (Array.isArray(query)) {
     return query.map((q: SqlQuery) => convertBooleans(q) as SqlQuery)
@@ -97,6 +87,13 @@ function convertBooleans(query: SqlQuery | SqlQuery[]): SqlQuery | SqlQuery[] {
   return query
 }
 
+function isSqs(table: Table): boolean {
+  return (
+    table.sourceType === TableSourceType.INTERNAL ||
+    table.sourceId === INTERNAL_TABLE_SOURCE_ID
+  )
+}
+
 class InternalBuilder {
   private readonly client: SqlClient
   private readonly query: QueryJson
@@ -178,15 +175,13 @@ class InternalBuilder {
   }
 
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { meta, endpoint, resource, tableAliases } = this.query
+    const { meta, endpoint, resource } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
     }
 
-    const alias = tableAliases?.[endpoint.entityId]
+    const alias = this.getTableName(endpoint.entityId)
-      ? tableAliases?.[endpoint.entityId]
-      : endpoint.entityId
     const schema = meta.table.schema
     if (!this.isFullSelectStatementRequired()) {
       return [this.knex.raw(`${this.quote(alias)}.*`)]
@@ -811,26 +806,88 @@ class InternalBuilder {
     return query
   }
 
+  isSqs(): boolean {
+    return isSqs(this.table)
+  }
+
+  getTableName(tableOrName?: Table | string): string {
+    let table: Table
+    if (typeof tableOrName === "string") {
+      const name = tableOrName
+      if (this.query.table?.name === name) {
+        table = this.query.table
+      } else if (this.query.meta.table?.name === name) {
+        table = this.query.meta.table
+      } else if (!this.query.meta.tables?.[name]) {
+        // This can legitimately happen in custom queries, where the user is
+        // querying against a table that may not have been imported into
+        // Budibase.
+        return name
+      } else {
+        table = this.query.meta.tables[name]
+      }
+    } else if (tableOrName) {
+      table = tableOrName
+    } else {
+      table = this.table
+    }
+
+    let name = table.name
+    if (isSqs(table) && table._id) {
+      // SQS uses the table ID rather than the table name
+      name = table._id
+    }
+    const aliases = this.query.tableAliases || {}
+    return aliases[name] ? aliases[name] : name
+  }
+
   addDistinctCount(query: Knex.QueryBuilder): Knex.QueryBuilder {
-    const primary = this.table.primary
+    if (!this.table.primary) {
-    const aliases = this.query.tableAliases
-    const aliased =
-      this.table.name && aliases?.[this.table.name]
-        ? aliases[this.table.name]
-        : this.table.name
-    if (!primary) {
       throw new Error("SQL counting requires primary key to be supplied")
     }
-    return query.countDistinct(`${aliased}.${primary[0]} as total`)
+    return query.countDistinct(
+      `${this.getTableName()}.${this.table.primary[0]} as __bb_total`
+    )
+  }
+
+  addAggregations(
+    query: Knex.QueryBuilder,
+    aggregations: Aggregation[]
+  ): Knex.QueryBuilder {
+    const fields = this.query.resource?.fields || []
+    const tableName = this.getTableName()
+    if (fields.length > 0) {
+      query = query.groupBy(fields.map(field => `${tableName}.${field}`))
+      query = query.select(fields.map(field => `${tableName}.${field}`))
+    }
+    for (const aggregation of aggregations) {
+      const op = aggregation.calculationType
+      const field = `${tableName}.${aggregation.field} as ${aggregation.name}`
+      switch (op) {
+        case CalculationType.COUNT:
+          query = query.count(field)
+          break
+        case CalculationType.SUM:
+          query = query.sum(field)
+          break
+        case CalculationType.AVG:
+          query = query.avg(field)
+          break
+        case CalculationType.MIN:
+          query = query.min(field)
+          break
+        case CalculationType.MAX:
+          query = query.max(field)
+          break
+      }
+    }
+    return query
   }
 
   addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder {
-    let { sort } = this.query
+    let { sort, resource } = this.query
     const primaryKey = this.table.primary
-    const tableName = getTableName(this.table)
+    const aliased = this.getTableName()
-    const aliases = this.query.tableAliases
-    const aliased =
-      tableName && aliases?.[tableName] ? aliases[tableName] : this.table?.name
     if (!Array.isArray(primaryKey)) {
       throw new Error("Sorting requires primary key to be specified for table")
     }
@@ -862,7 +919,8 @@ class InternalBuilder {
 
     // add sorting by the primary key if the result isn't already sorted by it,
     // to make sure result is deterministic
-    if (!sort || sort[primaryKey[0]] === undefined) {
+    const hasAggregations = (resource?.aggregations?.length ?? 0) > 0
+    if (!hasAggregations && (!sort || sort[primaryKey[0]] === undefined)) {
       query = query.orderBy(`${aliased}.${primaryKey[0]}`)
     }
     return query
@@ -1246,10 +1304,15 @@ class InternalBuilder {
       }
     }
 
-    // if counting, use distinct count, else select
+    const aggregations = this.query.resource?.aggregations || []
-    query = !counting
+    if (counting) {
-      ? query.select(this.generateSelectStatement())
+      query = this.addDistinctCount(query)
-      : this.addDistinctCount(query)
+    } else if (aggregations.length > 0) {
+      query = this.addAggregations(query, aggregations)
+    } else {
+      query = query.select(this.generateSelectStatement())
+    }
+
     // have to add after as well (this breaks MS-SQL)
     if (!counting) {
       query = this.addSorting(query)
@@ -1468,23 +1531,40 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     return results.length ? results : [{ [operation.toLowerCase()]: true }]
   }
 
+  private getTableName(
+    table: Table,
+    aliases?: Record<string, string>
+  ): string | undefined {
+    let name = table.name
+    if (
+      table.sourceType === TableSourceType.INTERNAL ||
+      table.sourceId === INTERNAL_TABLE_SOURCE_ID
+    ) {
+      if (!table._id) {
+        return
+      }
+      // SQS uses the table ID rather than the table name
+      name = table._id
+    }
+    return aliases?.[name] || name
+  }
+
   convertJsonStringColumns<T extends Record<string, any>>(
     table: Table,
     results: T[],
     aliases?: Record<string, string>
   ): T[] {
-    const tableName = getTableName(table)
+    const tableName = this.getTableName(table, aliases)
     for (const [name, field] of Object.entries(table.schema)) {
       if (!this._isJsonColumn(field)) {
         continue
       }
-      const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
+      const fullName = `${tableName}.${name}` as keyof T
-      const fullName = `${aliasedTableName}.${name}`
       for (let row of results) {
-        if (typeof row[fullName as keyof T] === "string") {
+        if (typeof row[fullName] === "string") {
-          row[fullName as keyof T] = JSON.parse(row[fullName])
+          row[fullName] = JSON.parse(row[fullName])
         }
-        if (typeof row[name as keyof T] === "string") {
+        if (typeof row[name] === "string") {
           row[name as keyof T] = JSON.parse(row[name])
         }
       }

packages/builder/assets/MagicWand.svelte

@@ -0,0 +1,29 @@
+<script>
+  export let width
+  export let height
+</script>
+
+<svg
+  {width}
+  {height}
+  viewBox="0 0 13 12"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="M9.4179 4.13222C9.4179 3.73121 9.26166 3.35428 8.97913 3.07175C8.41342 2.50538 7.4239 2.50408 6.85753 3.07175L5.64342 4.28586C5.6291 4.30018 5.61543 4.3158 5.60305 4.33143C5.58678 4.3438 5.5718 4.35747 5.55683 4.37244L0.491426 9.43785C0.208245 9.72103 0.052002 10.098 0.052002 10.4983C0.052002 10.8987 0.208245 11.2756 0.491426 11.5588C0.774607 11.842 1.15153 11.9982 1.5519 11.9982C1.95227 11.9982 2.32919 11.842 2.61238 11.5588L8.97848 5.1927C9.26166 4.90952 9.4179 4.53259 9.4179 4.13222ZM1.90539 10.8518C1.7166 11.0406 1.3872 11.0406 1.1984 10.8518C1.10401 10.7574 1.05193 10.6318 1.05193 10.4983C1.05193 10.3649 1.104 10.2392 1.1984 10.1448L5.99821 5.34503L6.70845 6.04875L1.90539 10.8518ZM8.2715 4.48571L7.41544 5.34178L6.7052 4.63805L7.56452 3.77873C7.7533 3.58995 8.08271 3.58929 8.2715 3.77939C8.36589 3.87313 8.41798 3.99877 8.41798 4.13223C8.41798 4.26569 8.3659 4.39132 8.2715 4.48571Z"
+    fill="#C8C8C8"
+  />
+  <path
+    d="M11.8552 6.55146L11.0144 6.21913L10.879 5.32449C10.8356 5.03919 10.3737 4.98776 10.2686 5.255L9.93606 6.09642L9.04143 6.23085C8.89951 6.25216 8.78884 6.36658 8.77257 6.50947C8.75629 6.65253 8.83783 6.78826 8.97193 6.84148L9.81335 7.17464L9.94794 8.06862C9.9691 8.21053 10.0835 8.32121 10.2266 8.33748C10.3695 8.35375 10.5052 8.27221 10.5586 8.13811L10.8914 7.29751L11.7855 7.1621C11.9283 7.1403 12.0381 7.02637 12.0544 6.88348C12.0707 6.74058 11.9887 6.60403 11.8552 6.55146Z"
+    fill="#F9634C"
+  />
+  <path
+    d="M8.94215 1.76145L9.78356 2.0946L9.91815 2.9885C9.93931 3.13049 10.0539 3.24117 10.1968 3.25744C10.3398 3.27371 10.4756 3.19218 10.5288 3.05807L10.8618 2.21739L11.7559 2.08207C11.8985 2.06034 12.0085 1.94633 12.0248 1.80344C12.0411 1.66054 11.959 1.524 11.8254 1.47143L10.9847 1.13909L10.8494 0.244456C10.806 -0.0409246 10.3439 -0.0922745 10.2388 0.174881L9.90643 1.0163L9.0118 1.15089C8.86972 1.17213 8.75905 1.28654 8.74278 1.42952C8.72651 1.57249 8.80804 1.70823 8.94215 1.76145Z"
+    fill="#8488FD"
+  />
+  <path
+    d="M3.2379 2.46066L3.92063 2.73091L4.02984 3.45637C4.04709 3.57151 4.14002 3.66135 4.25606 3.67453C4.37194 3.6878 4.48212 3.62163 4.52541 3.51276L4.79557 2.83059L5.52094 2.72074C5.63682 2.70316 5.72601 2.61072 5.73936 2.49468C5.75254 2.37864 5.68597 2.26797 5.57758 2.22533L4.89533 1.95565L4.78548 1.22963C4.75016 0.998038 4.37535 0.956375 4.29007 1.17315L4.0204 1.85597L3.29437 1.96517C3.17915 1.98235 3.08931 2.07527 3.07613 2.19131C3.06294 2.30727 3.12902 2.41737 3.2379 2.46066Z"
+    fill="#F7D804"
+  />
+</svg>

packages/builder/package.json

@@ -69,6 +69,7 @@
     "@xyflow/svelte": "^0.1.18",
     "@zerodevx/svelte-json-view": "^1.0.7",
     "codemirror": "^5.65.16",
+    "cron-parser": "^4.9.0",
     "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -648,6 +648,8 @@
       let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
       if (isLoopBlock) {
         runtimeName = `loop.${name}`
+      } else if (idx === 0) {
+        runtimeName = `trigger.${name}`
       } else if (block.name.startsWith("JS")) {
         runtimeName = hasUserDefinedName
           ? `stepsByName["${bindingName}"].${name}`
@@ -657,7 +659,7 @@
           ? `stepsByName.${bindingName}.${name}`
           : `steps.${idx - loopBlockCount}.${name}`
       }
-      return idx === 0 ? `trigger.${name}` : runtimeName
+      return runtimeName
     }
 
     const determineCategoryName = (idx, isLoopBlock, bindingName) => {
@@ -684,7 +686,7 @@
       )
       return {
         readableBinding:
-          bindingName && !isLoopBlock
+          bindingName && !isLoopBlock && idx !== 0
             ? `steps.${bindingName}.${name}`
             : runtimeBinding,
         runtimeBinding,
@@ -1060,7 +1062,7 @@
             {:else if value.customType === "cron"}
               <CronBuilder
                 on:change={e => onChange({ [key]: e.detail })}
-                value={inputData[key]}
+                cronExpression={inputData[key]}
               />
             {:else if value.customType === "automationFields"}
               <AutomationSelector

packages/builder/src/components/automation/SetupPanel/CronBuilder.svelte

@@ -1,41 +1,70 @@
 <script>
-  import { Button, Select, Input, Label } from "@budibase/bbui"
+  import {
+    Select,
+    InlineAlert,
+    Input,
+    Label,
+    Layout,
+    notifications,
+  } from "@budibase/bbui"
   import { onMount, createEventDispatcher } from "svelte"
   import { flags } from "stores/builder"
+  import { licensing } from "stores/portal"
+  import { API } from "api"
+  import MagicWand from "../../../../assets/MagicWand.svelte"
+
   import { helpers, REBOOT_CRON } from "@budibase/shared-core"
 
   const dispatch = createEventDispatcher()
 
-  export let value
+  export let cronExpression
+
   let error
+  let nextExecutions
 
+  // AI prompt
+  let aiCronPrompt = ""
+  let loadingAICronExpression = false
+
+  $: aiEnabled =
+    $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled
   $: {
-    const exists = CRON_EXPRESSIONS.some(cron => cron.value === value)
+    if (cronExpression) {
-    const customIndex = CRON_EXPRESSIONS.findIndex(
+      try {
-      cron => cron.label === "Custom"
+        nextExecutions = helpers.cron
-    )
+          .getNextExecutionDates(cronExpression)
-
+          .join("\n")
-    if (!exists && customIndex === -1) {
+      } catch (err) {
-      CRON_EXPRESSIONS[0] = { label: "Custom", value: value }
+        nextExecutions = null
-    } else if (exists && customIndex !== -1) {
+      }
-      CRON_EXPRESSIONS.splice(customIndex, 1)
     }
   }
 
   const onChange = e => {
-    if (value !== REBOOT_CRON) {
+    if (e.detail !== REBOOT_CRON) {
       error = helpers.cron.validate(e.detail).err
     }
-    if (e.detail === value || error) {
+    if (e.detail === cronExpression || error) {
       return
     }
 
-    value = e.detail
+    cronExpression = e.detail
     dispatch("change", e.detail)
   }
 
+  const updatePreset = e => {
+    aiCronPrompt = ""
+    onChange(e)
+  }
+
+  const updateCronExpression = e => {
+    aiCronPrompt = ""
+    cronExpression = null
+    nextExecutions = null
+    onChange(e)
+  }
+
   let touched = false
-  let presets = false
 
   const CRON_EXPRESSIONS = [
     {
@@ -64,45 +93,130 @@
       })
     }
   })
+
+  async function generateAICronExpression() {
+    loadingAICronExpression = true
+    try {
+      const response = await API.generateCronExpression({
+        prompt: aiCronPrompt,
+      })
+      cronExpression = response.message
+      dispatch("change", response.message)
+    } catch (err) {
+      notifications.error(err.message)
+    } finally {
+      loadingAICronExpression = false
+    }
+  }
 </script>
 
-<div class="block-field">
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<Layout noPadding gap="S">
+  <Select
+    on:change={updatePreset}
+    value={cronExpression || "Custom"}
+    secondary
+    extraThin
+    label="Use a Preset (Optional)"
+    options={CRON_EXPRESSIONS}
+  />
+  {#if aiEnabled}
+    <div class="cron-ai-generator">
+      <Input
+        bind:value={aiCronPrompt}
+        label="Generate Cron Expression with AI"
+        size="S"
+        placeholder="Run every hour between 1pm to 4pm everyday of the week"
+      />
+      {#if aiCronPrompt}
+        <div
+          class="icon"
+          class:pulsing-text={loadingAICronExpression}
+          on:click={generateAICronExpression}
+        >
+          <MagicWand height="17" width="17" />
+        </div>
+      {/if}
+    </div>
+  {/if}
   <Input
+    label="Cron Expression"
     {error}
-    on:change={onChange}
+    on:change={updateCronExpression}
-    {value}
+    value={cronExpression}
     on:blur={() => (touched = true)}
     updateOnChange={false}
   />
-  {#if touched && !value}
+  {#if touched && !cronExpression}
     <Label><div class="error">Please specify a CRON expression</div></Label>
   {/if}
-  <div class="presets">
+  {#if nextExecutions}
-    <Button on:click={() => (presets = !presets)}
+    <InlineAlert
-      >{presets ? "Hide" : "Show"} Presets</Button
+      type="info"
-    >
+      header="Next Executions"
-    {#if presets}
+      message={nextExecutions}
-      <Select
+    />
-        on:change={onChange}
+  {/if}
-        value={value || "Custom"}
+</Layout>
-        secondary
-        extraThin
-        label="Presets"
-        options={CRON_EXPRESSIONS}
-      />
-    {/if}
-  </div>
-</div>
 
 <style>
-  .presets {
+  .cron-ai-generator {
-    margin-top: var(--spacing-m);
+    flex: 1;
+    position: relative;
   }
-  .block-field {
+  .icon {
-    padding-top: var(--spacing-s);
+    right: 1px;
+    bottom: 1px;
+    position: absolute;
+    justify-content: center;
+    align-items: center;
+    display: flex;
+    flex-direction: row;
+    box-sizing: border-box;
+    border-left: 1px solid var(--spectrum-alias-border-color);
+    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
+    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
+    width: 31px;
+    color: var(--spectrum-alias-text-color);
+    background-color: var(--spectrum-global-color-gray-75);
+    transition: background-color
+        var(--spectrum-global-animation-duration-100, 130ms),
+      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
+      border-color var(--spectrum-global-animation-duration-100, 130ms);
+    height: calc(var(--spectrum-alias-item-height-m) - 2px);
   }
+
+  .icon:hover {
+    cursor: pointer;
+    color: var(--spectrum-alias-text-color-hover);
+    background-color: var(--spectrum-global-color-gray-50);
+    border-color: var(--spectrum-alias-border-color-hover);
+  }
+
   .error {
     padding-top: var(--spacing-xs);
     color: var(--spectrum-global-color-red-500);
   }
+
+  .pulsing-text {
+    font-size: 24px;
+    font-weight: bold;
+    animation: pulse 1.5s infinite;
+  }
+
+  @keyframes pulse {
+    0% {
+      opacity: 0.3;
+      transform: scale(1);
+    }
+    50% {
+      opacity: 1;
+      transform: scale(1.05);
+    }
+    100% {
+      opacity: 0.3;
+      transform: scale(1);
+    }
+  }
 </style>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -21,6 +21,7 @@
     PROTECTED_EXTERNAL_COLUMNS,
     canHaveDefaultColumn,
   } from "@budibase/shared-core"
+  import { makePropSafe } from "@budibase/string-templates"
   import { createEventDispatcher, getContext, onMount } from "svelte"
   import { cloneDeep } from "lodash/fp"
   import { tables, datasources } from "stores/builder"
@@ -46,6 +47,7 @@
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
   import { isEnabled } from "helpers/featureFlags"
+  import { getUserBindings } from "dataBinding"
 
   const AUTO_TYPE = FieldType.AUTO
   const FORMULA_TYPE = FieldType.FORMULA
@@ -191,6 +193,19 @@
     fieldId: makeFieldId(t.type, t.subtype),
     ...t,
   }))
+  $: defaultValueBindings = [
+    {
+      type: "context",
+      runtimeBinding: `${makePropSafe("now")}`,
+      readableBinding: `Date`,
+      category: "Date",
+      icon: "Date",
+      display: {
+        name: "Server date",
+      },
+    },
+    ...getUserBindings(),
+  ]
 
   const fieldDefinitions = Object.values(FIELDS).reduce(
     // Storing the fields by complex field id
@@ -781,9 +796,8 @@
             setRequired(false)
           }
         }}
-        bindings={getBindings({ table })}
+        bindings={defaultValueBindings}
         allowJS
-        context={rowGoldenSample}
       />
     </div>
   {/if}

packages/builder/src/components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte

@@ -2,7 +2,12 @@
   import { getContext } from "svelte"
   import CreateEditColumn from "components/backend/DataTable/modals/CreateEditColumn.svelte"
 
-  const { datasource } = getContext("grid")
+  const { datasource, rows } = getContext("grid")
+
+  const onUpdate = async () => {
+    await datasource.actions.refreshDefinition()
+    await rows.actions.refreshData()
+  }
 </script>
 
-<CreateEditColumn on:updatecolumns={datasource.actions.refreshDefinition} />
+<CreateEditColumn on:updatecolumns={onUpdate} />

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/[viewId]/index.svelte

@@ -3,6 +3,7 @@
   import { admin, themeStore } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
+  import { notifications } from "@budibase/bbui"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
   import GridFilterButton from "components/backend/DataTable/buttons/grid/GridFilterButton.svelte"
   import GridManageAccessButton from "components/backend/DataTable/buttons/grid/GridManageAccessButton.svelte"
@@ -31,6 +32,7 @@
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)
+        notifications.success("Row action triggered successfully")
       },
     }))
   }

packages/builder/src/pages/builder/app/[application]/data/table/[tableId]/index.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { Banner } from "@budibase/bbui"
+  import { Banner, notifications } from "@budibase/bbui"
   import {
     datasources,
     tables,
@@ -72,6 +72,7 @@
       text: action.name,
       onClick: async row => {
         await rowActions.trigger(id, action.id, row._id)
+        notifications.success("Row action triggered successfully")
       },
     }))
   }

packages/frontend-core/src/api/ai.js

@@ -0,0 +1,11 @@
+export const buildAIEndpoints = API => ({
+  /**
+   * Generates a cron expression from a prompt
+   */
+  generateCronExpression: async ({ prompt }) => {
+    return await API.post({
+      url: "/api/ai/cron",
+      body: { prompt },
+    })
+  },
+})

packages/frontend-core/src/api/index.js

@@ -2,6 +2,7 @@ import { Helpers } from "@budibase/bbui"
 import { Header } from "@budibase/shared-core"
 import { ApiVersion } from "../constants"
 import { buildAnalyticsEndpoints } from "./analytics"
+import { buildAIEndpoints } from "./ai"
 import { buildAppEndpoints } from "./app"
 import { buildAttachmentEndpoints } from "./attachments"
 import { buildAuthEndpoints } from "./auth"
@@ -269,6 +270,7 @@ export const createAPIClient = config => {
   // Attach all endpoints
   return {
     ...API,
+    ...buildAIEndpoints(API),
     ...buildAnalyticsEndpoints(API),
     ...buildAppEndpoints(API),
     ...buildAttachmentEndpoints(API),

packages/frontend-core/src/components/grid/layout/ButtonColumn.svelte

@@ -98,7 +98,6 @@
                   align="right"
                   offset={5}
                   size="S"
-                  quiet
                   animate={false}
                   on:mouseenter={() => ($hoveredRowId = row._id)}
                 />

packages/pro

@@ -1 +1 @@
-Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733
+Subproject commit aca9828117bb97f54f40ee359f1a3f6e259174e7

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,5 +1,6 @@
 import dayjs from "dayjs"
 import {
+  Aggregation,
   AutoFieldSubType,
   AutoReason,
   Datasource,
@@ -19,6 +20,7 @@ import {
   SortJson,
   SortType,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import {
   breakExternalTableId,
@@ -46,7 +48,7 @@ import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import env from "../../../environment"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import { dataFilters } from "@budibase/shared-core"
+import { dataFilters, helpers } from "@budibase/shared-core"
 
 export interface ManyRelationship {
   tableId?: string
@@ -159,17 +161,41 @@ function isEditableColumn(column: FieldSchema) {
 
 export class ExternalRequest<T extends Operation> {
   private readonly operation: T
-  private readonly tableId: string
+  private readonly source: Table | ViewV2
-  private datasource?: Datasource
+  private datasource: Datasource
-  private tables: { [key: string]: Table } = {}
 
-  constructor(operation: T, tableId: string, datasource?: Datasource) {
+  public static async for<T extends Operation>(
-    this.operation = operation
+    operation: T,
-    this.tableId = tableId
+    source: Table | ViewV2,
-    this.datasource = datasource
+    opts: { datasource?: Datasource } = {}
-    if (datasource && datasource.entities) {
+  ) {
-      this.tables = datasource.entities
+    if (!opts.datasource) {
+      if (sdk.views.isView(source)) {
+        const table = await sdk.views.getTable(source.id)
+        opts.datasource = await sdk.datasources.get(table.sourceId!)
+      } else {
+        opts.datasource = await sdk.datasources.get(source.sourceId!)
+      }
     }
+
+    return new ExternalRequest(operation, source, opts.datasource)
+  }
+
+  private get tables(): { [key: string]: Table } {
+    if (!this.datasource.entities) {
+      throw new Error("Datasource does not have entities")
+    }
+    return this.datasource.entities
+  }
+
+  private constructor(
+    operation: T,
+    source: Table | ViewV2,
+    datasource: Datasource
+  ) {
+    this.operation = operation
+    this.source = source
+    this.datasource = datasource
   }
 
   private prepareFilters(
@@ -290,20 +316,6 @@ export class ExternalRequest<T extends Operation> {
     return this.tables[tableName]
   }
 
-  // seeds the object with table and datasource information
-  async retrieveMetadata(
-    datasourceId: string
-  ): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
-    if (!this.datasource) {
-      this.datasource = await sdk.datasources.get(datasourceId)
-      if (!this.datasource || !this.datasource.entities) {
-        throw "No tables found, fetch tables before query."
-      }
-      this.tables = this.datasource.entities
-    }
-    return { tables: this.tables, datasource: this.datasource }
-  }
-
   async getRow(table: Table, rowId: string): Promise<Row> {
     const response = await getDatasourceAndQuery({
       endpoint: getEndpoint(table._id!, Operation.READ),
@@ -619,24 +631,16 @@ export class ExternalRequest<T extends Operation> {
   }
 
   async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
-    const { operation, tableId } = this
+    const { operation } = this
-    if (!tableId) {
+    let table: Table
-      throw new Error("Unable to run without a table ID")
+    if (sdk.views.isView(this.source)) {
-    }
+      table = await sdk.views.getTable(this.source.id)
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    } else {
-    let datasource = this.datasource
+      table = this.source
-    if (!datasource) {
-      const { datasource: ds } = await this.retrieveMetadata(datasourceId)
-      datasource = ds
-    }
-    const tables = this.tables
-    const table = tables[tableName]
-    let isSql = isSQL(datasource)
-    if (!table) {
-      throw new Error(
-        `Unable to process query, table "${tableName}" not defined.`
-      )
     }
+
+    let isSql = isSQL(this.datasource)
+
     // look for specific components of config which may not be considered acceptable
     let { id, row, filters, sort, paginate, rows } = cleanupConfig(
       config,
@@ -679,25 +683,40 @@ export class ExternalRequest<T extends Operation> {
         }
       }
     }
+
     if (
       operation === Operation.DELETE &&
       (filters == null || Object.keys(filters).length === 0)
     ) {
       throw "Deletion must be filtered"
     }
+
+    let aggregations: Aggregation[] = []
+    if (sdk.views.isView(this.source)) {
+      const calculationFields = helpers.views.calculationFields(this.source)
+      for (const [key, field] of Object.entries(calculationFields)) {
+        aggregations.push({
+          name: key,
+          field: field.field,
+          calculationType: field.calculationType,
+        })
+      }
+    }
+
     let json: QueryJson = {
       endpoint: {
-        datasourceId: datasourceId!,
+        datasourceId: this.datasource._id!,
-        entityId: tableName,
+        entityId: table.name,
         operation,
       },
       resource: {
         // have to specify the fields to avoid column overlap (for SQL)
         fields: isSql
-          ? buildSqlFieldList(table, this.tables, {
+          ? await buildSqlFieldList(this.source, this.tables, {
               relationships: incRelationships,
             })
           : [],
+        aggregations,
       },
       filters,
       sort,
@@ -714,7 +733,7 @@ export class ExternalRequest<T extends Operation> {
       },
       meta: {
         table,
-        tables: tables,
+        tables: this.tables,
       },
     }
 
@@ -745,7 +764,7 @@ export class ExternalRequest<T extends Operation> {
     }
     const output = await sqlOutputProcessing(
       response,
-      table,
+      this.source,
       this.tables,
       relationships
     )

packages/server/src/api/controllers/row/external.ts

@@ -17,6 +17,7 @@ import {
   Row,
   Table,
   UserCtx,
+  ViewV2,
 } from "@budibase/types"
 import sdk from "../../../sdk"
 import * as utils from "./utils"
@@ -29,39 +30,40 @@ import { generateIdForRow } from "./utils"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
-  tableId: string,
+  source: Table | ViewV2,
   opts?: RunConfig
 ): Promise<ExternalRequestReturnType<T>> {
-  return new ExternalRequest<T>(operation, tableId, opts?.datasource).run(
+  return (
-    opts || {}
+    await ExternalRequest.for<T>(operation, source, {
-  )
+      datasource: opts?.datasource,
+    })
+  ).run(opts || {})
 }
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const { tableId, viewId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
-
+  const table = await utils.getTableFromSource(source)
   const { _id, ...rowData } = ctx.request.body
-  const table = await sdk.tables.getTable(tableId)
 
-  const { row: dataToUpdate } = await inputProcessing(
+  const dataToUpdate = await inputProcessing(
     ctx.user?._id,
-    cloneDeep(table),
+    cloneDeep(source),
     rowData
   )
 
   const validateResult = await sdk.rows.utils.validate({
     row: dataToUpdate,
-    tableId,
+    source,
   })
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
 
-  const beforeRow = await sdk.rows.external.getRow(tableId, _id, {
+  const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
     relationships: true,
   })
 
-  const response = await handleRequest(Operation.UPDATE, tableId, {
+  const response = await handleRequest(Operation.UPDATE, source, {
     id: breakRowIdField(_id),
     row: dataToUpdate,
   })
@@ -69,17 +71,16 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   // The id might have been changed, so the refetching would fail. Recalculating the id just in case
   const updatedId =
     generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
-  const row = await sdk.rows.external.getRow(tableId, updatedId, {
+  const row = await sdk.rows.external.getRow(table._id!, updatedId, {
     relationships: true,
   })
 
   const [enrichedRow, oldRow] = await Promise.all([
-    outputProcessing(table, row, {
+    outputProcessing(source, row, {
       squash: true,
       preserveLinks: true,
-      fromViewId: viewId,
     }),
-    outputProcessing(table, beforeRow, {
+    outputProcessing(source, beforeRow, {
       squash: true,
       preserveLinks: true,
     }),
@@ -94,9 +95,9 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 }
 
 export async function destroy(ctx: UserCtx) {
-  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
   const _id = ctx.request.body._id
-  const { row } = await handleRequest(Operation.DELETE, tableId, {
+  const { row } = await handleRequest(Operation.DELETE, source, {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
@@ -105,11 +106,11 @@ export async function destroy(ctx: UserCtx) {
 
 export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
-  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
   let promises: Promise<{ row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
-      handleRequest(Operation.DELETE, tableId, {
+      handleRequest(Operation.DELETE, source, {
         id: breakRowIdField(row._id),
         includeSqlRelationships: IncludeRelationship.EXCLUDE,
       })
@@ -124,6 +125,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
+  const source = await utils.getSource(ctx)
   const { tableId } = utils.getSourceId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   const datasource: Datasource = await sdk.datasources.get(datasourceId)
@@ -131,7 +133,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
   const tables = datasource.entities
-  const response = await handleRequest(Operation.READ, tableId, {
+  const response = await handleRequest(Operation.READ, source, {
     id,
     datasource,
     includeSqlRelationships: IncludeRelationship.INCLUDE,
@@ -155,7 +157,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     // don't support composite keys right now
     const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
     const primaryLink = linkedTable.primary?.[0] as string
-    const relatedRows = await handleRequest(Operation.READ, linkedTableId!, {
+    const relatedRows = await handleRequest(Operation.READ, linkedTable, {
       tables,
       filters: {
         oneOf: {

packages/server/src/api/controllers/row/index.ts

@@ -207,7 +207,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 }
 
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
-  const { tableId } = utils.getSourceId(ctx)
+  const { tableId, viewId } = utils.getSourceId(ctx)
 
   await context.ensureSnippetContext(true)
 
@@ -222,6 +222,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
     ...ctx.request.body,
     query: enrichedQuery,
     tableId,
+    viewId,
   }
 
   ctx.status = 200
@@ -229,14 +230,15 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
 }
 
 export async function validate(ctx: Ctx<Row, ValidateResponse>) {
-  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
+  const table = await utils.getTableFromSource(source)
   // external tables are hard to validate currently
-  if (isExternalTableID(tableId)) {
+  if (isExternalTableID(table._id!)) {
     ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
-      tableId,
+      source,
     })
   }
 }

packages/server/src/api/controllers/row/internal.ts

@@ -21,18 +21,19 @@ import {
 import sdk from "../../../sdk"
 import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
 import { flatten } from "lodash"
+import { findRow } from "../../../sdk/app/rows/internal"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const { tableId, viewId } = utils.getSourceId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
+  const source = await utils.getSource(ctx)
+  const table = sdk.views.isView(source)
+    ? await sdk.views.getTable(source.id)
+    : source
   const inputs = ctx.request.body
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
-  const dbTable = await sdk.tables.getTable(tableId)
   try {
-    oldRow = await outputProcessing(
+    oldRow = await outputProcessing(source, await findRow(tableId, inputs._id!))
-      dbTable,
-      await utils.findRow(tableId, inputs._id!)
-    )
   } catch (err) {
     if (isUserTable) {
       // don't include the rev, it'll be the global rev
@@ -48,22 +49,15 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   // need to build up full patch fields before coerce
   let combinedRow: any = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
-    if (!dbTable.schema[key]) continue
+    if (!table.schema[key]) continue
     combinedRow[key] = inputs[key]
   }
 
-  // need to copy the table so it can be differenced on way out
-  const tableClone = cloneDeep(dbTable)
-
   // this returns the table and row incase they have been updated
-  let { table, row } = await inputProcessing(
+  let row = await inputProcessing(ctx.user?._id, source, combinedRow)
-    ctx.user?._id,
-    tableClone,
-    combinedRow
-  )
   const validateResult = await sdk.rows.utils.validate({
     row,
-    table,
+    source,
   })
 
   if (!validateResult.valid) {
@@ -87,10 +81,8 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     return { row: ctx.body as Row, table, oldRow }
   }
 
-  const result = await finaliseRow(table, row, {
+  const result = await finaliseRow(source, row, {
-    oldTable: dbTable,
     updateFormula: true,
-    fromViewId: viewId,
   })
 
   return { ...result, oldRow }
@@ -186,7 +178,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
     sdk.tables.getTable(tableId),
     linkRows.getLinkDocuments({ tableId, rowId, fieldName }),
   ])
-  let row = await utils.findRow(tableId, rowId)
+  let row = await findRow(tableId, rowId)
   row = await outputProcessing(table, row)
   const linkVals = links as LinkDocumentValue[]
 

packages/server/src/api/controllers/row/staticFormula.ts

@@ -4,10 +4,11 @@ import {
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
-import { Table, Row, FormulaType, FieldType } from "@budibase/types"
+import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep } from "lodash/fp"
+import sdk from "../../../sdk"
 
 /**
  * This function runs through a list of enriched rows, looks at the rows which
@@ -121,33 +122,26 @@ export async function updateAllFormulasInTable(table: Table) {
  * expects the row to be totally enriched/contain all relationships.
  */
 export async function finaliseRow(
-  table: Table,
+  source: Table | ViewV2,
   row: Row,
-  {
+  opts?: { updateFormula: boolean }
-    oldTable,
-    updateFormula,
-    fromViewId,
-  }: { oldTable?: Table; updateFormula: boolean; fromViewId?: string } = {
-    updateFormula: true,
-  }
 ) {
   const db = context.getAppDB()
+  const { updateFormula = true } = opts || {}
+  const table = sdk.views.isView(source)
+    ? await sdk.views.getTable(source.id)
+    : source
+
   row.type = "row"
   // process the row before return, to include relationships
-  let enrichedRow = (await outputProcessing(table, cloneDeep(row), {
+  let enrichedRow = await outputProcessing(source, cloneDeep(row), {
     squash: false,
-  })) as Row
+  })
   // use enriched row to generate formulas for saving, specifically only use as context
   row = await processFormulas(table, row, {
     dynamic: false,
     contextRows: [enrichedRow],
   })
-  // don't worry about rev, tables handle rev/lastID updates
-  // if another row has been written since processing this will
-  // handle the auto ID clash
-  if (oldTable && !isEqual(oldTable, table)) {
-    await db.put(table)
-  }
   const response = await db.put(row)
   // for response, calculate the formulas for the enriched row
   enrichedRow._rev = response.rev
@@ -158,8 +152,6 @@ export async function finaliseRow(
   if (updateFormula) {
     await updateRelatedFormula(table, enrichedRow)
   }
-  const squashed = await linkRows.squashLinks(table, enrichedRow, {
+  const squashed = await linkRows.squashLinks(source, enrichedRow)
-    fromViewId,
-  })
   return { row: enrichedRow, squashed, table }
 }

packages/server/src/api/controllers/row/utils/basic.ts

@@ -1,11 +1,19 @@
 // need to handle table name + field or just field, depending on if relationships used
-import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types"
+import {
+  FieldSchema,
+  FieldType,
+  Row,
+  Table,
+  JsonTypes,
+  ViewV2,
+} from "@budibase/types"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
 import { generateRowIdField } from "../../../../integrations/utils"
+import sdk from "../../../../sdk"
 
 function extractFieldValue({
   row,
@@ -78,20 +86,30 @@ function fixJsonTypes(row: Row, table: Table) {
   return row
 }
 
-export function basicProcessing({
+export async function basicProcessing({
   row,
-  table,
+  source,
   tables,
   isLinked,
   sqs,
 }: {
   row: Row
-  table: Table
+  source: Table | ViewV2
   tables: Table[]
   isLinked: boolean
   sqs?: boolean
-}): Row {
+}): Promise<Row> {
+  let table: Table
+  let isCalculationView = false
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+    isCalculationView = helpers.views.isCalculationView(source)
+  } else {
+    table = source
+  }
+
   const thisRow: Row = {}
+
   // filter the row down to what is actually the row (not joined)
   for (let fieldName of Object.keys(table.schema)) {
     let value = extractFieldValue({
@@ -108,13 +126,20 @@ export function basicProcessing({
       thisRow[fieldName] = value
     }
   }
+
+  if (sdk.views.isView(source)) {
+    for (const key of Object.keys(helpers.views.calculationFields(source))) {
+      thisRow[key] = row[key]
+    }
+  }
+
   let columns: string[] = Object.keys(table.schema)
-  if (!sqs) {
+  if (!sqs && !isCalculationView) {
     thisRow._id = generateIdForRow(row, table, isLinked)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
     columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
-  } else {
+  } else if (!isCalculationView) {
     columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
     for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
       thisRow[internalColumn] = extractFieldValue({
@@ -149,28 +174,30 @@ export function basicProcessing({
       thisRow[col] = array
       // make sure all of them have an _id
       const sortField = relatedTable.primaryDisplay || relatedTable.primary![0]!
-      thisRow[col] = (thisRow[col] as Row[])
+      thisRow[col] = (
-        .map(relatedRow =>
+        await Promise.all(
-          basicProcessing({
+          (thisRow[col] as Row[]).map(relatedRow =>
-            row: relatedRow,
+            basicProcessing({
-            table: relatedTable,
+              row: relatedRow,
-            tables,
+              source: relatedTable,
-            isLinked: false,
+              tables,
-            sqs,
+              isLinked: false,
-          })
+              sqs,
+            })
+          )
         )
-        .sort((a, b) => {
+      ).sort((a, b) => {
-          const aField = a?.[sortField],
+        const aField = a?.[sortField],
-            bField = b?.[sortField]
+          bField = b?.[sortField]
-          if (!aField) {
+        if (!aField) {
-            return 1
+          return 1
-          } else if (!bField) {
+        } else if (!bField) {
-            return -1
+          return -1
-          }
+        }
-          return aField.localeCompare
+        return aField.localeCompare
-            ? aField.localeCompare(bField)
+          ? aField.localeCompare(bField)
-            : aField - bField
+          : aField - bField
-        })
+      })
     }
   }
   return fixJsonTypes(thisRow, table)

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -7,10 +7,14 @@ import {
   ManyToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
   RelationshipsJson,
+  Row,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
+import sdk from "../../../../sdk"
+import { helpers } from "@budibase/shared-core"
 
 type TableMap = Record<string, Table>
 
@@ -108,11 +112,12 @@ export function buildInternalRelationships(
  * Creating the specific list of fields that we desire, and excluding the ones that are no use to us
  * is more performant and has the added benefit of protecting against this scenario.
  */
-export function buildSqlFieldList(
+export async function buildSqlFieldList(
-  table: Table,
+  source: Table | ViewV2,
   tables: TableMap,
   opts?: { relationships: boolean }
 ) {
+  const { relationships } = opts || {}
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
@@ -123,22 +128,33 @@ export function buildSqlFieldList(
       )
       .map(column => `${table.name}.${column[0]}`)
   }
-  let fields = extractRealFields(table)
+
+  let fields: string[] = []
+  if (sdk.views.isView(source)) {
+    fields = Object.keys(helpers.views.basicFields(source)).filter(
+      key => source.schema?.[key]?.visible !== false
+    )
+  } else {
+    fields = extractRealFields(source)
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   for (let field of Object.values(table.schema)) {
-    if (
+    if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
-      field.type !== FieldType.LINK ||
-      !opts?.relationships ||
-      !field.tableId
-    ) {
       continue
     }
-    const { tableName: linkTableName } = breakExternalTableId(field.tableId)
+    const { tableName } = breakExternalTableId(field.tableId)
-    const linkTable = tables[linkTableName]
+    if (tables[tableName]) {
-    if (linkTable) {
+      fields = fields.concat(extractRealFields(tables[tableName], fields))
-      const linkedFields = extractRealFields(linkTable, fields)
-      fields = fields.concat(linkedFields)
     }
   }
+
   return fields
 }
 
@@ -149,3 +165,7 @@ export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {
     (DSPlusOperation.READ in resp[0] && resp[0].read === true)
   )
 }
+
+export function isKnexRows(resp: DatasourcePlusQueryResponse): resp is Row[] {
+  return !isKnexEmptyReadResponse(resp)
+}

packages/server/src/api/controllers/row/utils/utils.ts

@@ -1,6 +1,6 @@
 import * as utils from "../../../../db/utils"
 
-import { context } from "@budibase/backend-core"
+import { docIds } from "@budibase/backend-core"
 import {
   Ctx,
   DatasourcePlusQueryResponse,
@@ -8,17 +8,18 @@ import {
   RelationshipsJson,
   Row,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import {
   processDates,
   processFormulas,
 } from "../../../../utilities/rowProcessor"
-import { isKnexEmptyReadResponse } from "./sqlUtils"
+import { isKnexRows } from "./sqlUtils"
 import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
 import sdk from "../../../../sdk"
 import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
-import { getFullUser } from "../../../../utilities/users"
+import { helpers } from "@budibase/shared-core"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -58,26 +59,11 @@ export async function processRelationshipFields(
   return row
 }
 
-export async function findRow(tableId: string, rowId: string) {
-  const db = context.getAppDB()
-  let row: Row
-  // TODO remove special user case in future
-  if (tableId === utils.InternalTables.USER_METADATA) {
-    row = await getFullUser(rowId)
-  } else {
-    row = await db.get(rowId)
-  }
-  if (row.tableId !== tableId) {
-    throw "Supplied tableId does not match the rows tableId"
-  }
-  return row
-}
-
 export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
   // top priority, use the URL first
   if (ctx.params?.sourceId) {
     const { sourceId } = ctx.params
-    if (utils.isViewID(sourceId)) {
+    if (docIds.isViewId(sourceId)) {
       return {
         tableId: utils.extractViewInfoFromID(sourceId).tableId,
         viewId: sourceId,
@@ -96,22 +82,22 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
   throw new Error("Unable to find table ID in request")
 }
 
-export async function validate(
+export async function getSource(ctx: Ctx): Promise<Table | ViewV2> {
-  opts: { row: Row } & ({ tableId: string } | { table: Table })
+  const { tableId, viewId } = getSourceId(ctx)
-) {
+  if (viewId) {
-  let fetchedTable: Table
+    return sdk.views.get(viewId)
-  if ("tableId" in opts) {
-    fetchedTable = await sdk.tables.getTable(opts.tableId)
-  } else {
-    fetchedTable = opts.table
   }
-  return sdk.rows.utils.validate({
+  return sdk.tables.getTable(tableId)
-    ...opts,
-    table: fetchedTable,
-  })
 }
 
-function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
+export async function getTableFromSource(source: Table | ViewV2) {
+  if (sdk.views.isView(source)) {
+    return await sdk.views.getTable(source.id)
+  }
+  return source
+}
+
+function fixBooleanFields(row: Row, table: Table) {
   for (let col of Object.values(table.schema)) {
     if (col.type === FieldType.BOOLEAN) {
       if (row[col.name] === 1) {
@@ -126,49 +112,45 @@ function fixBooleanFields({ row, table }: { row: Row; table: Table }) {
 
 export async function sqlOutputProcessing(
   rows: DatasourcePlusQueryResponse,
-  table: Table,
+  source: Table | ViewV2,
   tables: Record<string, Table>,
   relationships: RelationshipsJson[],
   opts?: { sqs?: boolean }
 ): Promise<Row[]> {
-  if (isKnexEmptyReadResponse(rows)) {
+  if (!isKnexRows(rows)) {
     return []
   }
-  let finalRows: { [key: string]: Row } = {}
+
-  for (let row of rows as Row[]) {
+  let table: Table
-    let rowId = row._id
+  let isCalculationView = false
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+    isCalculationView = helpers.views.isCalculationView(source)
+  } else {
+    table = source
+  }
+
+  let processedRows: Row[] = []
+  for (let row of rows) {
     if (opts?.sqs) {
-      rowId = getInternalRowId(row, table)
+      row._id = getInternalRowId(row, table)
-      row._id = rowId
+    } else if (row._id == null && !isCalculationView) {
-    } else if (!rowId) {
+      row._id = generateIdForRow(row, table)
-      rowId = generateIdForRow(row, table)
-      row._id = rowId
     }
-    const thisRow = basicProcessing({
+
+    row = await basicProcessing({
       row,
-      table,
+      source,
       tables: Object.values(tables),
       isLinked: false,
       sqs: opts?.sqs,
     })
-    if (thisRow._id == null) {
+    row = fixBooleanFields(row, table)
-      throw new Error("Unable to generate row ID for SQL rows")
+    row = await processRelationshipFields(table, tables, row, relationships)
-    }
+    processedRows.push(row)
-
-    finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
   }
 
-  // make sure all related rows are correct
+  return processDates(table, processedRows)
-  let finalRowArray = []
-  for (let row of Object.values(finalRows)) {
-    finalRowArray.push(
-      await processRelationshipFields(table, tables, row, relationships)
-    )
-  }
-
-  // process some additional types
-  finalRowArray = processDates(table, finalRowArray)
-  return finalRowArray
 }
 
 export function isUserMetadataTable(tableId: string) {

packages/server/src/api/controllers/row/views.ts

@@ -3,17 +3,9 @@ import {
   ViewV2,
   SearchRowResponse,
   SearchViewRowRequest,
-  RequiredKeys,
-  RowSearchParams,
-  SearchFilterKey,
-  LogicalOperator,
-  SearchFilter,
 } from "@budibase/types"
-import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../../sdk"
-import { db, context, features } from "@budibase/backend-core"
+import { context } from "@budibase/backend-core"
-import { enrichSearchContext } from "./utils"
-import { isExternalTableID } from "../../../integrations/utils"
 
 export async function searchView(
   ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
@@ -28,81 +20,26 @@ export async function searchView(
     ctx.throw(400, `This method only supports viewsV2`)
   }
 
-  const viewFields = Object.entries(view.schema || {})
-    .filter(([_, value]) => value.visible)
-    .map(([key]) => key)
   const { body } = ctx.request
 
-  const sqsEnabled = await features.flags.isEnabled("SQS")
-  const supportsLogicalOperators = isExternalTableID(view.tableId) || sqsEnabled
-
-  // Enrich saved query with ephemeral query params.
-  // We prevent searching on any fields that are saved as part of the query, as
-  // that could let users find rows they should not be allowed to access.
-  let query = supportsLogicalOperators
-    ? dataFilters.buildQuery(view.query)
-    : dataFilters.buildQueryLegacy(view.query)
-
-  delete query?.onEmptyFilter
-
-  if (body.query) {
-    // Delete extraneous search params that cannot be overridden
-    delete body.query.onEmptyFilter
-
-    if (!supportsLogicalOperators) {
-      // In the unlikely event that a Grouped Filter is in a non-SQS environment
-      // It needs to be ignored entirely
-      let queryFilters: SearchFilter[] = Array.isArray(view.query)
-        ? view.query
-        : []
-
-      // Extract existing fields
-      const existingFields =
-        queryFilters
-          ?.filter(filter => filter.field)
-          .map(filter => db.removeKeyNumbering(filter.field)) || []
-
-      // Carry over filters for unused fields
-      Object.keys(body.query).forEach(key => {
-        const operator = key as Exclude<SearchFilterKey, LogicalOperator>
-        Object.keys(body.query[operator] || {}).forEach(field => {
-          if (query && !existingFields.includes(db.removeKeyNumbering(field))) {
-            query[operator]![field] = body.query[operator]![field]
-          }
-        })
-      })
-    } else {
-      const conditions = query ? [query] : []
-      query = {
-        $and: {
-          conditions: [...conditions, body.query],
-        },
-      }
-    }
-  }
-
   await context.ensureSnippetContext(true)
 
-  const enrichedQuery = await enrichSearchContext(query || {}, {
+  const result = await sdk.rows.search(
-    user: sdk.users.getUserContextBindings(ctx.user),
+    {
-  })
+      viewId: view.id,
+      tableId: view.tableId,
+      query: body.query,
+      ...getSortOptions(body, view),
+      limit: body.limit,
+      bookmark: body.bookmark,
+      paginate: body.paginate,
+      countRows: body.countRows,
+    },
+    {
+      user: sdk.users.getUserContextBindings(ctx.user),
+    }
+  )
 
-  const searchOptions: RequiredKeys<SearchViewRowRequest> &
-    RequiredKeys<
-      Pick<RowSearchParams, "tableId" | "viewId" | "query" | "fields">
-    > = {
-    tableId: view.tableId,
-    viewId: view.id,
-    query: enrichedQuery,
-    fields: viewFields,
-    ...getSortOptions(body, view),
-    limit: body.limit,
-    bookmark: body.bookmark,
-    paginate: body.paginate,
-    countRows: body.countRows,
-  }
-
-  const result = await sdk.rows.search(searchOptions)
   result.rows.forEach(r => (r._viewId = view.id))
   ctx.body = result
 }

packages/server/src/api/controllers/table/external.ts

@@ -113,11 +113,10 @@ export async function bulkImport(
     const processed = await inputProcessing(ctx.user?._id, table, row, {
       noAutoRelationships: true,
     })
-    parsedRows.push(processed.row)
+    parsedRows.push(processed)
-    table = processed.table
   }
 
-  await handleRequest(Operation.BULK_UPSERT, table._id!, {
+  await handleRequest(Operation.BULK_UPSERT, table, {
     rows: parsedRows,
   })
   await events.rows.imported(table, parsedRows.length)

packages/server/src/api/controllers/table/index.ts

@@ -33,7 +33,7 @@ import {
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
 import { builderSocket } from "../../../websockets"
-import { cloneDeep, isEqual } from "lodash"
+import { cloneDeep } from "lodash"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -149,12 +149,7 @@ export async function bulkImport(
   ctx: UserCtx<BulkImportRequest, BulkImportResponse>
 ) {
   const tableId = ctx.params.tableId
-  let tableBefore = await sdk.tables.getTable(tableId)
+  await pickApi({ tableId }).bulkImport(ctx)
-  let tableAfter = await pickApi({ tableId }).bulkImport(ctx)
-
-  if (!isEqual(tableBefore, tableAfter)) {
-    await sdk.tables.saveTable(tableAfter)
-  }
 
   // right now we don't trigger anything for bulk import because it
   // can only be done in the builder, but in the future we may need to

packages/server/src/api/controllers/table/internal.ts

@@ -3,7 +3,6 @@ import { handleDataImport } from "./utils"
 import {
   BulkImportRequest,
   BulkImportResponse,
-  FieldType,
   RenameColumn,
   SaveTableRequest,
   SaveTableResponse,
@@ -70,22 +69,10 @@ export async function bulkImport(
 ) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows, identifierFields } = ctx.request.body
-  await handleDataImport(
+  await handleDataImport(table, {
-    {
+    importRows: rows,
-      ...table,
+    identifierFields,
-      schema: {
+    user: ctx.user,
-        _id: {
+  })
-          name: "_id",
-          type: FieldType.STRING,
-        },
-        ...table.schema,
-      },
-    },
-    {
-      importRows: rows,
-      identifierFields,
-      user: ctx.user,
-    }
-  )
   return table
 }

packages/server/src/api/controllers/table/utils.ts

@@ -139,8 +139,7 @@ export async function importToRows(
     const processed = await inputProcessing(user?._id, table, row, {
       noAutoRelationships: true,
     })
-    row = processed.row
+    row = processed
-    table = processed.table
 
     // However here we must reference the original table, as we want to mutate
     // the real schema of the table passed in, not the clone used for

packages/server/src/api/controllers/view/viewsV2.ts

@@ -7,10 +7,49 @@ import {
   ViewResponse,
   ViewResponseEnriched,
   ViewV2,
-  ViewFieldMetadata,
+  BasicViewFieldMetadata,
+  ViewCalculationFieldMetadata,
   RelationSchemaField,
+  ViewFieldMetadata,
 } from "@budibase/types"
 import { builderSocket, gridSocket } from "../../../websockets"
+import { helpers } from "@budibase/shared-core"
+
+function stripUnknownFields(
+  field: BasicViewFieldMetadata
+): RequiredKeys<BasicViewFieldMetadata> {
+  if (helpers.views.isCalculationField(field)) {
+    const strippedField: RequiredKeys<ViewCalculationFieldMetadata> = {
+      order: field.order,
+      width: field.width,
+      visible: field.visible,
+      readonly: field.readonly,
+      icon: field.icon,
+      calculationType: field.calculationType,
+      field: field.field,
+      columns: field.columns,
+    }
+    return strippedField
+  } else {
+    const strippedField: RequiredKeys<BasicViewFieldMetadata> = {
+      order: field.order,
+      width: field.width,
+      visible: field.visible,
+      readonly: field.readonly,
+      icon: field.icon,
+      columns: field.columns,
+    }
+    return strippedField
+  }
+}
+
+function stripUndefinedFields(obj: Record<string, any>): void {
+  Object.keys(obj)
+    .filter(key => obj[key] === undefined)
+    .forEach(key => {
+      delete obj[key]
+    })
+}
 
 async function parseSchema(view: CreateViewRequest) {
   if (!view.schema) {
@@ -22,6 +61,7 @@ async function parseSchema(view: CreateViewRequest) {
       let fieldRelatedSchema:
         | Record<string, RequiredKeys<RelationSchemaField>>
         | undefined
+
       if (schemaValue.columns) {
         fieldRelatedSchema = Object.entries(schemaValue.columns).reduce<
           NonNullable<typeof fieldRelatedSchema>
@@ -35,25 +75,12 @@ async function parseSchema(view: CreateViewRequest) {
           }
           return acc
         }, {})
+        schemaValue.columns = fieldRelatedSchema
       }
 
-      const fieldSchema: RequiredKeys<
+      const fieldSchema = stripUnknownFields(schemaValue)
-        ViewFieldMetadata & {
+      stripUndefinedFields(fieldSchema)
-          columns: typeof fieldRelatedSchema
+
-        }
-      > = {
-        order: schemaValue.order,
-        width: schemaValue.width,
-        visible: schemaValue.visible,
-        readonly: schemaValue.readonly,
-        icon: schemaValue.icon,
-        columns: fieldRelatedSchema,
-      }
-      Object.entries(fieldSchema)
-        .filter(([, val]) => val === undefined)
-        .forEach(([key]) => {
-          delete fieldSchema[key as keyof ViewFieldMetadata]
-        })
       p[fieldName] = fieldSchema
       return p
     }, {} as Record<string, RequiredKeys<ViewFieldMetadata>>)

packages/server/src/api/routes/index.ts

@@ -33,6 +33,7 @@ import rowActionRoutes from "./rowAction"
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
+const aiRoutes = pro.ai
 const appBackupRoutes = pro.appBackups
 const environmentVariableRoutes = pro.environmentVariables
 
@@ -67,6 +68,7 @@ export const mainRoutes: Router[] = [
   debugRoutes,
   environmentVariableRoutes,
   rowActionRoutes,
+  aiRoutes,
   // these need to be handled last as they still use /api/:tableId
   // this could be breaking as koa may recognise other routes as this
   tableRoutes,

packages/server/src/api/routes/tests/row.spec.ts

@@ -76,7 +76,7 @@ async function waitForEvent(
 }
 
 describe.each([
-  ["internal", undefined],
+  ["lucene", undefined],
   ["sqs", undefined],
   [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
@@ -2453,9 +2453,15 @@ describe.each([
       let flagCleanup: (() => void) | undefined
 
       beforeAll(async () => {
-        flagCleanup = setCoreEnv({
+        const env = {
           TENANT_FEATURE_FLAGS: `*:${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
-        })
+        }
+        if (isSqs) {
+          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:SQS`
+        } else {
+          env.TENANT_FEATURE_FLAGS = `${env.TENANT_FEATURE_FLAGS},*:!SQS`
+        }
+        flagCleanup = setCoreEnv(env)
 
         const aux2Table = await config.api.table.save(saveTableRequest())
         const aux2Data = await config.api.row.save(aux2Table._id!, {})

packages/server/src/api/routes/tests/search.spec.ts

@@ -157,7 +157,11 @@ describe.each([
       if (isInMemory) {
         return dataFilters.search(_.cloneDeep(rows), this.query)
       } else {
-        return config.api.row.search(this.query.tableId, this.query)
+        const sourceId = this.query.viewId || this.query.tableId
+        if (!sourceId) {
+          throw new Error("No source ID provided")
+        }
+        return config.api.row.search(sourceId, this.query)
       }
     }
 
@@ -404,7 +408,6 @@ describe.each([
     })
   })
 
-  // We've decided not to try and support binding for in-memory search just now.
   !isInMemory &&
     describe("bindings", () => {
       let globalUsers: any = []
@@ -524,6 +527,20 @@ describe.each([
         ])
       })
 
+      !isLucene &&
+        it("should return all rows matching the session user firstname when logical operator used", async () => {
+          await expectQuery({
+            $and: {
+              conditions: [{ equal: { name: "{{ [user].firstName }}" } }],
+            },
+          }).toContainExactly([
+            {
+              name: config.getUser().firstName,
+              appointment: future.toISOString(),
+            },
+          ])
+        })
+
       it("should parse the date binding and return all rows after the resolved value", async () => {
         await tk.withFreeze(serverTime, async () => {
           await expectQuery({

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -18,10 +18,11 @@ import {
   ViewV2,
   SearchResponse,
   BasicOperator,
+  CalculationType,
   RelationshipType,
   TableSchema,
-  ViewFieldMetadata,
   RenameColumn,
+  ViewFieldMetadata,
   FeatureFlag,
   BBReferenceFieldSubType,
 } from "@budibase/types"
@@ -36,7 +37,6 @@ import {
   setEnv as setCoreEnv,
   env,
 } from "@budibase/backend-core"
-import sdk from "../../../sdk"
 
 describe.each([
   ["lucene", undefined],
@@ -1738,6 +1738,40 @@ describe.each([
         })
       })
 
+      it("views filters are respected even if the column is hidden", async () => {
+        await config.api.row.save(table._id!, {
+          one: "foo",
+          two: "bar",
+        })
+        const two = await config.api.row.save(table._id!, {
+          one: "foo2",
+          two: "bar2",
+        })
+
+        const view = await config.api.viewV2.create({
+          tableId: table._id!,
+          name: generator.guid(),
+          query: [
+            {
+              operator: BasicOperator.EQUAL,
+              field: "two",
+              value: "bar2",
+            },
+          ],
+          schema: {
+            id: { visible: true },
+            one: { visible: false },
+            two: { visible: false },
+          },
+        })
+
+        const response = await config.api.viewV2.search(view.id)
+        expect(response.rows).toHaveLength(1)
+        expect(response.rows).toEqual([
+          expect.objectContaining({ _id: two._id }),
+        ])
+      })
+
       it("views without data can be returned", async () => {
         const response = await config.api.viewV2.search(view.id)
         expect(response.rows).toHaveLength(0)
@@ -2196,28 +2230,6 @@ describe.each([
           expect(response.rows).toHaveLength(0)
         })
 
-      it("queries the row api passing the view fields only", async () => {
-        const searchSpy = jest.spyOn(sdk.rows, "search")
-
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
-          name: generator.guid(),
-          schema: {
-            id: { visible: true },
-            one: { visible: false },
-          },
-        })
-
-        await config.api.viewV2.search(view.id, { query: {} })
-        expect(searchSpy).toHaveBeenCalledTimes(1)
-
-        expect(searchSpy).toHaveBeenCalledWith(
-          expect.objectContaining({
-            fields: ["id"],
-          })
-        )
-      })
-
       describe("foreign relationship columns", () => {
         let envCleanup: () => void
         beforeAll(() => {
@@ -2382,6 +2394,158 @@ describe.each([
           ])
         })
       })
+
+      !isLucene &&
+        describe("calculations", () => {
+          let table: Table
+          let rows: Row[]
+
+          beforeAll(async () => {
+            table = await config.api.table.save(
+              saveTableRequest({
+                schema: {
+                  quantity: {
+                    type: FieldType.NUMBER,
+                    name: "quantity",
+                  },
+                  price: {
+                    type: FieldType.NUMBER,
+                    name: "price",
+                  },
+                },
+              })
+            )
+
+            rows = await Promise.all(
+              Array.from({ length: 10 }, () =>
+                config.api.row.save(table._id!, {
+                  quantity: generator.natural({ min: 1, max: 10 }),
+                  price: generator.natural({ min: 1, max: 10 }),
+                })
+              )
+            )
+          })
+
+          it("should be able to search by calculations", async () => {
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              schema: {
+                "Quantity Sum": {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "quantity",
+                },
+              },
+            })
+
+            const response = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+
+            expect(response.rows).toHaveLength(1)
+            expect(response.rows).toEqual(
+              expect.arrayContaining([
+                expect.objectContaining({
+                  "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0),
+                }),
+              ])
+            )
+
+            // Calculation views do not return rows that can be linked back to
+            // the source table, and so should not have an _id field.
+            for (const row of response.rows) {
+              expect("_id" in row).toBe(false)
+            }
+          })
+
+          it("should be able to group by a basic field", async () => {
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              schema: {
+                quantity: {
+                  visible: true,
+                  field: "quantity",
+                },
+                "Total Price": {
+                  visible: true,
+                  calculationType: CalculationType.SUM,
+                  field: "price",
+                },
+              },
+            })
+
+            const response = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+
+            const priceByQuantity: Record<number, number> = {}
+            for (const row of rows) {
+              priceByQuantity[row.quantity] ??= 0
+              priceByQuantity[row.quantity] += row.price
+            }
+
+            for (const row of response.rows) {
+              expect(row["Total Price"]).toEqual(priceByQuantity[row.quantity])
+            }
+          })
+
+          it.each([
+            CalculationType.COUNT,
+            CalculationType.SUM,
+            CalculationType.AVG,
+            CalculationType.MIN,
+            CalculationType.MAX,
+          ])("should be able to calculate $type", async type => {
+            const view = await config.api.viewV2.create({
+              tableId: table._id!,
+              name: generator.guid(),
+              schema: {
+                aggregate: {
+                  visible: true,
+                  calculationType: type,
+                  field: "price",
+                },
+              },
+            })
+
+            const response = await config.api.viewV2.search(view.id, {
+              query: {},
+            })
+
+            function calculate(
+              type: CalculationType,
+              numbers: number[]
+            ): number {
+              switch (type) {
+                case CalculationType.COUNT:
+                  return numbers.length
+                case CalculationType.SUM:
+                  return numbers.reduce((a, b) => a + b, 0)
+                case CalculationType.AVG:
+                  return numbers.reduce((a, b) => a + b, 0) / numbers.length
+                case CalculationType.MIN:
+                  return Math.min(...numbers)
+                case CalculationType.MAX:
+                  return Math.max(...numbers)
+              }
+            }
+
+            const prices = rows.map(row => row.price)
+            const expected = calculate(type, prices)
+            const actual = response.rows[0].aggregate
+
+            if (type === CalculationType.AVG) {
+              // The average calculation can introduce floating point rounding
+              // errors, so we need to compare to within a small margin of
+              // error.
+              expect(actual).toBeCloseTo(expected)
+            } else {
+              expect(actual).toEqual(expected)
+            }
+          })
+        })
     })
 
     describe("permissions", () => {

packages/server/src/automations/tests/scenarios/branching.spec.ts

@@ -17,44 +17,65 @@ describe("Branching automations", () => {
   afterAll(setup.afterAll)
 
   it("should run a multiple nested branching automation", async () => {
+    const firstLogId = "11111111-1111-1111-1111-111111111111"
+    const branch1LogId = "22222222-2222-2222-2222-222222222222"
+    const branch2LogId = "33333333-3333-3333-3333-333333333333"
+    const branch2Id = "44444444-4444-4444-4444-444444444444"
+
     const builder = createAutomationBuilder({
       name: "Test Trigger with Loop and Create Row",
     })
 
     const results = await builder
       .appAction({ fields: {} })
-      .serverLog({ text: "Starting automation" })
+      .serverLog(
+        { text: "Starting automation" },
+        { stepName: "FirstLog", stepId: firstLogId }
+      )
       .branch({
         topLevelBranch1: {
           steps: stepBuilder =>
-            stepBuilder.serverLog({ text: "Branch 1" }).branch({
+            stepBuilder
-              branch1: {
+              .serverLog(
-                steps: stepBuilder =>
+                { text: "Branch 1" },
-                  stepBuilder.serverLog({ text: "Branch 1.1" }),
+                { stepId: "66666666-6666-6666-6666-666666666666" }
-                condition: {
+              )
-                  equal: { "{{steps.1.success}}": true },
+              .branch({
+                branch1: {
+                  steps: stepBuilder =>
+                    stepBuilder.serverLog(
+                      { text: "Branch 1.1" },
+                      { stepId: branch1LogId }
+                    ),
+                  condition: {
+                    equal: { [`{{ steps.${firstLogId}.success }}`]: true },
+                  },
                 },
-              },
+                branch2: {
-              branch2: {
+                  steps: stepBuilder =>
-                steps: stepBuilder =>
+                    stepBuilder.serverLog(
-                  stepBuilder.serverLog({ text: "Branch 1.2" }),
+                      { text: "Branch 1.2" },
-                condition: {
+                      { stepId: branch2LogId }
-                  equal: { "{{steps.1.success}}": false },
+                    ),
+                  condition: {
+                    equal: { [`{{ steps.${firstLogId}.success }}`]: false },
+                  },
                 },
-              },
+              }),
-            }),
           condition: {
-            equal: { "{{steps.1.success}}": true },
+            equal: { [`{{ steps.${firstLogId}.success }}`]: true },
           },
         },
         topLevelBranch2: {
-          steps: stepBuilder => stepBuilder.serverLog({ text: "Branch 2" }),
+          steps: stepBuilder =>
+            stepBuilder.serverLog({ text: "Branch 2" }, { stepId: branch2Id }),
           condition: {
-            equal: { "{{steps.1.success}}": false },
+            equal: { [`{{ steps.${firstLogId}.success }}`]: false },
           },
         },
       })
       .run()
+
     expect(results.steps[3].outputs.status).toContain("branch1 branch taken")
     expect(results.steps[4].outputs.message).toContain("Branch 1.1")
   })

packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts

@@ -64,18 +64,18 @@ class BaseStepBuilder {
     stepId: TStep,
     stepSchema: Omit<AutomationStep, "id" | "stepId" | "inputs">,
     inputs: AutomationStepInputs<TStep>,
-    stepName?: string
+    opts?: { stepName?: string; stepId?: string }
   ): this {
-    const id = uuidv4()
+    const id = opts?.stepId || uuidv4()
     this.steps.push({
       ...stepSchema,
       inputs: inputs as any,
       id,
       stepId,
-      name: stepName || stepSchema.name,
+      name: opts?.stepName || stepSchema.name,
     })
-    if (stepName) {
+    if (opts?.stepName) {
-      this.stepNames[id] = stepName
+      this.stepNames[id] = opts.stepName
     }
     return this
   }
@@ -95,7 +95,6 @@ class BaseStepBuilder {
       })
       branchStepInputs.children![key] = stepBuilder.build()
     })
-
     const branchStep: AutomationStep = {
       ...definition,
       id: uuidv4(),
@@ -106,80 +105,98 @@ class BaseStepBuilder {
   }
 
   // STEPS
-  createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this {
+  createRow(
+    inputs: CreateRowStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.CREATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
-  updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this {
+  updateRow(
+    inputs: UpdateRowStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.UPDATE_ROW,
       BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
-  deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this {
+  deleteRow(
+    inputs: DeleteRowStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.DELETE_ROW,
       BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
   sendSmtpEmail(
     inputs: SmtpEmailStepInputs,
-    opts?: { stepName?: string }
+    opts?: { stepName?: string; stepId?: string }
   ): this {
     return this.step(
       AutomationActionStepId.SEND_EMAIL_SMTP,
       BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
   executeQuery(
     inputs: ExecuteQueryStepInputs,
-    opts?: { stepName?: string }
+    opts?: { stepName?: string; stepId?: string }
   ): this {
     return this.step(
       AutomationActionStepId.EXECUTE_QUERY,
       BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
-  queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this {
+  queryRows(
+    inputs: QueryRowsStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.QUERY_ROWS,
       BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
       inputs,
-      opts?.stepName
+      opts
     )
   }
-  loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this {
+  loop(
+    inputs: LoopStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.LOOP,
       BUILTIN_ACTION_DEFINITIONS.LOOP,
       inputs,
-      opts?.stepName
+      opts
     )
   }
 
-  serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this {
+  serverLog(
+    input: ServerLogStepInputs,
+    opts?: { stepName?: string; stepId?: string }
+  ): this {
     return this.step(
       AutomationActionStepId.SERVER_LOG,
       BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
       input,
-      opts?.stepName
+      opts
     )
   }
 

packages/server/src/db/linkedRows/index.ts

@@ -27,6 +27,7 @@ import {
   ViewV2,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import { helpers } from "@budibase/shared-core"
 
 export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils"
 
@@ -247,26 +248,36 @@ function getPrimaryDisplayValue(row: Row, table?: Table) {
 export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
 
 /**
- * This function will take the given enriched rows and squash the links to only contain the primary display field.
+ * This function will take the given enriched rows and squash the links to only
- * @returns The rows after having their links squashed to only contain the ID and primary display.
+ * contain the primary display field.
+ *
+ * @returns The rows after having their links squashed to only contain the ID
+ * and primary display.
  */
 export async function squashLinks<T = Row[] | Row>(
-  table: Table,
+  source: Table | ViewV2,
-  enriched: T,
+  enriched: T
-  options?: {
-    fromViewId?: string
-  }
 ): Promise<T> {
   const allowRelationshipSchemas = await features.flags.isEnabled(
     FeatureFlag.ENRICHED_RELATIONSHIPS
   )
 
   let viewSchema: Record<string, ViewFieldMetadata> = {}
-  if (options?.fromViewId && allowRelationshipSchemas) {
+  if (sdk.views.isView(source)) {
-    const view = Object.values(table.views || {}).find(
+    if (helpers.views.isCalculationView(source)) {
-      (v): v is ViewV2 => sdk.views.isV2(v) && v.id === options?.fromViewId
+      return enriched
-    )
+    }
-    viewSchema = view?.schema || {}
+
+    if (allowRelationshipSchemas) {
+      viewSchema = source.schema || {}
+    }
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
   }
 
   // will populate this as we find them

packages/server/src/db/utils.ts

@@ -1,4 +1,4 @@
-import { context, db as dbCore, utils } from "@budibase/backend-core"
+import { context, db as dbCore, docIds, utils } from "@budibase/backend-core"
 import {
   DatabaseQueryOpts,
   Datasource,
@@ -318,12 +318,8 @@ export function generateViewID(tableId: string) {
   }${SEPARATOR}${tableId}${SEPARATOR}${newid()}`
 }
 
-export function isViewID(viewId: string) {
-  return viewId?.split(SEPARATOR)[0] === VirtualDocumentType.VIEW
-}
-
 export function extractViewInfoFromID(viewId: string) {
-  if (!isViewID(viewId)) {
+  if (!docIds.isViewId(viewId)) {
     throw new Error("Unable to extract table ID, is not a view ID")
   }
   const split = viewId.split(SEPARATOR)

packages/server/src/definitions/automations.ts

@@ -15,7 +15,8 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
-  stepsByName?: Record<string, any>
+  stepsById: Record<string, any>
+  stepsByName: Record<string, any>
   env?: Record<string, string>
   trigger: any
 }

packages/server/src/middleware/triggerRowActionAuthorised.ts

@@ -15,7 +15,7 @@ export function triggerRowActionAuthorised(
       const rowActionId: string = ctx.params[actionPath]
 
       const isTableId = docIds.isTableId(sourceId)
-      const isViewId = utils.isViewID(sourceId)
+      const isViewId = docIds.isViewId(sourceId)
       if (!isTableId && !isViewId) {
         ctx.throw(400, `'${sourceId}' is not a valid source id`)
       }

packages/server/src/sdk/app/permissions/index.ts

@@ -1,4 +1,4 @@
-import { db, roles, context } from "@budibase/backend-core"
+import { db, roles, context, docIds } from "@budibase/backend-core"
 import {
   PermissionLevel,
   PermissionSource,
@@ -6,11 +6,7 @@ import {
   Role,
   Database,
 } from "@budibase/types"
-import {
+import { extractViewInfoFromID, getRoleParams } from "../../../db/utils"
-  extractViewInfoFromID,
-  isViewID,
-  getRoleParams,
-} from "../../../db/utils"
 import {
   CURRENTLY_SUPPORTED_LEVELS,
   getBasePermissions,
@@ -32,7 +28,7 @@ export const enum PermissionUpdateType {
 export async function getInheritablePermissions(
   resourceId: string
 ): Promise<ResourcePermissions | undefined> {
-  if (isViewID(resourceId)) {
+  if (docIds.isViewId(resourceId)) {
     return await getResourcePerms(extractViewInfoFromID(resourceId).tableId)
   }
 }

packages/server/src/sdk/app/rowActions.ts

@@ -1,11 +1,11 @@
-import { context, HTTPError, utils } from "@budibase/backend-core"
+import { context, docIds, HTTPError, utils } from "@budibase/backend-core"
 import {
   AutomationTriggerStepId,
   SEPARATOR,
   TableRowActions,
   VirtualDocumentType,
 } from "@budibase/types"
-import { generateRowActionsID, isViewID } from "../../db/utils"
+import { generateRowActionsID } from "../../db/utils"
 import automations from "./automations"
 import { definitions as TRIGGER_DEFINITIONS } from "../../automations/triggerInfo"
 import * as triggers from "../../automations/triggers"
@@ -155,7 +155,7 @@ export async function update(
 
 async function guardView(tableId: string, viewId: string) {
   let view
-  if (isViewID(viewId)) {
+  if (docIds.isViewId(viewId)) {
     view = await sdk.views.get(viewId)
   }
   if (!view || view.tableId !== tableId) {

packages/server/src/sdk/app/rows/external.ts

@@ -1,5 +1,11 @@
-import { IncludeRelationship, Operation, Row } from "@budibase/types"
+import {
-import { HTTPError } from "@budibase/backend-core"
+  IncludeRelationship,
+  Operation,
+  Row,
+  Table,
+  ViewV2,
+} from "@budibase/types"
+import { docIds, HTTPError } from "@budibase/backend-core"
 import { handleRequest } from "../../../api/controllers/row/external"
 import { breakRowIdField } from "../../../integrations/utils"
 import sdk from "../../../sdk"
@@ -8,15 +14,24 @@ import {
   outputProcessing,
 } from "../../../utilities/rowProcessor"
 import cloneDeep from "lodash/fp/cloneDeep"
-import isEqual from "lodash/fp/isEqual"
 import { tryExtractingTableAndViewId } from "./utils"
 
 export async function getRow(
-  tableId: string,
+  sourceId: string | Table | ViewV2,
   rowId: string,
   opts?: { relationships?: boolean }
 ) {
-  const response = await handleRequest(Operation.READ, tableId, {
+  let source: Table | ViewV2
+  if (typeof sourceId === "string") {
+    if (docIds.isViewId(sourceId)) {
+      source = await sdk.views.get(sourceId)
+    } else {
+      source = await sdk.tables.getTable(sourceId)
+    }
+  } else {
+    source = sourceId
+  }
+  const response = await handleRequest(Operation.READ, source, {
     id: breakRowIdField(rowId),
     includeSqlRelationships: opts?.relationships
       ? IncludeRelationship.INCLUDE
@@ -27,45 +42,42 @@ export async function getRow(
 }
 
 export async function save(
-  tableOrViewId: string,
+  sourceId: string,
   inputs: Row,
   userId: string | undefined
 ) {
-  const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
+  const { tableId, viewId } = tryExtractingTableAndViewId(sourceId)
-  const table = await sdk.tables.getTable(tableId)
+  let source: Table | ViewV2
-  const { table: updatedTable, row } = await inputProcessing(
+  if (viewId) {
-    userId,
+    source = await sdk.views.get(viewId)
-    cloneDeep(table),
+  } else {
-    inputs
+    source = await sdk.tables.getTable(tableId)
-  )
+  }
+
+  const row = await inputProcessing(userId, cloneDeep(source), inputs)
 
   const validateResult = await sdk.rows.utils.validate({
     row,
-    tableId,
+    source,
   })
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
 
-  const response = await handleRequest(Operation.CREATE, tableId, {
+  const response = await handleRequest(Operation.CREATE, source, {
     row,
   })
 
-  if (!isEqual(table, updatedTable)) {
-    await sdk.tables.saveTable(updatedTable)
-  }
-
   const rowId = response.row._id
   if (rowId) {
-    const row = await getRow(tableId, rowId, {
+    const row = await getRow(source, rowId, {
       relationships: true,
     })
     return {
       ...response,
-      row: await outputProcessing(table, row, {
+      row: await outputProcessing(source, row, {
         preserveLinks: true,
         squash: true,
-        fromViewId: viewId,
       }),
     }
   } else {
@@ -76,7 +88,14 @@ export async function save(
 export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
   const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
 
-  const row = await getRow(tableId, rowId, {
+  let source: Table | ViewV2
+  if (viewId) {
+    source = await sdk.views.get(viewId)
+  } else {
+    source = await sdk.tables.getTable(tableId)
+  }
+
+  const row = await getRow(source, rowId, {
     relationships: true,
   })
 
@@ -84,11 +103,10 @@ export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
     throw new HTTPError("Row not found", 404)
   }
 
-  const table = await sdk.tables.getTable(tableId)
+  // Preserving links, as the outputProcessing does not support external rows
-  // Preserving links, as the outputProcessing does not support external rows yet and we don't need it in this use case
+  // yet and we don't need it in this use case
-  return await outputProcessing(table, row, {
+  return await outputProcessing(source, row, {
     squash: true,
     preserveLinks: true,
-    fromViewId: viewId,
   })
 }

packages/server/src/sdk/app/rows/internal.ts

@@ -1,7 +1,6 @@
 import { context, db } from "@budibase/backend-core"
-import { Row } from "@budibase/types"
+import { Row, Table, ViewV2 } from "@budibase/types"
 import sdk from "../../../sdk"
-import cloneDeep from "lodash/fp/cloneDeep"
 import { finaliseRow } from "../../../api/controllers/row/staticFormula"
 import {
   inputProcessing,
@@ -10,7 +9,7 @@ import {
 import * as linkRows from "../../../db/linkedRows"
 import { InternalTables } from "../../../db/utils"
 import { getFullUser } from "../../../utilities/users"
-import { tryExtractingTableAndViewId } from "./utils"
+import { getSource, tryExtractingTableAndViewId } from "./utils"
 
 export async function save(
   tableOrViewId: string,
@@ -20,21 +19,25 @@ export async function save(
   const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
   inputs.tableId = tableId
 
+  let source: Table | ViewV2
+  let table: Table
+  if (viewId) {
+    source = await sdk.views.get(viewId)
+    table = await sdk.views.getTable(viewId)
+  } else {
+    source = await sdk.tables.getTable(tableId)
+    table = source
+  }
+
   if (!inputs._rev && !inputs._id) {
     inputs._id = db.generateRowID(inputs.tableId)
   }
 
-  // this returns the table and row incase they have been updated
+  let row = await inputProcessing(userId, source, inputs)
-  const dbTable = await sdk.tables.getTable(inputs.tableId)
-
-  // need to copy the table so it can be differenced on way out
-  const tableClone = cloneDeep(dbTable)
-
-  let { table, row } = await inputProcessing(userId, tableClone, inputs)
 
   const validateResult = await sdk.rows.utils.validate({
     row,
-    table,
+    source,
   })
 
   if (!validateResult.valid) {
@@ -49,24 +52,18 @@ export async function save(
     table,
   })) as Row
 
-  return finaliseRow(table, row, {
+  return finaliseRow(source, row, { updateFormula: true })
-    oldTable: dbTable,
+}
-    updateFormula: true,
+
-    fromViewId: viewId,
+export async function find(sourceId: string, rowId: string): Promise<Row> {
+  const source = await getSource(sourceId)
+  return await outputProcessing(source, await findRow(sourceId, rowId), {
+    squash: true,
   })
 }
 
-export async function find(tableOrViewId: string, rowId: string): Promise<Row> {
+export async function findRow(sourceId: string, rowId: string) {
-  const { tableId, viewId } = tryExtractingTableAndViewId(tableOrViewId)
+  const { tableId } = tryExtractingTableAndViewId(sourceId)
-
-  const table = await sdk.tables.getTable(tableId)
-  let row = await findRow(tableId, rowId)
-
-  row = await outputProcessing(table, row, { squash: true, fromViewId: viewId })
-  return row
-}
-
-async function findRow(tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row: Row
   // TODO remove special user case in future

packages/server/src/sdk/app/rows/queryUtils.ts

@@ -18,11 +18,11 @@ export const removeInvalidFilters = (
   for (const filterKey of Object.keys(
     result || {}
   ) as (keyof SearchFilters)[]) {
-    const filter = result[filterKey]
-    if (!filter || typeof filter !== "object") {
-      continue
-    }
     if (isLogicalSearchOperator(filterKey)) {
+      const filter = result[filterKey]
+      if (!filter || typeof filter !== "object") {
+        continue
+      }
       const resultingConditions: SearchFilters[] = []
       for (const condition of filter.conditions) {
         const resultingCondition = removeInvalidFilters(condition, validFields)
@@ -38,6 +38,11 @@ export const removeInvalidFilters = (
       continue
     }
 
+    const filter = result[filterKey]
+    if (!filter || typeof filter !== "object") {
+      continue
+    }
+
     for (const columnKey of Object.keys(filter)) {
       const possibleKeys = [columnKey, db.removeKeyNumbering(columnKey)].map(
         c => c.toLowerCase()
@@ -55,8 +60,8 @@ export const removeInvalidFilters = (
 }
 
 export const getQueryableFields = async (
-  fields: string[],
+  table: Table,
-  table: Table
+  fields?: string[]
 ): Promise<string[]> => {
   const extractTableFields = async (
     table: Table,
@@ -112,6 +117,9 @@ export const getQueryableFields = async (
     "_id", // Querying by _id is always allowed, even if it's never part of the schema
   ]
 
+  if (fields == null) {
+    fields = Object.keys(table.schema)
+  }
   result.push(...(await extractTableFields(table, fields, [table._id!])))
 
   return result

packages/server/src/sdk/app/rows/rows.ts

@@ -1,10 +1,6 @@
-import { db as dbCore, context } from "@budibase/backend-core"
+import { db as dbCore, context, docIds } from "@budibase/backend-core"
 import { Database, Row } from "@budibase/types"
-import {
+import { extractViewInfoFromID, getRowParams } from "../../../db/utils"
-  extractViewInfoFromID,
-  getRowParams,
-  isViewID,
-} from "../../../db/utils"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./internal"
 import * as external from "./external"
@@ -26,7 +22,7 @@ export async function getAllInternalRows(appId?: string) {
 
 function pickApi(tableOrViewId: string) {
   let tableId = tableOrViewId
-  if (isViewID(tableOrViewId)) {
+  if (docIds.isViewId(tableOrViewId)) {
     tableId = extractViewInfoFromID(tableOrViewId).tableId
   }
 
@@ -37,13 +33,13 @@ function pickApi(tableOrViewId: string) {
 }
 
 export async function save(
-  tableOrViewId: string,
+  sourceId: string,
   row: Row,
   userId: string | undefined
 ) {
-  return pickApi(tableOrViewId).save(tableOrViewId, row, userId)
+  return pickApi(sourceId).save(sourceId, row, userId)
 }
 
-export async function find(tableOrViewId: string, rowId: string) {
+export async function find(sourceId: string, rowId: string) {
-  return pickApi(tableOrViewId).find(tableOrViewId, rowId)
+  return pickApi(sourceId).find(sourceId, rowId)
 }

packages/server/src/sdk/app/rows/search.ts

@@ -1,9 +1,16 @@
 import {
   EmptyFilterOption,
+  LogicalOperator,
   Row,
   RowSearchParams,
+  SearchFilter,
+  SearchFilterGroup,
+  SearchFilterKey,
+  SearchFilters,
   SearchResponse,
   SortOrder,
+  Table,
+  ViewV2,
 } from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
@@ -12,9 +19,10 @@ import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
-import { features } from "@budibase/backend-core"
+import { db, features } from "@budibase/backend-core"
 import tracer from "dd-trace"
 import { getQueryableFields, removeInvalidFilters } from "./queryUtils"
+import { enrichSearchContext } from "../../../api/controllers/row/utils"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -32,11 +40,13 @@ function pickApi(tableId: any) {
 }
 
 export async function search(
-  options: RowSearchParams
+  options: RowSearchParams,
+  context?: Record<string, any>
 ): Promise<SearchResponse<Row>> {
   return await tracer.trace("search", async span => {
     span?.addTags({
       tableId: options.tableId,
+      viewId: options.viewId,
       query: options.query,
       sort: options.sort,
       sortOrder: options.sortOrder,
@@ -48,20 +58,85 @@ export async function search(
       countRows: options.countRows,
     })
 
-    const isExternalTable = isExternalTableID(options.tableId)
+    let source: Table | ViewV2
-    options.query = dataFilters.cleanupQuery(options.query || {})
+    let table: Table
+    if (options.viewId) {
+      source = await sdk.views.get(options.viewId)
+      table = await sdk.views.getTable(source)
+      options = searchInputMapping(table, options)
+    } else if (options.tableId) {
+      source = await sdk.tables.getTable(options.tableId)
+      table = source
+    } else {
+      throw new Error(`Must supply either a view ID or a table ID`)
+    }
+
+    const isExternalTable = isExternalTableID(table._id!)
+
+    if (options.query) {
+      const visibleFields = (
+        options.fields || Object.keys(table.schema)
+      ).filter(field => table.schema[field].visible !== false)
+
+      const queryableFields = await getQueryableFields(table, visibleFields)
+      options.query = removeInvalidFilters(options.query, queryableFields)
+    } else {
+      options.query = {}
+    }
+
+    if (options.viewId) {
+      const view = await sdk.views.get(options.viewId)
+      // Enrich saved query with ephemeral query params.
+      // We prevent searching on any fields that are saved as part of the query, as
+      // that could let users find rows they should not be allowed to access.
+      let viewQuery = dataFilters.buildQuery(view.query || [])
+
+      if (!isExternalTable && !(await features.flags.isEnabled("SQS"))) {
+        // Lucene does not accept conditional filters, so we need to keep the old logic
+        const query: SearchFilters = viewQuery || {}
+        const viewFilters = view.query as SearchFilter[]
+
+        // Extract existing fields
+        const existingFields =
+          viewFilters
+            ?.filter(filter => filter.field)
+            .map(filter => db.removeKeyNumbering(filter.field)) || []
+
+        // Carry over filters for unused fields
+        Object.keys(options.query || {}).forEach(key => {
+          const operator = key as Exclude<SearchFilterKey, LogicalOperator>
+          Object.keys(options.query[operator] || {}).forEach(field => {
+            if (!existingFields.includes(db.removeKeyNumbering(field))) {
+              query[operator]![field] = options.query[operator]![field]
+            }
+          })
+        })
+        options.query = query
+      } else {
+        options.query = {
+          $and: {
+            conditions: [viewQuery as SearchFilterGroup, options.query],
+          },
+        }
+      }
+    }
+
+    if (context) {
+      options.query = await enrichSearchContext(options.query, context)
+    }
+
+    options.query = dataFilters.cleanupQuery(options.query)
     options.query = dataFilters.fixupFilterArrays(options.query)
 
-    span?.addTags({
+    span.addTags({
       cleanedQuery: options.query,
-      isExternalTable,
     })
 
     if (
       !dataFilters.hasFilters(options.query) &&
       options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
     ) {
-      span?.addTags({ emptyQuery: true })
+      span.addTags({ emptyQuery: true })
       return {
         rows: [],
       }
@@ -71,34 +146,21 @@ export async function search(
       options.sortOrder = options.sortOrder.toLowerCase() as SortOrder
     }
 
-    const table = await sdk.tables.getTable(options.tableId)
     options = searchInputMapping(table, options)
 
-    if (options.query) {
-      const tableFields = Object.keys(table.schema).filter(
-        f => table.schema[f].visible !== false
-      )
-
-      const queriableFields = await getQueryableFields(
-        options.fields?.filter(f => tableFields.includes(f)) ?? tableFields,
-        table
-      )
-      options.query = removeInvalidFilters(options.query, queriableFields)
-    }
-
     let result: SearchResponse<Row>
     if (isExternalTable) {
       span?.addTags({ searchType: "external" })
-      result = await external.search(options, table)
+      result = await external.search(options, source)
     } else if (await features.flags.isEnabled("SQS")) {
       span?.addTags({ searchType: "sqs" })
-      result = await internal.sqs.search(options, table)
+      result = await internal.sqs.search(options, source)
     } else {
       span?.addTags({ searchType: "lucene" })
-      result = await internal.lucene.search(options, table)
+      result = await internal.lucene.search(options, source)
     }
 
-    span?.addTags({
+    span.addTags({
       foundRows: result.rows.length,
       totalRows: result.totalRows,
     })

packages/server/src/sdk/app/rows/search/external.ts

@@ -9,6 +9,7 @@ import {
   SortJson,
   SortOrder,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import { handleRequest } from "../../../../api/controllers/row/external"
@@ -60,9 +61,8 @@ function getPaginationAndLimitParameters(
 
 export async function search(
   options: RowSearchParams,
-  table: Table
+  source: Table | ViewV2
 ): Promise<SearchResponse<Row>> {
-  const { tableId } = options
   const { countRows, paginate, query, ...params } = options
   const { limit } = params
   let bookmark =
@@ -106,16 +106,15 @@ export async function search(
       includeSqlRelationships: IncludeRelationship.INCLUDE,
     }
     const [{ rows, rawResponseSize }, totalRows] = await Promise.all([
-      handleRequest(Operation.READ, tableId, parameters),
+      handleRequest(Operation.READ, source, parameters),
       countRows
-        ? handleRequest(Operation.COUNT, tableId, parameters)
+        ? handleRequest(Operation.COUNT, source, parameters)
         : Promise.resolve(undefined),
     ])
 
-    let processed = await outputProcessing(table, rows, {
+    let processed = await outputProcessing(source, rows, {
       preserveLinks: true,
       squash: true,
-      fromViewId: options.viewId,
     })
 
     let hasNextPage = false
@@ -128,10 +127,13 @@ export async function search(
       }
     }
 
-    if (options.fields) {
+    const visibleFields =
-      const fields = [...options.fields, ...PROTECTED_EXTERNAL_COLUMNS]
+      options.fields ||
-      processed = processed.map((r: any) => pick(r, fields))
+      Object.keys(source.schema || {}).filter(
-    }
+        key => source.schema?.[key].visible !== false
+      )
+    const allowedFields = [...visibleFields, ...PROTECTED_EXTERNAL_COLUMNS]
+    processed = processed.map((r: any) => pick(r, allowedFields))
 
     // need wrapper object for bookmarks etc when paginating
     const response: SearchResponse<Row> = { rows: processed, hasNextPage }
@@ -201,7 +203,7 @@ export async function exportRows(
   }
 
   let result = await search(
-    { tableId, query: requestQuery, sort, sortOrder },
+    { tableId: table._id!, query: requestQuery, sort, sortOrder },
     table
   )
   let rows: Row[] = []
@@ -257,10 +259,10 @@ export async function exportRows(
 }
 
 export async function fetch(tableId: string): Promise<Row[]> {
-  const response = await handleRequest(Operation.READ, tableId, {
+  const table = await sdk.tables.getTable(tableId)
+  const response = await handleRequest(Operation.READ, table, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
-  const table = await sdk.tables.getTable(tableId)
   return await outputProcessing(table, response.rows, {
     preserveLinks: true,
     squash: true,
@@ -268,7 +270,8 @@ export async function fetch(tableId: string): Promise<Row[]> {
 }
 
 export async function fetchRaw(tableId: string): Promise<Row[]> {
-  const response = await handleRequest(Operation.READ, tableId, {
+  const table = await sdk.tables.getTable(tableId)
+  const response = await handleRequest(Operation.READ, table, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
   return response.rows

packages/server/src/sdk/app/rows/search/internal/lucene.ts

@@ -8,21 +8,29 @@ import {
   SortType,
   Table,
   User,
+  ViewV2,
 } from "@budibase/types"
 import { getGlobalUsersFromMetadata } from "../../../../../utilities/global"
 import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
+import sdk from "../../../../"
 
 export async function search(
   options: RowSearchParams,
-  table: Table
+  source: Table | ViewV2
 ): Promise<SearchResponse<Row>> {
-  const { tableId } = options
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
 
   const { paginate, query } = options
 
   const params: RowSearchParams = {
     tableId: options.tableId,
+    viewId: options.viewId,
     sort: options.sort,
     sortOrder: options.sortOrder,
     sortType: options.sortType,
@@ -50,18 +58,20 @@ export async function search(
   // Enrich search results with relationships
   if (response.rows && response.rows.length) {
     // enrich with global users if from users table
-    if (tableId === InternalTables.USER_METADATA) {
+    if (table._id === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
     }
 
-    if (options.fields) {
+    const visibleFields =
-      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
+      options.fields ||
-      response.rows = response.rows.map((r: any) => pick(r, fields))
+      Object.keys(source.schema || {}).filter(
-    }
+        key => source.schema?.[key].visible !== false
+      )
+    const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
+    response.rows = response.rows.map((r: any) => pick(r, allowedFields))
 
-    response.rows = await outputProcessing(table, response.rows, {
+    response.rows = await outputProcessing(source, response.rows, {
       squash: true,
-      fromViewId: options.viewId,
     })
   }
 

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -1,4 +1,5 @@
 import {
+  Aggregation,
   Datasource,
   DocumentType,
   FieldType,
@@ -15,6 +16,7 @@ import {
   SortType,
   SqlClient,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import {
   buildInternalRelationships,
@@ -44,10 +46,12 @@ import {
 import {
   dataFilters,
   helpers,
+  isInternalColumnName,
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
 import { isSearchingByRowID } from "../utils"
 import tracer from "dd-trace"
+import { cloneDeep } from "lodash"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const SQLITE_COLUMN_LIMIT = 2000
@@ -55,11 +59,34 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
 const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
 
-function buildInternalFieldList(
+async function buildInternalFieldList(
-  table: Table,
+  source: Table | ViewV2,
   tables: Table[],
-  opts?: { relationships?: RelationshipsJson[] }
+  opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] }
 ) {
+  const { relationships, allowedFields } = opts || {}
+  let schemaFields: string[] = []
+  if (sdk.views.isView(source)) {
+    schemaFields = Object.keys(helpers.views.basicFields(source)).filter(
+      key => source.schema?.[key]?.visible !== false
+    )
+  } else {
+    schemaFields = Object.keys(source.schema).filter(
+      key => source.schema[key].visible !== false
+    )
+  }
+
+  if (allowedFields) {
+    schemaFields = schemaFields.filter(field => allowedFields.includes(field))
+  }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   let fieldList: string[] = []
   const getJunctionFields = (relatedTable: Table, fields: string[]) => {
     const junctionFields: string[] = []
@@ -70,13 +97,18 @@ function buildInternalFieldList(
     })
     return junctionFields
   }
-  fieldList = fieldList.concat(
+  if (sdk.tables.isTable(source)) {
-    PROTECTED_INTERNAL_COLUMNS.map(col => `${table._id}.${col}`)
+    for (const key of PROTECTED_INTERNAL_COLUMNS) {
-  )
+      if (allowedFields && !allowedFields.includes(key)) {
-  for (let key of Object.keys(table.schema)) {
+        continue
+      }
+      fieldList.push(`${table._id}.${key}`)
+    }
+  }
+  for (let key of schemaFields) {
     const col = table.schema[key]
     const isRelationship = col.type === FieldType.LINK
-    if (!opts?.relationships && isRelationship) {
+    if (!relationships && isRelationship) {
       continue
     }
     if (!isRelationship) {
@@ -87,7 +119,9 @@ function buildInternalFieldList(
       if (!relatedTable) {
         continue
       }
-      const relatedFields = buildInternalFieldList(relatedTable, tables).concat(
+      const relatedFields = (
+        await buildInternalFieldList(relatedTable, tables)
+      ).concat(
         getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
       )
       // break out of the loop if we have reached the max number of columns
@@ -128,15 +162,22 @@ function cleanupFilters(
   // generate a map of all possible column names (these can be duplicated across tables
   // the map of them will always be the same
   const userColumnMap: Record<string, string> = {}
-  allTables.forEach(table =>
+  for (const table of allTables) {
-    Object.keys(table.schema).forEach(
+    for (const key of Object.keys(table.schema)) {
-      key => (userColumnMap[key] = mapToUserColumn(key))
+      if (isInternalColumnName(key)) {
-    )
+        continue
-  )
+      }
+      userColumnMap[key] = mapToUserColumn(key)
+    }
+  }
 
   // update the keys of filters to manage user columns
-  const keyInAnyTable = (key: string): boolean =>
+  const keyInAnyTable = (key: string): boolean => {
-    allTables.some(table => table.schema[key])
+    if (isInternalColumnName(key)) {
+      return false
+    }
+    return allTables.some(table => table.schema[key])
+  }
 
   const splitter = new dataFilters.ColumnSplitter(allTables)
 
@@ -291,16 +332,23 @@ function resyncDefinitionsRequired(status: number, message: string) {
 
 export async function search(
   options: RowSearchParams,
-  table: Table,
+  source: Table | ViewV2,
   opts?: { retrying?: boolean }
 ): Promise<SearchResponse<Row>> {
-  let { paginate, query, ...params } = options
+  let { paginate, query, ...params } = cloneDeep(options)
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
 
   const allTables = await sdk.tables.getAllInternalTables()
   const allTablesMap = buildTableMap(allTables)
   // make sure we have the mapped/latest table
-  if (table?._id) {
+  if (table._id) {
-    table = allTablesMap[table?._id]
+    table = allTablesMap[table._id]
   }
   if (!table) {
     throw new Error("Unable to find table")
@@ -312,6 +360,23 @@ export async function search(
     ...cleanupFilters(query, table, allTables),
     documentType: DocumentType.ROW,
   }
+
+  let aggregations: Aggregation[] = []
+  if (sdk.views.isView(source)) {
+    const calculationFields = helpers.views.calculationFields(source)
+    for (const [key, field] of Object.entries(calculationFields)) {
+      if (options.fields && !options.fields.includes(key)) {
+        continue
+      }
+
+      aggregations.push({
+        name: key,
+        field: mapToUserColumn(field.field),
+        calculationType: field.calculationType,
+      })
+    }
+  }
+
   const request: QueryJson = {
     endpoint: {
       // not important, we query ourselves
@@ -327,7 +392,11 @@ export async function search(
       columnPrefix: USER_COLUMN_PREFIX,
     },
     resource: {
-      fields: buildInternalFieldList(table, allTables, { relationships }),
+      fields: await buildInternalFieldList(source, allTables, {
+        relationships,
+        allowedFields: options.fields,
+      }),
+      aggregations,
     },
     relationships,
   }
@@ -372,7 +441,7 @@ export async function search(
     // make sure JSON columns corrected
     const processed = builder.convertJsonStringColumns<Row>(
       table,
-      await sqlOutputProcessing(rows, table!, allTablesMap, relationships, {
+      await sqlOutputProcessing(rows, source, allTablesMap, relationships, {
         sqs: true,
       })
     )
@@ -388,17 +457,18 @@ export async function search(
     }
 
     // get the rows
-    let finalRows = await outputProcessing(table, processed, {
+    let finalRows = await outputProcessing(source, processed, {
       preserveLinks: true,
       squash: true,
-      fromViewId: options.viewId,
     })
 
-    // check if we need to pick specific rows out
+    const visibleFields =
-    if (options.fields) {
+      options.fields ||
-      const fields = [...options.fields, ...PROTECTED_INTERNAL_COLUMNS]
+      Object.keys(source.schema || {}).filter(
-      finalRows = finalRows.map((r: any) => pick(r, fields))
+        key => source.schema?.[key].visible !== false
-    }
+      )
+    const allowedFields = [...visibleFields, ...PROTECTED_INTERNAL_COLUMNS]
+    finalRows = finalRows.map((r: any) => pick(r, allowedFields))
 
     const response: SearchResponse<Row> = {
       rows: finalRows,
@@ -419,7 +489,7 @@ export async function search(
     const msg = typeof err === "string" ? err : err.message
     if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) {
       await sdk.tables.sqs.syncDefinition()
-      return search(options, table, { retrying: true })
+      return search(options, source, { retrying: true })
     }
     // previously the internal table didn't error when a column didn't exist in search
     if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {

packages/server/src/sdk/app/rows/search/tests/utils.spec.ts

@@ -90,10 +90,8 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
     })
 
     it("shouldn't error if no query supplied", () => {
-      const params: any = {
+      // @ts-expect-error - intentionally passing in a bad type
-        tableId,
+      const output = searchInputMapping(col, { tableId })
-      }
-      const output = searchInputMapping(col, params)
       expect(output.query).toBeUndefined()
     })
   }

packages/server/src/sdk/app/rows/search/utils.ts

@@ -11,7 +11,7 @@ import {
   RowSearchParams,
 } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
-import { utils } from "@budibase/shared-core"
+import { utils, dataFilters } from "@budibase/shared-core"
 
 export async function paginatedSearch(
   query: SearchFilters,
@@ -31,13 +31,13 @@ export async function fullSearch(
 
 function findColumnInQueries(
   column: string,
-  options: RowSearchParams,
+  filters: SearchFilters,
   callback: (filter: any) => any
 ) {
-  if (!options.query) {
+  if (!filters) {
     return
   }
-  for (let filterBlock of Object.values(options.query)) {
+  for (let filterBlock of Object.values(filters)) {
     if (typeof filterBlock !== "object") {
       continue
     }
@@ -49,8 +49,8 @@ function findColumnInQueries(
   }
 }
 
-function userColumnMapping(column: string, options: RowSearchParams) {
+function userColumnMapping(column: string, filters: SearchFilters) {
-  findColumnInQueries(column, options, (filterValue: any): any => {
+  findColumnInQueries(column, filters, (filterValue: any): any => {
     const isArray = Array.isArray(filterValue),
       isString = typeof filterValue === "string"
     if (!isString && !isArray) {
@@ -83,29 +83,31 @@ function userColumnMapping(column: string, options: RowSearchParams) {
 // maps through the search parameters to check if any of the inputs are invalid
 // based on the table schema, converts them to something that is valid.
 export function searchInputMapping(table: Table, options: RowSearchParams) {
-  if (!table?.schema) {
+  // need an internal function to loop over filters, because this takes the full options
-    return options
+  function checkFilters(filters: SearchFilters) {
-  }
+    for (let [key, column] of Object.entries(table.schema || {})) {
-  for (let [key, column] of Object.entries(table.schema)) {
+      switch (column.type) {
-    switch (column.type) {
+        case FieldType.BB_REFERENCE_SINGLE: {
-      case FieldType.BB_REFERENCE_SINGLE: {
+          const subtype = column.subtype
-        const subtype = column.subtype
+          switch (subtype) {
-        switch (subtype) {
+            case BBReferenceFieldSubType.USER:
-          case BBReferenceFieldSubType.USER:
+              userColumnMapping(key, filters)
-            userColumnMapping(key, options)
+              break
-            break
 
-          default:
+            default:
-            utils.unreachable(subtype)
+              utils.unreachable(subtype)
+          }
+          break
+        }
+        case FieldType.BB_REFERENCE: {
+          userColumnMapping(key, filters)
+          break
         }
-        break
-      }
-      case FieldType.BB_REFERENCE: {
-        userColumnMapping(key, options)
-        break
       }
     }
+    return dataFilters.recurseLogicalOperators(filters, checkFilters)
   }
+  options.query = checkFilters(options.query)
   return options
 }
 

packages/server/src/sdk/app/rows/tests/queryUtils.spec.ts

@@ -203,7 +203,7 @@ describe("query utils", () => {
         },
       })
 
-      const result = await getQueryableFields(Object.keys(table.schema), table)
+      const result = await getQueryableFields(table)
       expect(result).toEqual(["_id", "name", "age"])
     })
 
@@ -216,7 +216,7 @@ describe("query utils", () => {
         },
       })
 
-      const result = await getQueryableFields(Object.keys(table.schema), table)
+      const result = await getQueryableFields(table)
       expect(result).toEqual(["_id", "name"])
     })
 
@@ -245,7 +245,7 @@ describe("query utils", () => {
       })
 
       const result = await config.doInContext(config.appId, () => {
-        return getQueryableFields(Object.keys(table.schema), table)
+        return getQueryableFields(table)
       })
       expect(result).toEqual([
         "_id",
@@ -282,7 +282,7 @@ describe("query utils", () => {
       })
 
       const result = await config.doInContext(config.appId, () => {
-        return getQueryableFields(Object.keys(table.schema), table)
+        return getQueryableFields(table)
       })
       expect(result).toEqual(["_id", "name", "aux.name", "auxTable.name"])
     })
@@ -313,7 +313,7 @@ describe("query utils", () => {
       })
 
       const result = await config.doInContext(config.appId, () => {
-        return getQueryableFields(Object.keys(table.schema), table)
+        return getQueryableFields(table)
       })
       expect(result).toEqual(["_id", "name"])
     })
@@ -381,7 +381,7 @@ describe("query utils", () => {
 
         it("includes nested relationship fields from main table", async () => {
           const result = await config.doInContext(config.appId, () => {
-            return getQueryableFields(Object.keys(table.schema), table)
+            return getQueryableFields(table)
           })
           expect(result).toEqual([
             "_id",
@@ -398,7 +398,7 @@ describe("query utils", () => {
 
         it("includes nested relationship fields from aux 1 table", async () => {
           const result = await config.doInContext(config.appId, () => {
-            return getQueryableFields(Object.keys(aux1.schema), aux1)
+            return getQueryableFields(aux1)
           })
           expect(result).toEqual([
             "_id",
@@ -420,7 +420,7 @@ describe("query utils", () => {
 
         it("includes nested relationship fields from aux 2 table", async () => {
           const result = await config.doInContext(config.appId, () => {
-            return getQueryableFields(Object.keys(aux2.schema), aux2)
+            return getQueryableFields(aux2)
           })
           expect(result).toEqual([
             "_id",
@@ -474,7 +474,7 @@ describe("query utils", () => {
 
         it("includes nested relationship fields from main table", async () => {
           const result = await config.doInContext(config.appId, () => {
-            return getQueryableFields(Object.keys(table.schema), table)
+            return getQueryableFields(table)
           })
           expect(result).toEqual([
             "_id",
@@ -488,7 +488,7 @@ describe("query utils", () => {
 
         it("includes nested relationship fields from aux table", async () => {
           const result = await config.doInContext(config.appId, () => {
-            return getQueryableFields(Object.keys(aux.schema), aux)
+            return getQueryableFields(aux)
           })
           expect(result).toEqual([
             "_id",

packages/server/src/sdk/app/rows/tests/utils.spec.ts

@@ -33,7 +33,7 @@ describe("validate", () => {
     it("should accept empty values", async () => {
       const row = {}
       const table = getTable()
-      const output = await validate({ table, tableId: table._id!, row })
+      const output = await validate({ source: table, row })
       expect(output.valid).toBe(true)
       expect(output.errors).toEqual({})
     })
@@ -43,7 +43,7 @@ describe("validate", () => {
         time: `${hour()}:${minute()}`,
       }
       const table = getTable()
-      const output = await validate({ table, tableId: table._id!, row })
+      const output = await validate({ source: table, row })
       expect(output.valid).toBe(true)
     })
 
@@ -52,7 +52,7 @@ describe("validate", () => {
         time: `${hour()}:${minute()}:${second()}`,
       }
       const table = getTable()
-      const output = await validate({ table, tableId: table._id!, row })
+      const output = await validate({ source: table, row })
       expect(output.valid).toBe(true)
     })
 
@@ -67,7 +67,7 @@ describe("validate", () => {
       table.schema.time.constraints = {
         presence: true,
       }
-      const output = await validate({ table, tableId: table._id!, row })
+      const output = await validate({ source: table, row })
       expect(output.valid).toBe(false)
       expect(output.errors).toEqual({ time: ['"time" is not a valid time'] })
     })
@@ -91,7 +91,7 @@ describe("validate", () => {
           `${generator.integer({ min: 11, max: 23 })}:${minute()}`,
         ])("should accept values after config value (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(true)
         })
 
@@ -100,7 +100,7 @@ describe("validate", () => {
           `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
         ])("should reject values before config value (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no earlier than 10:00"],
@@ -125,7 +125,7 @@ describe("validate", () => {
           `${generator.integer({ min: 0, max: 12 })}:${minute()}`,
         ])("should accept values before config value (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(true)
         })
 
@@ -134,7 +134,7 @@ describe("validate", () => {
           `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
         ])("should reject values after config value (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no later than 15:16:17"],
@@ -156,7 +156,7 @@ describe("validate", () => {
           "should accept values in range (%s)",
           async time => {
             const row = { time }
-            const output = await validate({ table, tableId: table._id!, row })
+            const output = await validate({ source: table, row })
             expect(output.valid).toBe(true)
           }
         )
@@ -166,7 +166,7 @@ describe("validate", () => {
           `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
         ])("should reject values before range (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no earlier than 10:00"],
@@ -178,7 +178,7 @@ describe("validate", () => {
           `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
         ])("should reject values after range (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no later than 15:00"],
@@ -199,7 +199,7 @@ describe("validate", () => {
             "should accept values in range (%s)",
             async time => {
               const row = { time }
-              const output = await validate({ table, tableId: table._id!, row })
+              const output = await validate({ source: table, row })
               expect(output.valid).toBe(true)
             }
           )
@@ -208,7 +208,7 @@ describe("validate", () => {
             "should reject values out range (%s)",
             async time => {
               const row = { time }
-              const output = await validate({ table, tableId: table._id!, row })
+              const output = await validate({ source: table, row })
               expect(output.valid).toBe(false)
               expect(output.errors).toEqual({
                 time: ["must be no later than 10:00"],
@@ -226,7 +226,7 @@ describe("validate", () => {
         table.schema.time.constraints = {
           presence: true,
         }
-        const output = await validate({ table, tableId: table._id!, row })
+        const output = await validate({ source: table, row })
         expect(output.valid).toBe(false)
         expect(output.errors).toEqual({ time: ["can't be blank"] })
       })
@@ -237,7 +237,7 @@ describe("validate", () => {
         table.schema.time.constraints = {
           presence: true,
         }
-        const output = await validate({ table, tableId: table._id!, row })
+        const output = await validate({ source: table, row })
         expect(output.valid).toBe(false)
         expect(output.errors).toEqual({ time: ["can't be blank"] })
       })
@@ -257,7 +257,7 @@ describe("validate", () => {
         "should accept values in range (%s)",
         async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(true)
         }
       )
@@ -267,7 +267,7 @@ describe("validate", () => {
         `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
       ])("should reject values before range (%s)", async time => {
         const row = { time }
-        const output = await validate({ table, tableId: table._id!, row })
+        const output = await validate({ source: table, row })
         expect(output.valid).toBe(false)
         expect(output.errors).toEqual({
           time: ["must be no earlier than 10:00"],
@@ -279,7 +279,7 @@ describe("validate", () => {
         `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
       ])("should reject values after range (%s)", async time => {
         const row = { time }
-        const output = await validate({ table, tableId: table._id!, row })
+        const output = await validate({ source: table, row })
         expect(output.valid).toBe(false)
         expect(output.errors).toEqual({
           time: ["must be no later than 15:00"],
@@ -301,7 +301,7 @@ describe("validate", () => {
           "should accept values in range (%s)",
           async time => {
             const row = { time }
-            const output = await validate({ table, tableId: table._id!, row })
+            const output = await validate({ source: table, row })
             expect(output.valid).toBe(true)
           }
         )
@@ -311,7 +311,7 @@ describe("validate", () => {
           `${generator.integer({ min: 0, max: 9 })}:${minute()}`,
         ])("should reject values before range (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no earlier than 10:00"],
@@ -323,7 +323,7 @@ describe("validate", () => {
           `${generator.integer({ min: 16, max: 23 })}:${minute()}`,
         ])("should reject values after range (%s)", async time => {
           const row = { time }
-          const output = await validate({ table, tableId: table._id!, row })
+          const output = await validate({ source: table, row })
           expect(output.valid).toBe(false)
           expect(output.errors).toEqual({
             time: ["must be no later than 15:00"],

packages/server/src/sdk/app/rows/utils.ts

@@ -13,16 +13,15 @@ import {
   TableSchema,
   SqlClient,
   ArrayOperator,
+  ViewV2,
 } from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
-import {
+import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
-  extractViewInfoFromID,
-  isRelationshipColumn,
-  isViewID,
-} from "../../../db/utils"
 import { isSQL } from "../../../integrations/utils"
+import { docIds } from "@budibase/backend-core"
+import { getTableFromSource } from "../../../api/controllers/row/utils"
 
 const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
   [SourceName.POSTGRES]: SqlClient.POSTGRES,
@@ -58,8 +57,8 @@ export function getSQLClient(datasource: Datasource): SqlClient {
 export function processRowCountResponse(
   response: DatasourcePlusQueryResponse
 ): number {
-  if (response && response.length === 1 && "total" in response[0]) {
+  if (response && response.length === 1 && "__bb_total" in response[0]) {
-    const total = response[0].total
+    const total = response[0].__bb_total
     return typeof total === "number" ? total : parseInt(total)
   } else {
     throw new Error("Unable to count rows in query - no count response")
@@ -142,37 +141,27 @@ function isForeignKey(key: string, table: Table) {
 }
 
 export async function validate({
-  tableId,
+  source,
   row,
-  table,
 }: {
-  tableId?: string
+  source: Table | ViewV2
   row: Row
-  table?: Table
 }): Promise<{
   valid: boolean
   errors: Record<string, any>
 }> {
-  let fetchedTable: Table | undefined
+  const table = await getTableFromSource(source)
-  if (!table && tableId) {
-    fetchedTable = await sdk.tables.getTable(tableId)
-  } else if (table) {
-    fetchedTable = table
-  }
-  if (fetchedTable === undefined) {
-    throw new Error("Unable to fetch table for validation")
-  }
   const errors: Record<string, any> = {}
   const disallowArrayTypes = [
     FieldType.ATTACHMENT_SINGLE,
     FieldType.BB_REFERENCE_SINGLE,
   ]
-  for (let fieldName of Object.keys(fetchedTable.schema)) {
+  for (let fieldName of Object.keys(table.schema)) {
-    const column = fetchedTable.schema[fieldName]
+    const column = table.schema[fieldName]
     const constraints = cloneDeep(column.constraints)
     const type = column.type
     // foreign keys are likely to be enriched
-    if (isForeignKey(fieldName, fetchedTable)) {
+    if (isForeignKey(fieldName, table)) {
       continue
     }
     // formulas shouldn't validated, data will be deleted anyway
@@ -323,7 +312,7 @@ export function isArrayFilter(operator: any): operator is ArrayOperator {
 }
 
 export function tryExtractingTableAndViewId(tableOrViewId: string) {
-  if (isViewID(tableOrViewId)) {
+  if (docIds.isViewId(tableOrViewId)) {
     return {
       tableId: extractViewInfoFromID(tableOrViewId).tableId,
       viewId: tableOrViewId,
@@ -332,3 +321,10 @@ export function tryExtractingTableAndViewId(tableOrViewId: string) {
 
   return { tableId: tableOrViewId }
 }
+
+export function getSource(tableOrViewId: string) {
+  if (docIds.isViewId(tableOrViewId)) {
+    return sdk.views.get(tableOrViewId)
+  }
+  return sdk.tables.getTable(tableOrViewId)
+}

packages/server/src/sdk/app/tables/utils.ts

@@ -1,5 +1,6 @@
 import { Table, TableSourceType } from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
+import { docIds } from "@budibase/backend-core"
 
 export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
   if (opts.table && opts.table.sourceType === TableSourceType.EXTERNAL) {
@@ -9,3 +10,7 @@ export function isExternal(opts: { table?: Table; tableId?: string }): boolean {
   }
   return false
 }
+
+export function isTable(table: any): table is Table {
+  return table._id && docIds.isTableId(table._id)
+}

packages/server/src/sdk/app/views/index.ts

@@ -10,7 +10,7 @@ import {
   ViewV2ColumnEnriched,
   ViewV2Enriched,
 } from "@budibase/types"
-import { HTTPError, roles } from "@budibase/backend-core"
+import { context, docIds, HTTPError, roles } from "@budibase/backend-core"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -21,7 +21,7 @@ import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./internal"
 import * as external from "./external"
 import sdk from "../../../sdk"
-import { updatePermissionOnRole, PermissionUpdateType } from "../permissions"
+import { PermissionUpdateType, updatePermissionOnRole } from "../permissions"
 
 function pickApi(tableId: any) {
   if (isExternalTableID(tableId)) {
@@ -40,16 +40,85 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
   return pickApi(tableId).getEnriched(viewId)
 }
 
+export async function getTable(view: string | ViewV2): Promise<Table> {
+  const viewId = typeof view === "string" ? view : view.id
+  const cached = context.getTableForView(viewId)
+  if (cached) {
+    return cached
+  }
+  const { tableId } = utils.extractViewInfoFromID(viewId)
+  const table = await sdk.tables.getTable(tableId)
+  context.setTableForView(viewId, table)
+  return table
+}
+
+export function isView(view: any): view is ViewV2 {
+  return view.id && docIds.isViewId(view.id) && view.version === 2
+}
+
+async function guardCalculationViewSchema(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const calculationFields = helpers.views.calculationFields(view)
+  for (const calculationFieldName of Object.keys(calculationFields)) {
+    const schema = calculationFields[calculationFieldName]
+    const targetSchema = table.schema[schema.field]
+    if (!targetSchema) {
+      throw new HTTPError(
+        `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
+        400
+      )
+    }
+
+    if (!helpers.schema.isNumeric(targetSchema)) {
+      throw new HTTPError(
+        `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
+        400
+      )
+    }
+  }
+
+  const groupByFields = helpers.views.basicFields(view)
+  for (const groupByFieldName of Object.keys(groupByFields)) {
+    const targetSchema = table.schema[groupByFieldName]
+    if (!targetSchema) {
+      throw new HTTPError(
+        `Group by field "${groupByFieldName}" does not exist in the table schema`,
+        400
+      )
+    }
+  }
+}
+
 async function guardViewSchema(
   tableId: string,
   view: Omit<ViewV2, "id" | "version">
 ) {
-  const viewSchema = view.schema || {}
   const table = await sdk.tables.getTable(tableId)
 
+  if (helpers.views.isCalculationView(view)) {
+    await guardCalculationViewSchema(table, view)
+  }
+
+  await checkReadonlyFields(table, view)
+  checkRequiredFields(table, view)
+  checkDisplayField(view)
+}
+
+async function checkReadonlyFields(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const viewSchema = view.schema || {}
   for (const field of Object.keys(viewSchema)) {
-    const tableSchemaField = table.schema[field]
+    const viewFieldSchema = viewSchema[field]
-    if (!tableSchemaField) {
+    if (helpers.views.isCalculationField(viewFieldSchema)) {
+      continue
+    }
+
+    const tableFieldSchema = table.schema[field]
+    if (!tableFieldSchema) {
       throw new HTTPError(
         `Field "${field}" is not valid for the requested table`,
         400
@@ -65,18 +134,33 @@ async function guardViewSchema(
       }
     }
   }
+}
 
-  const existingView =
+function checkDisplayField(view: Omit<ViewV2, "id" | "version">) {
-    table?.views && (table.views[view.name] as ViewV2 | undefined)
+  if (view.primaryDisplay) {
+    const viewSchemaField = view.schema?.[view.primaryDisplay]
 
+    if (!viewSchemaField?.visible) {
+      throw new HTTPError(
+        `You can't hide "${view.primaryDisplay}" because it is the display column.`,
+        400
+      )
+    }
+  }
+}
+
+function checkRequiredFields(
+  table: Table,
+  view: Omit<ViewV2, "id" | "version">
+) {
+  const existingView = table.views?.[view.name] as ViewV2 | undefined
   for (const field of Object.values(table.schema)) {
     if (!helpers.schema.isRequired(field.constraints)) {
       continue
     }
 
-    const viewSchemaField = viewSchema[field.name]
+    const viewSchemaField = view.schema?.[field.name]
-    const existingViewSchema =
+    const existingViewSchema = existingView?.schema?.[field.name]
-      existingView?.schema && existingView.schema[field.name]
     if (!viewSchemaField && !existingViewSchema?.visible) {
       // Supporting existing configs with required columns but hidden in views
       continue
@@ -89,24 +173,16 @@ async function guardViewSchema(
       )
     }
 
-    if (viewSchemaField.readonly) {
+    if (
+      helpers.views.isBasicViewField(viewSchemaField) &&
+      viewSchemaField.readonly
+    ) {
       throw new HTTPError(
         `You can't make "${field.name}" readonly because it is a required field.`,
         400
       )
     }
   }
-
-  if (view.primaryDisplay) {
-    const viewSchemaField = viewSchema[view.primaryDisplay]
-
-    if (!viewSchemaField?.visible) {
-      throw new HTTPError(
-        `You can't hide "${view.primaryDisplay}" because it is the display column.`,
-        400
-      )
-    }
-  }
 }
 
 export async function create(

packages/server/src/sdk/users/utils.ts

@@ -130,6 +130,26 @@ export function getUserContextBindings(user: ContextUser) {
     return {}
   }
   // Current user context for bindable search
-  const { _id, _rev, firstName, lastName, email, status, roleId } = user
+  const {
-  return { _id, _rev, firstName, lastName, email, status, roleId }
+    _id,
+    _rev,
+    firstName,
+    lastName,
+    email,
+    status,
+    roleId,
+    globalId,
+    userId,
+  } = user
+  return {
+    _id,
+    _rev,
+    firstName,
+    lastName,
+    email,
+    status,
+    roleId,
+    globalId,
+    userId,
+  }
 }

packages/server/src/threads/automation.ts

@@ -74,7 +74,7 @@ class Orchestrator {
   private job: Job
   private loopStepOutputs: LoopStep[]
   private stopped: boolean
-  private executionOutput: AutomationContext
+  private executionOutput: Omit<AutomationContext, "stepsByName" | "stepsById">
 
   constructor(job: AutomationJob) {
     let automation = job.data.automation
@@ -91,6 +91,7 @@ class Orchestrator {
     // step zero is never used as the template string is zero indexed for customer facing
     this.context = {
       steps: [{}],
+      stepsById: {},
       stepsByName: {},
       trigger: triggerOutput,
     }
@@ -457,8 +458,9 @@ class Orchestrator {
         inputs: steps[stepToLoopIndex].inputs,
       })
 
+      this.context.stepsById[steps[stepToLoopIndex].id] = tempOutput
       const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id
-      this.context.stepsByName![stepName] = tempOutput
+      this.context.stepsByName[stepName] = tempOutput
       this.context.steps[this.context.steps.length] = tempOutput
       this.context.steps = this.context.steps.filter(
         item => !item.hasOwnProperty.call(item, "currentItem")
@@ -517,7 +519,10 @@ class Orchestrator {
         Object.entries(filter).forEach(([_, value]) => {
           Object.entries(value).forEach(([field, _]) => {
             const updatedField = field.replace("{{", "{{ literal ")
-            const fromContext = processStringSync(updatedField, this.context)
+            const fromContext = processStringSync(
+              updatedField,
+              this.processContext(this.context)
+            )
             toFilter[field] = fromContext
           })
         })
@@ -563,9 +568,9 @@ class Orchestrator {
         }
 
         const stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await this.addContextAndProcess(
+        let inputs = await processObject(
           originalStepInput,
-          this.context
+          this.processContext(this.context)
         )
 
         inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
@@ -594,16 +599,16 @@ class Orchestrator {
     return null
   }
 
-  private async addContextAndProcess(inputs: any, context: any) {
+  private processContext(context: AutomationContext) {
     const processContext = {
       ...context,
       steps: {
         ...context.steps,
+        ...context.stepsById,
         ...context.stepsByName,
       },
     }
-
+    return processContext
-    return processObject(inputs, processContext)
   }
 
   private handleStepOutput(
@@ -623,6 +628,7 @@ class Orchestrator {
     } else {
       this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
       this.context.steps[this.context.steps.length] = outputs
+      this.context.stepsById![step.id] = outputs
       const stepName = step.name || step.id
       this.context.stepsByName![stepName] = outputs
     }

packages/server/src/utilities/rowProcessor/index.ts

@@ -18,6 +18,7 @@ import {
   RowAttachment,
   Table,
   User,
+  ViewV2,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import {
@@ -33,7 +34,11 @@ import {
   PROTECTED_INTERNAL_COLUMNS,
 } from "@budibase/shared-core"
 import { processString } from "@budibase/string-templates"
-import { isUserMetadataTable } from "../../api/controllers/row/utils"
+import {
+  getTableFromSource,
+  isUserMetadataTable,
+} from "../../api/controllers/row/utils"
+import sdk from "../../sdk"
 
 export * from "./utils"
 export * from "./attachments"
@@ -67,6 +72,7 @@ export async function processAutoColumn(
   // check its not user table, or whether any of the processing options have been disabled
   const shouldUpdateUserFields =
     !isUserTable && !opts?.reprocessing && !opts?.noAutoRelationships && !noUser
+  let tableMutated = false
   for (let [key, schema] of Object.entries(table.schema)) {
     if (!schema.autocolumn) {
       continue
@@ -99,10 +105,17 @@ export async function processAutoColumn(
           row[key] = schema.lastID + 1
           schema.lastID++
           table.schema[key] = schema
+          tableMutated = true
         }
         break
     }
   }
+
+  if (tableMutated) {
+    const db = context.getAppDB()
+    const resp = await db.put(table)
+    table._rev = resp.rev
+  }
 }
 
 async function processDefaultValues(table: Table, row: Row) {
@@ -169,11 +182,12 @@ export function coerce(row: any, type: string) {
  */
 export async function inputProcessing(
   userId: string | null | undefined,
-  table: Table,
+  source: Table | ViewV2,
   row: Row,
   opts?: AutoColumnProcessingOpts
 ) {
   const clonedRow = cloneDeep(row)
+  const table = await getTableFromSource(source)
 
   const dontCleanseKeys = ["type", "_id", "_rev", "tableId"]
   for (const [key, value] of Object.entries(clonedRow)) {
@@ -228,8 +242,7 @@ export async function inputProcessing(
 
   await processAutoColumn(userId, table, clonedRow, opts)
   await processDefaultValues(table, clonedRow)
-
+  return clonedRow
-  return { table, row: clonedRow }
 }
 
 /**
@@ -242,14 +255,13 @@ export async function inputProcessing(
  * @returns the enriched rows will be returned.
  */
 export async function outputProcessing<T extends Row[] | Row>(
-  table: Table,
+  source: Table | ViewV2,
   rows: T,
   opts: {
     squash?: boolean
     preserveLinks?: boolean
     fromRow?: Row
     skipBBReferences?: boolean
-    fromViewId?: string
   } = {
     squash: true,
     preserveLinks: false,
@@ -264,6 +276,14 @@ export async function outputProcessing<T extends Row[] | Row>(
   } else {
     safeRows = rows
   }
+
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   // SQS returns the rows with full relationship contents
   // attach any linked row information
   let enriched = !opts.preserveLinks
@@ -276,25 +296,25 @@ export async function outputProcessing<T extends Row[] | Row>(
     opts.squash = true
   }
 
-  enriched = await coreOutputProcessing(table, enriched, opts)
+  enriched = await coreOutputProcessing(source, enriched, opts)
 
   if (opts.squash) {
-    enriched = await linkRows.squashLinks(table, enriched, {
+    enriched = await linkRows.squashLinks(source, enriched)
-      fromViewId: opts?.fromViewId,
-    })
   }
 
   return (wasArray ? enriched : enriched[0]) as T
 }
 
 /**
- * This function is similar to the outputProcessing function above, it makes sure that all the provided
+ * This function is similar to the outputProcessing function above, it makes
- * rows are ready for output, but does not have enrichment for squash capabilities which can cause performance issues.
+ * sure that all the provided rows are ready for output, but does not have
- * outputProcessing should be used when responding from the API, while this should be used when internally processing
+ * enrichment for squash capabilities which can cause performance issues.
- * rows for any reason (like part of view operations).
+ * outputProcessing should be used when responding from the API, while this
+ * should be used when internally processing rows for any reason (like part of
+ * view operations).
  */
 export async function coreOutputProcessing(
-  table: Table,
+  source: Table | ViewV2,
   rows: Row[],
   opts: {
     preserveLinks?: boolean
@@ -305,6 +325,13 @@ export async function coreOutputProcessing(
     skipBBReferences: false,
   }
 ): Promise<Row[]> {
+  let table: Table
+  if (sdk.views.isView(source)) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   // process complex types: attachments, bb references...
   for (const [property, column] of Object.entries(table.schema)) {
     if (
@@ -399,6 +426,25 @@ export async function coreOutputProcessing(
         }
       }
     }
+
+    if (sdk.views.isView(source)) {
+      const calculationFields = Object.keys(
+        helpers.views.calculationFields(source)
+      )
+
+      // We ensure all calculation fields are returned as numbers.  During the
+      // testing of this feature it was discovered that the COUNT operation
+      // returns a string for MySQL, MariaDB, and Postgres. But given that all
+      // calculation fields should be numbers, we blanket make sure of that
+      // here.
+      for (const key of calculationFields) {
+        for (const row of rows) {
+          if (typeof row[key] === "string") {
+            row[key] = parseFloat(row[key])
+          }
+        }
+      }
+    }
   }
 
   if (!isUserMetadataTable(table._id!)) {
@@ -409,9 +455,18 @@ export async function coreOutputProcessing(
     const tableFields = Object.keys(table.schema).filter(
       f => table.schema[f].visible !== false
     )
+
     const fields = [...tableFields, ...protectedColumns].map(f =>
       f.toLowerCase()
     )
+
+    if (sdk.views.isView(source)) {
+      const aggregations = helpers.views.calculationFields(source)
+      for (const key of Object.keys(aggregations)) {
+        fields.push(key.toLowerCase())
+      }
+    }
+
     for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (!fields.includes(key.toLowerCase())) {

packages/server/src/utilities/rowProcessor/tests/inputProcessing.spec.ts

@@ -65,7 +65,7 @@ describe("rowProcessor - inputProcessing", () => {
 
     processInputBBReferenceMock.mockResolvedValue(user)
 
-    const { row } = await inputProcessing(userId, table, newRow)
+    const row = await inputProcessing(userId, table, newRow)
 
     expect(bbReferenceProcessor.processInputBBReference).toHaveBeenCalledTimes(
       1
@@ -117,7 +117,7 @@ describe("rowProcessor - inputProcessing", () => {
 
     processInputBBReferencesMock.mockResolvedValue(user)
 
-    const { row } = await inputProcessing(userId, table, newRow)
+    const row = await inputProcessing(userId, table, newRow)
 
     expect(bbReferenceProcessor.processInputBBReferences).toHaveBeenCalledTimes(
       1
@@ -164,7 +164,7 @@ describe("rowProcessor - inputProcessing", () => {
       name: "Jack",
     }
 
-    const { row } = await inputProcessing(userId, table, newRow)
+    const row = await inputProcessing(userId, table, newRow)
 
     expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled()
     expect(row).toEqual({ ...newRow, user: undefined })
@@ -207,7 +207,7 @@ describe("rowProcessor - inputProcessing", () => {
         user: userValue,
       }
 
-      const { row } = await inputProcessing(userId, table, newRow)
+      const row = await inputProcessing(userId, table, newRow)
 
       if (userValue === undefined) {
         // The 'user' field is omitted
@@ -262,7 +262,7 @@ describe("rowProcessor - inputProcessing", () => {
       user: "123",
     }
 
-    const { row } = await inputProcessing(userId, table, newRow)
+    const row = await inputProcessing(userId, table, newRow)
 
     expect(bbReferenceProcessor.processInputBBReferences).not.toHaveBeenCalled()
     expect(row).toEqual({

packages/server/src/utilities/schema.ts

@@ -148,9 +148,16 @@ export function parse(rows: Rows, table: Table): Rows {
 
     Object.keys(row).forEach(columnName => {
       const columnData = row[columnName]
+
+      if (columnName === "_id") {
+        parsedRow[columnName] = columnData
+        return
+      }
+
       const schema = table.schema
       if (!(columnName in schema)) {
-        // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+        // Objects can be present in the row data but not in the schema, so make
+        // sure we don't proceed in such a case
         return
       }
 

packages/shared-core/src/filters.ts

@@ -127,7 +127,7 @@ export function recurseLogicalOperators(
   fn: (f: SearchFilters) => SearchFilters
 ) {
   for (const logical of LOGICAL_OPERATORS) {
-    if (filters[logical]) {
+    if (filters?.[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
       )

packages/shared-core/src/helpers/cron.ts

@@ -1,4 +1,5 @@
 import cronValidate from "cron-validate"
+import cronParser from "cron-parser"
 
 const INPUT_CRON_START = "(Input cron: "
 const ERROR_SWAPS = {
@@ -30,6 +31,19 @@ function improveErrors(errors: string[]): string[] {
   return finalErrors
 }
 
+export function getNextExecutionDates(
+  cronExpression: string,
+  limit: number = 4
+): string[] {
+  const parsed = cronParser.parseExpression(cronExpression)
+  const nextRuns = []
+  for (let i = 0; i < limit; i++) {
+    nextRuns.push(parsed.next().toString())
+  }
+
+  return nextRuns
+}
+
 export function validate(
   cronExpression: string
 ): { valid: false; err: string[] } | { valid: true } {

packages/shared-core/src/helpers/index.ts

@@ -2,3 +2,4 @@ export * from "./helpers"
 export * from "./integrations"
 export * as cron from "./cron"
 export * as schema from "./schema"
+export * as views from "./views"

packages/shared-core/src/helpers/schema.ts

@@ -45,3 +45,7 @@ export function decodeNonAscii(str: string): string {
     String.fromCharCode(parseInt(p1, 16))
   )
 }
+
+export function isNumeric(field: FieldSchema) {
+  return field.type === FieldType.NUMBER || field.type === FieldType.BIGINT
+}

packages/shared-core/src/helpers/views.ts

@@ -0,0 +1,33 @@
+import {
+  BasicViewFieldMetadata,
+  ViewCalculationFieldMetadata,
+  ViewFieldMetadata,
+  ViewV2,
+} from "@budibase/types"
+import { pickBy } from "lodash"
+
+export function isCalculationField(
+  field: ViewFieldMetadata
+): field is ViewCalculationFieldMetadata {
+  return "calculationType" in field
+}
+
+export function isBasicViewField(
+  field: ViewFieldMetadata
+): field is BasicViewFieldMetadata {
+  return !isCalculationField(field)
+}
+
+type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
+
+export function isCalculationView(view: UnsavedViewV2) {
+  return Object.values(view.schema || {}).some(isCalculationField)
+}
+
+export function calculationFields(view: UnsavedViewV2) {
+  return pickBy(view.schema || {}, isCalculationField)
+}
+
+export function basicFields(view: UnsavedViewV2) {
+  return pickBy(view.schema || {}, field => !isCalculationField(field))
+}

packages/types/src/documents/app/view.ts

@@ -33,15 +33,24 @@ export interface View {
   groupBy?: string
 }
 
-export type ViewFieldMetadata = UIFieldMetadata & {
+export interface BasicViewFieldMetadata extends UIFieldMetadata {
   readonly?: boolean
   columns?: Record<string, RelationSchemaField>
 }
 
-export type RelationSchemaField = UIFieldMetadata & {
+export interface RelationSchemaField extends UIFieldMetadata {
   readonly?: boolean
 }
 
+export interface ViewCalculationFieldMetadata extends BasicViewFieldMetadata {
+  calculationType: CalculationType
+  field: string
+}
+
+export type ViewFieldMetadata =
+  | BasicViewFieldMetadata
+  | ViewCalculationFieldMetadata
+
 export enum CalculationType {
   SUM = "sum",
   AVG = "avg",
@@ -50,11 +59,6 @@ export enum CalculationType {
   MAX = "max",
 }
 
-export type ViewCalculationFieldMetadata = ViewFieldMetadata & {
-  calculationType: CalculationType
-  field: string
-}
-
 export interface ViewV2 {
   version: 2
   id: string
@@ -67,7 +71,7 @@ export interface ViewV2 {
     order?: SortOrder
     type?: SortType
   }
-  schema?: Record<string, ViewFieldMetadata | ViewCalculationFieldMetadata>
+  schema?: Record<string, ViewFieldMetadata>
 }
 
 export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema

packages/types/src/sdk/row.ts

@@ -1,8 +1,14 @@
 import { SortOrder, SortType } from "../api"
 import { SearchFilters } from "./search"
-import { Row } from "../documents"
+import { CalculationType, Row } from "../documents"
 import { WithRequired } from "../shared"
 
+export interface Aggregation {
+  name: string
+  calculationType: CalculationType
+  field: string
+}
+
 export interface SearchParams {
   tableId?: string
   viewId?: string

packages/types/src/sdk/search.ts

@@ -2,6 +2,7 @@ import { Operation } from "./datasources"
 import { Row, Table, DocumentType } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
+import { Aggregation } from "./row"
 
 export enum BasicOperator {
   EQUAL = "equal",
@@ -67,6 +68,8 @@ type RangeFilter = Record<
   [InternalSearchFilterOperator.COMPLEX_ID_OPERATOR]?: never
 }
 
+type LogicalFilter = { conditions: SearchFilters[] }
+
 export type AnySearchFilter = BasicFilter | ArrayFilter | RangeFilter
 
 export interface SearchFilters {
@@ -91,12 +94,8 @@ export interface SearchFilters {
   // specific document type (such as just rows)
   documentType?: DocumentType
 
-  [LogicalOperator.AND]?: {
+  [LogicalOperator.AND]?: LogicalFilter
-    conditions: SearchFilters[]
+  [LogicalOperator.OR]?: LogicalFilter
-  }
-  [LogicalOperator.OR]?: {
-    conditions: SearchFilters[]
-  }
 }
 
 export type SearchFilterKey = keyof Omit<
@@ -154,6 +153,7 @@ export interface QueryJson {
   }
   resource?: {
     fields: string[]
+    aggregations?: Aggregation[]
   }
   filters?: SearchFilters
   sort?: SortJson

packages/types/src/shared/typeUtils.ts

@@ -4,6 +4,29 @@ export type DeepPartial<T> = {
 
 export type ISO8601 = string
 
+/**
+ * RequiredKeys make it such that you _must_ assign a value to every key in the
+ * type. It differs subtly from Required<T> in that it doesn't change the type
+ * of the fields, you can specify undefined as a value and that's fine.
+ *
+ * Example:
+ *
+ * ```ts
+ * interface Foo {
+ *   bar: string
+ *   baz?: string
+ * }
+ *
+ * type FooRequiredKeys = RequiredKeys<Foo>
+ * type FooRequired = Required<Foo>
+ *
+ * const a: FooRequiredKeys = { bar: "hello", baz: undefined }
+ * const b: FooRequired = { bar: "hello", baz: undefined }
+ * ```
+ *
+ * In this code, a passes type checking whereas b does not. This is because
+ * Required<Foo> makes baz non-optional.
+ */
 export type RequiredKeys<T> = {
   [K in keyof Required<T>]: T[K]
 }

yarn.lock

@@ -817,23 +817,23 @@
     tslib "^2.2.0"
 
 "@azure/msal-browser@^3.11.1":
-  version "3.23.0"
+  version "3.24.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.23.0.tgz#446aaf268247e5943f464f007d3aa3a04abfe95b"
+  resolved "https://registry.yarnpkg.com/@azure/msal-browser/-/msal-browser-3.24.0.tgz#3208047672d0b0c943b0bef5f995d510d6582ae4"
-  integrity sha512-+QgdMvaeEpdtgRTD7AHHq9aw8uga7mXVHV1KshO1RQ2uI5B55xJ4aEpGlg/ga3H+0arEVcRfT4ZVmX7QLXiCVw==
+  integrity sha512-JGNV9hTYAa7lsum9IMIibn2kKczAojNihGo1hi7pG0kNrcKej530Fl6jxwM05A44/6I079CSn6WxYxbVhKUmWg==
   dependencies:
-    "@azure/msal-common" "14.14.2"
+    "@azure/msal-common" "14.15.0"
 
-"@azure/msal-common@14.14.2":
+"@azure/msal-common@14.15.0":
-  version "14.14.2"
+  version "14.15.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.14.2.tgz#583b4ac9c089953718d7a5e2f3b8df2d4dbb17f4"
+  resolved "https://registry.yarnpkg.com/@azure/msal-common/-/msal-common-14.15.0.tgz#0e27ac0bb88fe100f4f8d1605b64d5c268636a55"
-  integrity sha512-XV0P5kSNwDwCA/SjIxTe9mEAsKB0NqGNSuaVrkCCE2lAyBr/D6YtD80Vkdp4tjWnPFwjzkwldjr1xU/facOJog==
+  integrity sha512-ImAQHxmpMneJ/4S8BRFhjt1MZ3bppmpRPYYNyzeQPeFN288YKbb8TmmISQEbtfkQ1BPASvYZU5doIZOPBAqENQ==
 
 "@azure/msal-node@^2.9.2":
-  version "2.13.1"
+  version "2.14.0"
-  resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.13.1.tgz#f144371275b7c3cbe564762b84772a9732457a47"
+  resolved "https://registry.yarnpkg.com/@azure/msal-node/-/msal-node-2.14.0.tgz#7881895d41b03d8b9b38a29550ba3bbb15f73b3c"
-  integrity sha512-sijfzPNorKt6+9g1/miHwhj6Iapff4mPQx1azmmZExgzUROqWTM1o3ACyxDja0g47VpowFy/sxTM/WsuCyXTiw==
+  integrity sha512-rrfzIpG3Q1rHjVYZmHAEDidWAZZ2cgkxlIcMQ8dHebRISaZ2KCV33Q8Vs+uaV6lxweROabNxKFlR2lIKagZqYg==
   dependencies:
-    "@azure/msal-common" "14.14.2"
+    "@azure/msal-common" "14.15.0"
     jsonwebtoken "^9.0.0"
     uuid "^8.3.0"
 
@@ -9250,6 +9250,13 @@ cron-parser@^4.2.1:
   dependencies:
     luxon "^3.2.1"
 
+cron-parser@^4.9.0:
+  version "4.9.0"
+  resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-4.9.0.tgz#0340694af3e46a0894978c6f52a6dbb5c0f11ad5"
+  integrity sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==
+  dependencies:
+    luxon "^3.2.1"
+
 cron-validate@1.4.5:
   version "1.4.5"
   resolved "https://registry.yarnpkg.com/cron-validate/-/cron-validate-1.4.5.tgz#eceb221f7558e6302e5f84c7b3a454fdf4d064c3"