Merge branch 'master' of github.com:budibase/budibase into budi-8960-issue-sorting-dates-when-using-mmddyyyy

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.39",
+  "version": "3.2.41",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/src/sql/sql.ts

@@ -272,17 +272,6 @@ class InternalBuilder {
     return parts.join(".")
   }
 
-  private isFullSelectStatementRequired(): boolean {
-    for (let column of Object.values(this.table.schema)) {
-      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
-        return true
-      } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
-        return true
-      }
-    }
-    return false
-  }
-
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
     const { table, resource } = this.query
 
@@ -292,11 +281,9 @@ class InternalBuilder {
 
     const alias = this.getTableName(table)
     const schema = this.table.schema
-    if (!this.isFullSelectStatementRequired()) {
+
-      return [this.knex.raw("??", [`${alias}.*`])]
-    }
     // get just the fields for this table
-    return resource.fields
+    const tableFields = resource.fields
       .map(field => {
         const parts = field.split(/\./g)
         let table: string | undefined = undefined
@@ -311,34 +298,33 @@ class InternalBuilder {
         return { table, column, field }
       })
       .filter(({ table }) => !table || table === alias)
-      .map(({ table, column, field }) => {
-        const columnSchema = schema[column]
 
-        if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
+    return tableFields.map(({ table, column, field }) => {
-          return this.knex.raw(`??::money::numeric as ??`, [
+      const columnSchema = schema[column]
-            this.rawQuotedIdentifier([table, column].join(".")),
-            this.knex.raw(this.quote(field)),
-          ])
-        }
 
-        if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
+      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
-          // Time gets returned as timestamp from mssql, not matching the expected
+        return this.knex.raw(`??::money::numeric as ??`, [
-          // HH:mm format
+          this.rawQuotedIdentifier([table, column].join(".")),
+          this.knex.raw(this.quote(field)),
+        ])
+      }
 
-          // TODO: figure out how to express this safely without string
+      if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
-          // interpolation.
+        // Time gets returned as timestamp from mssql, not matching the expected
-          return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
+        // HH:mm format
-            this.rawQuotedIdentifier(field),
-            this.knex.raw(this.quote(field)),
-          ])
-        }
 
-        if (table) {
+        return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
-          return this.rawQuotedIdentifier(`${table}.${column}`)
+          this.rawQuotedIdentifier(field),
-        } else {
+          this.knex.raw(this.quote(field)),
-          return this.rawQuotedIdentifier(field)
+        ])
-        }
+      }
-      })
+
+      if (table) {
+        return this.rawQuotedIdentifier(`${table}.${column}`)
+      } else {
+        return this.rawQuotedIdentifier(field)
+      }
+    })
   }
 
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
@@ -1291,6 +1277,7 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
+
       const relatedTable = tables[toTable]
       if (!relatedTable) {
         throw new Error(`related table "${toTable}" not found in datasource`)
@@ -1319,6 +1306,10 @@ class InternalBuilder {
       const fieldList = relationshipFields.map(field =>
         this.buildJsonField(relatedTable, field)
       )
+      if (!fieldList.length) {
+        continue
+      }
+
       const fieldListFormatted = fieldList
         .map(f => {
           const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","
@@ -1359,7 +1350,9 @@ class InternalBuilder {
       )
 
       const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => {
-        subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit())
+        subQuery = subQuery
+          .select(relationshipFields)
+          .limit(getRelationshipLimit())
         // @ts-ignore - the from alias syntax isn't in Knex typing
         return knex.select(select).from({
           [toAlias]: subQuery,
@@ -1589,11 +1582,12 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { operation, filters, paginate, relationships, table } = this.query
+    const { operation, filters, paginate, relationships, table } = this.query
     const { limits } = opts
 
     // start building the query
     let query = this.qualifiedKnex()
+
     // handle pagination
     let foundOffset: number | null = null
     let foundLimit = limits?.query || limits?.base
@@ -1642,7 +1636,7 @@ class InternalBuilder {
       const mainTable = this.query.tableAliases?.[table.name] || table.name
       const cte = this.addSorting(
         this.knex
-          .with("paginated", query)
+          .with("paginated", query.clone().clearSelect().select("*"))
           .select(this.generateSelectStatement())
           .from({
             [mainTable]: "paginated",

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -49,7 +49,6 @@
   import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core"
   import ServerBindingPanel from "@/components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
-  import { isEnabled } from "@/helpers/featureFlags"
   import { getUserBindings } from "@/dataBinding"
 
   export let field
@@ -168,7 +167,6 @@
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
-  $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
   $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== FieldType.LINK &&
@@ -300,7 +298,7 @@
     }
 
     // Ensure we don't have a default value if we can't have one
-    if (!canHaveDefault || !defaultValuesEnabled) {
+    if (!canHaveDefault) {
       delete saveColumn.default
     }
 
@@ -848,51 +846,49 @@
     </div>
   {/if}
 
-  {#if defaultValuesEnabled}
+  {#if editableColumn.type === FieldType.OPTIONS}
-    {#if editableColumn.type === FieldType.OPTIONS}
+    <Select
-      <Select
+      disabled={!canHaveDefault}
-        disabled={!canHaveDefault}
+      options={editableColumn.constraints?.inclusion || []}
-        options={editableColumn.constraints?.inclusion || []}
+      label="Default value"
-        label="Default value"
+      value={editableColumn.default}
-        value={editableColumn.default}
+      on:change={e => (editableColumn.default = e.detail)}
-        on:change={e => (editableColumn.default = e.detail)}
+      placeholder="None"
-        placeholder="None"
+    />
-      />
+  {:else if editableColumn.type === FieldType.ARRAY}
-    {:else if editableColumn.type === FieldType.ARRAY}
+    <Multiselect
-      <Multiselect
+      disabled={!canHaveDefault}
-        disabled={!canHaveDefault}
+      options={editableColumn.constraints?.inclusion || []}
-        options={editableColumn.constraints?.inclusion || []}
+      label="Default value"
-        label="Default value"
+      value={editableColumn.default}
-        value={editableColumn.default}
+      on:change={e =>
-        on:change={e =>
+        (editableColumn.default = e.detail?.length ? e.detail : undefined)}
-          (editableColumn.default = e.detail?.length ? e.detail : undefined)}
+      placeholder="None"
-        placeholder="None"
+    />
-      />
+  {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
-    {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
+    {@const defaultValue =
-      {@const defaultValue =
+      editableColumn.type === FieldType.BB_REFERENCE_SINGLE
-        editableColumn.type === FieldType.BB_REFERENCE_SINGLE
+        ? SingleUserDefault
-          ? SingleUserDefault
+        : MultiUserDefault}
-          : MultiUserDefault}
+    <Toggle
-      <Toggle
+      disabled={!canHaveDefault}
-        disabled={!canHaveDefault}
+      text="Default to current user"
-        text="Default to current user"
+      value={editableColumn.default === defaultValue}
-        value={editableColumn.default === defaultValue}
+      on:change={e =>
-        on:change={e =>
+        (editableColumn.default = e.detail ? defaultValue : undefined)}
-          (editableColumn.default = e.detail ? defaultValue : undefined)}
+    />
-      />
+  {:else}
-    {:else}
+    <ModalBindableInput
-      <ModalBindableInput
+      disabled={!canHaveDefault}
-        disabled={!canHaveDefault}
+      panel={ServerBindingPanel}
-        panel={ServerBindingPanel}
+      title="Default value"
-        title="Default value"
+      label="Default value"
-        label="Default value"
+      placeholder="None"
-        placeholder="None"
+      value={editableColumn.default}
-        value={editableColumn.default}
+      on:change={e => (editableColumn.default = e.detail)}
-        on:change={e => (editableColumn.default = e.detail)}
+      bindings={defaultValueBindings}
-        bindings={defaultValueBindings}
+      allowJS
-        allowJS
+    />
-      />
-    {/if}
   {/if}
 </Layout>
 

packages/builder/src/components/common/bindings/SnippetDrawer.svelte

@@ -28,7 +28,9 @@
   let loading = false
   let deleteConfirmationDialog
 
-  $: defaultName = getSequentialName($snippets, "MySnippet", x => x.name)
+  $: defaultName = getSequentialName($snippets, "MySnippet", {
+    getName: x => x.name,
+  })
   $: key = snippet?.name
   $: name = snippet?.name || defaultName
   $: code = snippet?.code ? encodeJSBinding(snippet.code) : ""

packages/builder/src/constants/backend/index.ts

@@ -16,7 +16,10 @@ export {
 
 export const AUTO_COLUMN_SUB_TYPES = AutoFieldSubType
 
-export const AUTO_COLUMN_DISPLAY_NAMES = {
+export const AUTO_COLUMN_DISPLAY_NAMES: Record<
+  keyof typeof AUTO_COLUMN_SUB_TYPES,
+  string
+> = {
   AUTO_ID: "Auto ID",
   CREATED_BY: "Created By",
   CREATED_AT: "Created At",
@@ -209,13 +212,6 @@ export const Roles = {
   BUILDER: "BUILDER",
 }
 
-export function isAutoColumnUserRelationship(subtype) {
-  return (
-    subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY ||
-    subtype === AUTO_COLUMN_SUB_TYPES.UPDATED_BY
-  )
-}
-
 export const PrettyRelationshipDefinitions = {
   MANY: "Many rows",
   ONE: "One row",

packages/builder/src/helpers/duplicate.ts

@@ -10,13 +10,13 @@
  *
  * Repl
  */
-export const duplicateName = (name, allNames) => {
+export const duplicateName = (name: string, allNames: string[]) => {
   const duplicatePattern = new RegExp(`\\s(\\d+)$`)
   const baseName = name.split(duplicatePattern)[0]
   const isDuplicate = new RegExp(`${baseName}\\s(\\d+)$`)
 
   // get the sequence from matched names
-  const sequence = []
+  const sequence: number[] = []
   allNames.filter(n => {
     if (n === baseName) {
       return true
@@ -70,12 +70,18 @@ export const duplicateName = (name, allNames) => {
  * @param getName optional function to extract the name for an item, if not a
  *  flat array of strings
  */
-export const getSequentialName = (
+export const getSequentialName = <T extends any>(
-  items,
+  items: T[] | null,
-  prefix,
+  prefix: string | null,
-  { getName = x => x, numberFirstItem = false } = {}
+  {
+    getName,
+    numberFirstItem,
+  }: {
+    getName?: (item: T) => string
+    numberFirstItem?: boolean
+  } = {}
 ) => {
-  if (!prefix?.length || !getName) {
+  if (!prefix?.length) {
     return null
   }
   const trimmedPrefix = prefix.trim()
@@ -85,7 +91,7 @@ export const getSequentialName = (
   }
   let max = 0
   items.forEach(item => {
-    const name = getName(item)
+    const name = getName?.(item) ?? item
     if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) {
       return
     }

packages/builder/src/helpers/featureFlags.ts

@@ -1,7 +1,8 @@
+import { FeatureFlag } from "@budibase/types"
 import { auth } from "../stores/portal"
 import { get } from "svelte/store"
 
-export const isEnabled = featureFlag => {
+export const isEnabled = (featureFlag: FeatureFlag | `${FeatureFlag}`) => {
   const user = get(auth).user
   return !!user?.flags?.[featureFlag]
 }

packages/builder/src/helpers/fetchData.ts

@@ -1,13 +1,21 @@
 import { writable } from "svelte/store"
 import { API } from "@/api"
 
-export default function (url) {
+export default function (url: string) {
-  const store = writable({ status: "LOADING", data: {}, error: {} })
+  const store = writable<{
+    status: "LOADING" | "SUCCESS" | "ERROR"
+    data: object
+    error?: unknown
+  }>({
+    status: "LOADING",
+    data: {},
+    error: {},
+  })
 
   async function get() {
     store.update(u => ({ ...u, status: "LOADING" }))
     try {
-      const data = await API.get({ url })
+      const data = await API.get<object>({ url })
       store.set({ data, status: "SUCCESS" })
     } catch (e) {
       store.set({ data: {}, error: e, status: "ERROR" })

packages/builder/src/helpers/helpers.js

@@ -1,46 +0,0 @@
-import { last, flow } from "lodash/fp"
-
-export const buildStyle = styles => {
-  let str = ""
-  for (let s in styles) {
-    if (styles[s]) {
-      let key = convertCamel(s)
-      str += `${key}: ${styles[s]}; `
-    }
-  }
-  return str
-}
-
-export const convertCamel = str => {
-  return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`)
-}
-
-export const pipe = (arg, funcs) => flow(funcs)(arg)
-
-export const capitalise = s => {
-  if (!s) {
-    return s
-  }
-  return s.substring(0, 1).toUpperCase() + s.substring(1)
-}
-
-export const lowercase = s => s.substring(0, 1).toLowerCase() + s.substring(1)
-
-export const lowercaseExceptFirst = s =>
-  s.charAt(0) + s.substring(1).toLowerCase()
-
-export const get_name = s => (!s ? "" : last(s.split("/")))
-
-export const get_capitalised_name = name => pipe(name, [get_name, capitalise])
-
-export const isBuilderInputFocused = e => {
-  const activeTag = document.activeElement?.tagName.toLowerCase()
-  const inCodeEditor = document.activeElement?.classList?.contains("cm-content")
-  if (
-    (inCodeEditor || ["input", "textarea"].indexOf(activeTag) !== -1) &&
-    e.key !== "Escape"
-  ) {
-    return true
-  }
-  return false
-}

packages/builder/src/helpers/helpers.ts

@@ -0,0 +1,50 @@
+import type { Many } from "lodash"
+import { last, flow } from "lodash/fp"
+
+export const buildStyle = (styles: Record<string, any>) => {
+  let str = ""
+  for (let s in styles) {
+    if (styles[s]) {
+      let key = convertCamel(s)
+      str += `${key}: ${styles[s]}; `
+    }
+  }
+  return str
+}
+
+export const convertCamel = (str: string) => {
+  return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`)
+}
+
+export const pipe = (arg: string, funcs: Many<(...args: any[]) => any>) =>
+  flow(funcs)(arg)
+
+export const capitalise = (s: string) => {
+  if (!s) {
+    return s
+  }
+  return s.substring(0, 1).toUpperCase() + s.substring(1)
+}
+
+export const lowercase = (s: string) =>
+  s.substring(0, 1).toLowerCase() + s.substring(1)
+
+export const lowercaseExceptFirst = (s: string) =>
+  s.charAt(0) + s.substring(1).toLowerCase()
+
+export const get_name = (s: string) => (!s ? "" : last(s.split("/")))
+
+export const get_capitalised_name = (name: string) =>
+  pipe(name, [get_name, capitalise])
+
+export const isBuilderInputFocused = (e: KeyboardEvent) => {
+  const activeTag = document.activeElement?.tagName.toLowerCase()
+  const inCodeEditor = document.activeElement?.classList?.contains("cm-content")
+  if (
+    (inCodeEditor || ["input", "textarea"].indexOf(activeTag!) !== -1) &&
+    e.key !== "Escape"
+  ) {
+    return true
+  }
+  return false
+}

packages/builder/src/helpers/keyUtils.js

@@ -1,7 +0,0 @@
-function handleEnter(fnc) {
-  return e => e.key === "Enter" && fnc()
-}
-
-export const keyUtils = {
-  handleEnter,
-}

packages/builder/src/helpers/keyUtils.ts

@@ -0,0 +1,7 @@
+function handleEnter(fnc: () => void) {
+  return (e: KeyboardEvent) => e.key === "Enter" && fnc()
+}
+
+export const keyUtils = {
+  handleEnter,
+}

packages/builder/src/helpers/pagination.ts

@@ -1,6 +1,16 @@
 import { writable } from "svelte/store"
 
-function defaultValue() {
+interface PaginationStore {
+  nextPage: string | null | undefined
+  page: string | null | undefined
+  hasPrevPage: boolean
+  hasNextPage: boolean
+  loading: boolean
+  pageNumber: number
+  pages: string[]
+}
+
+function defaultValue(): PaginationStore {
   return {
     nextPage: null,
     page: undefined,
@@ -29,13 +39,13 @@ export function createPaginationStore() {
     update(state => {
       state.pageNumber++
       state.page = state.nextPage
-      state.pages.push(state.page)
+      state.pages.push(state.page!)
       state.hasPrevPage = state.pageNumber > 1
       return state
     })
   }
 
-  function fetched(hasNextPage, nextPage) {
+  function fetched(hasNextPage: boolean, nextPage: string) {
     update(state => {
       state.hasNextPage = hasNextPage
       state.nextPage = nextPage

packages/builder/src/helpers/planTitle.ts

@@ -1,6 +1,6 @@
 import { PlanType } from "@budibase/types"
 
-export function getFormattedPlanName(userPlanType) {
+export function getFormattedPlanName(userPlanType: PlanType) {
   let planName
   switch (userPlanType) {
     case PlanType.PRO:
@@ -29,6 +29,6 @@ export function getFormattedPlanName(userPlanType) {
   return `${planName} Plan`
 }
 
-export function isPremiumOrAbove(userPlanType) {
+export function isPremiumOrAbove(userPlanType: PlanType) {
   return ![PlanType.PRO, PlanType.TEAM, PlanType.FREE].includes(userPlanType)
 }

packages/builder/src/helpers/sanitizeUrl.ts

@@ -1,4 +1,4 @@
-export default function (url) {
+export default function (url: string) {
   return url
     .split("/")
     .map(part => {

packages/builder/src/helpers/utils.js

@@ -1,75 +0,0 @@
-import { FieldType } from "@budibase/types"
-import { ActionStepID } from "@/constants/backend/automations"
-import { TableNames } from "@/constants"
-import {
-  AUTO_COLUMN_DISPLAY_NAMES,
-  AUTO_COLUMN_SUB_TYPES,
-  FIELDS,
-  isAutoColumnUserRelationship,
-} from "@/constants/backend"
-import { isEnabled } from "@/helpers/featureFlags"
-
-export function getAutoColumnInformation(enabled = true) {
-  let info = {}
-  for (const [key, subtype] of Object.entries(AUTO_COLUMN_SUB_TYPES)) {
-    // Because it's possible to replicate the functionality of CREATED_AT and
-    // CREATED_BY columns, we disable their creation when the DEFAULT_VALUES
-    // feature flag is enabled.
-    if (isEnabled("DEFAULT_VALUES")) {
-      if (
-        subtype === AUTO_COLUMN_SUB_TYPES.CREATED_AT ||
-        subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY
-      ) {
-        continue
-      }
-    }
-    info[subtype] = { enabled, name: AUTO_COLUMN_DISPLAY_NAMES[key] }
-  }
-  return info
-}
-
-export function buildAutoColumn(tableName, name, subtype) {
-  let type, constraints
-  switch (subtype) {
-    case AUTO_COLUMN_SUB_TYPES.UPDATED_BY:
-    case AUTO_COLUMN_SUB_TYPES.CREATED_BY:
-      type = FieldType.LINK
-      constraints = FIELDS.LINK.constraints
-      break
-    case AUTO_COLUMN_SUB_TYPES.AUTO_ID:
-      type = FieldType.NUMBER
-      constraints = FIELDS.NUMBER.constraints
-      break
-    case AUTO_COLUMN_SUB_TYPES.UPDATED_AT:
-    case AUTO_COLUMN_SUB_TYPES.CREATED_AT:
-      type = FieldType.DATETIME
-      constraints = FIELDS.DATETIME.constraints
-      break
-    default:
-      type = FieldType.STRING
-      constraints = FIELDS.STRING.constraints
-      break
-  }
-  if (Object.values(AUTO_COLUMN_SUB_TYPES).indexOf(subtype) === -1) {
-    throw "Cannot build auto column with supplied subtype"
-  }
-  const base = {
-    name,
-    type,
-    subtype,
-    icon: "ri-magic-line",
-    autocolumn: true,
-    constraints,
-  }
-  if (isAutoColumnUserRelationship(subtype)) {
-    base.tableId = TableNames.USERS
-    base.fieldName = `${tableName}-${name}`
-  }
-  return base
-}
-
-export function checkForCollectStep(automation) {
-  return automation.definition.steps.some(
-    step => step.stepId === ActionStepID.COLLECT
-  )
-}

packages/builder/src/helpers/utils.ts

@@ -0,0 +1,96 @@
+import {
+  AutoFieldSubType,
+  Automation,
+  DateFieldMetadata,
+  FieldType,
+  NumberFieldMetadata,
+  RelationshipFieldMetadata,
+  RelationshipType,
+} from "@budibase/types"
+import { ActionStepID } from "@/constants/backend/automations"
+import { TableNames } from "@/constants"
+import {
+  AUTO_COLUMN_DISPLAY_NAMES,
+  AUTO_COLUMN_SUB_TYPES,
+  FIELDS,
+} from "@/constants/backend"
+import { utils } from "@budibase/shared-core"
+
+type AutoColumnInformation = Partial<
+  Record<AutoFieldSubType, { enabled: boolean; name: string }>
+>
+
+export function getAutoColumnInformation(
+  enabled = true
+): AutoColumnInformation {
+  const info: AutoColumnInformation = {}
+  for (const [key, subtype] of Object.entries(AUTO_COLUMN_SUB_TYPES)) {
+    // Because it's possible to replicate the functionality of CREATED_AT and
+    // CREATED_BY columns with user column default values, we disable their creation
+    if (
+      subtype === AUTO_COLUMN_SUB_TYPES.CREATED_AT ||
+      subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY
+    ) {
+      continue
+    }
+    const typedKey = key as keyof typeof AUTO_COLUMN_SUB_TYPES
+    info[subtype] = {
+      enabled,
+      name: AUTO_COLUMN_DISPLAY_NAMES[typedKey],
+    }
+  }
+  return info
+}
+
+export function buildAutoColumn(
+  tableName: string,
+  name: string,
+  subtype: AutoFieldSubType
+): RelationshipFieldMetadata | NumberFieldMetadata | DateFieldMetadata {
+  const base = {
+    name,
+    icon: "ri-magic-line",
+    autocolumn: true,
+  }
+
+  switch (subtype) {
+    case AUTO_COLUMN_SUB_TYPES.UPDATED_BY:
+    case AUTO_COLUMN_SUB_TYPES.CREATED_BY:
+      return {
+        ...base,
+        type: FieldType.LINK,
+        subtype,
+        constraints: FIELDS.LINK.constraints,
+        tableId: TableNames.USERS,
+        fieldName: `${tableName}-${name}`,
+        relationshipType: RelationshipType.MANY_TO_ONE,
+      }
+
+    case AUTO_COLUMN_SUB_TYPES.AUTO_ID:
+      return {
+        ...base,
+        type: FieldType.NUMBER,
+        subtype,
+        constraints: FIELDS.NUMBER.constraints,
+      }
+    case AUTO_COLUMN_SUB_TYPES.UPDATED_AT:
+    case AUTO_COLUMN_SUB_TYPES.CREATED_AT:
+      return {
+        ...base,
+        type: FieldType.DATETIME,
+        subtype,
+        constraints: FIELDS.DATETIME.constraints,
+      }
+
+    default:
+      throw utils.unreachable(subtype, {
+        message: "Cannot build auto column with supplied subtype",
+      })
+  }
+}
+
+export function checkForCollectStep(automation: Automation) {
+  return automation.definition.steps.some(
+    step => step.stepId === ActionStepID.COLLECT
+  )
+}

packages/builder/src/helpers/warnings.ts

@@ -1,4 +1,4 @@
-export const suppressWarnings = warnings => {
+export const suppressWarnings = (warnings: string[]) => {
   if (!warnings?.length) {
     return
   }

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -442,13 +442,11 @@
 
   const onUpdateUserInvite = async (invite, role) => {
     let updateBody = {
-      code: invite.code,
       apps: {
         ...invite.apps,
         [prodAppId]: role,
       },
     }
-
     if (role === Constants.Roles.CREATOR) {
       updateBody.builder = updateBody.builder || {}
       updateBody.builder.apps = [...(updateBody.builder.apps ?? []), prodAppId]
@@ -456,7 +454,7 @@
     } else if (role !== Constants.Roles.CREATOR && invite?.builder?.apps) {
       invite.builder.apps = []
     }
-    await users.updateInvite(updateBody)
+    await users.updateInvite(invite.code, updateBody)
     await filterInvites(query)
   }
 
@@ -470,8 +468,7 @@
     let updated = { ...invite }
     delete updated.info.apps[prodAppId]
 
-    return await users.updateInvite({
+    return await users.updateInvite(updated.code, {
-      code: updated.code,
       apps: updated.apps,
     })
   }

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -191,8 +191,14 @@
           ? "View errors"
           : "View error"}
         on:dismiss={async () => {
-          await automationStore.actions.clearLogErrors({ appId })
+          const automationId = Object.keys(automationErrors[appId] || {})[0]
-          await appsStore.load()
+          if (automationId) {
+            await automationStore.actions.clearLogErrors({
+              appId,
+              automationId,
+            })
+            await appsStore.load()
+          }
         }}
         message={automationErrorMessage(appId)}
       />

packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte

@@ -52,7 +52,7 @@
   ]
 
   const removeUser = async id => {
-    await groups.actions.removeUser(groupId, id)
+    await groups.removeUser(groupId, id)
     fetchGroupUsers.refresh()
   }
 

packages/builder/src/pages/builder/portal/users/users/index.svelte

@@ -251,6 +251,7 @@
       passwordModal.show()
       await fetch.refresh()
     } catch (error) {
+      console.error(error)
       notifications.error("Error creating user")
     }
   }

packages/builder/src/stores/builder/rowActions.ts

@@ -62,7 +62,7 @@ export class RowActionStore extends BudiStore<RowActionState> {
       const existingRowActions = get(this)[tableId] || []
       name = getSequentialName(existingRowActions, "New row action ", {
         getName: x => x.name,
-      })
+      })!
     }
 
     if (!name) {

packages/builder/src/stores/portal/users.ts

@@ -1,41 +1,71 @@
-import { writable } from "svelte/store"
 import { API } from "@/api"
-import { update } from "lodash"
 import { licensing } from "."
 import { sdk } from "@budibase/shared-core"
 import { Constants } from "@budibase/frontend-core"
+import {
+  DeleteInviteUsersRequest,
+  InviteUsersRequest,
+  SearchUsersRequest,
+  SearchUsersResponse,
+  UpdateInviteRequest,
+  User,
+  UserIdentifier,
+  UnsavedUser,
+} from "@budibase/types"
+import { BudiStore } from "../BudiStore"
 
-export function createUsersStore() {
+interface UserInfo {
-  const { subscribe, set } = writable({})
+  email: string
+  password: string
+  forceResetPassword?: boolean
+  role: keyof typeof Constants.BudibaseRoles
+}
 
-  // opts can contain page and search params
+type UserState = SearchUsersResponse & SearchUsersRequest
-  async function search(opts = {}) {
+
+class UserStore extends BudiStore<UserState> {
+  constructor() {
+    super({
+      data: [],
+    })
+  }
+
+  async search(opts: SearchUsersRequest = {}) {
     const paged = await API.searchUsers(opts)
-    set({
+    this.set({
       ...paged,
       ...opts,
     })
     return paged
   }
 
-  async function get(userId) {
+  async get(userId: string) {
     try {
       return await API.getUser(userId)
     } catch (err) {
       return null
     }
   }
-  const fetch = async () => {
+
+  async fetch() {
     return await API.getUsers()
   }
 
-  // One or more users.
+  async onboard(payload: InviteUsersRequest) {
-  async function onboard(payload) {
     return await API.onboardUsers(payload)
   }
 
-  async function invite(payload) {
+  async invite(
-    const users = payload.map(user => {
+    payload: {
+      admin?: boolean
+      builder?: boolean
+      creator?: boolean
+      email: string
+      apps?: any[]
+      groups?: any[]
+    }[]
+  ) {
+    const users: InviteUsersRequest = payload.map(user => {
       let builder = undefined
       if (user.admin || user.builder) {
         builder = { global: true }
@@ -55,11 +85,16 @@ export function createUsersStore() {
     return API.inviteUsers(users)
   }
 
-  async function removeInvites(payload) {
+  async removeInvites(payload: DeleteInviteUsersRequest) {
     return API.removeUserInvites(payload)
   }
 
-  async function acceptInvite(inviteCode, password, firstName, lastName) {
+  async acceptInvite(
+    inviteCode: string,
+    password: string,
+    firstName: string,
+    lastName?: string
+  ) {
     return API.acceptInvite({
       inviteCode,
       password,
@@ -68,21 +103,25 @@ export function createUsersStore() {
     })
   }
 
-  async function fetchInvite(inviteCode) {
+  async fetchInvite(inviteCode: string) {
     return API.getUserInvite(inviteCode)
   }
 
-  async function getInvites() {
+  async getInvites() {
     return API.getUserInvites()
   }
 
-  async function updateInvite(invite) {
+  async updateInvite(code: string, invite: UpdateInviteRequest) {
-    return API.updateUserInvite(invite.code, invite)
+    return API.updateUserInvite(code, invite)
   }
 
-  async function create(data) {
+  async getUserCountByApp(appId: string) {
-    let mappedUsers = data.users.map(user => {
+    return await API.getUserCountByApp(appId)
-      const body = {
+  }
+
+  async create(data: { users: UserInfo[]; groups: any[] }) {
+    let mappedUsers: UnsavedUser[] = data.users.map((user: any) => {
+      const body: UnsavedUser = {
         email: user.email,
         password: user.password,
         roles: {},
@@ -92,17 +131,17 @@ export function createUsersStore() {
       }
 
       switch (user.role) {
-        case "appUser":
+        case Constants.BudibaseRoles.AppUser:
           body.builder = { global: false }
           body.admin = { global: false }
           break
-        case "developer":
+        case Constants.BudibaseRoles.Developer:
           body.builder = { global: true }
           break
-        case "creator":
+        case Constants.BudibaseRoles.Creator:
           body.builder = { creator: true, global: false }
           break
-        case "admin":
+        case Constants.BudibaseRoles.Admin:
           body.admin = { global: true }
           body.builder = { global: true }
           break
@@ -111,43 +150,47 @@ export function createUsersStore() {
       return body
     })
     const response = await API.createUsers(mappedUsers, data.groups)
+    licensing.setQuotaUsage()
 
     // re-search from first page
-    await search()
+    await this.search()
     return response
   }
 
-  async function del(id) {
+  async delete(id: string) {
     await API.deleteUser(id)
-    update(users => users.filter(user => user._id !== id))
+    licensing.setQuotaUsage()
   }
 
-  async function getUserCountByApp(appId) {
+  async bulkDelete(users: UserIdentifier[]) {
-    return await API.getUserCountByApp(appId)
+    const res = API.deleteUsers(users)
+    licensing.setQuotaUsage()
+    return res
   }
 
-  async function bulkDelete(users) {
+  async save(user: User) {
-    return API.deleteUsers(users)
+    const res = await API.saveUser(user)
+    licensing.setQuotaUsage()
+    return res
   }
 
-  async function save(user) {
+  async addAppBuilder(userId: string, appId: string) {
-    return await API.saveUser(user)
-  }
-
-  async function addAppBuilder(userId, appId) {
     return await API.addAppBuilder(userId, appId)
   }
 
-  async function removeAppBuilder(userId, appId) {
+  async removeAppBuilder(userId: string, appId: string) {
     return await API.removeAppBuilder(userId, appId)
   }
 
-  async function getAccountHolder() {
+  async getAccountHolder() {
     return await API.getAccountHolder()
   }
 
-  const getUserRole = user => {
+  getUserRole(user?: User & { tenantOwnerEmail?: string }) {
-    if (user && user.email === user.tenantOwnerEmail) {
+    if (!user) {
+      return Constants.BudibaseRoles.AppUser
+    }
+    if (user.email === user.tenantOwnerEmail) {
       return Constants.BudibaseRoles.Owner
     } else if (sdk.users.isAdmin(user)) {
       return Constants.BudibaseRoles.Admin
@@ -159,38 +202,6 @@ export function createUsersStore() {
       return Constants.BudibaseRoles.AppUser
     }
   }
-
-  const refreshUsage =
-    fn =>
-    async (...args) => {
-      const response = await fn(...args)
-      await licensing.setQuotaUsage()
-      return response
-    }
-
-  return {
-    subscribe,
-    search,
-    get,
-    getUserRole,
-    fetch,
-    invite,
-    onboard,
-    fetchInvite,
-    getInvites,
-    removeInvites,
-    updateInvite,
-    getUserCountByApp,
-    addAppBuilder,
-    removeAppBuilder,
-    // any operation that adds or deletes users
-    acceptInvite,
-    create: refreshUsage(create),
-    save: refreshUsage(save),
-    bulkDelete: refreshUsage(bulkDelete),
-    delete: refreshUsage(del),
-    getAccountHolder,
-  }
 }
 
-export const users = createUsersStore()
+export const users = new UserStore()

packages/client/manifest.json

@@ -5139,7 +5139,8 @@
       {
         "type": "text",
         "label": "File name",
-        "key": "key"
+        "key": "key",
+        "nested": true
       },
       {
         "type": "event",

packages/client/src/api/patches.ts

@@ -1,19 +1,20 @@
-import { Constants } from "@budibase/frontend-core"
+import { Constants, APIClient } from "@budibase/frontend-core"
 import { FieldTypes } from "../constants"
+import { Row, Table } from "@budibase/types"
 
-export const patchAPI = API => {
+export const patchAPI = (API: APIClient) => {
   /**
    * Enriches rows which contain certain field types so that they can
    * be properly displayed.
    * The ability to create these bindings has been removed, but they will still
    * exist in client apps to support backwards compatibility.
    */
-  const enrichRows = async (rows, tableId) => {
+  const enrichRows = async (rows: Row[], tableId: string) => {
     if (!Array.isArray(rows)) {
       return []
     }
     if (rows.length) {
-      const tables = {}
+      const tables: Record<string, Table> = {}
       for (let row of rows) {
         // Fall back to passed in tableId if row doesn't have it specified
         let rowTableId = row.tableId || tableId
@@ -54,7 +55,7 @@ export const patchAPI = API => {
   const fetchSelf = API.fetchSelf
   API.fetchSelf = async () => {
     const user = await fetchSelf()
-    if (user && user._id) {
+    if (user && "_id" in user && user._id) {
       if (user.roleId === "PUBLIC") {
         // Don't try to enrich a public user as it will 403
         return user
@@ -90,13 +91,14 @@ export const patchAPI = API => {
     return await enrichRows(rows, tableId)
   }
 
-  // Wipe any HBS formulae from table definitions, as these interfere with
+  // Wipe any HBS formulas from table definitions, as these interfere with
   // handlebars enrichment
   const fetchTableDefinition = API.fetchTableDefinition
   API.fetchTableDefinition = async tableId => {
     const definition = await fetchTableDefinition(tableId)
     Object.keys(definition?.schema || {}).forEach(field => {
       if (definition.schema[field]?.type === "formula") {
+        // @ts-expect-error TODO check what use case removing that would break
         delete definition.schema[field].formula
       }
     })

packages/client/src/components/Block.svelte

@@ -1,7 +1,7 @@
 <script>
   import { getContext, onDestroy, onMount, setContext } from "svelte"
   import { builderStore } from "stores/builder.js"
-  import { blockStore } from "stores/blocks.js"
+  import { blockStore } from "stores/blocks"
 
   const component = getContext("component")
   const { styleable } = getContext("sdk")

packages/client/src/components/Component.svelte

@@ -39,7 +39,7 @@
     getActionContextKey,
     getActionDependentContextKeys,
   } from "../utils/buttonActions.js"
-  import { gridLayout } from "utils/grid.js"
+  import { gridLayout } from "utils/grid"
 
   export let instance = {}
   export let parent = null

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -3,7 +3,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { get } from "svelte/store"
 
   export let title

packages/client/src/components/app/deprecated/TableBlock.svelte

@@ -5,7 +5,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { Utils } from "@budibase/frontend-core"
 
   export let title

packages/client/src/components/app/forms/S3Upload.svelte

@@ -2,6 +2,8 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
+  import { builderStore } from "stores/builder.js"
+  import { processStringSync } from "@budibase/string-templates"
 
   export let datasourceId
   export let bucket
@@ -12,6 +14,8 @@
   export let validation
   export let onChange
 
+  const context = getContext("context")
+
   let fieldState
   let fieldApi
   let localFiles = []
@@ -42,6 +46,9 @@
   // Process the file input and return a serializable structure expected by
   // the dropzone component to display the file
   const processFiles = async fileList => {
+    if ($builderStore.inBuilder) {
+      return []
+    }
     return await new Promise(resolve => {
       if (!fileList?.length) {
         return []
@@ -78,9 +85,15 @@
   }
 
   const upload = async () => {
+    const processedFileKey = processStringSync(key, $context)
     loading = true
     try {
-      const res = await API.externalUpload(datasourceId, bucket, key, data)
+      const res = await API.externalUpload(
+        datasourceId,
+        bucket,
+        processedFileKey,
+        data
+      )
       notificationStore.actions.success("File uploaded successfully")
       loading = false
       return res
@@ -126,7 +139,7 @@
   bind:fieldApi
   defaultValue={[]}
 >
-  <div class="content">
+  <div class="content" class:builder={$builderStore.inBuilder}>
     {#if fieldState}
       <CoreDropzone
         value={localFiles}
@@ -149,6 +162,9 @@
 </Field>
 
 <style>
+  .content.builder :global(.spectrum-Dropzone) {
+    pointer-events: none;
+  }
   .content {
     position: relative;
   }

packages/client/src/sdk.js

@@ -74,6 +74,7 @@ export default {
   fetchData,
   QueryUtils,
   ContextScopes: Constants.ContextScopes,
+  // This is not used internally but exposed to users to be used in plugins
   getAPIKey,
   enrichButtonActions,
   processStringSync,

packages/client/src/utils/blocks.ts

@@ -1,10 +1,16 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
-import { API } from "../api/index.js"
+import { API } from "../api"
-import { UILogicalOperator } from "@budibase/types"
+import {
+  BasicOperator,
+  LegacyFilter,
+  UIColumn,
+  UILogicalOperator,
+  UISearchFilter,
+} from "@budibase/types"
 import { Constants } from "@budibase/frontend-core"
 
 // Map of data types to component types for search fields inside blocks
-const schemaComponentMap = {
+const schemaComponentMap: Record<string, string> = {
   string: "stringfield",
   options: "optionsfield",
   number: "numberfield",
@@ -19,7 +25,16 @@ const schemaComponentMap = {
  * @param searchColumns the search columns to use
  * @param schema the datasource schema
  */
-export const enrichSearchColumns = async (searchColumns, schema) => {
+export const enrichSearchColumns = async (
+  searchColumns: string[],
+  schema: Record<
+    string,
+    {
+      tableId: string
+      type: string
+    }
+  >
+) => {
   if (!searchColumns?.length || !schema) {
     return []
   }
@@ -61,12 +76,16 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
  * @param columns the enriched search column structure
  * @param formId the ID of the form containing the search fields
  */
-export const enrichFilter = (filter, columns, formId) => {
+export const enrichFilter = (
+  filter: UISearchFilter,
+  columns: UIColumn[],
+  formId: string
+) => {
   if (!columns?.length) {
     return filter
   }
 
-  let newFilters = []
+  const newFilters: LegacyFilter[] = []
   columns?.forEach(column => {
     const safePath = column.name.split(".").map(safe).join(".")
     const stringType = column.type === "string" || column.type === "formula"
@@ -99,7 +118,7 @@ export const enrichFilter = (filter, columns, formId) => {
       newFilters.push({
         field: column.name,
         type: column.type,
-        operator: stringType ? "string" : "equal",
+        operator: stringType ? BasicOperator.STRING : BasicOperator.EQUAL,
         valueType: "Binding",
         value: `{{ ${binding} }}`,
       })

packages/client/src/utils/grid.ts

@@ -1,7 +1,27 @@
-import { GridSpacing, GridRowHeight } from "constants"
+import { GridSpacing, GridRowHeight } from "@/constants"
 import { builderStore } from "stores"
 import { buildStyleString } from "utils/styleable.js"
 
+interface GridMetadata {
+  id: string
+  styles: Record<string, string | number> & {
+    "--default-width"?: number
+    "--default-height"?: number
+  }
+  interactive: boolean
+  errored: boolean
+  definition?: {
+    size?: {
+      width: number
+      height: number
+    }
+    grid?: { hAlign: string; vAlign: string }
+  }
+  draggable: boolean
+  insideGrid: boolean
+  ignoresLayout: boolean
+}
+
 /**
  * We use CSS variables on components to control positioning and layout of
  * components inside grids.
@@ -44,14 +64,17 @@ export const GridDragModes = {
 }
 
 // Builds a CSS variable name for a certain piece of grid metadata
-export const getGridVar = (device, param) => `--grid-${device}-${param}`
+export const getGridVar = (device: string, param: string) =>
+  `--grid-${device}-${param}`
 
 // Determines whether a JS event originated from immediately within a grid
-export const isGridEvent = e => {
+export const isGridEvent = (e: Event & { target: HTMLElement }): boolean => {
   return (
     e.target.dataset?.indicator === "true" ||
+    // @ts-expect-error: api is not properly typed
     e.target
       .closest?.(".component")
+      // @ts-expect-error
       ?.parentNode.closest(".component")
       ?.childNodes[0]?.classList?.contains("grid")
   )
@@ -59,11 +82,11 @@ export const isGridEvent = e => {
 
 // Svelte action to apply required class names and styles to our component
 // wrappers
-export const gridLayout = (node, metadata) => {
+export const gridLayout = (node: HTMLDivElement, metadata: GridMetadata) => {
-  let selectComponent
+  let selectComponent: ((e: Event) => void) | null
 
   // Applies the required listeners, CSS and classes to a component DOM node
-  const applyMetadata = metadata => {
+  const applyMetadata = (metadata: GridMetadata) => {
     const {
       id,
       styles,
@@ -86,7 +109,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Callback to select the component when clicking on the wrapper
-    selectComponent = e => {
+    selectComponent = (e: Event) => {
       e.stopPropagation()
       builderStore.actions.selectComponent(id)
     }
@@ -100,7 +123,7 @@ export const gridLayout = (node, metadata) => {
     }
     width += 2 * GridSpacing
     height += 2 * GridSpacing
-    let vars = {
+    const vars: Record<string, string | number> = {
       "--default-width": width,
       "--default-height": height,
     }
@@ -135,7 +158,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Apply some metadata to data attributes to speed up lookups
-    const addDataTag = (tagName, device, param) => {
+    const addDataTag = (tagName: string, device: string, param: string) => {
       const val = `${vars[getGridVar(device, param)]}`
       if (node.dataset[tagName] !== val) {
         node.dataset[tagName] = val
@@ -147,11 +170,12 @@ export const gridLayout = (node, metadata) => {
     addDataTag("gridMobileHAlign", Devices.Mobile, GridParams.HAlign)
     addDataTag("gridDesktopVAlign", Devices.Desktop, GridParams.VAlign)
     addDataTag("gridMobileVAlign", Devices.Mobile, GridParams.VAlign)
-    if (node.dataset.insideGrid !== true) {
+    if (node.dataset.insideGrid !== "true") {
-      node.dataset.insideGrid = true
+      node.dataset.insideGrid = "true"
     }
 
     // Apply all CSS variables to the wrapper
+    // @ts-expect-error TODO
     node.style = buildStyleString(vars)
 
     // Add a listener to select this node on click
@@ -160,7 +184,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Add draggable attribute
-    node.setAttribute("draggable", !!draggable)
+    node.setAttribute("draggable", (!!draggable).toString())
   }
 
   // Removes the previously set up listeners
@@ -176,7 +200,7 @@ export const gridLayout = (node, metadata) => {
   applyMetadata(metadata)
 
   return {
-    update(newMetadata) {
+    update(newMetadata: GridMetadata) {
       removeListeners()
       applyMetadata(newMetadata)
     },

packages/client/src/utils/linkable.ts

@@ -1,8 +1,8 @@
 import { get } from "svelte/store"
-import { link } from "svelte-spa-router"
+import { link, LinkActionOpts } from "svelte-spa-router"
 import { builderStore } from "stores"
 
-export const linkable = (node, href) => {
+export const linkable = (node: HTMLElement, href?: LinkActionOpts) => {
   if (get(builderStore).inBuilder) {
     node.onclick = e => {
       e.preventDefault()

packages/client/tsconfig.json

@@ -14,6 +14,7 @@
         "../*",
         "../../node_modules/@budibase/*"
       ],
+      "@/*": ["./src/*"],
       "*": ["./src/*"]
     }
   }

packages/client/vite.config.mjs

@@ -67,6 +67,10 @@ export default defineConfig(({ mode }) => {
           find: "constants",
           replacement: path.resolve("./src/constants"),
         },
+        {
+          find: "@/constants",
+          replacement: path.resolve("./src/constants"),
+        },
         {
           find: "sdk",
           replacement: path.resolve("./src/sdk"),

packages/frontend-core/src/api/attachments.ts

@@ -100,6 +100,7 @@ export const buildAttachmentEndpoints = (
         body: data,
         json: false,
         external: true,
+        parseResponse: response => response as any,
       })
       return { publicUrl }
     },

packages/frontend-core/src/api/index.ts

@@ -46,6 +46,8 @@ import { buildLogsEndpoints } from "./logs"
 import { buildMigrationEndpoints } from "./migrations"
 import { buildRowActionEndpoints } from "./rowActions"
 
+export type { APIClient } from "./types"
+
 /**
  * Random identifier to uniquely identify a session in a tab. This is
  * used to determine the originator of calls to the API, which is in

packages/frontend-core/src/api/self.ts

@@ -13,7 +13,7 @@ export interface SelfEndpoints {
   generateAPIKey: () => Promise<string | undefined>
   fetchDeveloperInfo: () => Promise<FetchAPIKeyResponse>
   fetchBuilderSelf: () => Promise<GetGlobalSelfResponse>
-  fetchSelf: () => Promise<AppSelfResponse>
+  fetchSelf: () => Promise<AppSelfResponse | null>
 }
 
 export const buildSelfEndpoints = (API: BaseAPIClient): SelfEndpoints => ({

packages/frontend-core/src/api/user.ts

@@ -21,11 +21,12 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   UpdateSelfMetadataRequest,
   UpdateSelfMetadataResponse,
-  User,
+  UserIdentifier,
 } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
@@ -38,14 +39,9 @@ export interface UserEndpoints {
   createAdminUser: (
     user: CreateAdminUserRequest
   ) => Promise<CreateAdminUserResponse>
-  saveUser: (user: User) => Promise<SaveUserResponse>
+  saveUser: (user: UnsavedUser) => Promise<SaveUserResponse>
   deleteUser: (userId: string) => Promise<DeleteUserResponse>
-  deleteUsers: (
+  deleteUsers: (users: UserIdentifier[]) => Promise<BulkUserDeleted | undefined>
-    users: Array<{
-      userId: string
-      email: string
-    }>
-  ) => Promise<BulkUserDeleted | undefined>
   onboardUsers: (data: InviteUsersRequest) => Promise<InviteUsersResponse>
   getUserInvite: (code: string) => Promise<CheckInviteResponse>
   getUserInvites: () => Promise<GetUserInvitesResponse>
@@ -60,7 +56,7 @@ export interface UserEndpoints {
   getAccountHolder: () => Promise<LookupAccountHolderResponse>
   searchUsers: (data: SearchUsersRequest) => Promise<SearchUsersResponse>
   createUsers: (
-    users: User[],
+    users: UnsavedUser[],
     groups: any[]
   ) => Promise<BulkUserCreated | undefined>
   updateUserInvite: (

packages/frontend-core/src/index.ts

@@ -1,4 +1,5 @@
 export { createAPIClient } from "./api"
+export type { APIClient } from "./api"
 export { fetchData, DataFetchMap } from "./fetch"
 export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"

packages/server/src/api/controllers/row/external.ts

@@ -45,6 +45,9 @@ export async function handleRequest<T extends Operation>(
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const source = await utils.getSource(ctx)
 
+  const { viewId, tableId } = utils.getSourceId(ctx)
+  const sourceId = viewId || tableId
+
   if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     ctx.throw(400, "Cannot update rows through a calculation view")
   }
@@ -86,7 +89,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   // The id might have been changed, so the refetching would fail. Recalculating the id just in case
   const updatedId =
     generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
-  const row = await sdk.rows.external.getRow(table._id!, updatedId, {
+  const row = await sdk.rows.external.getRow(sourceId, updatedId, {
     relationships: true,
   })
 

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -14,7 +14,8 @@ import {
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
 import sdk from "../../../../sdk"
-import { helpers } from "@budibase/shared-core"
+import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import { sql } from "@budibase/backend-core"
 
 type TableMap = Record<string, Table>
 
@@ -118,45 +119,131 @@ export async function buildSqlFieldList(
   opts?: { relationships: boolean }
 ) {
   const { relationships } = opts || {}
+
+  const nonMappedColumns = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
+
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
         ([columnName, column]) =>
-          column.type !== FieldType.LINK &&
+          !nonMappedColumns.includes(column.type) &&
-          column.type !== FieldType.FORMULA &&
+          !existing.find((field: string) => field === columnName)
-          column.type !== FieldType.AI &&
-          !existing.find(
-            (field: string) => field === `${table.name}.${columnName}`
-          )
       )
-      .map(([columnName]) => `${table.name}.${columnName}`)
+      .map(([columnName]) => columnName)
+  }
+
+  function getRequiredFields(table: Table, existing: string[] = []) {
+    const requiredFields: string[] = []
+    if (table.primary) {
+      requiredFields.push(...table.primary)
+    }
+    if (table.primaryDisplay) {
+      requiredFields.push(table.primaryDisplay)
+    }
+
+    if (!sql.utils.isExternalTable(table)) {
+      requiredFields.push(...PROTECTED_INTERNAL_COLUMNS)
+    }
+
+    return requiredFields.filter(
+      column =>
+        !existing.find((field: string) => field === column) &&
+        table.schema[column] &&
+        !nonMappedColumns.includes(table.schema[column].type)
+    )
   }
 
   let fields: string[] = []
-  if (sdk.views.isView(source)) {
+
-    fields = Object.keys(helpers.views.basicFields(source))
+  const isView = sdk.views.isView(source)
-  } else {
-    fields = extractRealFields(source)
-  }
 
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
+
+    fields = Object.keys(helpers.views.basicFields(source)).filter(
+      f => table.schema[f].type !== FieldType.LINK
+    )
   } else {
     table = source
+    fields = extractRealFields(source).filter(
+      f => table.schema[f].visible !== false
+    )
   }
 
-  for (let field of Object.values(table.schema)) {
+  const containsFormula = (isView ? fields : Object.keys(table.schema)).some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (containsFormula) {
+    fields = extractRealFields(table)
+  }
+
+  if (!isView || !helpers.views.isCalculationView(source)) {
+    fields.push(
+      ...getRequiredFields(
+        {
+          ...table,
+          primaryDisplay: source.primaryDisplay || table.primaryDisplay,
+        },
+        fields
+      )
+    )
+  }
+
+  fields = fields.map(c => `${table.name}.${c}`)
+
+  for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue
     }
-    const { tableName } = breakExternalTableId(field.tableId)
+
-    if (tables[tableName]) {
+    if (
-      fields = fields.concat(extractRealFields(tables[tableName], fields))
+      isView &&
+      (!source.schema?.[field.name] ||
+        !helpers.views.isVisible(source.schema[field.name])) &&
+      !containsFormula
+    ) {
+      continue
     }
+
+    const { tableName } = breakExternalTableId(field.tableId)
+    const relatedTable = tables[tableName]
+    if (!relatedTable) {
+      continue
+    }
+
+    const viewFields = new Set<string>()
+    if (containsFormula) {
+      extractRealFields(relatedTable).forEach(f => viewFields.add(f))
+    } else {
+      relatedTable.primary?.forEach(f => viewFields.add(f))
+      if (relatedTable.primaryDisplay) {
+        viewFields.add(relatedTable.primaryDisplay)
+      }
+
+      if (isView) {
+        Object.entries(source.schema?.[field.name]?.columns || {})
+          .filter(
+            ([columnName, columnConfig]) =>
+              relatedTable.schema[columnName] &&
+              helpers.views.isVisible(columnConfig) &&
+              ![FieldType.LINK, FieldType.FORMULA].includes(
+                relatedTable.schema[columnName].type
+              )
+          )
+          .forEach(([field]) => viewFields.add(field))
+      }
+    }
+
+    const fieldsToAdd = Array.from(viewFields)
+      .filter(f => !nonMappedColumns.includes(relatedTable.schema[f].type))
+      .map(f => `${relatedTable.name}.${f}`)
+      .filter(f => !fields.includes(f))
+    fields.push(...fieldsToAdd)
   }
 
-  return fields
+  return [...new Set(fields)]
 }
 
 export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {

packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts

@@ -0,0 +1,511 @@
+import {
+  AIOperationEnum,
+  CalculationType,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  ViewV2,
+  ViewV2Type,
+} from "@budibase/types"
+import { buildSqlFieldList } from "../sqlUtils"
+import { structures } from "../../../../routes/tests/utilities"
+import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import { generateViewID } from "../../../../../db/utils"
+
+import sdk from "../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
+
+jest.mock("../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
+
+describe("buildSqlFieldList", () => {
+  let allTables: Record<string, Table>
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor(name: string) {
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables[name] = this._table
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withPrimary(field: string) {
+      this._table.primary = [field]
+      return this
+    }
+
+    withDisplay(field: string) {
+      this._table.primaryDisplay = field
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    allTables = {}
+  })
+
+  describe("table", () => {
+    it("extracts fields from table schema", async () => {
+      const table = new TableConfig("table").create()
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .create()
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual(["table.name", "table.amount"])
+    })
+
+    it("excludes non-sql fields fields", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.id",
+        "linkedTable.name",
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.hidden",
+      ])
+    })
+
+    it("never includes non-sql columns from relationships", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
+  })
+
+  describe("view", () => {
+    it("extracts fields from table schema", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "linkedTable.id",
+        "linkedTable.amount",
+      ])
+    })
+
+    it("excludes relationships fields when view is not included in the view", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("amount")
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .withPrimary("id")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
+  })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+  })
+})

packages/server/src/automations/steps/createRow.ts

@@ -97,7 +97,7 @@ export async function run({
   const ctx: any = buildCtx(appId, emitter, {
     body: inputs.row,
     params: {
-      tableId: inputs.row.tableId,
+      tableId: decodeURIComponent(inputs.row.tableId),
     },
   })
   try {

packages/server/src/automations/steps/deleteRow.ts

@@ -85,7 +85,7 @@ export async function run({
       _rev: inputs.revision,
     },
     params: {
-      tableId: inputs.tableId,
+      tableId: decodeURIComponent(inputs.tableId),
     },
   })
 

packages/server/src/automations/steps/queryRows.ts

@@ -122,9 +122,10 @@ export async function run({
     sortType =
       fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
   }
+  // when passing the tableId in the Ctx it needs to be decoded
   const ctx = buildCtx(appId, null, {
     params: {
-      tableId,
+      tableId: decodeURIComponent(tableId),
     },
     body: {
       sortType,

packages/server/src/automations/steps/updateRow.ts

@@ -90,6 +90,8 @@ export async function run({
     }
   }
   const tableId = inputs.row.tableId
+    ? decodeURIComponent(inputs.row.tableId)
+    : inputs.row.tableId
 
   // Base update
   let rowUpdate: Record<string, any>
@@ -157,7 +159,7 @@ export async function run({
       },
       params: {
         rowId: inputs.rowId,
-        tableId,
+        tableId: tableId,
       },
     })
     await rowController.patch(ctx)

packages/server/src/automations/tests/queryRows.spec.ts

@@ -2,6 +2,7 @@ import { EmptyFilterOption, SortOrder, Table } from "@budibase/types"
 import * as setup from "./utilities"
 import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
 import * as automation from "../index"
+import { basicTable } from "../../tests/utilities/structures"
 
 const NAME = "Test"
 
@@ -13,6 +14,7 @@ describe("Test a query step automation", () => {
     await automation.init()
     await config.init()
     table = await config.createTable()
+
     const row = {
       name: NAME,
       description: "original description",
@@ -153,4 +155,32 @@ describe("Test a query step automation", () => {
     expect(result.steps[0].outputs.rows).toBeDefined()
     expect(result.steps[0].outputs.rows.length).toBe(2)
   })
+
+  it("return rows when querying a table with a space in the name", async () => {
+    const tableWithSpaces = await config.createTable({
+      ...basicTable(),
+      name: "table with spaces",
+    })
+    await config.createRow({
+      name: NAME,
+      tableId: tableWithSpaces._id,
+    })
+    const result = await createAutomationBuilder({
+      name: "Return All Test",
+      config,
+    })
+      .appAction({ fields: {} })
+      .queryRows(
+        {
+          tableId: tableWithSpaces._id!,
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+          filters: {},
+        },
+        { stepName: "Query table with spaces" }
+      )
+      .run()
+    expect(result.steps[0].outputs.success).toBe(true)
+    expect(result.steps[0].outputs.rows).toBeDefined()
+    expect(result.steps[0].outputs.rows.length).toBe(1)
+  })
 })

packages/server/src/integration-test/postgres.spec.ts

@@ -13,7 +13,7 @@ const mainDescriptions = datasourceDescribe({
 
 if (mainDescriptions.length) {
   describe.each(mainDescriptions)(
-    "/postgres integrations",
+    "/postgres integrations ($dbName)",
     ({ config, dsProvider }) => {
       let datasource: Datasource
       let client: Knex

packages/server/src/integrations/tests/sqlAlias.spec.ts

@@ -73,6 +73,27 @@ describe("Captures of real examples", () => {
   })
 
   describe("read", () => {
+    it("should retrieve all fields if non are specified", () => {
+      const queryJson = getJson("basicFetch.json")
+      delete queryJson.resource
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select * from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
+    it("should retrieve only requested fields", () => {
+      const queryJson = getJson("basicFetch.json")
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select "a"."year", "a"."firstname", "a"."personid", "a"."age", "a"."type", "a"."lastname" from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
@@ -112,9 +133,9 @@ describe("Captures of real examples", () => {
         bindings: [primaryLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
+            `with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
+                 select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
-                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
+                 from (select "b"."executorid", "b"."qaid", "b"."taskid", "b"."completed", "b"."taskname" from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
         ),
@@ -130,9 +151,9 @@ describe("Captures of real examples", () => {
       expect(query).toEqual({
         bindings: [...filters, relationshipLimit, relationshipLimit],
         sql: multiline(
-          `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
+          `with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
-               select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
+               select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
-               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
+               from (select "b"."productid", "b"."productname" from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
                where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
         ),
       })
@@ -209,7 +230,7 @@ describe("Captures of real examples", () => {
         bindings: ["ddd", ""],
         sql: multiline(`delete from "compositetable" as "a" 
           where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
-          returning "a".*`),
+          returning "a"."keyparttwo", "a"."keypartone", "a"."name"`),
       })
     })
   })

packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json

@@ -0,0 +1,137 @@
+{
+  "operation": "READ",
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.age",
+      "a.type",
+      "a.lastname"
+    ]
+  },
+  "filters": {},
+  "sort": {
+    "firstname": {
+      "direction": "ascending"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [],
+  "extra": {
+    "idFilter": {}
+  },
+  "table": {
+    "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
+    "name": "persons",
+    "schema": {
+      "year": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "year",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "firstname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}

packages/server/src/sdk/app/rows/search/internal/sqs.ts

@@ -55,32 +55,44 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
 const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
 
-async function buildInternalFieldList(
+export async function buildInternalFieldList(
   source: Table | ViewV2,
   tables: Table[],
-  opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] }
+  opts?: {
+    relationships?: RelationshipsJson[]
+    allowedFields?: string[]
+    includeHiddenFields?: boolean
+  }
 ) {
-  const { relationships, allowedFields } = opts || {}
+  const { relationships, allowedFields, includeHiddenFields } = opts || {}
   let schemaFields: string[] = []
-  if (sdk.views.isView(source)) {
-    schemaFields = Object.keys(helpers.views.basicFields(source))
-  } else {
-    schemaFields = Object.keys(source.schema).filter(
-      key => source.schema[key].visible !== false
-    )
-  }
-
-  if (allowedFields) {
-    schemaFields = schemaFields.filter(field => allowedFields.includes(field))
-  }
 
+  const isView = sdk.views.isView(source)
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
   } else {
     table = source
   }
 
+  if (isView) {
+    schemaFields = Object.keys(helpers.views.basicFields(source))
+  } else {
+    schemaFields = Object.keys(source.schema).filter(
+      key => includeHiddenFields || source.schema[key].visible !== false
+    )
+  }
+
+  const containsFormula = schemaFields.some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (containsFormula) {
+    schemaFields = Object.keys(table.schema)
+  } else if (allowedFields) {
+    schemaFields = schemaFields.filter(field => allowedFields.includes(field))
+  }
+
   let fieldList: string[] = []
   const getJunctionFields = (relatedTable: Table, fields: string[]) => {
     const junctionFields: string[] = []
@@ -101,10 +113,12 @@ async function buildInternalFieldList(
   }
   for (let key of schemaFields) {
     const col = table.schema[key]
+
     const isRelationship = col.type === FieldType.LINK
     if (!relationships && isRelationship) {
       continue
     }
+
     if (!isRelationship) {
       fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
     } else {
@@ -113,8 +127,17 @@ async function buildInternalFieldList(
       if (!relatedTable) {
         continue
       }
+
+      // a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full
+      // structure - essentially all relationships between two tables will be inserted into a single "table"
+      // we don't use an independent junction table ID for each separate relationship between two tables. For
+      // example if we have table A and B, with two relationships between them, all the junction documents will
+      // end up in the same junction table ID. We need to retrieve the field name property of the junction documents
+      // as part of the relationship to tell us which relationship column the junction is related to.
       const relatedFields = (
-        await buildInternalFieldList(relatedTable, tables)
+        await buildInternalFieldList(relatedTable, tables, {
+          includeHiddenFields: containsFormula,
+        })
       ).concat(
         getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
       )
@@ -125,6 +148,13 @@ async function buildInternalFieldList(
       fieldList = fieldList.concat(relatedFields)
     }
   }
+
+  if (!isView || !helpers.views.isCalculationView(source)) {
+    for (const field of PROTECTED_INTERNAL_COLUMNS) {
+      fieldList.push(`${table._id}.${field}`)
+    }
+  }
+
   return [...new Set(fieldList)]
 }
 
@@ -323,8 +353,9 @@ export async function search(
   }
 
   let aggregations: Aggregation[] = []
-  if (sdk.views.isView(source)) {
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     const calculationFields = helpers.views.calculationFields(source)
+
     for (const [key, field] of Object.entries(calculationFields)) {
       if (options.fields && !options.fields.includes(key)) {
         continue

packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts

@@ -0,0 +1,618 @@
+import {
+  AIOperationEnum,
+  CalculationType,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  ViewV2,
+  ViewV2Type,
+} from "@budibase/types"
+import { buildInternalFieldList } from "../sqs"
+import { structures } from "../../../../../../api/routes/tests/utilities"
+import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import {
+  generateJunctionTableID,
+  generateViewID,
+} from "../../../../../../db/utils"
+
+import sdk from "../../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
+
+jest.mock("../../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
+
+describe("buildInternalFieldList", () => {
+  let allTables: Table[]
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor() {
+      const name = generator.word()
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables.push(this._table)
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withEmptySchema() {
+      this._table.schema = {}
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    allTables = []
+  })
+
+  describe("table", () => {
+    it("includes internal columns by default", async () => {
+      const table = new TableConfig().withEmptySchema().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const table = new TableConfig().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table = new TableConfig().withHiddenField("description").create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable = new TableConfig()
+        .withHiddenField("description")
+        .create()
+
+      const table = new TableConfig()
+        .withHiddenField("amount")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("view", () => {
+    it("includes internal columns by default", async () => {
+      const view = new ViewConfig(new TableConfig().create()).create()
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_name`,
+        `${view.tableId}.data_description`,
+        `${view.tableId}.data_amount`,
+        `${view.tableId}.data_formula`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields", async () => {
+      const otherTable = new TableConfig().create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withHidden("amount")
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_formula`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}.data_formula`,
+        `${otherTable._id}.data_ai`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([`${view.tableId}.data_amount`])
+    })
+  })
+})

packages/types/src/api/web/user.ts

@@ -22,6 +22,8 @@ export interface UserDetails {
   password?: string
 }
 
+export type UnsavedUser = Omit<User, "tenantId">
+
 export interface BulkUserRequest {
   delete?: {
     users: Array<{
@@ -31,7 +33,7 @@ export interface BulkUserRequest {
   }
   create?: {
     roles?: any[]
-    users: User[]
+    users: UnsavedUser[]
     groups: any[]
   }
 }
@@ -124,7 +126,7 @@ export interface AcceptUserInviteRequest {
   inviteCode: string
   password: string
   firstName: string
-  lastName: string
+  lastName?: string
 }
 
 export interface AcceptUserInviteResponse {

packages/worker/src/api/controllers/global/users.ts

@@ -33,6 +33,7 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   User,
@@ -49,6 +50,7 @@ import {
   tenancy,
   db,
   locks,
+  context,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
@@ -66,10 +68,11 @@ const generatePassword = (length: number) => {
     .slice(0, length)
 }
 
-export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
+export const save = async (ctx: UserCtx<UnsavedUser, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
-    const requestUser = ctx.request.body
+    const tenantId = context.getTenantId()
+    const requestUser: User = { ...ctx.request.body, tenantId }
 
     // Do not allow the account holder role to be changed
     if (
@@ -151,7 +154,12 @@ export const bulkUpdate = async (
   let created, deleted
   try {
     if (input.create) {
-      created = await bulkCreate(input.create.users, input.create.groups)
+      const tenantId = context.getTenantId()
+      const users: User[] = input.create.users.map(user => ({
+        ...user,
+        tenantId,
+      }))
+      created = await bulkCreate(users, input.create.groups)
     }
     if (input.delete) {
       deleted = await bulkDelete(input.delete.users, currentUserId)