From 77587c86868b9970daa9664db3a38d3befe6b847 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 12:23:32 +0200
Subject: [PATCH 01/57] Add related columns

---
 .../GridColumnConfiguration/getColumns.js     | 44 ++++++++++++++++---
 .../src/components/app/GridBlock.svelte       |  1 +
 .../src/components/grid/stores/datasource.js  | 14 ++++++
 3 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index 148055d727..384f20cf65 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -1,3 +1,5 @@
+import { FieldType } from "@budibase/types"
+
 const modernize = columns => {
   if (!columns) {
     return []
@@ -8,6 +10,7 @@ const modernize = columns => {
       label: column.displayName,
       field: column.name,
       active: true,
+      related: column.related,
     }))
   }
 
@@ -50,12 +53,35 @@ const removeInvalidAddMissing = (
 const getDefault = (schema = {}) => {
   const defaultValues = Object.values(schema)
     .filter(column => !column.nestedJSON)
-    .map(column => ({
-      label: column.name,
-      field: column.name,
-      active: column.visible ?? true,
-      order: column.visible ? column.order ?? -1 : Number.MAX_SAFE_INTEGER,
-    }))
+    .flatMap(column => {
+      const order = column.visible
+        ? column.order ?? -1
+        : Number.MAX_SAFE_INTEGER
+      const columns = [
+        {
+          label: column.name,
+          field: column.name,
+          active: column.visible ?? true,
+          order,
+        },
+      ]
+
+      if (column.columns) {
+        for (const relColumn of Object.keys(column.columns).filter(
+          relColumn => column.columns[relColumn].visible !== false
+        )) {
+          columns.push({
+            label: `${relColumn} (${column.name})`,
+            field: `${column.name}.${relColumn}`,
+            active: column.visible ?? true,
+            order,
+            related: true,
+          })
+        }
+      }
+
+      return columns
+    })
 
   defaultValues.sort((a, b) => a.order - b.order)
 
@@ -69,6 +95,7 @@ const toGridFormat = draggableListColumns => {
     active: entry.active,
     width: entry.width,
     conditions: entry.conditions,
+    related: entry.related,
   }))
 }
 
@@ -82,9 +109,12 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
         active: column.active,
         field: column.field,
         label: column.label,
-        columnType: schema[column.field].type,
+        columnType: column.related
+          ? FieldType.FORMULA
+          : schema[column.field]?.type,
         width: column.width,
         conditions: column.conditions,
+        related: column.related,
       },
       {}
     )
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 30a35b0713..45aba66bab 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -98,6 +98,7 @@
         order: idx,
         conditions: column.conditions,
         visible: !!column.active,
+        related: column.related,
       }
       if (column.width) {
         overrides[column.field].width = column.width
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 68053f38ae..3af6948650 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -1,6 +1,7 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { memo } from "../../../utils"
+import { FieldType } from "@budibase/types"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -73,6 +74,19 @@ export const deriveStores = context => {
           }
         }
       })
+      if ($schemaOverrides) {
+        Object.keys($schemaOverrides).forEach(field => {
+          if (!$schemaOverrides[field].related) {
+            return
+          }
+          enrichedSchema[field] = {
+            ...$schemaOverrides[field],
+            name: field,
+            type: FieldType.FORMULA,
+            related: true,
+          }
+        })
+      }
       return enrichedSchema
     }
   )

From 55190883763eb2b58f64a10d79fe557c8ee7872a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 12:36:04 +0200
Subject: [PATCH 02/57] Display related

---
 .../GridColumnConfiguration/getColumns.js     |  2 +-
 .../src/components/grid/stores/datasource.js  |  1 -
 .../src/components/grid/stores/rows.js        | 32 +++++++++++++++----
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index 384f20cf65..419628ead2 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -75,7 +75,7 @@ const getDefault = (schema = {}) => {
             field: `${column.name}.${relColumn}`,
             active: column.visible ?? true,
             order,
-            related: true,
+            related: { field: column.name, subField: relColumn },
           })
         }
       }
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 3af6948650..6e0b31de4b 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -83,7 +83,6 @@ export const deriveStores = context => {
             ...$schemaOverrides[field],
             name: field,
             type: FieldType.FORMULA,
-            related: true,
           }
         })
       }
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index fb7a487c8e..e13767cbca 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -6,6 +6,7 @@ import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
 import { FieldType } from "@budibase/types"
+import { processStringSync } from "@budibase/string-templates"
 
 export const createStores = () => {
   const rows = writable([])
@@ -42,15 +43,32 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { rows } = context
+  const { rows, enrichedSchema } = context
 
   // Enrich rows with an index property and any pending changes
-  const enrichedRows = derived(rows, $rows => {
-    return $rows.map((row, idx) => ({
-      ...row,
-      __idx: idx,
-    }))
-  })
+  const enrichedRows = derived(
+    [rows, enrichedSchema],
+    ([$rows, $enrichedSchema]) => {
+      const customColumns = Object.values($enrichedSchema || {}).filter(
+        f => f.related
+      )
+      return $rows.map((row, idx) => ({
+        ...row,
+        __idx: idx,
+        ...customColumns.reduce((acc, c) => {
+          try {
+            acc[c.name] = processStringSync(
+              `{{ join (pluck ${c.related.field} '${c.related.subField}') ', ' }}`,
+              row
+            )
+          } catch {
+            // It might be some formula not set, or anything being incorrect
+          }
+          return acc
+        }, {}),
+      }))
+    }
+  )
 
   // Generate a lookup map to quick find a row by ID
   const rowLookupMap = derived(enrichedRows, $enrichedRows => {

From 49502afbeed80088fd5e161e4ab6bd886b3cbe9e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:04:57 +0200
Subject: [PATCH 03/57] Default off

---
 .../settings/controls/GridColumnConfiguration/getColumns.js     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index 419628ead2..22b42dd427 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -73,7 +73,7 @@ const getDefault = (schema = {}) => {
           columns.push({
             label: `${relColumn} (${column.name})`,
             field: `${column.name}.${relColumn}`,
-            active: column.visible ?? true,
+            active: false,
             order,
             related: { field: column.name, subField: relColumn },
           })

From 8ee6e52f08c133231e94dd6b607830421be4de6a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:24:10 +0200
Subject: [PATCH 04/57] Use types

---
 .../frontend-core/src/components/grid/stores/datasource.js  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 6e0b31de4b..ed84dc478a 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -79,10 +79,14 @@ export const deriveStores = context => {
           if (!$schemaOverrides[field].related) {
             return
           }
+
+          const { field: relField, subField: relSubField } =
+            $schemaOverrides[field].related
+
           enrichedSchema[field] = {
             ...$schemaOverrides[field],
             name: field,
-            type: FieldType.FORMULA,
+            type: $schema[relField]?.columns?.[relSubField]?.type,
           }
         })
       }

From 574b2e5a3c16668a710df57d2625e30906a6329b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:34:23 +0200
Subject: [PATCH 05/57] Map types

---
 .../GridColumnConfiguration/getColumns.js     | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index 22b42dd427..7380e40de0 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -1,5 +1,3 @@
-import { FieldType } from "@budibase/types"
-
 const modernize = columns => {
   if (!columns) {
     return []
@@ -20,7 +18,8 @@ const modernize = columns => {
 const removeInvalidAddMissing = (
   columns = [],
   defaultColumns,
-  primaryDisplayColumnName
+  primaryDisplayColumnName,
+  schema
 ) => {
   const defaultColumnNames = defaultColumns.map(column => column.field)
   const columnNames = columns.map(column => column.field)
@@ -47,6 +46,16 @@ const removeInvalidAddMissing = (
     combinedColumns[primaryDisplayIndex].active = true
   }
 
+  for (const column of combinedColumns) {
+    if (!column.related) {
+      column.columnType = schema[column.field]?.type
+      continue
+    }
+
+    const { field: relField, subField: relSubField } = column.related
+    column.columnType = schema[relField]?.columns?.[relSubField]?.type
+  }
+
   return combinedColumns
 }
 
@@ -99,7 +108,7 @@ const toGridFormat = draggableListColumns => {
   }))
 }
 
-const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
+const toDraggableListFormat = (gridFormatColumns, createComponent) => {
   return gridFormatColumns.map(column => {
     return createComponent(
       "@budibase/standard-components/labelfield",
@@ -109,9 +118,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
         active: column.active,
         field: column.field,
         label: column.label,
-        columnType: column.related
-          ? FieldType.FORMULA
-          : schema[column.field]?.type,
+        columnType: column.columnType,
         width: column.width,
         conditions: column.conditions,
         related: column.related,
@@ -131,13 +138,10 @@ const getColumns = ({
   const validatedColumns = removeInvalidAddMissing(
     modernize(columns),
     getDefault(schema),
-    primaryDisplayColumnName
-  )
-  const draggableList = toDraggableListFormat(
-    validatedColumns,
-    createComponent,
+    primaryDisplayColumnName,
     schema
   )
+  const draggableList = toDraggableListFormat(validatedColumns, createComponent)
   const primary = draggableList
     .filter(entry => entry.field === primaryDisplayColumnName)
     .map(instance => ({ ...instance, schema }))[0]

From d799bfacb60022c44c95ace62ca24f08fd9dfd04 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:35:11 +0200
Subject: [PATCH 06/57] Readonly

---
 packages/frontend-core/src/components/grid/stores/datasource.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index ed84dc478a..399ec341ee 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -87,6 +87,7 @@ export const deriveStores = context => {
             ...$schemaOverrides[field],
             name: field,
             type: $schema[relField]?.columns?.[relSubField]?.type,
+            readonly: true,
           }
         })
       }

From 4aa9e08ec564083ca6ff0e38f0feb45767631fa3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:39:04 +0200
Subject: [PATCH 07/57] Don't show non visible even if previously configured

---
 .../frontend-core/src/components/grid/stores/datasource.js    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 399ec341ee..65e925fdf2 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -83,6 +83,10 @@ export const deriveStores = context => {
           const { field: relField, subField: relSubField } =
             $schemaOverrides[field].related
 
+          if (!$schema[relField]?.columns?.[relSubField]?.visible) {
+            return
+          }
+
           enrichedSchema[field] = {
             ...$schemaOverrides[field],
             name: field,

From e9db3d64e7a8914cb154c654417e74ddd642e799 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 13:50:11 +0200
Subject: [PATCH 08/57] Extra enrichement

---
 .../frontend-core/src/components/grid/stores/datasource.js   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 65e925fdf2..a7a3083cde 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -83,7 +83,10 @@ export const deriveStores = context => {
           const { field: relField, subField: relSubField } =
             $schemaOverrides[field].related
 
-          if (!$schema[relField]?.columns?.[relSubField]?.visible) {
+          if (
+            !$schema[relField].visible ||
+            !$schema[relField]?.columns?.[relSubField]?.visible
+          ) {
             return
           }
 

From 488165d85985056ad3b6034bb10aa87ec47347d3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 6 Sep 2024 14:00:53 +0200
Subject: [PATCH 09/57] Lint

---
 packages/frontend-core/src/components/grid/stores/datasource.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index a7a3083cde..bdd338618c 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -1,7 +1,6 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
 import { memo } from "../../../utils"
-import { FieldType } from "@budibase/types"
 
 export const createStores = () => {
   const definition = memo(null)

From eed82075fd6c678a4b509a76a9e3e7ef1060603f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 09:52:09 +0200
Subject: [PATCH 10/57] Enrich visible columns

---
 .../src/components/grid/stores/columns.js     | 39 ++++++++++++++++---
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 0073754a5d..9648b367f9 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -31,7 +31,7 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { columns } = context
+  const { columns, enrichedSchema } = context
 
   // Derive a lookup map for all columns by name
   const columnLookupMap = derived(columns, $columns => {
@@ -43,9 +43,38 @@ export const deriveStores = context => {
   })
 
   // Derived list of columns which have not been explicitly hidden
-  const visibleColumns = derived(columns, $columns => {
-    return $columns.filter(col => col.visible)
-  })
+  const visibleColumns = derived(
+    [columns, enrichedSchema],
+    ([$columns, $enrichedSchema]) => {
+      return $columns
+        .filter(col => col.visible)
+        .flatMap(c => {
+          const relatedColumns = []
+
+          const schemaColumns = $enrichedSchema?.[c.name]?.columns
+          if (schemaColumns) {
+            for (const relColumn of Object.keys(schemaColumns)) {
+              const relFieldSchema = schemaColumns[relColumn]
+              if (!relFieldSchema.visible) {
+                continue
+              }
+              relatedColumns.push({
+                name: `${c.name}.${relColumn}`,
+                label: `${relColumn} (${c.name})`,
+                schema: relFieldSchema,
+                width: relFieldSchema.width || DefaultColumnWidth,
+                visible: relFieldSchema.visible ?? true,
+                readonly: relFieldSchema.readonly,
+                order: relFieldSchema.order,
+                conditions: relFieldSchema.conditions,
+              })
+            }
+          }
+
+          return [c, ...relatedColumns]
+        })
+    }
+  )
 
   // Split visible columns into their discrete types
   const displayColumn = derived(visibleColumns, $visibleColumns => {
@@ -136,7 +165,7 @@ export const initialise = context => {
         .map(field => {
           const fieldSchema = $enrichedSchema[field]
           const oldColumn = $columns?.find(col => col.name === field)
-          let column = {
+          const column = {
             name: field,
             label: fieldSchema.displayName || field,
             schema: fieldSchema,

From 4272b614e254927ddbea2bce2762ca4128854522 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 09:58:43 +0200
Subject: [PATCH 11/57] Cleanup

---
 .../GridColumnConfiguration/getColumns.js       | 17 -----------------
 .../client/src/components/app/GridBlock.svelte  |  1 -
 2 files changed, 18 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index 7380e40de0..f5ac106162 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -8,7 +8,6 @@ const modernize = columns => {
       label: column.displayName,
       field: column.name,
       active: true,
-      related: column.related,
     }))
   }
 
@@ -75,20 +74,6 @@ const getDefault = (schema = {}) => {
         },
       ]
 
-      if (column.columns) {
-        for (const relColumn of Object.keys(column.columns).filter(
-          relColumn => column.columns[relColumn].visible !== false
-        )) {
-          columns.push({
-            label: `${relColumn} (${column.name})`,
-            field: `${column.name}.${relColumn}`,
-            active: false,
-            order,
-            related: { field: column.name, subField: relColumn },
-          })
-        }
-      }
-
       return columns
     })
 
@@ -104,7 +89,6 @@ const toGridFormat = draggableListColumns => {
     active: entry.active,
     width: entry.width,
     conditions: entry.conditions,
-    related: entry.related,
   }))
 }
 
@@ -121,7 +105,6 @@ const toDraggableListFormat = (gridFormatColumns, createComponent) => {
         columnType: column.columnType,
         width: column.width,
         conditions: column.conditions,
-        related: column.related,
       },
       {}
     )
diff --git a/packages/client/src/components/app/GridBlock.svelte b/packages/client/src/components/app/GridBlock.svelte
index 45aba66bab..30a35b0713 100644
--- a/packages/client/src/components/app/GridBlock.svelte
+++ b/packages/client/src/components/app/GridBlock.svelte
@@ -98,7 +98,6 @@
         order: idx,
         conditions: column.conditions,
         visible: !!column.active,
-        related: column.related,
       }
       if (column.width) {
         overrides[column.field].width = column.width

From 6d8505cee617cd3ebdc50d5edaf4d6733ba97688 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 10:18:43 +0200
Subject: [PATCH 12/57] Add tablecolumns

---
 .../grid/controls/ColumnsSettingButton.svelte | 12 ++--
 .../src/components/grid/stores/columns.js     | 69 ++++++++++---------
 2 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
index 2a3ca139fc..75108870bd 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingButton.svelte
@@ -4,13 +4,15 @@
   import ColumnsSettingContent from "./ColumnsSettingContent.svelte"
   import { FieldPermissions } from "../../../constants"
 
-  const { columns, datasource } = getContext("grid")
+  const { tableColumns, datasource } = getContext("grid")
 
   let open = false
   let anchor
 
-  $: anyRestricted = $columns.filter(col => !col.visible || col.readonly).length
-  $: text = anyRestricted ? `Columns: (${anyRestricted} restricted)` : "Columns"
+  $: anyRestricted = $tableColumns.filter(
+    col => !col.visible || col.readonly
+  ).length
+  $: text = anyRestricted ? `Columns (${anyRestricted} restricted)` : "Columns"
   $: permissions =
     $datasource.type === "viewV2"
       ? [
@@ -28,12 +30,12 @@
     size="M"
     on:click={() => (open = !open)}
     selected={open || anyRestricted}
-    disabled={!$columns.length}
+    disabled={!$tableColumns.length}
   >
     {text}
   </ActionButton>
 </div>
 
 <Popover bind:open {anchor} align="left">
-  <ColumnsSettingContent columns={$columns} {permissions} />
+  <ColumnsSettingContent columns={$tableColumns} {permissions} />
 </Popover>
diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index 9648b367f9..b20f17fd4d 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -31,7 +31,7 @@ export const createStores = () => {
 }
 
 export const deriveStores = context => {
-  const { columns, enrichedSchema } = context
+  const { columns } = context
 
   // Derive a lookup map for all columns by name
   const columnLookupMap = derived(columns, $columns => {
@@ -42,39 +42,15 @@ export const deriveStores = context => {
     return map
   })
 
+  // Derived list of columns which are direct part of the table
+  const tableColumns = derived(columns, $columns => {
+    return $columns.filter(col => !col.related)
+  })
+
   // Derived list of columns which have not been explicitly hidden
-  const visibleColumns = derived(
-    [columns, enrichedSchema],
-    ([$columns, $enrichedSchema]) => {
-      return $columns
-        .filter(col => col.visible)
-        .flatMap(c => {
-          const relatedColumns = []
-
-          const schemaColumns = $enrichedSchema?.[c.name]?.columns
-          if (schemaColumns) {
-            for (const relColumn of Object.keys(schemaColumns)) {
-              const relFieldSchema = schemaColumns[relColumn]
-              if (!relFieldSchema.visible) {
-                continue
-              }
-              relatedColumns.push({
-                name: `${c.name}.${relColumn}`,
-                label: `${relColumn} (${c.name})`,
-                schema: relFieldSchema,
-                width: relFieldSchema.width || DefaultColumnWidth,
-                visible: relFieldSchema.visible ?? true,
-                readonly: relFieldSchema.readonly,
-                order: relFieldSchema.order,
-                conditions: relFieldSchema.conditions,
-              })
-            }
-          }
-
-          return [c, ...relatedColumns]
-        })
-    }
-  )
+  const visibleColumns = derived(columns, $columns => {
+    return $columns.filter(col => col.visible)
+  })
 
   // Split visible columns into their discrete types
   const displayColumn = derived(visibleColumns, $visibleColumns => {
@@ -93,6 +69,7 @@ export const deriveStores = context => {
   })
 
   return {
+    tableColumns,
     displayColumn,
     columnLookupMap,
     visibleColumns,
@@ -183,6 +160,32 @@ export const initialise = context => {
           }
           return column
         })
+        .flatMap(field => {
+          const relatedColumns = []
+
+          const schemaColumns = $enrichedSchema?.[field.name]?.columns
+          if (field.visible && schemaColumns) {
+            for (const relColumn of Object.keys(schemaColumns)) {
+              const relFieldSchema = schemaColumns[relColumn]
+              if (!relFieldSchema.visible) {
+                continue
+              }
+              relatedColumns.push({
+                name: `${field.name}.${relColumn}`,
+                label: `${relColumn} (${field.name})`,
+                schema: relFieldSchema,
+                width: relFieldSchema.width || DefaultColumnWidth,
+                visible: relFieldSchema.visible ?? true,
+                readonly: relFieldSchema.readonly,
+                order: relFieldSchema.order,
+                conditions: relFieldSchema.conditions,
+                related: true,
+              })
+            }
+          }
+
+          return [field, ...relatedColumns]
+        })
         .sort((a, b) => {
           // Display column should always come first
           if (a.name === primaryDisplay) {

From 083e6ae15bb1a3d87b1cf48ff5a07dfd4241f918 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 12:31:00 +0200
Subject: [PATCH 13/57] Enrich at schema level

---
 .../src/components/grid/stores/columns.js     | 27 +---------
 .../src/components/grid/stores/datasource.js  | 49 +++++++++----------
 2 files changed, 24 insertions(+), 52 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index b20f17fd4d..b4eba300ad 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -151,6 +151,7 @@ export const initialise = context => {
             readonly: fieldSchema.readonly,
             order: fieldSchema.order ?? oldColumn?.order,
             conditions: fieldSchema.conditions,
+            related: fieldSchema.related,
           }
           // Override a few properties for primary display
           if (field === primaryDisplay) {
@@ -160,32 +161,6 @@ export const initialise = context => {
           }
           return column
         })
-        .flatMap(field => {
-          const relatedColumns = []
-
-          const schemaColumns = $enrichedSchema?.[field.name]?.columns
-          if (field.visible && schemaColumns) {
-            for (const relColumn of Object.keys(schemaColumns)) {
-              const relFieldSchema = schemaColumns[relColumn]
-              if (!relFieldSchema.visible) {
-                continue
-              }
-              relatedColumns.push({
-                name: `${field.name}.${relColumn}`,
-                label: `${relColumn} (${field.name})`,
-                schema: relFieldSchema,
-                width: relFieldSchema.width || DefaultColumnWidth,
-                visible: relFieldSchema.visible ?? true,
-                readonly: relFieldSchema.readonly,
-                order: relFieldSchema.order,
-                conditions: relFieldSchema.conditions,
-                related: true,
-              })
-            }
-          }
-
-          return [field, ...relatedColumns]
-        })
         .sort((a, b) => {
           // Display column should always come first
           if (a.name === primaryDisplay) {
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index bdd338618c..5ec94874dd 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -53,10 +53,31 @@ export const deriveStores = context => {
       if (!$schema) {
         return null
       }
+
+      const schemaWithRelatedColumns = Object.keys($schema || {}).reduce(
+        (acc, c) => {
+          const field = $schema[c]
+          acc[c] = field
+
+          if (field.columns) {
+            for (const relColumn of Object.keys(field.columns)) {
+              const name = `${field.name}.${relColumn}`
+              acc[name] = {
+                ...field.columns[relColumn],
+                name,
+                related: { field: c, subField: relColumn },
+              }
+            }
+          }
+          return acc
+        },
+        {}
+      )
+
       let enrichedSchema = {}
-      Object.keys($schema).forEach(field => {
+      Object.keys(schemaWithRelatedColumns).forEach(field => {
         enrichedSchema[field] = {
-          ...$schema[field],
+          ...schemaWithRelatedColumns[field],
           ...$schemaOverrides?.[field],
           ...$schemaMutations[field],
         }
@@ -73,30 +94,6 @@ export const deriveStores = context => {
           }
         }
       })
-      if ($schemaOverrides) {
-        Object.keys($schemaOverrides).forEach(field => {
-          if (!$schemaOverrides[field].related) {
-            return
-          }
-
-          const { field: relField, subField: relSubField } =
-            $schemaOverrides[field].related
-
-          if (
-            !$schema[relField].visible ||
-            !$schema[relField]?.columns?.[relSubField]?.visible
-          ) {
-            return
-          }
-
-          enrichedSchema[field] = {
-            ...$schemaOverrides[field],
-            name: field,
-            type: $schema[relField]?.columns?.[relSubField]?.type,
-            readonly: true,
-          }
-        })
-      }
       return enrichedSchema
     }
   )

From 4e83daf5d96fc734f74fe62150545a1130de729d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 12:41:59 +0200
Subject: [PATCH 14/57] Allow selecting columns in design

---
 .../GridColumnConfiguration.svelte            |  4 ++-
 .../src/components/grid/stores/datasource.js  | 24 +++--------------
 packages/frontend-core/src/utils/index.js     |  1 +
 packages/frontend-core/src/utils/schema.js    | 27 +++++++++++++++++++
 4 files changed, 34 insertions(+), 22 deletions(-)
 create mode 100644 packages/frontend-core/src/utils/schema.js

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
index 17cb171da5..b89c45046e 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/GridColumnConfiguration.svelte
@@ -1,4 +1,5 @@
 <script>
+  import { enrichSchemaWithRelColumns } from "@budibase/frontend-core"
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import { selectedScreen, componentStore } from "stores/builder"
   import DraggableList from "../DraggableList/DraggableList.svelte"
@@ -27,7 +28,8 @@
       delete schema._rev
     }
 
-    return schema
+    const result = enrichSchemaWithRelColumns(schema)
+    return result
   }
 
   $: datasource = getDatasourceForProvider($selectedScreen, componentInstance)
diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 5ec94874dd..0174c82c4f 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -1,6 +1,6 @@
 import { derived, get } from "svelte/store"
 import { getDatasourceDefinition, getDatasourceSchema } from "../../../fetch"
-import { memo } from "../../../utils"
+import { enrichSchemaWithRelColumns, memo } from "../../../utils"
 
 export const createStores = () => {
   const definition = memo(null)
@@ -54,27 +54,9 @@ export const deriveStores = context => {
         return null
       }
 
-      const schemaWithRelatedColumns = Object.keys($schema || {}).reduce(
-        (acc, c) => {
-          const field = $schema[c]
-          acc[c] = field
+      const schemaWithRelatedColumns = enrichSchemaWithRelColumns($schema)
 
-          if (field.columns) {
-            for (const relColumn of Object.keys(field.columns)) {
-              const name = `${field.name}.${relColumn}`
-              acc[name] = {
-                ...field.columns[relColumn],
-                name,
-                related: { field: c, subField: relColumn },
-              }
-            }
-          }
-          return acc
-        },
-        {}
-      )
-
-      let enrichedSchema = {}
+      const enrichedSchema = {}
       Object.keys(schemaWithRelatedColumns).forEach(field => {
         enrichedSchema[field] = {
           ...schemaWithRelatedColumns[field],
diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js
index 5f21e7db99..6f1b42c32e 100644
--- a/packages/frontend-core/src/utils/index.js
+++ b/packages/frontend-core/src/utils/index.js
@@ -10,3 +10,4 @@ export { createWebsocket } from "./websocket"
 export * from "./download"
 export * from "./theme"
 export * from "./settings"
+export * from "./schema"
diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
new file mode 100644
index 0000000000..ca33f75a60
--- /dev/null
+++ b/packages/frontend-core/src/utils/schema.js
@@ -0,0 +1,27 @@
+export function enrichSchemaWithRelColumns(schema) {
+  if (!schema) {
+    return
+  }
+  const result = Object.keys(schema).reduce((acc, c) => {
+    const field = schema[c]
+    acc[c] = field
+
+    if (field.columns) {
+      for (const relColumn of Object.keys(field.columns)) {
+        const relField = field.columns[relColumn]
+        if (!relField.visible) {
+          continue
+        }
+        const name = `${field.name}.${relColumn}`
+        acc[name] = {
+          ...relField,
+          name,
+          related: { field: c, subField: relColumn },
+        }
+      }
+    }
+    return acc
+  }, {})
+
+  return result
+}

From 48f4dceddc034a07137210db0d891dafb6f8e9ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 13:32:40 +0200
Subject: [PATCH 15/57] Link

---
 packages/frontend-core/src/utils/schema.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
index ca33f75a60..774e650175 100644
--- a/packages/frontend-core/src/utils/schema.js
+++ b/packages/frontend-core/src/utils/schema.js
@@ -6,7 +6,7 @@ export function enrichSchemaWithRelColumns(schema) {
     const field = schema[c]
     acc[c] = field
 
-    if (field.columns) {
+    if (field.visible !== false && field.columns) {
       for (const relColumn of Object.keys(field.columns)) {
         const relField = field.columns[relColumn]
         if (!relField.visible) {

From d4555c00d455f3b7e6b49ddbcb888f1bdb32a8ca Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 13:33:20 +0200
Subject: [PATCH 16/57] Fix resize

---
 .../src/components/grid/stores/resize.js              | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/resize.js b/packages/frontend-core/src/components/grid/stores/resize.js
index 86ee0d298f..c242dd9a60 100644
--- a/packages/frontend-core/src/components/grid/stores/resize.js
+++ b/packages/frontend-core/src/components/grid/stores/resize.js
@@ -38,6 +38,7 @@ export const createActions = context => {
       initialWidth: column.width,
       initialMouseX: x,
       column: column.name,
+      related: column.related,
     })
 
     // Add mouse event listeners to handle resizing
@@ -50,7 +51,7 @@ export const createActions = context => {
 
   // Handler for moving the mouse to resize columns
   const onResizeMouseMove = e => {
-    const { initialMouseX, initialWidth, width, column } = get(resize)
+    const { initialMouseX, initialWidth, width, column, related } = get(resize)
     const { x } = parseEventLocation(e)
     const dx = x - initialMouseX
     const newWidth = Math.round(Math.max(MinColumnWidth, initialWidth + dx))
@@ -61,7 +62,13 @@ export const createActions = context => {
     }
 
     // Update column state
-    datasource.actions.addSchemaMutation(column, { width })
+    if (!related) {
+      datasource.actions.addSchemaMutation(column, { width })
+    } else {
+      datasource.actions.addSubSchemaMutation(related.subField, related.field, {
+        width,
+      })
+    }
 
     // Update state
     resize.update(state => ({

From b7b932788b74584187ecd74386541dd750a95426 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 13:44:06 +0200
Subject: [PATCH 17/57] Handle reordering

---
 .../src/components/grid/stores/reorder.js         | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/reorder.js b/packages/frontend-core/src/components/grid/stores/reorder.js
index c15c7fbb16..8153ee7084 100644
--- a/packages/frontend-core/src/components/grid/stores/reorder.js
+++ b/packages/frontend-core/src/components/grid/stores/reorder.js
@@ -214,11 +214,20 @@ export const createActions = context => {
     })
 
     // Extract new orders as schema mutations
-    let mutations = {}
     get(columns).forEach((column, idx) => {
-      mutations[column.name] = { order: idx }
+      const { related } = column
+      const mutation = { order: idx }
+      if (!related) {
+        datasource.actions.addSchemaMutation(column.name, mutation)
+      } else {
+        datasource.actions.addSubSchemaMutation(
+          related.subField,
+          related.field,
+          mutation
+        )
+      }
     })
-    datasource.actions.addSchemaMutations(mutations)
+
     await datasource.actions.saveSchemaMutations()
   }
 

From 6096f106b3ad2332f34aaf1ab6b23e01d45c1844 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 13:51:58 +0200
Subject: [PATCH 18/57] Handle resize all

---
 .../src/components/grid/stores/columns.js     | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/columns.js b/packages/frontend-core/src/components/grid/stores/columns.js
index b4eba300ad..4823f2159c 100644
--- a/packages/frontend-core/src/components/grid/stores/columns.js
+++ b/packages/frontend-core/src/components/grid/stores/columns.js
@@ -79,16 +79,24 @@ export const deriveStores = context => {
 }
 
 export const createActions = context => {
-  const { columns, datasource, schema } = context
+  const { columns, datasource } = context
 
   // Updates the width of all columns
   const changeAllColumnWidths = async width => {
-    const $schema = get(schema)
-    let mutations = {}
-    Object.keys($schema).forEach(field => {
-      mutations[field] = { width }
+    const $columns = get(columns)
+    $columns.forEach(column => {
+      const { related } = column
+      const mutation = { width }
+      if (!related) {
+        datasource.actions.addSchemaMutation(column.name, mutation)
+      } else {
+        datasource.actions.addSubSchemaMutation(
+          related.subField,
+          related.field,
+          mutation
+        )
+      }
     })
-    datasource.actions.addSchemaMutations(mutations)
     await datasource.actions.saveSchemaMutations()
   }
 

From 7f6236852e61fb8cec5e22a697152ef91dc8a972 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 13:52:25 +0200
Subject: [PATCH 19/57] Clean deprecated code

---
 .../src/components/grid/stores/datasource.js  | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasource.js b/packages/frontend-core/src/components/grid/stores/datasource.js
index 0174c82c4f..1454a8d7db 100644
--- a/packages/frontend-core/src/components/grid/stores/datasource.js
+++ b/packages/frontend-core/src/components/grid/stores/datasource.js
@@ -205,24 +205,6 @@ export const createActions = context => {
     })
   }
 
-  // Adds schema mutations for multiple fields at once
-  const addSchemaMutations = mutations => {
-    const fields = Object.keys(mutations || {})
-    if (!fields.length) {
-      return
-    }
-    schemaMutations.update($schemaMutations => {
-      let newSchemaMutations = { ...$schemaMutations }
-      fields.forEach(field => {
-        newSchemaMutations[field] = {
-          ...newSchemaMutations[field],
-          ...mutations[field],
-        }
-      })
-      return newSchemaMutations
-    })
-  }
-
   // Saves schema changes to the server, if possible
   const saveSchemaMutations = async () => {
     // If we can't save schema changes then we just want to keep this in memory
@@ -312,7 +294,6 @@ export const createActions = context => {
         changePrimaryDisplay,
         addSchemaMutation,
         addSubSchemaMutation,
-        addSchemaMutations,
         saveSchemaMutations,
         resetSchemaMutations,
       },

From 33a5075f100bf8f00b63aacd9255945e08a4c31a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 14:13:14 +0200
Subject: [PATCH 20/57] Map values

---
 .../src/components/grid/stores/rows.js        | 30 ++++++++++++++-----
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index e13767cbca..6e2fa1bcb8 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -42,6 +42,27 @@ export const createStores = () => {
   }
 }
 
+function getRelatedTableValues(row, field) {
+  let result = ""
+  const separator = ", "
+
+  switch (field.type) {
+    case FieldType.STRING:
+    case FieldType.NUMBER:
+    default:
+      result = processStringSync(
+        `{{ join (pluck ${field.related.field} '${field.related.subField}') '${separator}' }}`,
+        row
+      )
+      break
+    case FieldType.ARRAY:
+    case FieldType.OPTIONS:
+      result = row[field.related.field].flatMap(r => r[field.related.subField])
+  }
+
+  return result
+}
+
 export const deriveStores = context => {
   const { rows, enrichedSchema } = context
 
@@ -56,14 +77,7 @@ export const deriveStores = context => {
         ...row,
         __idx: idx,
         ...customColumns.reduce((acc, c) => {
-          try {
-            acc[c.name] = processStringSync(
-              `{{ join (pluck ${c.related.field} '${c.related.subField}') ', ' }}`,
-              row
-            )
-          } catch {
-            // It might be some formula not set, or anything being incorrect
-          }
+          acc[c.name] = getRelatedTableValues(row, c)
           return acc
         }, {}),
       }))

From a998e46014b05abc3f4240dfb0a2e4a5cc4eea47 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 15:14:30 +0200
Subject: [PATCH 21/57] Remove duplicates

---
 packages/frontend-core/src/components/grid/stores/rows.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 6e2fa1bcb8..5af3ef9d17 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -57,7 +57,11 @@ function getRelatedTableValues(row, field) {
       break
     case FieldType.ARRAY:
     case FieldType.OPTIONS:
-      result = row[field.related.field].flatMap(r => r[field.related.subField])
+      result = Array.from(
+        new Set(
+          row[field.related.field].flatMap(r => r[field.related.subField])
+        )
+      )
   }
 
   return result

From 18084b16f7bb84d76ee877ac19d32e1c6f197246 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 9 Sep 2024 15:37:25 +0200
Subject: [PATCH 22/57] Undo unnecessary changes

---
 .../GridColumnConfiguration/getColumns.js     | 37 ++++---------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index f5ac106162..dcfed9053f 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -17,8 +17,7 @@ const modernize = columns => {
 const removeInvalidAddMissing = (
   columns = [],
   defaultColumns,
-  primaryDisplayColumnName,
-  schema
+  primaryDisplayColumnName
 ) => {
   const defaultColumnNames = defaultColumns.map(column => column.field)
   const columnNames = columns.map(column => column.field)
@@ -45,37 +44,18 @@ const removeInvalidAddMissing = (
     combinedColumns[primaryDisplayIndex].active = true
   }
 
-  for (const column of combinedColumns) {
-    if (!column.related) {
-      column.columnType = schema[column.field]?.type
-      continue
-    }
-
-    const { field: relField, subField: relSubField } = column.related
-    column.columnType = schema[relField]?.columns?.[relSubField]?.type
-  }
-
   return combinedColumns
 }
 
 const getDefault = (schema = {}) => {
   const defaultValues = Object.values(schema)
     .filter(column => !column.nestedJSON)
-    .flatMap(column => {
-      const order = column.visible
-        ? column.order ?? -1
-        : Number.MAX_SAFE_INTEGER
-      const columns = [
-        {
-          label: column.name,
-          field: column.name,
-          active: column.visible ?? true,
-          order,
-        },
-      ]
-
-      return columns
-    })
+    .map(column => ({
+      label: column.name,
+      field: column.name,
+      active: column.visible ?? true,
+      order: column.visible ? column.order ?? -1 : Number.MAX_SAFE_INTEGER,
+    }))
 
   defaultValues.sort((a, b) => a.order - b.order)
 
@@ -121,8 +101,7 @@ const getColumns = ({
   const validatedColumns = removeInvalidAddMissing(
     modernize(columns),
     getDefault(schema),
-    primaryDisplayColumnName,
-    schema
+    primaryDisplayColumnName
   )
   const draggableList = toDraggableListFormat(validatedColumns, createComponent)
   const primary = draggableList

From b9c3838312768d95702685ef9832c751ef214d47 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 10 Sep 2024 13:48:58 +0200
Subject: [PATCH 23/57] Handle single

---
 .../src/components/grid/stores/rows.js        | 61 +++++++++++++------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 5af3ef9d17..d501520705 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -5,8 +5,7 @@ import { getCellID, parseCellID } from "../lib/utils"
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
-import { FieldType } from "@budibase/types"
-import { processStringSync } from "@budibase/string-templates"
+import { FieldType, RelationshipType } from "@budibase/types"
 
 export const createStores = () => {
   const rows = writable([])
@@ -42,26 +41,50 @@ export const createStores = () => {
   }
 }
 
-function getRelatedTableValues(row, field) {
-  let result = ""
-  const separator = ", "
+function getRelatedTableValues(row, field, isSingle) {
+  let result = "Not rendable"
+  try {
+    if (isSingle) {
+      result = row[field.related.field]?.[0]?.[field.related.subField]
+      switch (field.type) {
+        case FieldType.JSON:
+        case FieldType.SIGNATURE_SINGLE:
+        case FieldType.ATTACHMENT_SINGLE:
+        case FieldType.ATTACHMENTS:
+        case FieldType.BB_REFERENCE:
+          result = JSON.parse(result)
+          break
 
-  switch (field.type) {
-    case FieldType.STRING:
-    case FieldType.NUMBER:
-    default:
-      result = processStringSync(
-        `{{ join (pluck ${field.related.field} '${field.related.subField}') '${separator}' }}`,
-        row
-      )
-      break
-    case FieldType.ARRAY:
-    case FieldType.OPTIONS:
+        case FieldType.LINK:
+          console.error(`${field.type} type is not rendable`)
+      }
+    } else {
+      // TODO: check all types
       result = Array.from(
         new Set(
           row[field.related.field].flatMap(r => r[field.related.subField])
         )
       )
+
+      switch (field.type) {
+        case FieldType.STRING:
+        case FieldType.NUMBER:
+        case FieldType.BIGINT:
+          result = result.join(", ")
+          break
+
+        case FieldType.JSON:
+        case FieldType.ATTACHMENTS:
+        case FieldType.SIGNATURE_SINGLE:
+          result = result.map(JSON.parse)
+          break
+
+        case FieldType.LINK:
+          console.error(`${field.type} type is not rendable`)
+      }
+    }
+  } catch (e) {
+    console.error(e.message)
   }
 
   return result
@@ -81,7 +104,11 @@ export const deriveStores = context => {
         ...row,
         __idx: idx,
         ...customColumns.reduce((acc, c) => {
-          acc[c.name] = getRelatedTableValues(row, c)
+          const isSingle =
+            $enrichedSchema[c.related.field].relationshipType ===
+            RelationshipType.ONE_TO_MANY
+
+          acc[c.name] = getRelatedTableValues(row, c, isSingle)
           return acc
         }, {}),
       }))

From 7dc54b3f263f68ca53b2c05e5cc6b5487acf8a93 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 10:21:05 +0200
Subject: [PATCH 24/57] Simplify parsing

---
 .../frontend-core/src/components/grid/stores/rows.js | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index d501520705..3ebf610a29 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -46,18 +46,6 @@ function getRelatedTableValues(row, field, isSingle) {
   try {
     if (isSingle) {
       result = row[field.related.field]?.[0]?.[field.related.subField]
-      switch (field.type) {
-        case FieldType.JSON:
-        case FieldType.SIGNATURE_SINGLE:
-        case FieldType.ATTACHMENT_SINGLE:
-        case FieldType.ATTACHMENTS:
-        case FieldType.BB_REFERENCE:
-          result = JSON.parse(result)
-          break
-
-        case FieldType.LINK:
-          console.error(`${field.type} type is not rendable`)
-      }
     } else {
       // TODO: check all types
       result = Array.from(

From d60588d3237ce81fe23627971662d118db9de716 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 13:05:53 +0200
Subject: [PATCH 25/57] Display signature on on views when dark mode

---
 .../src/components/backend/DataTable/ViewV2DataTable.svelte | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
index 90e5e216f3..a35e1b034e 100644
--- a/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
+++ b/packages/builder/src/components/backend/DataTable/ViewV2DataTable.svelte
@@ -1,6 +1,6 @@
 <script>
   import { viewsV2 } from "stores/builder"
-  import { admin } from "stores/portal"
+  import { admin, themeStore } from "stores/portal"
   import { Grid } from "@budibase/frontend-core"
   import { API } from "api"
   import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
@@ -16,6 +16,9 @@
     tableId: $viewsV2.selected?.tableId,
   }
 
+  $: currentTheme = $themeStore?.theme
+  $: darkMode = !currentTheme.includes("light")
+
   const handleGridViewUpdate = async e => {
     viewsV2.replaceView(id, e.detail)
   }
@@ -25,6 +28,7 @@
   <Grid
     {API}
     {datasource}
+    {darkMode}
     allowAddRows
     allowDeleteRows
     showAvatars={false}

From e4259b4d61e214ab7e166a5c75fc85977d1d68c5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 13:08:10 +0200
Subject: [PATCH 26/57] ENRICHED_RELATIONSHIPS on for dev

---
 packages/backend-core/src/features/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index 0765d09036..1d100c5b16 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -275,5 +275,5 @@ export const flags = new FlagSet({
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
   SQS: Flag.boolean(env.isDev()),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(false),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
 })

From c351a9bced3211346a8f4409a61404f699685274 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 13:11:25 +0200
Subject: [PATCH 27/57] Fix icons

---
 .../src/components/grid/controls/ColumnsSettingContent.svelte   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte
index 456eb50a9c..bbc3d55f04 100644
--- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte
+++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte
@@ -122,8 +122,10 @@
       label: name,
       schema: {
         type: column.type,
+        subtype: column.subtype,
         visible: column.visible,
         readonly: column.readonly,
+        constraints: column.constraints, // This is needed to properly display "users" column
       },
     }
   })

From 475775206cae52626d6da06841214b19d114f8e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 14:20:03 +0200
Subject: [PATCH 28/57] Fix test

---
 .../controls/GridColumnConfiguration/getColumns.js     | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
index dcfed9053f..d3f081756d 100644
--- a/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
+++ b/packages/builder/src/components/design/settings/controls/GridColumnConfiguration/getColumns.js
@@ -72,7 +72,7 @@ const toGridFormat = draggableListColumns => {
   }))
 }
 
-const toDraggableListFormat = (gridFormatColumns, createComponent) => {
+const toDraggableListFormat = (gridFormatColumns, createComponent, schema) => {
   return gridFormatColumns.map(column => {
     return createComponent(
       "@budibase/standard-components/labelfield",
@@ -82,7 +82,7 @@ const toDraggableListFormat = (gridFormatColumns, createComponent) => {
         active: column.active,
         field: column.field,
         label: column.label,
-        columnType: column.columnType,
+        columnType: column.columnType || schema[column.field].type,
         width: column.width,
         conditions: column.conditions,
       },
@@ -103,7 +103,11 @@ const getColumns = ({
     getDefault(schema),
     primaryDisplayColumnName
   )
-  const draggableList = toDraggableListFormat(validatedColumns, createComponent)
+  const draggableList = toDraggableListFormat(
+    validatedColumns,
+    createComponent,
+    schema
+  )
   const primary = draggableList
     .filter(entry => entry.field === primaryDisplayColumnName)
     .map(instance => ({ ...instance, schema }))[0]

From 33f42e10bd07ce961863d5e0b10562e6d005e998 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 14:20:52 +0200
Subject: [PATCH 29/57] Typings

---
 .../src/components/grid/stores/rows.js            | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 3ebf610a29..11bddc3a58 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -42,36 +42,27 @@ export const createStores = () => {
 }
 
 function getRelatedTableValues(row, field, isSingle) {
-  let result = "Not rendable"
+  let result = ""
   try {
     if (isSingle) {
       result = row[field.related.field]?.[0]?.[field.related.subField]
     } else {
-      // TODO: check all types
       result = Array.from(
         new Set(
           row[field.related.field].flatMap(r => r[field.related.subField])
         )
       )
-
       switch (field.type) {
         case FieldType.STRING:
         case FieldType.NUMBER:
         case FieldType.BIGINT:
+        case FieldType.BARCODEQR:
           result = result.join(", ")
           break
-
-        case FieldType.JSON:
-        case FieldType.ATTACHMENTS:
-        case FieldType.SIGNATURE_SINGLE:
-          result = result.map(JSON.parse)
-          break
-
-        case FieldType.LINK:
-          console.error(`${field.type} type is not rendable`)
       }
     }
   } catch (e) {
+    result = "Not rendable"
     console.error(e.message)
   }
 

From 337ac90906fd138f27f9be04a8d7641300bbe79b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 16:39:13 +0200
Subject: [PATCH 30/57] Move and unify code

---
 .../src/components/grid/stores/rows.js        | 29 +---------
 packages/frontend-core/src/utils/index.js     |  2 +-
 .../frontend-core/src/utils/relatedColumns.js | 57 +++++++++++++++++++
 packages/frontend-core/src/utils/schema.js    | 27 ---------
 4 files changed, 59 insertions(+), 56 deletions(-)
 create mode 100644 packages/frontend-core/src/utils/relatedColumns.js
 delete mode 100644 packages/frontend-core/src/utils/schema.js

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 11bddc3a58..9d4608776f 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -6,6 +6,7 @@ import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
 import { FieldType, RelationshipType } from "@budibase/types"
+import { getRelatedTableValues } from "../../../utils"
 
 export const createStores = () => {
   const rows = writable([])
@@ -41,34 +42,6 @@ export const createStores = () => {
   }
 }
 
-function getRelatedTableValues(row, field, isSingle) {
-  let result = ""
-  try {
-    if (isSingle) {
-      result = row[field.related.field]?.[0]?.[field.related.subField]
-    } else {
-      result = Array.from(
-        new Set(
-          row[field.related.field].flatMap(r => r[field.related.subField])
-        )
-      )
-      switch (field.type) {
-        case FieldType.STRING:
-        case FieldType.NUMBER:
-        case FieldType.BIGINT:
-        case FieldType.BARCODEQR:
-          result = result.join(", ")
-          break
-      }
-    }
-  } catch (e) {
-    result = "Not rendable"
-    console.error(e.message)
-  }
-
-  return result
-}
-
 export const deriveStores = context => {
   const { rows, enrichedSchema } = context
 
diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js
index 6f1b42c32e..7663f1fc0b 100644
--- a/packages/frontend-core/src/utils/index.js
+++ b/packages/frontend-core/src/utils/index.js
@@ -10,4 +10,4 @@ export { createWebsocket } from "./websocket"
 export * from "./download"
 export * from "./theme"
 export * from "./settings"
-export * from "./schema"
+export * from "./relatedColumns"
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
new file mode 100644
index 0000000000..d2eca29608
--- /dev/null
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -0,0 +1,57 @@
+import { FieldType } from "@budibase/types"
+
+export function enrichSchemaWithRelColumns(schema) {
+  if (!schema) {
+    return
+  }
+  const result = Object.keys(schema).reduce((acc, c) => {
+    const field = schema[c]
+    acc[c] = field
+
+    if (field.visible !== false && field.columns) {
+      for (const relColumn of Object.keys(field.columns)) {
+        const relField = field.columns[relColumn]
+        if (!relField.visible) {
+          continue
+        }
+        const name = `${field.name}.${relColumn}`
+        acc[name] = {
+          ...relField,
+          name,
+          related: { field: c, subField: relColumn },
+        }
+      }
+    }
+    return acc
+  }, {})
+
+  return result
+}
+
+export function getRelatedTableValues(row, field, isSingle) {
+  let result = ""
+  try {
+    if (isSingle) {
+      result = row[field.related.field]?.[0]?.[field.related.subField]
+    } else {
+      result = Array.from(
+        new Set(
+          row[field.related.field].flatMap(r => r[field.related.subField])
+        )
+      )
+      switch (field.type) {
+        case FieldType.STRING:
+        case FieldType.NUMBER:
+        case FieldType.BIGINT:
+        case FieldType.BARCODEQR:
+          result = result.join(", ")
+          break
+      }
+    }
+  } catch (e) {
+    result = "Not rendable"
+    console.error(e.message)
+  }
+
+  return result
+}
diff --git a/packages/frontend-core/src/utils/schema.js b/packages/frontend-core/src/utils/schema.js
deleted file mode 100644
index 774e650175..0000000000
--- a/packages/frontend-core/src/utils/schema.js
+++ /dev/null
@@ -1,27 +0,0 @@
-export function enrichSchemaWithRelColumns(schema) {
-  if (!schema) {
-    return
-  }
-  const result = Object.keys(schema).reduce((acc, c) => {
-    const field = schema[c]
-    acc[c] = field
-
-    if (field.visible !== false && field.columns) {
-      for (const relColumn of Object.keys(field.columns)) {
-        const relField = field.columns[relColumn]
-        if (!relField.visible) {
-          continue
-        }
-        const name = `${field.name}.${relColumn}`
-        acc[name] = {
-          ...relField,
-          name,
-          related: { field: c, subField: relColumn },
-        }
-      }
-    }
-    return acc
-  }, {})
-
-  return result
-}

From 863ff4d9f9724bd2061438b6ad0f2f754bbabb91 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 17:29:44 +0200
Subject: [PATCH 31/57] Format dates

---
 .../src/components/grid/lib/renderers.js      |  6 ++-
 .../frontend-core/src/utils/relatedColumns.js | 40 ++++++++++++++-----
 2 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.js
index 2e22ee1871..6af21e65f0 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.js
@@ -35,5 +35,9 @@ const TypeComponentMap = {
   [FieldType.BB_REFERENCE_SINGLE]: BBReferenceSingleCell,
 }
 export const getCellRenderer = column => {
-  return TypeComponentMap[column?.schema?.type] || TextCell
+  return (
+    TypeComponentMap[column?.schema?.cellRenderType] ||
+    TypeComponentMap[column?.schema?.type] ||
+    TextCell
+  )
 }
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index d2eca29608..a0ac01a21c 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -1,4 +1,25 @@
 import { FieldType } from "@budibase/types"
+import { Helpers } from "@budibase/bbui"
+
+const columnTypeManyOverrides = {
+  [FieldType.DATETIME]: {
+    overridedType: FieldType.STRING,
+    parser: (value, field) => {
+      const { timeOnly, dateOnly, ignoreTimezones } = field
+      const enableTime = !dateOnly
+      const parsedValue = Helpers.parseDate(value, {
+        timeOnly,
+        enableTime,
+        ignoreTimezones,
+      })
+      const result = Helpers.getDateDisplayValue(parsedValue, {
+        enableTime,
+        timeOnly,
+      })
+      return result
+    },
+  },
+}
 
 export function enrichSchemaWithRelColumns(schema) {
   if (!schema) {
@@ -19,6 +40,9 @@ export function enrichSchemaWithRelColumns(schema) {
           ...relField,
           name,
           related: { field: c, subField: relColumn },
+          cellRenderType:
+            columnTypeManyOverrides[relField.type]?.overridedType ||
+            relField.type,
         }
       }
     }
@@ -34,19 +58,15 @@ export function getRelatedTableValues(row, field, isSingle) {
     if (isSingle) {
       result = row[field.related.field]?.[0]?.[field.related.subField]
     } else {
+      const parser =
+        columnTypeManyOverrides[field.type]?.parser || (value => value)
       result = Array.from(
         new Set(
-          row[field.related.field].flatMap(r => r[field.related.subField])
+          row[field.related.field].flatMap(r =>
+            parser(r[field.related.subField], field)
+          )
         )
-      )
-      switch (field.type) {
-        case FieldType.STRING:
-        case FieldType.NUMBER:
-        case FieldType.BIGINT:
-        case FieldType.BARCODEQR:
-          result = result.join(", ")
-          break
-      }
+      ).join(", ")
     }
   } catch (e) {
     result = "Not rendable"

From 58a51207f3f5d390104f95f13a92843b277bcade Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 17:31:16 +0200
Subject: [PATCH 32/57] Display multi boolean

---
 packages/frontend-core/src/utils/relatedColumns.js | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index a0ac01a21c..e7271e5a26 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -19,6 +19,15 @@ const columnTypeManyOverrides = {
       return result
     },
   },
+  [FieldType.BOOLEAN]: {
+    overridedType: FieldType.STRING,
+    parser: value => {
+      if (value) {
+        return true
+      }
+      return false
+    },
+  },
 }
 
 export function enrichSchemaWithRelColumns(schema) {

From cbd82bcf6a6266e3aec5e619c1f4b0e8b43fc555 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 17:42:21 +0200
Subject: [PATCH 33/57] Format strings

---
 .../src/components/grid/stores/rows.js          |  4 ++--
 .../frontend-core/src/utils/relatedColumns.js   | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 9d4608776f..3e2b59c633 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -56,11 +56,11 @@ export const deriveStores = context => {
         ...row,
         __idx: idx,
         ...customColumns.reduce((acc, c) => {
-          const isSingle =
+          const fromSingle =
             $enrichedSchema[c.related.field].relationshipType ===
             RelationshipType.ONE_TO_MANY
 
-          acc[c.name] = getRelatedTableValues(row, c, isSingle)
+          acc[c.name] = getRelatedTableValues(row, c, fromSingle)
           return acc
         }, {}),
       }))
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index e7271e5a26..f7298614ed 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -61,10 +61,10 @@ export function enrichSchemaWithRelColumns(schema) {
   return result
 }
 
-export function getRelatedTableValues(row, field, isSingle) {
+export function getRelatedTableValues(row, field, fromSingle) {
   let result = ""
   try {
-    if (isSingle) {
+    if (fromSingle) {
       result = row[field.related.field]?.[0]?.[field.related.subField]
     } else {
       const parser =
@@ -75,7 +75,18 @@ export function getRelatedTableValues(row, field, isSingle) {
             parser(r[field.related.subField], field)
           )
         )
-      ).join(", ")
+      )
+
+      if (
+        [
+          FieldType.STRING,
+          FieldType.NUMBER,
+          FieldType.BIGINT,
+          FieldType.BOOLEAN,
+        ].includes(field.type)
+      ) {
+        result = result.join(", ")
+      }
     }
   } catch (e) {
     result = "Not rendable"

From d68a2ac41e02db0478f750a7e3663558930c23b7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 17:48:33 +0200
Subject: [PATCH 34/57] Display signatures

---
 .../src/components/grid/stores/rows.js        |  7 ++----
 .../frontend-core/src/utils/relatedColumns.js | 23 +++++++++++--------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 3e2b59c633..7317611b5e 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -56,11 +56,8 @@ export const deriveStores = context => {
         ...row,
         __idx: idx,
         ...customColumns.reduce((acc, c) => {
-          const fromSingle =
-            $enrichedSchema[c.related.field].relationshipType ===
-            RelationshipType.ONE_TO_MANY
-
-          acc[c.name] = getRelatedTableValues(row, c, fromSingle)
+          const fromField = $enrichedSchema[c.related.field]
+          acc[c.name] = getRelatedTableValues(row, c, fromField)
           return acc
         }, {}),
       }))
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index f7298614ed..0b04ea9b69 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -1,4 +1,4 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, RelationshipType } from "@budibase/types"
 import { Helpers } from "@budibase/bbui"
 
 const columnTypeManyOverrides = {
@@ -21,12 +21,10 @@ const columnTypeManyOverrides = {
   },
   [FieldType.BOOLEAN]: {
     overridedType: FieldType.STRING,
-    parser: value => {
-      if (value) {
-        return true
-      }
-      return false
-    },
+    parser: value => !!value,
+  },
+  [FieldType.SIGNATURE_SINGLE]: {
+    overridedType: FieldType.ATTACHMENTS,
   },
 }
 
@@ -39,6 +37,9 @@ export function enrichSchemaWithRelColumns(schema) {
     acc[c] = field
 
     if (field.visible !== false && field.columns) {
+      const fromSingle =
+        field?.relationshipType === RelationshipType.ONE_TO_MANY
+
       for (const relColumn of Object.keys(field.columns)) {
         const relField = field.columns[relColumn]
         if (!relField.visible) {
@@ -50,7 +51,8 @@ export function enrichSchemaWithRelColumns(schema) {
           name,
           related: { field: c, subField: relColumn },
           cellRenderType:
-            columnTypeManyOverrides[relField.type]?.overridedType ||
+            (!fromSingle &&
+              columnTypeManyOverrides[relField.type]?.overridedType) ||
             relField.type,
         }
       }
@@ -61,7 +63,10 @@ export function enrichSchemaWithRelColumns(schema) {
   return result
 }
 
-export function getRelatedTableValues(row, field, fromSingle) {
+export function getRelatedTableValues(row, field, fromField) {
+  const fromSingle =
+    fromField?.relationshipType === RelationshipType.ONE_TO_MANY
+
   let result = ""
   try {
     if (fromSingle) {

From 7f5f4d9ecbf341de3867c3a68f4c836cc8dd82e6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 18:10:23 +0200
Subject: [PATCH 35/57] Handle unique values

---
 .../frontend-core/src/utils/relatedColumns.js | 21 +++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index 0b04ea9b69..2685a2ce98 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -26,6 +26,15 @@ const columnTypeManyOverrides = {
   [FieldType.SIGNATURE_SINGLE]: {
     overridedType: FieldType.ATTACHMENTS,
   },
+  [FieldType.BB_REFERENCE_SINGLE]: {
+    parser: value => [...new Map(value.map(i => [i._id, i])).values()],
+  },
+  [FieldType.BB_REFERENCE]: {
+    parser: value => [...new Map(value.map(i => [i._id, i])).values()],
+  },
+  [FieldType.ARRAY]: {
+    parser: value => Array.from(new Set(value)),
+  },
 }
 
 export function enrichSchemaWithRelColumns(schema) {
@@ -74,12 +83,9 @@ export function getRelatedTableValues(row, field, fromField) {
     } else {
       const parser =
         columnTypeManyOverrides[field.type]?.parser || (value => value)
-      result = Array.from(
-        new Set(
-          row[field.related.field].flatMap(r =>
-            parser(r[field.related.subField], field)
-          )
-        )
+
+      result = parser(
+        row[field.related.field].flatMap(r => r[field.related.subField], field)
       )
 
       if (
@@ -88,6 +94,9 @@ export function getRelatedTableValues(row, field, fromField) {
           FieldType.NUMBER,
           FieldType.BIGINT,
           FieldType.BOOLEAN,
+          FieldType.DATETIME,
+          FieldType.LONGFORM,
+          FieldType.BARCODEQR,
         ].includes(field.type)
       ) {
         result = result.join(", ")

From 680cf5803d3af03feeb81bd30ece1a2dfe9ca24e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 25 Sep 2024 18:17:55 +0200
Subject: [PATCH 36/57] Fix formattings

---
 .../frontend-core/src/utils/relatedColumns.js | 56 +++++++++----------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index 2685a2ce98..616b4f4b8c 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -1,40 +1,39 @@
 import { FieldType, RelationshipType } from "@budibase/types"
 import { Helpers } from "@budibase/bbui"
 
-const columnTypeManyOverrides = {
-  [FieldType.DATETIME]: {
-    overridedType: FieldType.STRING,
-    parser: (value, field) => {
-      const { timeOnly, dateOnly, ignoreTimezones } = field
+const columnTypeManyTypeOverrides = {
+  [FieldType.DATETIME]: FieldType.STRING,
+  [FieldType.BOOLEAN]: FieldType.STRING,
+  [FieldType.SIGNATURE_SINGLE]: FieldType.ATTACHMENTS,
+}
+
+const columnTypeManyParser = {
+  [FieldType.DATETIME]: (value, field) => {
+    function parseDate(value) {
+      const { timeOnly, dateOnly, ignoreTimezones } = field || {}
       const enableTime = !dateOnly
       const parsedValue = Helpers.parseDate(value, {
         timeOnly,
         enableTime,
         ignoreTimezones,
       })
-      const result = Helpers.getDateDisplayValue(parsedValue, {
+      const parsed = Helpers.getDateDisplayValue(parsedValue, {
         enableTime,
         timeOnly,
       })
-      return result
-    },
-  },
-  [FieldType.BOOLEAN]: {
-    overridedType: FieldType.STRING,
-    parser: value => !!value,
-  },
-  [FieldType.SIGNATURE_SINGLE]: {
-    overridedType: FieldType.ATTACHMENTS,
-  },
-  [FieldType.BB_REFERENCE_SINGLE]: {
-    parser: value => [...new Map(value.map(i => [i._id, i])).values()],
-  },
-  [FieldType.BB_REFERENCE]: {
-    parser: value => [...new Map(value.map(i => [i._id, i])).values()],
-  },
-  [FieldType.ARRAY]: {
-    parser: value => Array.from(new Set(value)),
+      return parsed
+    }
+
+    return value?.map(v => parseDate(v))
   },
+  [FieldType.BOOLEAN]: value => value?.map(v => !!v),
+  [FieldType.BB_REFERENCE_SINGLE]: value => [
+    ...new Map(value.map(i => [i._id, i])).values(),
+  ],
+  [FieldType.BB_REFERENCE]: value => [
+    ...new Map(value.map(i => [i._id, i])).values(),
+  ],
+  [FieldType.ARRAY]: value => Array.from(new Set(value)),
 }
 
 export function enrichSchemaWithRelColumns(schema) {
@@ -60,8 +59,7 @@ export function enrichSchemaWithRelColumns(schema) {
           name,
           related: { field: c, subField: relColumn },
           cellRenderType:
-            (!fromSingle &&
-              columnTypeManyOverrides[relField.type]?.overridedType) ||
+            (!fromSingle && columnTypeManyTypeOverrides[relField.type]) ||
             relField.type,
         }
       }
@@ -81,11 +79,11 @@ export function getRelatedTableValues(row, field, fromField) {
     if (fromSingle) {
       result = row[field.related.field]?.[0]?.[field.related.subField]
     } else {
-      const parser =
-        columnTypeManyOverrides[field.type]?.parser || (value => value)
+      const parser = columnTypeManyParser[field.type] || (value => value)
 
       result = parser(
-        row[field.related.field].flatMap(r => r[field.related.subField], field)
+        row[field.related.field].flatMap(r => r[field.related.subField]),
+        field
       )
 
       if (

From 0c44f584665a8c200f7916ff292e0ac39c12ef14 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 13:33:14 +0200
Subject: [PATCH 37/57] Fix attachment undefined references

---
 .../components/grid/cells/AttachmentCell.svelte    | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index f485593c46..285577a71c 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -80,12 +80,14 @@
   bind:this={anchor}
 >
   {#each value || [] as attachment}
-    {#if isImage(attachment.extension)}
-      <img src={attachment.url} alt={attachment.extension} />
-    {:else}
-      <div class="file" title={attachment.name}>
-        {attachment.extension}
-      </div>
+    {#if attachment}
+      {#if isImage(attachment.extension)}
+        <img src={attachment.url} alt={attachment.extension} />
+      {:else}
+        <div class="file" title={attachment.name}>
+          {attachment.extension}
+        </div>
+      {/if}
     {/if}
   {/each}
 </div>

From 031bdcfeb52f37f717b87e5de79502a80da6f2a2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 13:41:54 +0200
Subject: [PATCH 38/57] Display signatures

---
 .../src/components/grid/cells/AttachmentCell.svelte | 13 ++++++++++++-
 .../src/components/grid/stores/rows.js              |  2 +-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index 285577a71c..029fdfd726 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -2,6 +2,7 @@
   import { onMount, getContext } from "svelte"
   import { Dropzone } from "@budibase/bbui"
   import GridPopover from "../overlays/GridPopover.svelte"
+  import { FieldType } from "@budibase/types"
 
   export let value
   export let focused = false
@@ -82,7 +83,12 @@
   {#each value || [] as attachment}
     {#if attachment}
       {#if isImage(attachment.extension)}
-        <img src={attachment.url} alt={attachment.extension} />
+        <img
+          class:light={!$props?.darkMode &&
+            schema.type === FieldType.SIGNATURE_SINGLE}
+          src={attachment.url}
+          alt={attachment.extension}
+        />
       {:else}
         <div class="file" title={attachment.name}>
           {attachment.extension}
@@ -142,4 +148,9 @@
     width: 320px;
     padding: var(--cell-padding);
   }
+
+  .attachment-cell img.light {
+    -webkit-filter: invert(100%);
+    filter: invert(100%);
+  }
 </style>
diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 7317611b5e..25acad6520 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -5,7 +5,7 @@ import { getCellID, parseCellID } from "../lib/utils"
 import { tick } from "svelte"
 import { Helpers } from "@budibase/bbui"
 import { sleep } from "../../../utils/utils"
-import { FieldType, RelationshipType } from "@budibase/types"
+import { FieldType } from "@budibase/types"
 import { getRelatedTableValues } from "../../../utils"
 
 export const createStores = () => {

From f53664a694f1af0816eee2bc306ccd5c2be28c19 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 13:59:42 +0200
Subject: [PATCH 39/57] Fix null references

---
 .../grid/cells/AttachmentCell.svelte          | 24 +++++++++----------
 .../frontend-core/src/utils/relatedColumns.js |  4 +++-
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
index 029fdfd726..848ea470dd 100644
--- a/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
@@ -81,19 +81,17 @@
   bind:this={anchor}
 >
   {#each value || [] as attachment}
-    {#if attachment}
-      {#if isImage(attachment.extension)}
-        <img
-          class:light={!$props?.darkMode &&
-            schema.type === FieldType.SIGNATURE_SINGLE}
-          src={attachment.url}
-          alt={attachment.extension}
-        />
-      {:else}
-        <div class="file" title={attachment.name}>
-          {attachment.extension}
-        </div>
-      {/if}
+    {#if isImage(attachment.extension)}
+      <img
+        class:light={!$props?.darkMode &&
+          schema.type === FieldType.SIGNATURE_SINGLE}
+        src={attachment.url}
+        alt={attachment.extension}
+      />
+    {:else}
+      <div class="file" title={attachment.name}>
+        {attachment.extension}
+      </div>
     {/if}
   {/each}
 </div>
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index 616b4f4b8c..55f1bbbb00 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -82,7 +82,9 @@ export function getRelatedTableValues(row, field, fromField) {
       const parser = columnTypeManyParser[field.type] || (value => value)
 
       result = parser(
-        row[field.related.field].flatMap(r => r[field.related.subField]),
+        row[field.related.field]
+          .flatMap(r => r[field.related.subField])
+          .filter(i => i !== undefined && i !== null),
         field
       )
 

From 4fe0af81ecd4fab1591a1c0fe69d6c4eeb45b3e6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 14:14:28 +0200
Subject: [PATCH 40/57] Fix test runs

---
 packages/backend-core/src/features/index.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index 2b915e5689..71918ef80c 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -277,5 +277,7 @@ export const flags = new FlagSet({
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
   SQS: Flag.boolean(env.isDev()),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(
+    env.isDev() && !env.isTest()
+  ),
 })

From b4da4529a35d63e28586377572947fccbf022d5f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 15:23:41 +0200
Subject: [PATCH 41/57] ENRICHED_RELATIONSHIPS on by default in tests

---
 packages/backend-core/src/features/index.ts      | 4 +---
 packages/server/src/api/routes/tests/row.spec.ts | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index 71918ef80c..2b915e5689 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -277,7 +277,5 @@ export const flags = new FlagSet({
   AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
   SQS: Flag.boolean(env.isDev()),
   [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
-  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(
-    env.isDev() && !env.isTest()
-  ),
+  [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
 })
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index dc03a21d6d..60efb664cd 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2684,7 +2684,7 @@ describe.each([
         async (__, retrieveDelegate) => {
           await withCoreEnv(
             {
-              TENANT_FEATURE_FLAGS: ``,
+              TENANT_FEATURE_FLAGS: `*:!${FeatureFlag.ENRICHED_RELATIONSHIPS}`,
             },
             async () => {
               const otherRows = _.sampleSize(auxData, 5)

From ee962380b3d72c40b4f337ebd573e7ed22ab8f30 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 16:18:19 +0200
Subject: [PATCH 42/57] Handle undefined properly

---
 .../frontend-core/src/utils/relatedColumns.js | 50 +++++++++----------
 1 file changed, 23 insertions(+), 27 deletions(-)

diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index 55f1bbbb00..9d389524fd 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -75,36 +75,32 @@ export function getRelatedTableValues(row, field, fromField) {
     fromField?.relationshipType === RelationshipType.ONE_TO_MANY
 
   let result = ""
-  try {
-    if (fromSingle) {
-      result = row[field.related.field]?.[0]?.[field.related.subField]
-    } else {
-      const parser = columnTypeManyParser[field.type] || (value => value)
 
-      result = parser(
-        row[field.related.field]
-          .flatMap(r => r[field.related.subField])
-          .filter(i => i !== undefined && i !== null),
-        field
-      )
+  if (fromSingle) {
+    result = row[field.related.field]?.[0]?.[field.related.subField]
+  } else {
+    const parser = columnTypeManyParser[field.type] || (value => value)
 
-      if (
-        [
-          FieldType.STRING,
-          FieldType.NUMBER,
-          FieldType.BIGINT,
-          FieldType.BOOLEAN,
-          FieldType.DATETIME,
-          FieldType.LONGFORM,
-          FieldType.BARCODEQR,
-        ].includes(field.type)
-      ) {
-        result = result.join(", ")
-      }
+    result = parser(
+      row[field.related.field]
+        ?.flatMap(r => r[field.related.subField])
+        ?.filter(i => i !== undefined && i !== null),
+      field
+    )
+
+    if (
+      [
+        FieldType.STRING,
+        FieldType.NUMBER,
+        FieldType.BIGINT,
+        FieldType.BOOLEAN,
+        FieldType.DATETIME,
+        FieldType.LONGFORM,
+        FieldType.BARCODEQR,
+      ].includes(field.type)
+    ) {
+      result = result?.join(", ")
     }
-  } catch (e) {
-    result = "Not rendable"
-    console.error(e.message)
   }
 
   return result

From d7715e4f5c84ec0ccdde40a20cc93836c96c3660 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 17:00:44 +0200
Subject: [PATCH 43/57] Don't allow related columns as display or sorting

---
 .../DataTable/modals/CreateEditColumn.svelte       |  5 ++---
 .../backend/TableNavigator/TableDataImport.svelte  |  7 ++++---
 .../src/components/grid/cells/HeaderCell.svelte    |  6 +++---
 packages/frontend-core/src/utils/index.js          |  1 +
 packages/frontend-core/src/utils/table.js          | 14 ++++++++++++++
 5 files changed, 24 insertions(+), 9 deletions(-)
 create mode 100644 packages/frontend-core/src/utils/table.js

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index a956d09ee6..0130c39715 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -19,7 +19,6 @@
     helpers,
     PROTECTED_INTERNAL_COLUMNS,
     PROTECTED_EXTERNAL_COLUMNS,
-    canBeDisplayColumn,
     canHaveDefaultColumn,
   } from "@budibase/shared-core"
   import { createEventDispatcher, getContext, onMount } from "svelte"
@@ -43,7 +42,7 @@
     SourceName,
   } from "@budibase/types"
   import RelationshipSelector from "components/common/RelationshipSelector.svelte"
-  import { RowUtils } from "@budibase/frontend-core"
+  import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core"
   import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
   import { isEnabled } from "helpers/featureFlags"
@@ -166,7 +165,7 @@
     : availableAutoColumns
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
-    canBeDisplayColumn(editableColumn.type) && !editableColumn.autocolumn
+    canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
   $: canHaveDefault =
     isEnabled("DEFAULT_VALUES") && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
diff --git a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
index 4c9f4dd10f..5804dc3172 100644
--- a/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
+++ b/packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte
@@ -1,7 +1,8 @@
 <script>
   import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
-  import { canBeDisplayColumn, utils } from "@budibase/shared-core"
+  import { utils } from "@budibase/shared-core"
+  import { canBeDisplayColumn } from "@budibase/frontend-core"
   import { API } from "api"
   import { parseFile } from "./utils"
 
@@ -100,10 +101,10 @@
   let rawRows = []
 
   $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
-    return validation[column] && canBeDisplayColumn(schema[column].type)
+    return validation[column] && canBeDisplayColumn(schema[column])
   })
 
-  $: if (displayColumn && !canBeDisplayColumn(schema[displayColumn].type)) {
+  $: if (displayColumn && !canBeDisplayColumn(schema[displayColumn])) {
     displayColumn = null
   }
 
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 3b6aa5d424..23e19af96b 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -1,6 +1,7 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
-  import { canBeDisplayColumn, canBeSortColumn } from "@budibase/shared-core"
+  import { canBeSortColumn } from "@budibase/shared-core"
+  import { canBeDisplayColumn } from "@budibase/frontend-core"
   import { Icon, Menu, MenuItem, Modal } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
@@ -347,8 +348,7 @@
         <MenuItem
           icon="Label"
           on:click={makeDisplayColumn}
-          disabled={column.primaryDisplay ||
-            !canBeDisplayColumn(column.schema.type)}
+          disabled={column.primaryDisplay || !canBeDisplayColumn(column.schema)}
         >
           Use as display column
         </MenuItem>
diff --git a/packages/frontend-core/src/utils/index.js b/packages/frontend-core/src/utils/index.js
index 7663f1fc0b..adfe3ded26 100644
--- a/packages/frontend-core/src/utils/index.js
+++ b/packages/frontend-core/src/utils/index.js
@@ -11,3 +11,4 @@ export * from "./download"
 export * from "./theme"
 export * from "./settings"
 export * from "./relatedColumns"
+export * from "./table"
diff --git a/packages/frontend-core/src/utils/table.js b/packages/frontend-core/src/utils/table.js
new file mode 100644
index 0000000000..5fddf1f641
--- /dev/null
+++ b/packages/frontend-core/src/utils/table.js
@@ -0,0 +1,14 @@
+import * as sharedCore from "@budibase/shared-core"
+
+export function canBeDisplayColumn(column) {
+  if (!sharedCore.canBeDisplayColumn(column.type)) {
+    return sharedCore
+  }
+
+  if (column.related) {
+    // If it's a related column (only available in the frontend), don't allow using it as display column
+    return false
+  }
+
+  return true
+}

From aa044b94cf4615f778d27b1cd6f422ff65bca68b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 21:09:47 +0200
Subject: [PATCH 44/57] Don't allow sort columns as display or sorting

---
 .../src/components/backend/DataTable/Table.svelte |  4 ++--
 .../settings/controls/SortableFieldSelect.svelte  |  4 ++--
 .../components/app/deprecated/table/Table.svelte  |  4 ++--
 .../src/components/grid/cells/HeaderCell.svelte   |  7 +++----
 .../components/grid/controls/SortButton.svelte    |  4 ++--
 packages/frontend-core/src/utils/table.js         | 15 ++++++++++++++-
 6 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte
index f7eccd5242..e09dd4bd39 100644
--- a/packages/builder/src/components/backend/DataTable/Table.svelte
+++ b/packages/builder/src/components/backend/DataTable/Table.svelte
@@ -6,7 +6,7 @@
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
   import { createEventDispatcher } from "svelte"
-  import { canBeSortColumn } from "@budibase/shared-core"
+  import { canBeSortColumn } from "@budibase/frontend-core"
 
   export let schema = {}
   export let data = []
@@ -31,7 +31,7 @@
       acc[key] =
         typeof schema[key] === "string" ? { type: schema[key] } : schema[key]
 
-      if (!canBeSortColumn(acc[key].type)) {
+      if (!canBeSortColumn(acc[key])) {
         acc[key].sortable = false
       }
       return acc
diff --git a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
index d51493616e..0f9290ec8d 100644
--- a/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
+++ b/packages/builder/src/components/design/settings/controls/SortableFieldSelect.svelte
@@ -3,7 +3,7 @@
   import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
   import { selectedScreen } from "stores/builder"
   import { createEventDispatcher } from "svelte"
-  import { canBeSortColumn } from "@budibase/shared-core"
+  import { canBeSortColumn } from "@budibase/frontend-core"
 
   export let componentInstance = {}
   export let value = ""
@@ -17,7 +17,7 @@
 
   const getSortableFields = schema => {
     return Object.entries(schema || {})
-      .filter(entry => canBeSortColumn(entry[1].type))
+      .filter(entry => canBeSortColumn(entry[1]))
       .map(entry => entry[0])
   }
 
diff --git a/packages/client/src/components/app/deprecated/table/Table.svelte b/packages/client/src/components/app/deprecated/table/Table.svelte
index ac3d88d29c..ca97a012bc 100644
--- a/packages/client/src/components/app/deprecated/table/Table.svelte
+++ b/packages/client/src/components/app/deprecated/table/Table.svelte
@@ -2,7 +2,7 @@
   import { getContext, onDestroy } from "svelte"
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
-  import { canBeSortColumn } from "@budibase/shared-core"
+  import { canBeSortColumn } from "@budibase/frontend-core"
   import Provider from "components/context/Provider.svelte"
 
   export let dataProvider
@@ -146,7 +146,7 @@
         return
       }
       newSchema[columnName] = schema[columnName]
-      if (!canBeSortColumn(schema[columnName].type)) {
+      if (!canBeSortColumn(schema[columnName])) {
         newSchema[columnName].sortable = false
       }
 
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 23e19af96b..00b2cbabf3 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -1,7 +1,6 @@
 <script>
   import { getContext, onMount, tick } from "svelte"
-  import { canBeSortColumn } from "@budibase/shared-core"
-  import { canBeDisplayColumn } from "@budibase/frontend-core"
+  import { canBeSortColumn, canBeDisplayColumn } from "@budibase/frontend-core"
   import { Icon, Menu, MenuItem, Modal } from "@budibase/bbui"
   import GridCell from "./GridCell.svelte"
   import { getColumnIcon } from "../lib/utils"
@@ -355,7 +354,7 @@
         <MenuItem
           icon="SortOrderUp"
           on:click={sortAscending}
-          disabled={!canBeSortColumn(column.schema.type) ||
+          disabled={!canBeSortColumn(column.schema) ||
             (column.name === $sort.column && $sort.order === "ascending")}
         >
           Sort {sortingLabels.ascending}
@@ -363,7 +362,7 @@
         <MenuItem
           icon="SortOrderDown"
           on:click={sortDescending}
-          disabled={!canBeSortColumn(column.schema.type) ||
+          disabled={!canBeSortColumn(column.schema) ||
             (column.name === $sort.column && $sort.order === "descending")}
         >
           Sort {sortingLabels.descending}
diff --git a/packages/frontend-core/src/components/grid/controls/SortButton.svelte b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
index 96e5481d7a..eb67c34651 100644
--- a/packages/frontend-core/src/components/grid/controls/SortButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
@@ -1,7 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { ActionButton, Popover, Select } from "@budibase/bbui"
-  import { canBeSortColumn } from "@budibase/shared-core"
+  import { canBeSortColumn } from "@budibase/frontend-core"
 
   const { sort, columns } = getContext("grid")
 
@@ -14,7 +14,7 @@
       value: col.name,
       type: col.schema?.type,
     }))
-    .filter(col => canBeSortColumn(col.type))
+    .filter(col => canBeSortColumn(col))
   $: orderOptions = getOrderOptions($sort.column, columnOptions)
 
   const getOrderOptions = (column, columnOptions) => {
diff --git a/packages/frontend-core/src/utils/table.js b/packages/frontend-core/src/utils/table.js
index 5fddf1f641..193848ec4d 100644
--- a/packages/frontend-core/src/utils/table.js
+++ b/packages/frontend-core/src/utils/table.js
@@ -2,7 +2,20 @@ import * as sharedCore from "@budibase/shared-core"
 
 export function canBeDisplayColumn(column) {
   if (!sharedCore.canBeDisplayColumn(column.type)) {
-    return sharedCore
+    return false
+  }
+
+  if (column.related) {
+    // If it's a related column (only available in the frontend), don't allow using it as display column
+    return false
+  }
+
+  return true
+}
+
+export function canBeSortColumn(column) {
+  if (!sharedCore.canBeSortColumn(column.type)) {
+    return false
   }
 
   if (column.related) {

From 3e294c3cb93cb8106520dd1ca5305fdf82ee6689 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 26 Sep 2024 21:15:14 +0200
Subject: [PATCH 45/57] Don't allow related fields in sort button

---
 .../frontend-core/src/components/grid/controls/SortButton.svelte | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/frontend-core/src/components/grid/controls/SortButton.svelte b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
index eb67c34651..5a1f6b221a 100644
--- a/packages/frontend-core/src/components/grid/controls/SortButton.svelte
+++ b/packages/frontend-core/src/components/grid/controls/SortButton.svelte
@@ -13,6 +13,7 @@
       label: col.label || col.name,
       value: col.name,
       type: col.schema?.type,
+      related: col.related,
     }))
     .filter(col => canBeSortColumn(col))
   $: orderOptions = getOrderOptions($sort.column, columnOptions)

From acdcd02fcdf85427d5c42aa4bc5dd5219f979c24 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 27 Sep 2024 09:12:50 +0100
Subject: [PATCH 46/57] Fix small issue with conditions in branches

---
 .../tests/scenarios/branching.spec.ts         | 34 +++++++++----------
 packages/server/src/threads/automation.ts     |  6 ++--
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/packages/server/src/automations/tests/scenarios/branching.spec.ts b/packages/server/src/automations/tests/scenarios/branching.spec.ts
index df513e98c6..ae89fc18b5 100644
--- a/packages/server/src/automations/tests/scenarios/branching.spec.ts
+++ b/packages/server/src/automations/tests/scenarios/branching.spec.ts
@@ -32,25 +32,25 @@ describe("Branching automations", () => {
                 steps: stepBuilder =>
                   stepBuilder.serverLog({ text: "Branch 1.1" }),
                 condition: {
-                  equal: { "steps.1.success": true },
+                  equal: { "{{steps.1.success}}": true },
                 },
               },
               branch2: {
                 steps: stepBuilder =>
                   stepBuilder.serverLog({ text: "Branch 1.2" }),
                 condition: {
-                  equal: { "steps.1.success": false },
+                  equal: { "{{steps.1.success}}": false },
                 },
               },
             }),
           condition: {
-            equal: { "steps.1.success": true },
+            equal: { "{{steps.1.success}}": true },
           },
         },
         topLevelBranch2: {
           steps: stepBuilder => stepBuilder.serverLog({ text: "Branch 2" }),
           condition: {
-            equal: { "steps.1.success": false },
+            equal: { "{{steps.1.success}}": false },
           },
         },
       })
@@ -70,14 +70,14 @@ describe("Branching automations", () => {
         activeBranch: {
           steps: stepBuilder => stepBuilder.serverLog({ text: "Active user" }),
           condition: {
-            equal: { "trigger.fields.status": "active" },
+            equal: { "{{trigger.fields.status}}": "active" },
           },
         },
         inactiveBranch: {
           steps: stepBuilder =>
             stepBuilder.serverLog({ text: "Inactive user" }),
           condition: {
-            equal: { "trigger.fields.status": "inactive" },
+            equal: { "{{trigger.fields.status}}": "inactive" },
           },
         },
       })
@@ -102,8 +102,8 @@ describe("Branching automations", () => {
           condition: {
             $and: {
               conditions: [
-                { equal: { "trigger.fields.status": "active" } },
-                { equal: { "trigger.fields.role": "admin" } },
+                { equal: { "{{trigger.fields.status}}": "active" } },
+                { equal: { "{{trigger.fields.role}}": "admin" } },
               ],
             },
           },
@@ -111,7 +111,7 @@ describe("Branching automations", () => {
         otherBranch: {
           steps: stepBuilder => stepBuilder.serverLog({ text: "Other user" }),
           condition: {
-            notEqual: { "trigger.fields.status": "active" },
+            notEqual: { "{{trigger.fields.status}}": "active" },
           },
         },
       })
@@ -133,8 +133,8 @@ describe("Branching automations", () => {
           condition: {
             $or: {
               conditions: [
-                { equal: { "trigger.fields.status": "test" } },
-                { equal: { "trigger.fields.role": "admin" } },
+                { equal: { "{{trigger.fields.status}}": "test" } },
+                { equal: { "{{trigger.fields.role}}": "admin" } },
               ],
             },
           },
@@ -144,8 +144,8 @@ describe("Branching automations", () => {
           condition: {
             $and: {
               conditions: [
-                { notEqual: { "trigger.fields.status": "active" } },
-                { notEqual: { "trigger.fields.role": "admin" } },
+                { notEqual: { "{{trigger.fields.status}}": "active" } },
+                { notEqual: { "{{trigger.fields.role}}": "admin" } },
               ],
             },
           },
@@ -170,8 +170,8 @@ describe("Branching automations", () => {
           condition: {
             $or: {
               conditions: [
-                { equal: { "trigger.fields.status": "new" } },
-                { equal: { "trigger.fields.role": "admin" } },
+                { equal: { "{{trigger.fields.status}}": "new" } },
+                { equal: { "{{trigger.fields.role}}": "admin" } },
               ],
             },
           },
@@ -181,8 +181,8 @@ describe("Branching automations", () => {
           condition: {
             $and: {
               conditions: [
-                { equal: { "trigger.fields.status": "active" } },
-                { equal: { "trigger.fields.role": "admin" } },
+                { equal: { "{{trigger.fields.status}}": "active" } },
+                { equal: { "{{trigger.fields.role}}": "admin" } },
               ],
             },
           },
diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index a59935c6aa..e2a5a1c192 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -516,10 +516,8 @@ class Orchestrator {
       filter => {
         Object.entries(filter).forEach(([_, value]) => {
           Object.entries(value).forEach(([field, _]) => {
-            const fromContext = processStringSync(
-              `{{ literal ${field} }}`,
-              this.context
-            )
+            const updatedField = field.replace("{{", "{{ literal ")
+            const fromContext = processStringSync(updatedField, this.context)
             toFilter[field] = fromContext
           })
         })

From 64f4497c76f768a4bcb1e314b789c57086508d18 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 27 Sep 2024 11:02:00 +0200
Subject: [PATCH 47/57] Prevent flashed while sorting

---
 .../grid/stores/datasources/viewV2.js         | 38 ++++++++++---------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
index 425519d97e..360d7a0e1c 100644
--- a/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
+++ b/packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
@@ -120,25 +120,29 @@ export const initialise = context => {
     // When sorting changes, ensure view definition is kept up to date
     unsubscribers.push(
       sort.subscribe(async $sort => {
+        // Ensure we're updating the correct view
+        const $view = get(definition)
+        if ($view?.id !== $datasource.id) {
+          return
+        }
+
+        // Skip if nothing actually changed
+        if (
+          $sort?.column === $view.sort?.field &&
+          $sort?.order === $view.sort?.order
+        ) {
+          return
+        }
+
         // If we can mutate schema then update the view definition
         if (get(config).canSaveSchema) {
-          // Ensure we're updating the correct view
-          const $view = get(definition)
-          if ($view?.id !== $datasource.id) {
-            return
-          }
-          if (
-            $sort?.column !== $view.sort?.field ||
-            $sort?.order !== $view.sort?.order
-          ) {
-            await datasource.actions.saveDefinition({
-              ...$view,
-              sort: {
-                field: $sort.column,
-                order: $sort.order || "ascending",
-              },
-            })
-          }
+          await datasource.actions.saveDefinition({
+            ...$view,
+            sort: {
+              field: $sort.column,
+              order: $sort.order || "ascending",
+            },
+          })
         }
 
         // Also update the fetch to ensure the new sort is respected.

From f03ed037945075b3c3a4858de3554852dc8087ba Mon Sep 17 00:00:00 2001
From: Peter Clement <PClmnt@users.noreply.github.com>
Date: Fri, 27 Sep 2024 10:05:05 +0100
Subject: [PATCH 48/57] remove space

Co-authored-by: Sam Rose <hello@samwho.dev>
---
 packages/server/src/threads/automation.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index e2a5a1c192..120a323f20 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -516,7 +516,7 @@ class Orchestrator {
       filter => {
         Object.entries(filter).forEach(([_, value]) => {
           Object.entries(value).forEach(([field, _]) => {
-            const updatedField = field.replace("{{", "{{ literal ")
+            const updatedField = field.replace("{{", "{{ literal")
             const fromContext = processStringSync(updatedField, this.context)
             toFilter[field] = fromContext
           })

From f73cf5983a919f977d8b29d0b606e2fb433f5863 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 27 Sep 2024 11:14:32 +0200
Subject: [PATCH 49/57] Hide column

---
 .../src/components/grid/cells/HeaderCell.svelte      | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index 00b2cbabf3..f516ba7d29 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -165,7 +165,17 @@
   }
 
   const hideColumn = () => {
-    datasource.actions.addSchemaMutation(column.name, { visible: false })
+    const { related } = column
+    const mutation = { visible: false }
+    if (!related) {
+      datasource.actions.addSchemaMutation(column.name, mutation)
+    } else {
+      datasource.actions.addSubSchemaMutation(
+        related.subField,
+        related.field,
+        mutation
+      )
+    }
     datasource.actions.saveSchemaMutations()
     open = false
   }

From 5ef0e48ee0b35423c93402265f452f2b22a3d4f0 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 27 Sep 2024 10:21:52 +0100
Subject: [PATCH 50/57] add back space

---
 packages/server/src/threads/automation.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts
index 120a323f20..e2a5a1c192 100644
--- a/packages/server/src/threads/automation.ts
+++ b/packages/server/src/threads/automation.ts
@@ -516,7 +516,7 @@ class Orchestrator {
       filter => {
         Object.entries(filter).forEach(([_, value]) => {
           Object.entries(value).forEach(([field, _]) => {
-            const updatedField = field.replace("{{", "{{ literal")
+            const updatedField = field.replace("{{", "{{ literal ")
             const fromContext = processStringSync(updatedField, this.context)
             toFilter[field] = fromContext
           })

From fdfa451124bb4c14e74b50275d0a2577f7eb7d80 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 27 Sep 2024 11:23:50 +0200
Subject: [PATCH 51/57] More verbose reduce params

---
 .../frontend-core/src/components/grid/stores/rows.js |  8 ++++----
 packages/frontend-core/src/utils/relatedColumns.js   | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/stores/rows.js b/packages/frontend-core/src/components/grid/stores/rows.js
index 25acad6520..4cfaa1cfd5 100644
--- a/packages/frontend-core/src/components/grid/stores/rows.js
+++ b/packages/frontend-core/src/components/grid/stores/rows.js
@@ -55,10 +55,10 @@ export const deriveStores = context => {
       return $rows.map((row, idx) => ({
         ...row,
         __idx: idx,
-        ...customColumns.reduce((acc, c) => {
-          const fromField = $enrichedSchema[c.related.field]
-          acc[c.name] = getRelatedTableValues(row, c, fromField)
-          return acc
+        ...customColumns.reduce((map, column) => {
+          const fromField = $enrichedSchema[column.related.field]
+          map[column.name] = getRelatedTableValues(row, column, fromField)
+          return map
         }, {}),
       }))
     }
diff --git a/packages/frontend-core/src/utils/relatedColumns.js b/packages/frontend-core/src/utils/relatedColumns.js
index 9d389524fd..f625e83e95 100644
--- a/packages/frontend-core/src/utils/relatedColumns.js
+++ b/packages/frontend-core/src/utils/relatedColumns.js
@@ -40,9 +40,9 @@ export function enrichSchemaWithRelColumns(schema) {
   if (!schema) {
     return
   }
-  const result = Object.keys(schema).reduce((acc, c) => {
-    const field = schema[c]
-    acc[c] = field
+  const result = Object.keys(schema).reduce((result, fieldName) => {
+    const field = schema[fieldName]
+    result[fieldName] = field
 
     if (field.visible !== false && field.columns) {
       const fromSingle =
@@ -54,17 +54,17 @@ export function enrichSchemaWithRelColumns(schema) {
           continue
         }
         const name = `${field.name}.${relColumn}`
-        acc[name] = {
+        result[name] = {
           ...relField,
           name,
-          related: { field: c, subField: relColumn },
+          related: { field: fieldName, subField: relColumn },
           cellRenderType:
             (!fromSingle && columnTypeManyTypeOverrides[relField.type]) ||
             relField.type,
         }
       }
     }
-    return acc
+    return result
   }, {})
 
   return result

From c41c25a6037f69797ca313eca602f211ff8a9a7f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Sep 2024 12:19:24 +0100
Subject: [PATCH 52/57] This improves the performance of views when the
 enriched relationships feature has been enabled. There was an issue that
 caused the squashLinks and outputProcessing to loop onto each other until
 things broke down - this fixes the issue.

---
 packages/server/src/db/linkedRows/index.ts    | 40 +++++-------
 .../src/utilities/rowProcessor/index.ts       | 65 +++++++++++++------
 2 files changed, 62 insertions(+), 43 deletions(-)

diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index c2b043785f..7faaa1c252 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -10,7 +10,11 @@ import flatten from "lodash/flatten"
 import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
-import { outputProcessing, processFormulas } from "../../utilities/rowProcessor"
+import {
+  coreOutputProcessing,
+  outputProcessing,
+  processFormulas,
+} from "../../utilities/rowProcessor"
 import { context, features } from "@budibase/backend-core"
 import {
   ContextUser,
@@ -156,9 +160,6 @@ export async function updateLinks(args: {
 /**
  * Given a table and a list of rows this will retrieve all of the attached docs and enrich them into the row.
  * This is required for formula fields, this may only be utilised internally (for now).
- * @param table The table from which the rows originated.
- * @param rows The rows which are to be enriched.
- * @param opts optional - options like passing in a base row to use for enrichment.
  * @return returns the rows with all of the enriched relationships on it.
  */
 export async function attachFullLinkedDocs(
@@ -248,9 +249,6 @@ export type SquashTableFields = Record<string, { visibleFieldNames: string[] }>
 
 /**
  * This function will take the given enriched rows and squash the links to only contain the primary display field.
- * @param table The table from which the rows originated.
- * @param enriched The pre-enriched rows (full docs) which are to be squashed.
- * @param squashFields Per link column (key) define which columns are allowed while squashing.
  * @returns The rows after having their links squashed to only contain the ID and primary display.
  */
 export async function squashLinks<T = Row[] | Row>(
@@ -283,21 +281,18 @@ export async function squashLinks<T = Row[] | Row>(
       if (schema.type !== FieldType.LINK || !Array.isArray(row[column])) {
         continue
       }
-      const newLinks = []
-      for (const link of row[column]) {
-        const linkTblId =
-          link.tableId || getRelatedTableForField(table.schema, column)
-        const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
+      const relatedTable = await getLinkedTable(schema.tableId, linkedTables)
+      if (viewSchema[column]?.columns) {
+        row[column] = await coreOutputProcessing(relatedTable, row[column])
+      }
+      row[column] = row[column].map((link: Row) => {
         const obj: any = { _id: link._id }
-        obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)
+        obj.primaryDisplay = getPrimaryDisplayValue(link, relatedTable)
 
         if (viewSchema[column]?.columns) {
-          const enrichedLink = await outputProcessing(linkedTable, link, {
-            squash: false,
-          })
-          const squashFields = Object.entries(viewSchema[column].columns)
+          const squashFields = Object.entries(viewSchema[column].columns || {})
             .filter(([columnName, viewColumnConfig]) => {
-              const tableColumn = linkedTable.schema[columnName]
+              const tableColumn = relatedTable.schema[columnName]
               if (!tableColumn) {
                 return false
               }
@@ -315,13 +310,14 @@ export async function squashLinks<T = Row[] | Row>(
             .map(([columnName]) => columnName)
 
           for (const relField of squashFields) {
-            obj[relField] = enrichedLink[relField]
+            if (link[relField] != null) {
+              obj[relField] = link[relField]
+            }
           }
         }
 
-        newLinks.push(obj)
-      }
-      row[column] = newLinks
+        return obj
+      })
     }
   }
   return (isArray ? enrichedArray : enrichedArray[0]) as T
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index f6cf44d6d6..f20a95c2fa 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -264,18 +264,46 @@ export async function outputProcessing<T extends Row[] | Row>(
   } else {
     safeRows = rows
   }
-  // attach any linked row information
-  let enriched = !opts.preserveLinks
-    ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
-        fromRow: opts?.fromRow,
-      })
-    : safeRows
+  let enriched: Row[]
+  // SQS returns the rows with full relationship contents
+  if (!(await features.flags.isEnabled("SQS"))) {
+    // attach any linked row information
+    enriched = !opts.preserveLinks
+      ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
+          fromRow: opts?.fromRow,
+        })
+      : safeRows
+  } else {
+    enriched = cloneDeep(safeRows)
+  }
 
-  // make sure squash is enabled if needed
   if (!opts.squash && utils.hasCircularStructure(rows)) {
     opts.squash = true
   }
 
+  enriched = await coreOutputProcessing(table, enriched, opts)
+
+  if (opts.squash) {
+    enriched = await linkRows.squashLinks(table, enriched, {
+      fromViewId: opts?.fromViewId,
+    })
+  }
+
+  return (wasArray ? enriched : enriched[0]) as T
+}
+
+export async function coreOutputProcessing(
+  table: Table,
+  rows: Row[],
+  opts: {
+    preserveLinks?: boolean
+    skipBBReferences?: boolean
+    fromViewId?: string
+  } = {
+    preserveLinks: false,
+    skipBBReferences: false,
+  }
+): Promise<Row[]> {
   // process complex types: attachments, bb references...
   for (const [property, column] of Object.entries(table.schema)) {
     if (
@@ -283,7 +311,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       column.type === FieldType.ATTACHMENT_SINGLE ||
       column.type === FieldType.SIGNATURE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] == null) {
           continue
         }
@@ -308,7 +336,7 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReferences(
           row[property],
           column.subtype
@@ -318,14 +346,14 @@ export async function outputProcessing<T extends Row[] | Row>(
       !opts.skipBBReferences &&
       column.type == FieldType.BB_REFERENCE_SINGLE
     ) {
-      for (const row of enriched) {
+      for (const row of rows) {
         row[property] = await processOutputBBReference(
           row[property],
           column.subtype
         )
       }
     } else if (column.type === FieldType.DATETIME && column.timeOnly) {
-      for (const row of enriched) {
+      for (const row of rows) {
         if (row[property] instanceof Date) {
           const hours = row[property].getUTCHours().toString().padStart(2, "0")
           const minutes = row[property]
@@ -340,7 +368,7 @@ export async function outputProcessing<T extends Row[] | Row>(
         }
       }
     } else if (column.type === FieldType.LINK) {
-      for (let row of enriched) {
+      for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time
         if (Array.isArray(row[property]) && row[property].length === 0) {
           delete row[property]
@@ -350,17 +378,12 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
 
   // process formulas after the complex types had been processed
-  enriched = await processFormulas(table, enriched, { dynamic: true })
+  rows = await processFormulas(table, rows, { dynamic: true })
 
-  if (opts.squash) {
-    enriched = await linkRows.squashLinks(table, enriched, {
-      fromViewId: opts?.fromViewId,
-    })
-  }
   // remove null properties to match internal API
   const isExternal = isExternalTableID(table._id!)
   if (isExternal || (await features.flags.isEnabled("SQS"))) {
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (row[key] === null) {
           delete row[key]
@@ -388,7 +411,7 @@ export async function outputProcessing<T extends Row[] | Row>(
     const fields = [...tableFields, ...protectedColumns].map(f =>
       f.toLowerCase()
     )
-    for (const row of enriched) {
+    for (const row of rows) {
       for (const key of Object.keys(row)) {
         if (!fields.includes(key.toLowerCase())) {
           delete row[key]
@@ -397,5 +420,5 @@ export async function outputProcessing<T extends Row[] | Row>(
     }
   }
 
-  return (wasArray ? enriched : enriched[0]) as T
+  return rows
 }

From aa4cc2079f2e36ead729b9f08f716b3dee5323f4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Sep 2024 12:24:40 +0100
Subject: [PATCH 53/57] Adding comment to explain new function.

---
 packages/server/src/utilities/rowProcessor/index.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index f20a95c2fa..2d0098ede6 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -292,6 +292,12 @@ export async function outputProcessing<T extends Row[] | Row>(
   return (wasArray ? enriched : enriched[0]) as T
 }
 
+/**
+ * This function is similar to the outputProcessing function above, it makes sure that all the provided
+ * rows are ready for output, but does not have enrichment for squash capabilities which can cause performance issues.
+ * outputProcessing should be used when responding from the API, while this should be used when internally processing
+ * rows for any reason (like part of view operations).
+ */
 export async function coreOutputProcessing(
   table: Table,
   rows: Row[],

From d71ba7d37f43cd6edb35845689bacf8c88c11138 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Sep 2024 12:27:41 +0100
Subject: [PATCH 54/57] Removing un-needed import.

---
 packages/server/src/db/linkedRows/index.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts
index 7faaa1c252..2f7f7fd44c 100644
--- a/packages/server/src/db/linkedRows/index.ts
+++ b/packages/server/src/db/linkedRows/index.ts
@@ -12,7 +12,6 @@ import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import {
   coreOutputProcessing,
-  outputProcessing,
   processFormulas,
 } from "../../utilities/rowProcessor"
 import { context, features } from "@budibase/backend-core"

From 2e65c8f5a03f8ab29896386ac16e790ae797112e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 27 Sep 2024 13:57:11 +0200
Subject: [PATCH 55/57] Update yarn.lock

---
 yarn.lock | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 0fe11b1f76..4f959a3a98 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2066,7 +2066,7 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.32.5":
+"@budibase/backend-core@2.32.8":
   version "0.0.0"
   dependencies:
     "@budibase/nano" "10.1.5"
@@ -2147,14 +2147,15 @@
     through2 "^2.0.0"
 
 "@budibase/pro@npm:@budibase/pro@latest":
-  version "2.32.5"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea"
-  integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA==
+  version "2.32.8"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.8.tgz#1b42da2ca7a496ba1ea8687cde6e7f3a8da47e48"
+  integrity sha512-NlY8DkD54FVcy6sL4T+wBJr/KQjXv6CGlqcHyjWVRBd8k/xLTzivrC5gVryDrkja/5ZxIm0qBKVlE81H6urLNA==
   dependencies:
-    "@budibase/backend-core" "2.32.5"
-    "@budibase/shared-core" "2.32.5"
-    "@budibase/string-templates" "2.32.5"
-    "@budibase/types" "2.32.5"
+    "@anthropic-ai/sdk" "^0.27.3"
+    "@budibase/backend-core" "2.32.8"
+    "@budibase/shared-core" "2.32.8"
+    "@budibase/string-templates" "2.32.8"
+    "@budibase/types" "2.32.8"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     dd-trace "5.2.0"
@@ -2163,16 +2164,17 @@
     lru-cache "^7.14.1"
     memorystream "^0.3.1"
     node-fetch "2.6.7"
+    openai "4.59.0"
     scim-patch "^0.8.1"
     scim2-parse-filter "^0.2.8"
 
-"@budibase/shared-core@2.32.5":
+"@budibase/shared-core@2.32.8":
   version "0.0.0"
   dependencies:
     "@budibase/types" "0.0.0"
     cron-validate "1.4.5"
 
-"@budibase/string-templates@2.32.5":
+"@budibase/string-templates@2.32.8":
   version "0.0.0"
   dependencies:
     "@budibase/handlebars-helpers" "^0.13.2"
@@ -2180,7 +2182,7 @@
     handlebars "^4.7.8"
     lodash.clonedeep "^4.5.0"
 
-"@budibase/types@2.32.5":
+"@budibase/types@2.32.8":
   version "0.0.0"
   dependencies:
     scim-patch "^0.8.1"

From 2a78409e8f81d6abc937769b8e5e699e509bf3f3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Sep 2024 13:07:17 +0100
Subject: [PATCH 56/57] Removing feature check.

---
 .../server/src/utilities/rowProcessor/index.ts   | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 2d0098ede6..14f2690231 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -266,16 +266,12 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
   let enriched: Row[]
   // SQS returns the rows with full relationship contents
-  if (!(await features.flags.isEnabled("SQS"))) {
-    // attach any linked row information
-    enriched = !opts.preserveLinks
-      ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
-          fromRow: opts?.fromRow,
-        })
-      : safeRows
-  } else {
-    enriched = cloneDeep(safeRows)
-  }
+  // attach any linked row information
+  enriched = !opts.preserveLinks
+    ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
+        fromRow: opts?.fromRow,
+      })
+    : safeRows
 
   if (!opts.squash && utils.hasCircularStructure(rows)) {
     opts.squash = true

From c30fe28fe8e487112e7dc14d33dcdc8729d0638e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 27 Sep 2024 13:07:44 +0100
Subject: [PATCH 57/57] Lint.

---
 packages/server/src/utilities/rowProcessor/index.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 14f2690231..2f227dd646 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -264,10 +264,9 @@ export async function outputProcessing<T extends Row[] | Row>(
   } else {
     safeRows = rows
   }
-  let enriched: Row[]
   // SQS returns the rows with full relationship contents
   // attach any linked row information
-  enriched = !opts.preserveLinks
+  let enriched = !opts.preserveLinks
     ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
         fromRow: opts?.fromRow,
       })