From 87bfbf0e93269fe866366bc1f9302608fb1ff2f9 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Fri, 13 Dec 2024 09:26:53 +0000
Subject: [PATCH 001/163] Fix for progress circle appearing in the builder and
 allow binding of S3Upload value in S3Upload settings

---
 packages/client/manifest.json                     |  3 ++-
 .../src/components/app/forms/S3Upload.svelte      | 15 +++++++++++++--
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/client/manifest.json b/packages/client/manifest.json
index 6dd1b71b75..c236dd1ad9 100644
--- a/packages/client/manifest.json
+++ b/packages/client/manifest.json
@@ -5139,7 +5139,8 @@
       {
         "type": "text",
         "label": "File name",
-        "key": "key"
+        "key": "key",
+        "nested": true
       },
       {
         "type": "event",
diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index 0147cbca6e..fc4c0f8f08 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -2,6 +2,8 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle, Helpers } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
+  import { builderStore } from "stores/builder.js"
+  import { processStringSync } from "@budibase/string-templates"
 
   export let datasourceId
   export let bucket
@@ -42,6 +44,9 @@
   // Process the file input and return a serializable structure expected by
   // the dropzone component to display the file
   const processFiles = async fileList => {
+    if ($builderStore.inBuilder) {
+      return []
+    }
     return await new Promise(resolve => {
       if (!fileList?.length) {
         return []
@@ -78,12 +83,15 @@
   }
 
   const upload = async () => {
+    const processedFileKey = processStringSync(key, {
+      [$component.id]: fieldState,
+    })
     loading = true
     try {
       const res = await API.externalUpload({
         datasourceId,
         bucket,
-        key,
+        key: processedFileKey,
         data,
       })
       notificationStore.actions.success("File uploaded successfully")
@@ -131,7 +139,7 @@
   bind:fieldApi
   defaultValue={[]}
 >
-  <div class="content">
+  <div class="content" class:builder={$builderStore.inBuilder}>
     {#if fieldState}
       <CoreDropzone
         value={localFiles}
@@ -154,6 +162,9 @@
 </Field>
 
 <style>
+  .content.builder :global(.spectrum-Dropzone) {
+    pointer-events: none;
+  }
   .content {
     position: relative;
   }

From 544302a12992b22eed209626d01fe12567f006e3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 2 Dec 2024 12:42:48 +0100
Subject: [PATCH 002/163] Request only needed fields

---
 packages/backend-core/src/sql/sql.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 5f462ee144..a4828a9f4a 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1537,11 +1537,16 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { operation, filters, paginate, relationships, table } = this.query
+    const { operation, filters, paginate, relationships, table, resource } =
+      this.query
     const { limits } = opts
 
     // start building the query
     let query = this.qualifiedKnex()
+    if (resource?.fields) {
+      query = query.columns(resource?.fields)
+    }
+
     // handle pagination
     let foundOffset: number | null = null
     let foundLimit = limits?.query || limits?.base

From 0e0b6471d64e93f2559ab51374c734f5d103eed9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Dec 2024 10:22:16 +0100
Subject: [PATCH 003/163] Generate proper select

---
 packages/backend-core/src/sql/sql.ts                      | 5 +----
 packages/server/src/api/controllers/row/utils/sqlUtils.ts | 1 +
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index a4828a9f4a..76d554d211 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -292,7 +292,7 @@ class InternalBuilder {
 
     const alias = this.getTableName(table)
     const schema = this.table.schema
-    if (!this.isFullSelectStatementRequired()) {
+    if (this.isFullSelectStatementRequired()) {
       return [this.knex.raw("??", [`${alias}.*`])]
     }
     // get just the fields for this table
@@ -1543,9 +1543,6 @@ class InternalBuilder {
 
     // start building the query
     let query = this.qualifiedKnex()
-    if (resource?.fields) {
-      query = query.columns(resource?.fields)
-    }
 
     // handle pagination
     let foundOffset: number | null = null
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index af696c0758..9729018bf7 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -142,6 +142,7 @@ export async function buildSqlFieldList(
   let table: Table
   if (sdk.views.isView(source)) {
     table = await sdk.views.getTable(source.id)
+    fields = fields.filter(f => table.schema[f].type !== FieldType.LINK)
   } else {
     table = source
   }

From 191b63270cabd2a76eb85c401f6308821d02de47 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Dec 2024 13:30:08 +0100
Subject: [PATCH 004/163] Dry

---
 packages/server/src/api/controllers/row/utils/sqlUtils.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 9729018bf7..19f251bef3 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -133,14 +133,17 @@ export async function buildSqlFieldList(
   }
 
   let fields: string[] = []
-  if (sdk.views.isView(source)) {
+
+  const isView = sdk.views.isView(source)
+
+  if (isView) {
     fields = Object.keys(helpers.views.basicFields(source))
   } else {
     fields = extractRealFields(source)
   }
 
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
     fields = fields.filter(f => table.schema[f].type !== FieldType.LINK)
   } else {

From a415095c470f2d8a59575c4b6a38c0e0ecc67a5b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 11 Dec 2024 14:09:11 +0100
Subject: [PATCH 005/163] Lint

---
 packages/backend-core/src/sql/sql.ts | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 76d554d211..758dfa7fec 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1537,8 +1537,7 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    const { operation, filters, paginate, relationships, table, resource } =
-      this.query
+    const { operation, filters, paginate, relationships, table } = this.query
     const { limits } = opts
 
     // start building the query

From 74e1bbc7c66faa507edac9e56eb1214aa0b93384 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 12 Dec 2024 13:57:44 +0100
Subject: [PATCH 006/163] Require only visible fields on views

---
 .../server/src/api/controllers/row/utils/sqlUtils.ts  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 19f251bef3..b67ef6be92 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -150,10 +150,19 @@ export async function buildSqlFieldList(
     table = source
   }
 
-  for (let field of Object.values(table.schema)) {
+  for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue
     }
+
+    if (
+      isView &&
+      source.schema?.[field.name] &&
+      !helpers.views.isVisible(source.schema[field.name])
+    ) {
+      continue
+    }
+
     const { tableName } = breakExternalTableId(field.tableId)
     if (tables[tableName]) {
       fields = fields.concat(extractRealFields(tables[tableName], fields))

From e3c9156aef80c593b35936f853fba064936e8b04 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 12 Dec 2024 13:58:37 +0100
Subject: [PATCH 007/163] Trim selected fields

---
 packages/backend-core/src/sql/sql.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 758dfa7fec..4691cd71ba 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1591,7 +1591,7 @@ class InternalBuilder {
       const mainTable = this.query.tableAliases?.[table.name] || table.name
       const cte = this.addSorting(
         this.knex
-          .with("paginated", query)
+          .with("paginated", query.clone().clearSelect().select("*"))
           .select(this.generateSelectStatement())
           .from({
             [mainTable]: "paginated",

From 00557e5f7ea87f44dfe0cc6ca7202a3af28653e9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 12 Dec 2024 14:01:56 +0100
Subject: [PATCH 008/163] Don't include unnecessary joins

---
 packages/backend-core/src/sql/sql.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 4691cd71ba..0fe7e0ca57 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1239,6 +1239,12 @@ class InternalBuilder {
       if (!toTable || !fromTable) {
         continue
       }
+
+      // Don't include if not required
+      if (relationship.from && !fields.find(f => f === relationship.from)) {
+        continue
+      }
+
       const relatedTable = tables[toTable]
       if (!relatedTable) {
         throw new Error(`related table "${toTable}" not found in datasource`)

From 910faa6f33720cc3b396449cf8e036a6f77534ca Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 12 Dec 2024 17:10:01 +0100
Subject: [PATCH 009/163] Fix getting relations

---
 packages/backend-core/src/sql/sql.ts | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 0fe7e0ca57..24f30d57fa 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1240,11 +1240,6 @@ class InternalBuilder {
         continue
       }
 
-      // Don't include if not required
-      if (relationship.from && !fields.find(f => f === relationship.from)) {
-        continue
-      }
-
       const relatedTable = tables[toTable]
       if (!relatedTable) {
         throw new Error(`related table "${toTable}" not found in datasource`)
@@ -1594,17 +1589,8 @@ class InternalBuilder {
 
     // handle relationships with a CTE for all others
     if (relationships?.length && aggregations.length === 0) {
-      const mainTable = this.query.tableAliases?.[table.name] || table.name
-      const cte = this.addSorting(
-        this.knex
-          .with("paginated", query.clone().clearSelect().select("*"))
-          .select(this.generateSelectStatement())
-          .from({
-            [mainTable]: "paginated",
-          })
-      )
       // add JSON aggregations attached to the CTE
-      return this.addJsonRelationships(cte, table.name, relationships)
+      return this.addJsonRelationships(query, table.name, relationships)
     }
 
     return query

From be0074105cc8126c2029acc76b03bad8846b998e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 12 Dec 2024 17:25:07 +0100
Subject: [PATCH 010/163] Request relation only when required

---
 packages/backend-core/src/sql/sql.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 24f30d57fa..c38c0bd2c8 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1268,6 +1268,10 @@ class InternalBuilder {
       const fieldList = relationshipFields.map(field =>
         this.buildJsonField(relatedTable, field)
       )
+      if (!fieldList.length) {
+        continue
+      }
+
       const fieldListFormatted = fieldList
         .map(f => {
           const separator = this.client === SqlClient.ORACLE ? " VALUE " : ","

From 112b7c70e23e6d2c2b3f0d1f655ff1d622391ddf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 13 Dec 2024 11:21:24 +0100
Subject: [PATCH 011/163] Fix sqs views

---
 .../server/src/sdk/app/rows/search/internal/sqs.ts    | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index aa799390b8..6eb9bd913b 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -234,6 +234,17 @@ async function runSqlQuery(
     json.operation = Operation.COUNT
   }
   const processSQLQuery = async (json: EnrichedQueryJson) => {
+    const fields = json.resource?.fields
+    if (fields) {
+      const tableId = json.tableAliases?.[json.table._id!] ?? json.table._id!
+      for (const key of ["_id", "_rev", "tableId"]) {
+        const field = `${tableId}.${key}`
+        if (fields.includes(field)) {
+          continue
+        }
+        fields.push(field)
+      }
+    }
     const query = builder._query(json, {
       disableReturning: true,
     })

From bbb60afdc2d01c5c0f9e6e1bc4196139a351645b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 13 Dec 2024 13:30:39 +0100
Subject: [PATCH 012/163] Relations, select only required fields

---
 .../src/api/controllers/row/utils/sqlUtils.ts | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index b67ef6be92..51fba4bb79 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -164,8 +164,24 @@ export async function buildSqlFieldList(
     }
 
     const { tableName } = breakExternalTableId(field.tableId)
-    if (tables[tableName]) {
-      fields = fields.concat(extractRealFields(tables[tableName], fields))
+    const relatedTable = tables[tableName]
+    if (relatedTable) {
+      const viewFields = new Set<string>()
+      relatedTable.primary?.forEach(f => viewFields.add(f))
+      if (relatedTable.primaryDisplay) {
+        viewFields.add(relatedTable.primaryDisplay)
+      }
+
+      if (isView) {
+        Object.entries(source.schema?.[field.name].columns || {})
+          .filter(([_, column]) => helpers.views.isVisible(column))
+          .forEach(([field]) => viewFields.add(field))
+      }
+
+      const fieldsToAdd = Array.from(viewFields)
+        .map(f => `${relatedTable.name}.${f}`)
+        .filter(f => !fields.includes(f))
+      fields.push(...fieldsToAdd)
     }
   }
 

From 80dcc519235d419ed3224ac729c747beebd86517 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 09:55:18 +0100
Subject: [PATCH 013/163] Fix sql alias tests

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 35 ++++++++-----------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 7c6f583762..f7dab0ff46 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [primaryLimit, relationshipLimit, relationshipLimit],
+        bindings: [relationshipLimit, relationshipLimit, primaryLimit],
         sql: expect.stringContaining(
           multiline(
             `select json_agg(json_build_object('executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'completed',"b"."completed",'completed',"b"."completed",'taskname',"b"."taskname",'taskname',"b"."taskname"`
@@ -94,10 +94,10 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: ["assembling", primaryLimit, relationshipLimit],
+        bindings: [relationshipLimit, "assembling", primaryLimit],
         sql: expect.stringContaining(
           multiline(
-            `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))`
+            `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $2, FALSE)))`
           )
         ),
       })
@@ -109,13 +109,13 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [primaryLimit, relationshipLimit],
+        bindings: [relationshipLimit, primaryLimit],
         sql: expect.stringContaining(
           multiline(
-            `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
-                 select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
-                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
-                 from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
+            `select "a"."productname",
+              "a"."productid",
+              (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1) as "b") as "tasks"
+              from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $2`
           )
         ),
       })
@@ -128,13 +128,8 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [...filters, relationshipLimit, relationshipLimit],
-        sql: multiline(
-          `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
-               select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
-               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
-               where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
-        ),
+        bindings: [relationshipLimit, ...filters, relationshipLimit],
+        sql: `select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4`,
       })
     })
 
@@ -151,6 +146,9 @@ describe("Captures of real examples", () => {
 
       expect(query).toEqual({
         bindings: [
+          relationshipLimit,
+          relationshipLimit,
+          relationshipLimit,
           rangeValue.low,
           rangeValue.high,
           rangeValue.low,
@@ -158,13 +156,10 @@ describe("Captures of real examples", () => {
           equalValue,
           notEqualsValue,
           primaryLimit,
-          relationshipLimit,
-          relationshipLimit,
-          relationshipLimit,
         ],
         sql: expect.stringContaining(
           multiline(
-            `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))`
+            `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $4 and $5))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $6 and $7))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $8, FALSE))))`
           )
         ),
       })
@@ -209,7 +204,7 @@ describe("Captures of real examples", () => {
         bindings: ["ddd", ""],
         sql: multiline(`delete from "compositetable" as "a" 
           where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
-          returning "a".*`),
+          returning "a"."keyparttwo", "a"."keypartone", "a"."name"`),
       })
     })
   })

From f92fcea42abab49afc413aba4f51cb3b4c650b14 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 10:41:37 +0100
Subject: [PATCH 014/163] Include * when having formulas

---
 packages/backend-core/src/sql/sql.ts          | 72 +++++++++++--------
 .../src/api/controllers/row/utils/sqlUtils.ts |  2 +-
 2 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index c38c0bd2c8..337c65ff0f 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -272,12 +272,17 @@ class InternalBuilder {
     return parts.join(".")
   }
 
-  private isFullSelectStatementRequired(): boolean {
-    for (let column of Object.values(this.table.schema)) {
+  private isFullSelectStatementRequired(includedFields: string[]): boolean {
+    for (const column of Object.values(this.table.schema)) {
       if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
         return true
       } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
         return true
+      } else if (
+        column.type === FieldType.FORMULA &&
+        includedFields.includes(column.name)
+      ) {
+        return true
       }
     }
     return false
@@ -292,11 +297,9 @@ class InternalBuilder {
 
     const alias = this.getTableName(table)
     const schema = this.table.schema
-    if (this.isFullSelectStatementRequired()) {
-      return [this.knex.raw("??", [`${alias}.*`])]
-    }
+
     // get just the fields for this table
-    return resource.fields
+    const tableFields = resource.fields
       .map(field => {
         const parts = field.split(/\./g)
         let table: string | undefined = undefined
@@ -311,34 +314,43 @@ class InternalBuilder {
         return { table, column, field }
       })
       .filter(({ table }) => !table || table === alias)
-      .map(({ table, column, field }) => {
-        const columnSchema = schema[column]
 
-        if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
-          return this.knex.raw(`??::money::numeric as ??`, [
-            this.rawQuotedIdentifier([table, column].join(".")),
-            this.knex.raw(this.quote(field)),
-          ])
-        }
+    if (
+      this.isFullSelectStatementRequired(
+        tableFields.map(({ column }) => column)
+      )
+    ) {
+      return [this.knex.raw("??", [`${alias}.*`])]
+    }
 
-        if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
-          // Time gets returned as timestamp from mssql, not matching the expected
-          // HH:mm format
+    return tableFields.map(({ table, column, field }) => {
+      const columnSchema = schema[column]
 
-          // TODO: figure out how to express this safely without string
-          // interpolation.
-          return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
-            this.rawQuotedIdentifier(field),
-            this.knex.raw(this.quote(field)),
-          ])
-        }
+      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
+        return this.knex.raw(`??::money::numeric as ??`, [
+          this.rawQuotedIdentifier([table, column].join(".")),
+          this.knex.raw(this.quote(field)),
+        ])
+      }
 
-        if (table) {
-          return this.rawQuotedIdentifier(`${table}.${column}`)
-        } else {
-          return this.rawQuotedIdentifier(field)
-        }
-      })
+      if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
+        // Time gets returned as timestamp from mssql, not matching the expected
+        // HH:mm format
+
+        // TODO: figure out how to express this safely without string
+        // interpolation.
+        return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
+          this.rawQuotedIdentifier(field),
+          this.knex.raw(this.quote(field)),
+        ])
+      }
+
+      if (table) {
+        return this.rawQuotedIdentifier(`${table}.${column}`)
+      } else {
+        return this.rawQuotedIdentifier(field)
+      }
+    })
   }
 
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 51fba4bb79..2d5fc35761 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -173,7 +173,7 @@ export async function buildSqlFieldList(
       }
 
       if (isView) {
-        Object.entries(source.schema?.[field.name].columns || {})
+        Object.entries(source.schema?.[field.name]?.columns || {})
           .filter(([_, column]) => helpers.views.isVisible(column))
           .forEach(([field]) => viewFields.add(field))
       }

From f8ece826488927c314fec3245a4d4224cd66c7c3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 10:53:45 +0100
Subject: [PATCH 015/163] Fix formulas on internal

---
 packages/backend-core/src/sql/sql.ts | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 337c65ff0f..ba3076e553 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -315,11 +315,10 @@ class InternalBuilder {
       })
       .filter(({ table }) => !table || table === alias)
 
-    if (
-      this.isFullSelectStatementRequired(
-        tableFields.map(({ column }) => column)
-      )
-    ) {
+    const requestedTableColumns = tableFields.map(({ column }) =>
+      column.replace(new RegExp(`^${this.query.meta?.columnPrefix}`), "")
+    )
+    if (this.isFullSelectStatementRequired(requestedTableColumns)) {
       return [this.knex.raw("??", [`${alias}.*`])]
     }
 

From e6a27ad4d50ebc4776091a0020c4afbc3a44ff17 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 12:25:29 +0100
Subject: [PATCH 016/163] Ensure required fields are included

---
 .../src/api/controllers/row/utils/sqlUtils.ts | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 2d5fc35761..0c3d2b8607 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -15,6 +15,7 @@ import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
 import sdk from "../../../../sdk"
 import { helpers } from "@budibase/shared-core"
+import { sql } from "@budibase/backend-core"
 
 type TableMap = Record<string, Table>
 
@@ -132,6 +133,27 @@ export async function buildSqlFieldList(
       .map(([columnName]) => `${table.name}.${columnName}`)
   }
 
+  function getRequiredFields(table: Table, existing: string[] = []) {
+    const requiredFields: string[] = []
+    if (table.primary) {
+      requiredFields.push(...table.primary)
+    }
+    if (table.primaryDisplay) {
+      requiredFields.push(table.primaryDisplay)
+    }
+
+    if (!sql.utils.isExternalTable(table)) {
+      requiredFields.push(...["_id", "_rev", "_tableId"])
+    }
+
+    return requiredFields
+      .filter(
+        column =>
+          !existing.find((field: string) => field === `${table.name}.${column}`)
+      )
+      .map(column => `${table.name}.${column}`)
+  }
+
   let fields: string[] = []
 
   const isView = sdk.views.isView(source)
@@ -150,6 +172,16 @@ export async function buildSqlFieldList(
     table = source
   }
 
+  fields.push(
+    ...getRequiredFields(
+      {
+        ...table,
+        primaryDisplay: source.primaryDisplay || table.primaryDisplay,
+      },
+      fields
+    )
+  )
+
   for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue

From 2d771c96dd81f663ea6360cb6e3997138f03e045 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 12:53:40 +0100
Subject: [PATCH 017/163] Fix wrong relationship select

---
 .../server/src/api/controllers/row/utils/sqlUtils.ts     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 0c3d2b8607..80c5f95b05 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -206,7 +206,14 @@ export async function buildSqlFieldList(
 
       if (isView) {
         Object.entries(source.schema?.[field.name]?.columns || {})
-          .filter(([_, column]) => helpers.views.isVisible(column))
+          .filter(
+            ([columnName, columnConfig]) =>
+              relatedTable.schema[columnName] &&
+              helpers.views.isVisible(columnConfig) &&
+              ![FieldType.LINK, FieldType.FORMULA].includes(
+                relatedTable.schema[columnName].type
+              )
+          )
           .forEach(([field]) => viewFields.add(field))
       }
 

From 14b5a4264700e66bf2a7aef0f130bc41d034e885 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 13:57:38 +0100
Subject: [PATCH 018/163] Fix patch issues

---
 packages/server/src/api/controllers/row/external.ts    | 7 +++++--
 packages/server/src/api/controllers/row/utils/utils.ts | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 082d07283b..6dbd4dbb81 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -45,6 +45,9 @@ export async function handleRequest<T extends Operation>(
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const source = await utils.getSource(ctx)
 
+  const { viewId, tableId } = utils.getSourceId(ctx)
+  const sourceId = viewId || tableId
+
   if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     ctx.throw(400, "Cannot update rows through a calculation view")
   }
@@ -66,7 +69,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
 
-  const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
+  const beforeRow = await sdk.rows.external.getRow(sourceId, _id, {
     relationships: true,
   })
 
@@ -78,7 +81,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   // The id might have been changed, so the refetching would fail. Recalculating the id just in case
   const updatedId =
     generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
-  const row = await sdk.rows.external.getRow(table._id!, updatedId, {
+  const row = await sdk.rows.external.getRow(sourceId, updatedId, {
     relationships: true,
   })
 
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index baa811fe90..8701ba9197 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -66,7 +66,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
     if (docIds.isViewId(sourceId)) {
       return {
         tableId: utils.extractViewInfoFromID(sourceId).tableId,
-        viewId: sourceId,
+        viewId: encodeURI(sourceId),
       }
     }
     return { tableId: sql.utils.encodeTableId(ctx.params.sourceId) }

From d60cc7aaf4c4c47ff80d2caecac0837ae1c9c8fd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 15:51:17 +0100
Subject: [PATCH 019/163] Fix encoding

---
 packages/server/src/api/controllers/row/utils/utils.ts |  2 +-
 packages/server/src/db/utils.ts                        | 10 ++--------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index 8701ba9197..021a6317ea 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -66,7 +66,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
     if (docIds.isViewId(sourceId)) {
       return {
         tableId: utils.extractViewInfoFromID(sourceId).tableId,
-        viewId: encodeURI(sourceId),
+        viewId: sql.utils.encodeTableId(sourceId),
       }
     }
     return { tableId: sql.utils.encodeTableId(ctx.params.sourceId) }
diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts
index 70c69b3c60..6c1065e847 100644
--- a/packages/server/src/db/utils.ts
+++ b/packages/server/src/db/utils.ts
@@ -1,10 +1,4 @@
-import {
-  context,
-  db as dbCore,
-  docIds,
-  utils,
-  sql,
-} from "@budibase/backend-core"
+import { context, db as dbCore, docIds, utils } from "@budibase/backend-core"
 import {
   DatabaseQueryOpts,
   Datasource,
@@ -334,7 +328,7 @@ export function extractViewInfoFromID(viewId: string) {
   const regex = new RegExp(`^(?<tableId>.+)${SEPARATOR}([^${SEPARATOR}]+)$`)
   const res = regex.exec(viewId)
   return {
-    tableId: sql.utils.encodeTableId(res!.groups!["tableId"]),
+    tableId: res!.groups!["tableId"],
   }
 }
 

From 0ef4a154ef9b98b41eed2fcfdf424ff6ba3a4724 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 16:02:10 +0100
Subject: [PATCH 020/163] Prevent repeated fields on select

---
 .../src/api/controllers/row/utils/sqlUtils.ts      | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 80c5f95b05..b6215cc5b6 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -158,12 +158,6 @@ export async function buildSqlFieldList(
 
   const isView = sdk.views.isView(source)
 
-  if (isView) {
-    fields = Object.keys(helpers.views.basicFields(source))
-  } else {
-    fields = extractRealFields(source)
-  }
-
   let table: Table
   if (isView) {
     table = await sdk.views.getTable(source.id)
@@ -172,6 +166,14 @@ export async function buildSqlFieldList(
     table = source
   }
 
+  if (isView) {
+    fields = Object.keys(helpers.views.basicFields(source)).map(
+      c => `${table.name}.${c}`
+    )
+  } else {
+    fields = extractRealFields(source)
+  }
+
   fields.push(
     ...getRequiredFields(
       {

From fc22db35ac5a5d341f617446789fdc6d06cb6655 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 16:16:22 +0100
Subject: [PATCH 021/163] Add back cte

---
 packages/backend-core/src/sql/sql.ts                  | 11 ++++++++++-
 .../server/src/api/controllers/row/utils/sqlUtils.ts  |  7 +++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index ba3076e553..1a7e3beba4 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1604,8 +1604,17 @@ class InternalBuilder {
 
     // handle relationships with a CTE for all others
     if (relationships?.length && aggregations.length === 0) {
+      const mainTable = this.query.tableAliases?.[table.name] || table.name
+      const cte = this.addSorting(
+        this.knex
+          .with("paginated", query.clone().clearSelect().select("*"))
+          .select(this.generateSelectStatement())
+          .from({
+            [mainTable]: "paginated",
+          })
+      )
       // add JSON aggregations attached to the CTE
-      return this.addJsonRelationships(query, table.name, relationships)
+      return this.addJsonRelationships(cte, table.name, relationships)
     }
 
     return query
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index b6215cc5b6..9f5e14d732 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -161,15 +161,14 @@ export async function buildSqlFieldList(
   let table: Table
   if (isView) {
     table = await sdk.views.getTable(source.id)
-    fields = fields.filter(f => table.schema[f].type !== FieldType.LINK)
   } else {
     table = source
   }
 
   if (isView) {
-    fields = Object.keys(helpers.views.basicFields(source)).map(
-      c => `${table.name}.${c}`
-    )
+    fields = Object.keys(helpers.views.basicFields(source))
+      .filter(f => table.schema[f].type !== FieldType.LINK)
+      .map(c => `${table.name}.${c}`)
   } else {
     fields = extractRealFields(source)
   }

From aa288966d89e40ca85a8286f39a8f8eb13cec607 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 16:23:05 +0100
Subject: [PATCH 022/163] Fix tests back

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 33 +++++++++++--------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index f7dab0ff46..1630e9b204 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [relationshipLimit, relationshipLimit, primaryLimit],
+        bindings: [primaryLimit, relationshipLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
             `select json_agg(json_build_object('executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'completed',"b"."completed",'completed',"b"."completed",'taskname',"b"."taskname",'taskname',"b"."taskname"`
@@ -94,10 +94,10 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [relationshipLimit, "assembling", primaryLimit],
+        bindings: ["assembling", primaryLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $2, FALSE)))`
+            `where (exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" and (COALESCE("b"."taskname" = $1, FALSE)))`
           )
         ),
       })
@@ -109,13 +109,13 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [relationshipLimit, primaryLimit],
+        bindings: [primaryLimit, relationshipLimit],
         sql: expect.stringContaining(
           multiline(
-            `select "a"."productname",
-              "a"."productid",
-              (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1) as "b") as "tasks"
-              from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $2`
+            `with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
+                 select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
+                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
+                 from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
         ),
       })
@@ -128,8 +128,13 @@ describe("Captures of real examples", () => {
         queryJson
       )
       expect(query).toEqual({
-        bindings: [relationshipLimit, ...filters, relationshipLimit],
-        sql: `select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4`,
+        bindings: [...filters, relationshipLimit, relationshipLimit],
+        sql: multiline(
+          `with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
+               select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
+               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
+               where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
+        ),
       })
     })
 
@@ -146,9 +151,6 @@ describe("Captures of real examples", () => {
 
       expect(query).toEqual({
         bindings: [
-          relationshipLimit,
-          relationshipLimit,
-          relationshipLimit,
           rangeValue.low,
           rangeValue.high,
           rangeValue.low,
@@ -156,10 +158,13 @@ describe("Captures of real examples", () => {
           equalValue,
           notEqualsValue,
           primaryLimit,
+          relationshipLimit,
+          relationshipLimit,
+          relationshipLimit,
         ],
         sql: expect.stringContaining(
           multiline(
-            `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $4 and $5))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $6 and $7))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $8, FALSE))))`
+            `where (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))) and (exists (select 1 from "persons" as "c" where "c"."personid" = "a"."qaid" and ("c"."year" between $3 and $4))) and (exists (select 1 from "products" as "b" inner join "products_tasks" as "d" on "b"."productid" = "d"."productid" where "d"."taskid" = "a"."taskid" and (COALESCE("b"."productname" = $5, FALSE))))`
           )
         ),
       })

From eb7fcd0219d33eeb9fc07696e5ece0b519e189b2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 16:33:43 +0100
Subject: [PATCH 023/163] Don't select * on relationships

---
 packages/backend-core/src/sql/sql.ts                    | 4 +++-
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 1a7e3beba4..394ab92fd3 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -1323,7 +1323,9 @@ class InternalBuilder {
       )
 
       const standardWrap = (select: Knex.Raw): Knex.QueryBuilder => {
-        subQuery = subQuery.select(`${toAlias}.*`).limit(getRelationshipLimit())
+        subQuery = subQuery
+          .select(relationshipFields)
+          .limit(getRelationshipLimit())
         // @ts-ignore - the from alias syntax isn't in Knex typing
         return knex.select(select).from({
           [toAlias]: subQuery,
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 1630e9b204..704c003994 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -114,7 +114,7 @@ describe("Captures of real examples", () => {
           multiline(
             `with "paginated" as (select * from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) 
                  select "a"."productname", "a"."productid", (select json_agg(json_build_object('executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'completed',"b"."completed",'taskname',"b"."taskname")) 
-                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
+                 from (select "b"."executorid", "b"."qaid", "b"."taskid", "b"."completed", "b"."taskname" from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" 
                  from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
           )
         ),
@@ -132,7 +132,7 @@ describe("Captures of real examples", () => {
         sql: multiline(
           `with "paginated" as (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) 
                select "a"."executorid", "a"."taskname", "a"."taskid", "a"."completed", "a"."qaid", (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) 
-               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
+               from (select "b"."productid", "b"."productname" from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
                where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
         ),
       })

From df62845cf9c492aec568e6903ceb0882aec83517 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 16:55:38 +0100
Subject: [PATCH 024/163] Fix encodings

---
 packages/backend-core/src/sql/utils.ts                 | 4 ++++
 packages/server/src/api/controllers/row/utils/utils.ts | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 14127a189f..16b352995b 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -70,6 +70,10 @@ export function encodeTableId(tableId: string) {
   }
 }
 
+export function encodeViewId(viewId: string) {
+  return encodeURIComponent(viewId)
+}
+
 export function breakExternalTableId(tableId: string) {
   const parts = tableId.split(DOUBLE_SEPARATOR)
   let datasourceId = parts.shift()
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index 021a6317ea..86986b64e8 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -66,7 +66,7 @@ export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
     if (docIds.isViewId(sourceId)) {
       return {
         tableId: utils.extractViewInfoFromID(sourceId).tableId,
-        viewId: sql.utils.encodeTableId(sourceId),
+        viewId: sql.utils.encodeViewId(sourceId),
       }
     }
     return { tableId: sql.utils.encodeTableId(ctx.params.sourceId) }

From 740069ea78a7866ca1373f0b423526d24792d232 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 17:01:02 +0100
Subject: [PATCH 025/163] Use table for get before row

---
 packages/server/src/api/controllers/row/external.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 6dbd4dbb81..0405203f2f 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -69,7 +69,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
 
-  const beforeRow = await sdk.rows.external.getRow(sourceId, _id, {
+  const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
     relationships: true,
   })
 

From 7cd412bfdcd15e6815c99f2a5b5e6a93ee1caf1c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 17:46:37 +0100
Subject: [PATCH 026/163] Add basic sql alias test

---
 .../src/integrations/tests/sqlAlias.spec.ts   |   9 ++
 .../tests/sqlQueryJson/basicFetch.json        | 137 ++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 704c003994..4d39fa7338 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -73,6 +73,15 @@ describe("Captures of real examples", () => {
   })
 
   describe("read", () => {
+    it("should retrieve only requested fields", () => {
+      const queryJson = getJson("basicFetch.json")
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select "a"."year", "a"."firstname", "a"."personid", "a"."age", "a"."type", "a"."lastname" from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json
new file mode 100644
index 0000000000..9d9026c922
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetch.json
@@ -0,0 +1,137 @@
+{
+  "operation": "READ",
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.age",
+      "a.type",
+      "a.lastname"
+    ]
+  },
+  "filters": {},
+  "sort": {
+    "firstname": {
+      "direction": "ascending"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [],
+  "extra": {
+    "idFilter": {}
+  },
+  "table": {
+    "type": "table",
+    "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+    "primary": ["personid"],
+    "name": "persons",
+    "schema": {
+      "year": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "year",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "firstname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "firstname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "personid": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": true,
+        "name": "personid",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "address": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "address",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "age": {
+        "type": "number",
+        "externalType": "integer",
+        "autocolumn": false,
+        "name": "age",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "type": {
+        "type": "options",
+        "externalType": "USER-DEFINED",
+        "autocolumn": false,
+        "name": "type",
+        "constraints": {
+          "presence": false,
+          "inclusion": ["support", "designer", "programmer", "qa"]
+        }
+      },
+      "city": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "city",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "lastname": {
+        "type": "string",
+        "externalType": "character varying",
+        "autocolumn": false,
+        "name": "lastname",
+        "constraints": {
+          "presence": false
+        }
+      },
+      "QA": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "QA",
+        "relationshipType": "many-to-one",
+        "fieldName": "qaid",
+        "type": "link",
+        "main": true,
+        "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+        "foreignKey": "personid"
+      },
+      "executor": {
+        "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+        "name": "executor",
+        "relationshipType": "many-to-one",
+        "fieldName": "executorid",
+        "type": "link",
+        "main": true,
+        "_id": "c89530b9770d94bec851e062b5cff3001",
+        "foreignKey": "personid",
+        "tableName": "persons"
+      }
+    },
+    "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "sourceType": "external",
+    "primaryDisplay": "firstname",
+    "views": {}
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}

From fc75728a1e70f86f214ba2071fc0eaaf6a32a008 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 16 Dec 2024 17:52:30 +0100
Subject: [PATCH 027/163] Add more tests

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 4d39fa7338..647c18d4ae 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -8,6 +8,7 @@ import {
   TableSchema,
   Table,
   TableSourceType,
+  FieldType,
 } from "@budibase/types"
 import { sql } from "@budibase/backend-core"
 import { join } from "path"
@@ -73,8 +74,20 @@ describe("Captures of real examples", () => {
   })
 
   describe("read", () => {
+    it("should retrieve all fields if non are specified", () => {
+      const queryJson = getJson("basicFetch.json")
+      delete queryJson.resource
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select * from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
     it("should retrieve only requested fields", () => {
       const queryJson = getJson("basicFetch.json")
+
       let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
       expect(query).toEqual({
         bindings: [primaryLimit],
@@ -82,6 +95,22 @@ describe("Captures of real examples", () => {
       })
     })
 
+    it("should retrieve all fields if a formula column is requested", () => {
+      const queryJson = getJson("basicFetch.json")
+      queryJson.table.schema["formula"] = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+      queryJson.resource!.fields.push("formula")
+
+      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [primaryLimit],
+        sql: `select "a".* from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
+      })
+    })
+
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(

From 7932ee76204b96bc19b1295403449ed87762dfc1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 10:42:11 +0100
Subject: [PATCH 028/163] Fix sqs calculations

---
 .../src/sdk/app/rows/search/internal/sqs.ts   | 26 +++++++++----------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 6eb9bd913b..e1ccb4fb85 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -62,7 +62,8 @@ async function buildInternalFieldList(
 ) {
   const { relationships, allowedFields } = opts || {}
   let schemaFields: string[] = []
-  if (sdk.views.isView(source)) {
+  const isView = sdk.views.isView(source)
+  if (isView) {
     schemaFields = Object.keys(helpers.views.basicFields(source))
   } else {
     schemaFields = Object.keys(source.schema).filter(
@@ -75,7 +76,7 @@ async function buildInternalFieldList(
   }
 
   let table: Table
-  if (sdk.views.isView(source)) {
+  if (isView) {
     table = await sdk.views.getTable(source.id)
   } else {
     table = source
@@ -125,6 +126,13 @@ async function buildInternalFieldList(
       fieldList = fieldList.concat(relatedFields)
     }
   }
+
+  if (isView && !helpers.views.isCalculationView(source)) {
+    for (const field of ["_id", "_rev", "tableId"]) {
+      fieldList.push(field)
+    }
+  }
+
   return [...new Set(fieldList)]
 }
 
@@ -234,17 +242,6 @@ async function runSqlQuery(
     json.operation = Operation.COUNT
   }
   const processSQLQuery = async (json: EnrichedQueryJson) => {
-    const fields = json.resource?.fields
-    if (fields) {
-      const tableId = json.tableAliases?.[json.table._id!] ?? json.table._id!
-      for (const key of ["_id", "_rev", "tableId"]) {
-        const field = `${tableId}.${key}`
-        if (fields.includes(field)) {
-          continue
-        }
-        fields.push(field)
-      }
-    }
     const query = builder._query(json, {
       disableReturning: true,
     })
@@ -334,8 +331,9 @@ export async function search(
   }
 
   let aggregations: Aggregation[] = []
-  if (sdk.views.isView(source)) {
+  if (sdk.views.isView(source) && helpers.views.isCalculationView(source)) {
     const calculationFields = helpers.views.calculationFields(source)
+
     for (const [key, field] of Object.entries(calculationFields)) {
       if (options.fields && !options.fields.includes(key)) {
         continue

From 499d3f204128393719fcc166b7c984e2fe17b4a2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 10:47:43 +0100
Subject: [PATCH 029/163] Fix sql calculations

---
 .../src/api/controllers/row/utils/sqlUtils.ts | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 9f5e14d732..ce611b502b 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -169,20 +169,22 @@ export async function buildSqlFieldList(
     fields = Object.keys(helpers.views.basicFields(source))
       .filter(f => table.schema[f].type !== FieldType.LINK)
       .map(c => `${table.name}.${c}`)
+
+    if (!helpers.views.isCalculationView(source)) {
+      fields.push(
+        ...getRequiredFields(
+          {
+            ...table,
+            primaryDisplay: source.primaryDisplay || table.primaryDisplay,
+          },
+          fields
+        )
+      )
+    }
   } else {
     fields = extractRealFields(source)
   }
 
-  fields.push(
-    ...getRequiredFields(
-      {
-        ...table,
-        primaryDisplay: source.primaryDisplay || table.primaryDisplay,
-      },
-      fields
-    )
-  )
-
   for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue

From 51ba1f072bd940b5c70f2edad483762dd7c7e12c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 11:02:54 +0100
Subject: [PATCH 030/163] Add required fields on table search as well

---
 packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index e1ccb4fb85..a5967acfb0 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -127,7 +127,7 @@ async function buildInternalFieldList(
     }
   }
 
-  if (isView && !helpers.views.isCalculationView(source)) {
+  if (!isView || !helpers.views.isCalculationView(source)) {
     for (const field of ["_id", "_rev", "tableId"]) {
       fieldList.push(field)
     }

From c398412e00b52eb1aa8bd9298010dc072ea8f0b7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 11:16:27 +0100
Subject: [PATCH 031/163] Fix pg money

---
 packages/backend-core/src/sql/sql.ts                  | 4 +---
 packages/server/src/integration-test/postgres.spec.ts | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 394ab92fd3..015b7c4751 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -274,9 +274,7 @@ class InternalBuilder {
 
   private isFullSelectStatementRequired(includedFields: string[]): boolean {
     for (const column of Object.values(this.table.schema)) {
-      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
-        return true
-      } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
+      if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
         return true
       } else if (
         column.type === FieldType.FORMULA &&
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 6e674aa58e..713605eadb 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -13,7 +13,7 @@ const mainDescriptions = datasourceDescribe({
 
 if (mainDescriptions.length) {
   describe.each(mainDescriptions)(
-    "/postgres integrations",
+    "/postgres integrations ($dbName)",
     ({ config, dsProvider }) => {
       let datasource: Datasource
       let client: Knex

From 95f7eeacce750c1b050f59e94bbdf140f58b3ae0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 11:30:26 +0100
Subject: [PATCH 032/163] Don't add breaking changes

---
 packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index a5967acfb0..4eca0e4c09 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -128,7 +128,7 @@ async function buildInternalFieldList(
   }
 
   if (!isView || !helpers.views.isCalculationView(source)) {
-    for (const field of ["_id", "_rev", "tableId"]) {
+    for (const field of PROTECTED_INTERNAL_COLUMNS) {
       fieldList.push(field)
     }
   }

From 8765a28f0412aa3cb5353c23d655c34ab0116864 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 13:02:34 +0100
Subject: [PATCH 033/163] Move responsability to sql utils

---
 packages/backend-core/src/sql/sql.ts          | 21 -------
 .../src/api/controllers/row/utils/sqlUtils.ts | 63 +++++++++----------
 .../src/integrations/tests/sqlAlias.spec.ts   | 17 -----
 3 files changed, 31 insertions(+), 70 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 015b7c4751..cc5f226d56 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -272,20 +272,6 @@ class InternalBuilder {
     return parts.join(".")
   }
 
-  private isFullSelectStatementRequired(includedFields: string[]): boolean {
-    for (const column of Object.values(this.table.schema)) {
-      if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
-        return true
-      } else if (
-        column.type === FieldType.FORMULA &&
-        includedFields.includes(column.name)
-      ) {
-        return true
-      }
-    }
-    return false
-  }
-
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
     const { table, resource } = this.query
 
@@ -313,13 +299,6 @@ class InternalBuilder {
       })
       .filter(({ table }) => !table || table === alias)
 
-    const requestedTableColumns = tableFields.map(({ column }) =>
-      column.replace(new RegExp(`^${this.query.meta?.columnPrefix}`), "")
-    )
-    if (this.isFullSelectStatementRequired(requestedTableColumns)) {
-      return [this.knex.raw("??", [`${alias}.*`])]
-    }
-
     return tableFields.map(({ table, column, field }) => {
       const columnSchema = schema[column]
 
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index ce611b502b..750b53861f 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -14,7 +14,7 @@ import {
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { generateJunctionTableID } from "../../../../db/utils"
 import sdk from "../../../../sdk"
-import { helpers } from "@budibase/shared-core"
+import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 import { sql } from "@budibase/backend-core"
 
 type TableMap = Record<string, Table>
@@ -126,11 +126,9 @@ export async function buildSqlFieldList(
           column.type !== FieldType.LINK &&
           column.type !== FieldType.FORMULA &&
           column.type !== FieldType.AI &&
-          !existing.find(
-            (field: string) => field === `${table.name}.${columnName}`
-          )
+          !existing.find((field: string) => field === columnName)
       )
-      .map(([columnName]) => `${table.name}.${columnName}`)
+      .map(([columnName]) => columnName)
   }
 
   function getRequiredFields(table: Table, existing: string[] = []) {
@@ -143,15 +141,12 @@ export async function buildSqlFieldList(
     }
 
     if (!sql.utils.isExternalTable(table)) {
-      requiredFields.push(...["_id", "_rev", "_tableId"])
+      requiredFields.push(...PROTECTED_INTERNAL_COLUMNS)
     }
 
-    return requiredFields
-      .filter(
-        column =>
-          !existing.find((field: string) => field === `${table.name}.${column}`)
-      )
-      .map(column => `${table.name}.${column}`)
+    return requiredFields.filter(
+      column => !existing.find((field: string) => field === column)
+    )
   }
 
   let fields: string[] = []
@@ -161,30 +156,34 @@ export async function buildSqlFieldList(
   let table: Table
   if (isView) {
     table = await sdk.views.getTable(source.id)
+
+    fields = Object.keys(helpers.views.basicFields(source)).filter(
+      f => table.schema[f].type !== FieldType.LINK
+    )
   } else {
     table = source
-  }
-
-  if (isView) {
-    fields = Object.keys(helpers.views.basicFields(source))
-      .filter(f => table.schema[f].type !== FieldType.LINK)
-      .map(c => `${table.name}.${c}`)
-
-    if (!helpers.views.isCalculationView(source)) {
-      fields.push(
-        ...getRequiredFields(
-          {
-            ...table,
-            primaryDisplay: source.primaryDisplay || table.primaryDisplay,
-          },
-          fields
-        )
-      )
-    }
-  } else {
     fields = extractRealFields(source)
   }
 
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (fields.find(f => table.schema[f]?.type === FieldType.FORMULA)) {
+    fields = extractRealFields(table)
+  }
+
+  if (!isView || !helpers.views.isCalculationView(source)) {
+    fields.push(
+      ...getRequiredFields(
+        {
+          ...table,
+          primaryDisplay: source.primaryDisplay || table.primaryDisplay,
+        },
+        fields
+      )
+    )
+  }
+
+  fields = fields.map(c => `${table.name}.${c}`)
+
   for (const field of Object.values(table.schema)) {
     if (field.type !== FieldType.LINK || !relationships || !field.tableId) {
       continue
@@ -227,7 +226,7 @@ export async function buildSqlFieldList(
     }
   }
 
-  return fields
+  return [...new Set(fields)]
 }
 
 export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) {
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 647c18d4ae..898ab9314a 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -8,7 +8,6 @@ import {
   TableSchema,
   Table,
   TableSourceType,
-  FieldType,
 } from "@budibase/types"
 import { sql } from "@budibase/backend-core"
 import { join } from "path"
@@ -95,22 +94,6 @@ describe("Captures of real examples", () => {
       })
     })
 
-    it("should retrieve all fields if a formula column is requested", () => {
-      const queryJson = getJson("basicFetch.json")
-      queryJson.table.schema["formula"] = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-      queryJson.resource!.fields.push("formula")
-
-      let query = new Sql(SqlClient.POSTGRES)._query(queryJson)
-      expect(query).toEqual({
-        bindings: [primaryLimit],
-        sql: `select "a".* from "persons" as "a" order by "a"."firstname" asc nulls first, "a"."personid" asc limit $1`,
-      })
-    })
-
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(

From 8eb82d3b0597cf6f8151f8e5ef871e8e4df2d3d7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 16:10:15 +0100
Subject: [PATCH 034/163] Fix mysql formula test

---
 .../src/api/controllers/row/utils/sqlUtils.ts | 28 +++++++++++++------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 750b53861f..0cfc429afe 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -165,8 +165,11 @@ export async function buildSqlFieldList(
     fields = extractRealFields(source)
   }
 
+  const containsFormula = (isView ? fields : Object.keys(table.schema)).some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
   // If are requesting for a formula field, we need to retrieve all fields
-  if (fields.find(f => table.schema[f]?.type === FieldType.FORMULA)) {
+  if (containsFormula) {
     fields = extractRealFields(table)
   }
 
@@ -192,15 +195,22 @@ export async function buildSqlFieldList(
     if (
       isView &&
       source.schema?.[field.name] &&
-      !helpers.views.isVisible(source.schema[field.name])
+      !helpers.views.isVisible(source.schema[field.name]) &&
+      !containsFormula
     ) {
       continue
     }
 
     const { tableName } = breakExternalTableId(field.tableId)
     const relatedTable = tables[tableName]
-    if (relatedTable) {
-      const viewFields = new Set<string>()
+    if (!relatedTable) {
+      continue
+    }
+
+    const viewFields = new Set<string>()
+    if (containsFormula) {
+      extractRealFields(relatedTable).forEach(f => viewFields.add(f))
+    } else {
       relatedTable.primary?.forEach(f => viewFields.add(f))
       if (relatedTable.primaryDisplay) {
         viewFields.add(relatedTable.primaryDisplay)
@@ -218,12 +228,12 @@ export async function buildSqlFieldList(
           )
           .forEach(([field]) => viewFields.add(field))
       }
-
-      const fieldsToAdd = Array.from(viewFields)
-        .map(f => `${relatedTable.name}.${f}`)
-        .filter(f => !fields.includes(f))
-      fields.push(...fieldsToAdd)
     }
+
+    const fieldsToAdd = Array.from(viewFields)
+      .map(f => `${relatedTable.name}.${f}`)
+      .filter(f => !fields.includes(f))
+    fields.push(...fieldsToAdd)
   }
 
   return [...new Set(fields)]

From 23531e15096fec99f100a71c406658ee47230978 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 16:10:21 +0100
Subject: [PATCH 035/163] Fix sqs formula

---
 .../src/sdk/app/rows/search/internal/sqs.ts   | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 4eca0e4c09..c92d186a37 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -62,7 +62,15 @@ async function buildInternalFieldList(
 ) {
   const { relationships, allowedFields } = opts || {}
   let schemaFields: string[] = []
+
   const isView = sdk.views.isView(source)
+  let table: Table
+  if (isView) {
+    table = await sdk.views.getTable(source.id)
+  } else {
+    table = source
+  }
+
   if (isView) {
     schemaFields = Object.keys(helpers.views.basicFields(source))
   } else {
@@ -71,17 +79,16 @@ async function buildInternalFieldList(
     )
   }
 
-  if (allowedFields) {
+  const containsFormula = schemaFields.some(
+    f => table.schema[f]?.type === FieldType.FORMULA
+  )
+  // If are requesting for a formula field, we need to retrieve all fields
+  if (containsFormula) {
+    schemaFields = Object.keys(table.schema)
+  } else if (allowedFields) {
     schemaFields = schemaFields.filter(field => allowedFields.includes(field))
   }
 
-  let table: Table
-  if (isView) {
-    table = await sdk.views.getTable(source.id)
-  } else {
-    table = source
-  }
-
   let fieldList: string[] = []
   const getJunctionFields = (relatedTable: Table, fields: string[]) => {
     const junctionFields: string[] = []

From bcc9bba254ff4e75ee275041f5488875de2575dd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 16:58:15 +0100
Subject: [PATCH 036/163] Fix test

---
 packages/server/src/api/routes/tests/search.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e97f48afbe..e94e567b43 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -3399,7 +3399,7 @@ if (descriptions.length) {
                       type: FieldType.LINK,
                       relationshipType: RelationshipType.MANY_TO_ONE,
                       tableId: toRelateTableId,
-                      fieldName: "link",
+                      fieldName: "main",
                     },
                   })
 
@@ -3408,7 +3408,7 @@ if (descriptions.length) {
                   )
                   await config.api.table.save({
                     ...toRelateTable,
-                    primaryDisplay: "link",
+                    primaryDisplay: "name",
                   })
                   const relatedRows = await Promise.all([
                     config.api.row.save(toRelateTable._id!, {

From b05b523a6e96169e23dc1cfc7a78b038493c2fb7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 17 Dec 2024 17:01:06 +0100
Subject: [PATCH 037/163] Fix issues with display names not being sql tables

---
 .../src/api/controllers/row/utils/sqlUtils.ts       | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index 0cfc429afe..f28075bf84 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -119,13 +119,14 @@ export async function buildSqlFieldList(
   opts?: { relationships: boolean }
 ) {
   const { relationships } = opts || {}
+
+  const nonMappedColumns = [FieldType.LINK, FieldType.FORMULA, FieldType.AI]
+
   function extractRealFields(table: Table, existing: string[] = []) {
     return Object.entries(table.schema)
       .filter(
         ([columnName, column]) =>
-          column.type !== FieldType.LINK &&
-          column.type !== FieldType.FORMULA &&
-          column.type !== FieldType.AI &&
+          !nonMappedColumns.includes(column.type) &&
           !existing.find((field: string) => field === columnName)
       )
       .map(([columnName]) => columnName)
@@ -145,7 +146,10 @@ export async function buildSqlFieldList(
     }
 
     return requiredFields.filter(
-      column => !existing.find((field: string) => field === column)
+      column =>
+        !existing.find((field: string) => field === column) &&
+        table.schema[column] &&
+        !nonMappedColumns.includes(table.schema[column].type)
     )
   }
 
@@ -231,6 +235,7 @@ export async function buildSqlFieldList(
     }
 
     const fieldsToAdd = Array.from(viewFields)
+      .filter(f => !nonMappedColumns.includes(relatedTable.schema[f].type))
       .map(f => `${relatedTable.name}.${f}`)
       .filter(f => !fields.includes(f))
     fields.push(...fieldsToAdd)

From 27d1929388de83c56798dd2433685ec3376e67e2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 14:09:26 +0100
Subject: [PATCH 038/163] Add basic sqlUtils test

---
 .../src/api/controllers/row/utils/sqlUtils.ts |  4 +-
 .../row/utils/tests/sqlUtils.spec.ts          | 79 +++++++++++++++++++
 2 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index f28075bf84..e66a0b5bf6 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -166,7 +166,9 @@ export async function buildSqlFieldList(
     )
   } else {
     table = source
-    fields = extractRealFields(source)
+    fields = extractRealFields(source).filter(
+      f => table.schema[f].visible !== false
+    )
   }
 
   const containsFormula = (isView ? fields : Object.keys(table.schema)).some(
diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
new file mode 100644
index 0000000000..2ab8e251ea
--- /dev/null
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -0,0 +1,79 @@
+import {
+  AIOperationEnum,
+  FieldType,
+  RelationshipType,
+  Table,
+} from "@budibase/types"
+import { buildSqlFieldList } from "../sqlUtils"
+import { structures } from "../../../../routes/tests/utilities"
+import { cloneDeep } from "lodash"
+
+describe("buildSqlFieldList", () => {
+  const basicTable: Table = {
+    ...structures.basicTable(),
+    name: "table",
+    schema: {
+      name: {
+        type: FieldType.STRING,
+        name: "name",
+      },
+      description: {
+        type: FieldType.STRING,
+        name: "description",
+      },
+      amount: {
+        type: FieldType.NUMBER,
+        name: "amount",
+      },
+    },
+  }
+
+  it("extracts fields from table schema", async () => {
+    const result = await buildSqlFieldList(basicTable, {})
+    expect(result).toEqual(["table.name", "table.description", "table.amount"])
+  })
+
+  it("excludes hidden fields", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.description.visible = false
+    const result = await buildSqlFieldList(table, {})
+    expect(result).toEqual(["table.name", "table.amount"])
+  })
+
+  it("excludes non-sql fields fields", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.formula = {
+      name: "formula",
+      type: FieldType.FORMULA,
+      formula: "any",
+    }
+    table.schema.ai = {
+      name: "ai",
+      type: FieldType.AI,
+      operation: AIOperationEnum.PROMPT,
+    }
+    table.schema.link = {
+      name: "link",
+      type: FieldType.LINK,
+      relationshipType: RelationshipType.ONE_TO_MANY,
+      fieldName: "link",
+      tableId: "otherTableId",
+    }
+
+    const result = await buildSqlFieldList(table, {})
+    expect(result).toEqual(["table.name", "table.description", "table.amount"])
+  })
+
+  it("includes hidden fields if there is a formula column", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.description.visible = false
+    table.schema.formula = {
+      name: "formula",
+      type: FieldType.FORMULA,
+      formula: "any",
+    }
+
+    const result = await buildSqlFieldList(table, {})
+    expect(result).toEqual(["table.name", "table.description", "table.amount"])
+  })
+})

From 875319e85c1fa97abc4e052845ea61ef1dfde449 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 15:50:51 +0100
Subject: [PATCH 039/163] Relationship tests

---
 .../row/utils/tests/sqlUtils.spec.ts          | 105 +++++++++++++++++-
 1 file changed, 101 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 2ab8e251ea..5a1b8ae49e 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -2,28 +2,34 @@ import {
   AIOperationEnum,
   FieldType,
   RelationshipType,
+  SourceName,
   Table,
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
 import { cloneDeep } from "lodash"
+import { sql } from "@budibase/backend-core"
 
 describe("buildSqlFieldList", () => {
   const basicTable: Table = {
-    ...structures.basicTable(),
+    ...structures.tableForDatasource({
+      type: "datasource",
+      source: SourceName.POSTGRES,
+    }),
     name: "table",
+    _id: sql.utils.buildExternalTableId("ds_id", "table"),
     schema: {
       name: {
-        type: FieldType.STRING,
         name: "name",
+        type: FieldType.STRING,
       },
       description: {
-        type: FieldType.STRING,
         name: "description",
+        type: FieldType.STRING,
       },
       amount: {
-        type: FieldType.NUMBER,
         name: "amount",
+        type: FieldType.NUMBER,
       },
     },
   }
@@ -76,4 +82,95 @@ describe("buildSqlFieldList", () => {
     const result = await buildSqlFieldList(table, {})
     expect(result).toEqual(["table.name", "table.description", "table.amount"])
   })
+
+  it("includes relationships fields when flag", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.link = {
+      name: "link",
+      type: FieldType.LINK,
+      relationshipType: RelationshipType.ONE_TO_MANY,
+      fieldName: "link",
+      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+    }
+
+    const otherTable: Table = {
+      ...cloneDeep(basicTable),
+      name: "linkedTable",
+      primary: ["id"],
+      primaryDisplay: "name",
+      schema: {
+        ...cloneDeep(basicTable).schema,
+        id: {
+          name: "id",
+          type: FieldType.NUMBER,
+        },
+      },
+    }
+
+    const allTables: Record<string, Table> = {
+      otherTableId: otherTable,
+    }
+
+    const result = await buildSqlFieldList(table, allTables, {
+      relationships: true,
+    })
+    expect(result).toEqual([
+      "table.name",
+      "table.description",
+      "table.amount",
+      "linkedTable.id",
+      "linkedTable.name",
+    ])
+  })
+
+  it("includes all relationship fields if there is a formula column", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.link = {
+      name: "link",
+      type: FieldType.LINK,
+      relationshipType: RelationshipType.ONE_TO_MANY,
+      fieldName: "link",
+      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+    }
+    table.schema.formula = {
+      name: "formula",
+      type: FieldType.FORMULA,
+      formula: "any",
+    }
+
+    const otherTable: Table = {
+      ...cloneDeep(basicTable),
+      name: "linkedTable",
+      schema: {
+        ...cloneDeep(basicTable).schema,
+        id: {
+          name: "id",
+          type: FieldType.NUMBER,
+        },
+        hidden: {
+          name: "other",
+          type: FieldType.STRING,
+          visible: false,
+        },
+      },
+    }
+
+    const allTables: Record<string, Table> = {
+      otherTableId: otherTable,
+    }
+
+    const result = await buildSqlFieldList(table, allTables, {
+      relationships: true,
+    })
+    expect(result).toEqual([
+      "table.name",
+      "table.description",
+      "table.amount",
+      "linkedTable.name",
+      "linkedTable.description",
+      "linkedTable.amount",
+      "linkedTable.id",
+      "linkedTable.hidden",
+    ])
+  })
 })

From 96bddbb545b8ddc1602e20e981d2999f9b5d46a0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 15:53:09 +0100
Subject: [PATCH 040/163] More relationship tests

---
 .../row/utils/tests/sqlUtils.spec.ts          | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 5a1b8ae49e..767a325c7a 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -152,6 +152,23 @@ describe("buildSqlFieldList", () => {
           type: FieldType.STRING,
           visible: false,
         },
+        formula: {
+          name: "formula",
+          type: FieldType.FORMULA,
+          formula: "any",
+        },
+        ai: {
+          name: "ai",
+          type: FieldType.AI,
+          operation: AIOperationEnum.PROMPT,
+        },
+        link: {
+          name: "link",
+          type: FieldType.LINK,
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          fieldName: "link",
+          tableId: "otherTableId",
+        },
       },
     }
 
@@ -173,4 +190,68 @@ describe("buildSqlFieldList", () => {
       "linkedTable.hidden",
     ])
   })
+
+  it("never includes non-sql columns from relationships", async () => {
+    const table = cloneDeep(basicTable)
+    table.schema.link = {
+      name: "link",
+      type: FieldType.LINK,
+      relationshipType: RelationshipType.ONE_TO_MANY,
+      fieldName: "link",
+      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+    }
+    table.schema.formula = {
+      name: "formula",
+      type: FieldType.FORMULA,
+      formula: "any",
+    }
+
+    const otherTable: Table = {
+      ...cloneDeep(basicTable),
+      name: "linkedTable",
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.NUMBER,
+        },
+        hidden: {
+          name: "other",
+          type: FieldType.STRING,
+          visible: false,
+        },
+        formula: {
+          name: "formula",
+          type: FieldType.FORMULA,
+          formula: "any",
+        },
+        ai: {
+          name: "ai",
+          type: FieldType.AI,
+          operation: AIOperationEnum.PROMPT,
+        },
+        link: {
+          name: "link",
+          type: FieldType.LINK,
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          fieldName: "link",
+          tableId: "otherTableId",
+        },
+      },
+    }
+
+    const allTables: Record<string, Table> = {
+      otherTableId: otherTable,
+    }
+
+    const result = await buildSqlFieldList(table, allTables, {
+      relationships: true,
+    })
+    expect(result).toEqual([
+      "table.name",
+      "table.description",
+      "table.amount",
+      "linkedTable.id",
+      "linkedTable.hidden",
+    ])
+  })
 })

From e23753ac6134c33482ddcd1802907ec3d9ab9f11 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 15:54:07 +0100
Subject: [PATCH 041/163] Add describe

---
 .../row/utils/tests/sqlUtils.spec.ts          | 440 +++++++++---------
 1 file changed, 227 insertions(+), 213 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 767a325c7a..41cc9f6a03 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -34,224 +34,238 @@ describe("buildSqlFieldList", () => {
     },
   }
 
-  it("extracts fields from table schema", async () => {
-    const result = await buildSqlFieldList(basicTable, {})
-    expect(result).toEqual(["table.name", "table.description", "table.amount"])
-  })
-
-  it("excludes hidden fields", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.description.visible = false
-    const result = await buildSqlFieldList(table, {})
-    expect(result).toEqual(["table.name", "table.amount"])
-  })
-
-  it("excludes non-sql fields fields", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.formula = {
-      name: "formula",
-      type: FieldType.FORMULA,
-      formula: "any",
-    }
-    table.schema.ai = {
-      name: "ai",
-      type: FieldType.AI,
-      operation: AIOperationEnum.PROMPT,
-    }
-    table.schema.link = {
-      name: "link",
-      type: FieldType.LINK,
-      relationshipType: RelationshipType.ONE_TO_MANY,
-      fieldName: "link",
-      tableId: "otherTableId",
-    }
-
-    const result = await buildSqlFieldList(table, {})
-    expect(result).toEqual(["table.name", "table.description", "table.amount"])
-  })
-
-  it("includes hidden fields if there is a formula column", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.description.visible = false
-    table.schema.formula = {
-      name: "formula",
-      type: FieldType.FORMULA,
-      formula: "any",
-    }
-
-    const result = await buildSqlFieldList(table, {})
-    expect(result).toEqual(["table.name", "table.description", "table.amount"])
-  })
-
-  it("includes relationships fields when flag", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.link = {
-      name: "link",
-      type: FieldType.LINK,
-      relationshipType: RelationshipType.ONE_TO_MANY,
-      fieldName: "link",
-      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-    }
-
-    const otherTable: Table = {
-      ...cloneDeep(basicTable),
-      name: "linkedTable",
-      primary: ["id"],
-      primaryDisplay: "name",
-      schema: {
-        ...cloneDeep(basicTable).schema,
-        id: {
-          name: "id",
-          type: FieldType.NUMBER,
-        },
-      },
-    }
-
-    const allTables: Record<string, Table> = {
-      otherTableId: otherTable,
-    }
-
-    const result = await buildSqlFieldList(table, allTables, {
-      relationships: true,
+  describe("table", () => {
+    it("extracts fields from table schema", async () => {
+      const result = await buildSqlFieldList(basicTable, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
     })
-    expect(result).toEqual([
-      "table.name",
-      "table.description",
-      "table.amount",
-      "linkedTable.id",
-      "linkedTable.name",
-    ])
-  })
 
-  it("includes all relationship fields if there is a formula column", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.link = {
-      name: "link",
-      type: FieldType.LINK,
-      relationshipType: RelationshipType.ONE_TO_MANY,
-      fieldName: "link",
-      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-    }
-    table.schema.formula = {
-      name: "formula",
-      type: FieldType.FORMULA,
-      formula: "any",
-    }
-
-    const otherTable: Table = {
-      ...cloneDeep(basicTable),
-      name: "linkedTable",
-      schema: {
-        ...cloneDeep(basicTable).schema,
-        id: {
-          name: "id",
-          type: FieldType.NUMBER,
-        },
-        hidden: {
-          name: "other",
-          type: FieldType.STRING,
-          visible: false,
-        },
-        formula: {
-          name: "formula",
-          type: FieldType.FORMULA,
-          formula: "any",
-        },
-        ai: {
-          name: "ai",
-          type: FieldType.AI,
-          operation: AIOperationEnum.PROMPT,
-        },
-        link: {
-          name: "link",
-          type: FieldType.LINK,
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          fieldName: "link",
-          tableId: "otherTableId",
-        },
-      },
-    }
-
-    const allTables: Record<string, Table> = {
-      otherTableId: otherTable,
-    }
-
-    const result = await buildSqlFieldList(table, allTables, {
-      relationships: true,
+    it("excludes hidden fields", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.description.visible = false
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual(["table.name", "table.amount"])
     })
-    expect(result).toEqual([
-      "table.name",
-      "table.description",
-      "table.amount",
-      "linkedTable.name",
-      "linkedTable.description",
-      "linkedTable.amount",
-      "linkedTable.id",
-      "linkedTable.hidden",
-    ])
-  })
 
-  it("never includes non-sql columns from relationships", async () => {
-    const table = cloneDeep(basicTable)
-    table.schema.link = {
-      name: "link",
-      type: FieldType.LINK,
-      relationshipType: RelationshipType.ONE_TO_MANY,
-      fieldName: "link",
-      tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-    }
-    table.schema.formula = {
-      name: "formula",
-      type: FieldType.FORMULA,
-      formula: "any",
-    }
+    it("excludes non-sql fields fields", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+      table.schema.ai = {
+        name: "ai",
+        type: FieldType.AI,
+        operation: AIOperationEnum.PROMPT,
+      }
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: "otherTableId",
+      }
 
-    const otherTable: Table = {
-      ...cloneDeep(basicTable),
-      name: "linkedTable",
-      schema: {
-        id: {
-          name: "id",
-          type: FieldType.NUMBER,
-        },
-        hidden: {
-          name: "other",
-          type: FieldType.STRING,
-          visible: false,
-        },
-        formula: {
-          name: "formula",
-          type: FieldType.FORMULA,
-          formula: "any",
-        },
-        ai: {
-          name: "ai",
-          type: FieldType.AI,
-          operation: AIOperationEnum.PROMPT,
-        },
-        link: {
-          name: "link",
-          type: FieldType.LINK,
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          fieldName: "link",
-          tableId: "otherTableId",
-        },
-      },
-    }
-
-    const allTables: Record<string, Table> = {
-      otherTableId: otherTable,
-    }
-
-    const result = await buildSqlFieldList(table, allTables, {
-      relationships: true,
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.description.visible = false
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      const result = await buildSqlFieldList(table, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("includes relationships fields when flag", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+
+      const otherTable: Table = {
+        ...cloneDeep(basicTable),
+        name: "linkedTable",
+        primary: ["id"],
+        primaryDisplay: "name",
+        schema: {
+          ...cloneDeep(basicTable).schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.id",
+        "linkedTable.name",
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      const otherTable: Table = {
+        ...cloneDeep(basicTable),
+        name: "linkedTable",
+        schema: {
+          ...cloneDeep(basicTable).schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+          hidden: {
+            name: "other",
+            type: FieldType.STRING,
+            visible: false,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any",
+          },
+          ai: {
+            name: "ai",
+            type: FieldType.AI,
+            operation: AIOperationEnum.PROMPT,
+          },
+          link: {
+            name: "link",
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "link",
+            tableId: "otherTableId",
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
+
+    it("never includes non-sql columns from relationships", async () => {
+      const table = cloneDeep(basicTable)
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      const otherTable: Table = {
+        ...cloneDeep(basicTable),
+        name: "linkedTable",
+        schema: {
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+          hidden: {
+            name: "other",
+            type: FieldType.STRING,
+            visible: false,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any",
+          },
+          ai: {
+            name: "ai",
+            type: FieldType.AI,
+            operation: AIOperationEnum.PROMPT,
+          },
+          link: {
+            name: "link",
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "link",
+            tableId: "otherTableId",
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
     })
-    expect(result).toEqual([
-      "table.name",
-      "table.description",
-      "table.amount",
-      "linkedTable.id",
-      "linkedTable.hidden",
-    ])
   })
 })

From 14da90296b4d44fa56146d0fef304a1bdee79eba Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 16:23:53 +0100
Subject: [PATCH 042/163] Regenerate table before each test

---
 .../row/utils/tests/sqlUtils.spec.ts          | 101 +++++++++---------
 1 file changed, 49 insertions(+), 52 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 41cc9f6a03..b1be459f75 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -7,11 +7,13 @@ import {
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
-import { cloneDeep } from "lodash"
 import { sql } from "@budibase/backend-core"
 
 describe("buildSqlFieldList", () => {
-  const basicTable: Table = {
+  let table: Table & { _id: string }
+
+  beforeEach(() => {
+    table = {
     ...structures.tableForDatasource({
       type: "datasource",
       source: SourceName.POSTGRES,
@@ -33,10 +35,11 @@ describe("buildSqlFieldList", () => {
       },
     },
   }
+  })
 
   describe("table", () => {
     it("extracts fields from table schema", async () => {
-      const result = await buildSqlFieldList(basicTable, {})
+      const result = await buildSqlFieldList(table, {})
       expect(result).toEqual([
         "table.name",
         "table.description",
@@ -45,14 +48,12 @@ describe("buildSqlFieldList", () => {
     })
 
     it("excludes hidden fields", async () => {
-      const table = cloneDeep(basicTable)
       table.schema.description.visible = false
       const result = await buildSqlFieldList(table, {})
       expect(result).toEqual(["table.name", "table.amount"])
     })
 
     it("excludes non-sql fields fields", async () => {
-      const table = cloneDeep(basicTable)
       table.schema.formula = {
         name: "formula",
         type: FieldType.FORMULA,
@@ -80,7 +81,6 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes hidden fields if there is a formula column", async () => {
-      const table = cloneDeep(basicTable)
       table.schema.description.visible = false
       table.schema.formula = {
         name: "formula",
@@ -97,7 +97,20 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes relationships fields when flag", async () => {
-      const table = cloneDeep(basicTable)
+      const otherTable: Table = {
+        ...table,
+        name: "linkedTable",
+        primary: ["id"],
+        primaryDisplay: "name",
+        schema: {
+          ...table.schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
       table.schema.link = {
         name: "link",
         type: FieldType.LINK,
@@ -106,20 +119,6 @@ describe("buildSqlFieldList", () => {
         tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
       }
 
-      const otherTable: Table = {
-        ...cloneDeep(basicTable),
-        name: "linkedTable",
-        primary: ["id"],
-        primaryDisplay: "name",
-        schema: {
-          ...cloneDeep(basicTable).schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-        },
-      }
-
       const allTables: Record<string, Table> = {
         otherTableId: otherTable,
       }
@@ -137,25 +136,11 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes all relationship fields if there is a formula column", async () => {
-      const table = cloneDeep(basicTable)
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-
       const otherTable: Table = {
-        ...cloneDeep(basicTable),
+        ...table,
         name: "linkedTable",
         schema: {
-          ...cloneDeep(basicTable).schema,
+          ...table.schema,
           id: {
             name: "id",
             type: FieldType.NUMBER,
@@ -185,6 +170,19 @@ describe("buildSqlFieldList", () => {
         },
       }
 
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
       const allTables: Record<string, Table> = {
         otherTableId: otherTable,
       }
@@ -205,22 +203,8 @@ describe("buildSqlFieldList", () => {
     })
 
     it("never includes non-sql columns from relationships", async () => {
-      const table = cloneDeep(basicTable)
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-
       const otherTable: Table = {
-        ...cloneDeep(basicTable),
+        ...table,
         name: "linkedTable",
         schema: {
           id: {
@@ -252,6 +236,19 @@ describe("buildSqlFieldList", () => {
         },
       }
 
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
       const allTables: Record<string, Table> = {
         otherTableId: otherTable,
       }

From 8da96ab271c09e8c805608d095116235b4a9f840 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 16:36:20 +0100
Subject: [PATCH 043/163] Add initial view tests

---
 .../row/utils/tests/sqlUtils.spec.ts          | 119 ++++++++++++++----
 1 file changed, 98 insertions(+), 21 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index b1be459f75..6b7f0d9ce0 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -4,37 +4,51 @@ import {
   RelationshipType,
   SourceName,
   Table,
+  ViewV2,
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
 import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import { generateViewID } from "../../../../../db/utils"
+
+import sdk from "../../../../../sdk"
+
+jest.mock("../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
 
 describe("buildSqlFieldList", () => {
   let table: Table & { _id: string }
 
   beforeEach(() => {
+    jest.clearAllMocks()
     table = {
-    ...structures.tableForDatasource({
-      type: "datasource",
-      source: SourceName.POSTGRES,
-    }),
-    name: "table",
-    _id: sql.utils.buildExternalTableId("ds_id", "table"),
-    schema: {
-      name: {
-        name: "name",
-        type: FieldType.STRING,
+      ...structures.tableForDatasource({
+        type: "datasource",
+        source: SourceName.POSTGRES,
+      }),
+      name: "table",
+      _id: sql.utils.buildExternalTableId("ds_id", "table"),
+      schema: {
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        description: {
+          name: "description",
+          type: FieldType.STRING,
+        },
+        amount: {
+          name: "amount",
+          type: FieldType.NUMBER,
+        },
       },
-      description: {
-        name: "description",
-        type: FieldType.STRING,
-      },
-      amount: {
-        name: "amount",
-        type: FieldType.NUMBER,
-      },
-    },
-  }
+    }
   })
 
   describe("table", () => {
@@ -96,7 +110,7 @@ describe("buildSqlFieldList", () => {
       ])
     })
 
-    it("includes relationships fields when flag", async () => {
+    it("includes relationships fields when flagged", async () => {
       const otherTable: Table = {
         ...table,
         name: "linkedTable",
@@ -265,4 +279,67 @@ describe("buildSqlFieldList", () => {
       ])
     })
   })
+
+  describe("view", () => {
+    let view: ViewV2
+
+    beforeEach(() => {
+      getTableMock.mockResolvedValueOnce(table)
+
+      view = {
+        version: 2,
+        id: generateViewID(table._id),
+        name: generator.word(),
+        tableId: table._id,
+      }
+    })
+
+    it("extracts fields from table schema", async () => {
+      view.schema = {
+        name: { visible: false },
+        amount: { visible: true },
+      }
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      view.schema = {
+        name: { visible: false },
+        amount: { visible: true },
+        formula: { visible: true },
+      }
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      view.schema = {
+        name: { visible: false },
+        amount: { visible: true },
+        formula: { visible: false },
+      }
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+  })
 })

From 460b5fd7dd5c8dc839c361d857a774009feda72a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 16:45:08 +0100
Subject: [PATCH 044/163] More tests

---
 .../row/utils/tests/sqlUtils.spec.ts          | 224 ++++++++++++++++++
 1 file changed, 224 insertions(+)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 6b7f0d9ce0..e1719d980b 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -341,5 +341,229 @@ describe("buildSqlFieldList", () => {
       const result = await buildSqlFieldList(view, {})
       expect(result).toEqual(["table.amount"])
     })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable: Table = {
+        ...table,
+        name: "linkedTable",
+        primary: ["id"],
+        primaryDisplay: "name",
+        schema: {
+          ...table.schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+
+      view.schema = {
+        name: { visible: true },
+        link: { visible: true },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "linkedTable.id",
+        "linkedTable.name",
+      ])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable: Table = {
+        ...table,
+        name: "linkedTable",
+        primary: ["id"],
+        schema: {
+          ...table.schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any",
+          },
+        },
+      }
+
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+
+      view.schema = {
+        name: { visible: true },
+        link: {
+          visible: true,
+          columns: {
+            name: { visible: false },
+            amount: { visible: true },
+            formula: { visible: false },
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "linkedTable.id",
+        "linkedTable.amount",
+      ])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable: Table = {
+        ...table,
+        name: "linkedTable",
+        primary: ["id"],
+        schema: {
+          ...table.schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any",
+          },
+        },
+      }
+
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+
+      view.schema = {
+        name: { visible: true },
+        link: {
+          visible: false,
+          columns: {
+            name: { visible: false },
+            amount: { visible: true },
+            formula: { visible: false },
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable: Table = {
+        ...table,
+        name: "linkedTable",
+        schema: {
+          ...table.schema,
+          id: {
+            name: "id",
+            type: FieldType.NUMBER,
+          },
+          hidden: {
+            name: "other",
+            type: FieldType.STRING,
+            visible: false,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: "any",
+          },
+          ai: {
+            name: "ai",
+            type: FieldType.AI,
+            operation: AIOperationEnum.PROMPT,
+          },
+          link: {
+            name: "link",
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            fieldName: "link",
+            tableId: "otherTableId",
+          },
+        },
+      }
+
+      table.schema.link = {
+        name: "link",
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
+      }
+      table.schema.formula = {
+        name: "formula",
+        type: FieldType.FORMULA,
+        formula: "any",
+      }
+
+      view.schema = {
+        name: { visible: true },
+        formula: { visible: true },
+        link: {
+          visible: false,
+          columns: {
+            name: { visible: false },
+            amount: { visible: true },
+            formula: { visible: false },
+          },
+        },
+      }
+
+      const allTables: Record<string, Table> = {
+        otherTableId: otherTable,
+      }
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.id",
+        "linkedTable.hidden",
+      ])
+    })
   })
 })

From 9396292c4a451d83bb76002cb816f56011dfbbe0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 18 Dec 2024 18:34:40 +0100
Subject: [PATCH 045/163] Create table test builder

---
 .../row/utils/tests/sqlUtils.spec.ts          | 681 ++++++++----------
 1 file changed, 293 insertions(+), 388 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index e1719d980b..eafdd170f8 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -13,6 +13,8 @@ import { generator } from "@budibase/backend-core/tests"
 import { generateViewID } from "../../../../../db/utils"
 
 import sdk from "../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
 
 jest.mock("../../../../../sdk/app/views", () => ({
   ...jest.requireActual("../../../../../sdk/app/views"),
@@ -23,36 +25,161 @@ const getTableMock = sdk.views.getTable as jest.MockedFunction<
 >
 
 describe("buildSqlFieldList", () => {
-  let table: Table & { _id: string }
+  let allTables: Record<string, Table>
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor(name: string) {
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables[name] = this._table
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withPrimary(field: string) {
+      this._table.primary = [field]
+      return this
+    }
+
+    withDisplay(field: string) {
+      this._table.primaryDisplay = field
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
 
   beforeEach(() => {
     jest.clearAllMocks()
-    table = {
-      ...structures.tableForDatasource({
-        type: "datasource",
-        source: SourceName.POSTGRES,
-      }),
-      name: "table",
-      _id: sql.utils.buildExternalTableId("ds_id", "table"),
-      schema: {
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-        },
-        description: {
-          name: "description",
-          type: FieldType.STRING,
-        },
-        amount: {
-          name: "amount",
-          type: FieldType.NUMBER,
-        },
-      },
-    }
+    allTables = {}
   })
 
   describe("table", () => {
     it("extracts fields from table schema", async () => {
+      const table = new TableConfig("table").create()
       const result = await buildSqlFieldList(table, {})
       expect(result).toEqual([
         "table.name",
@@ -62,29 +189,19 @@ describe("buildSqlFieldList", () => {
     })
 
     it("excludes hidden fields", async () => {
-      table.schema.description.visible = false
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .create()
       const result = await buildSqlFieldList(table, {})
       expect(result).toEqual(["table.name", "table.amount"])
     })
 
     it("excludes non-sql fields fields", async () => {
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-      table.schema.ai = {
-        name: "ai",
-        type: FieldType.AI,
-        operation: AIOperationEnum.PROMPT,
-      }
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: "otherTableId",
-      }
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
 
       const result = await buildSqlFieldList(table, {})
       expect(result).toEqual([
@@ -95,12 +212,10 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes hidden fields if there is a formula column", async () => {
-      table.schema.description.visible = false
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
+      const table = new TableConfig("table")
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
 
       const result = await buildSqlFieldList(table, {})
       expect(result).toEqual([
@@ -111,31 +226,15 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes relationships fields when flagged", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        primary: ["id"],
-        primaryDisplay: "name",
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
 
       const result = await buildSqlFieldList(table, allTables, {
         relationships: true,
@@ -150,56 +249,42 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-          hidden: {
-            name: "other",
-            type: FieldType.STRING,
-            visible: false,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: "any",
-          },
-          ai: {
-            name: "ai",
-            type: FieldType.AI,
-            operation: AIOperationEnum.PROMPT,
-          },
-          link: {
-            name: "link",
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            fieldName: "link",
-            tableId: "otherTableId",
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
 
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const result = await buildSqlFieldList(table, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual([
+        "table.name",
+        "table.description",
+        "table.amount",
+        "linkedTable.name",
+        "linkedTable.description",
+        "linkedTable.amount",
+        "linkedTable.hidden",
+      ])
+    })
+
+    it("never includes non-sql columns from relationships", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
 
       const result = await buildSqlFieldList(table, allTables, {
         relationships: true,
@@ -215,107 +300,28 @@ describe("buildSqlFieldList", () => {
         "linkedTable.hidden",
       ])
     })
-
-    it("never includes non-sql columns from relationships", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        schema: {
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-          hidden: {
-            name: "other",
-            type: FieldType.STRING,
-            visible: false,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: "any",
-          },
-          ai: {
-            name: "ai",
-            type: FieldType.AI,
-            operation: AIOperationEnum.PROMPT,
-          },
-          link: {
-            name: "link",
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            fieldName: "link",
-            tableId: "otherTableId",
-          },
-        },
-      }
-
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
-
-      const result = await buildSqlFieldList(table, allTables, {
-        relationships: true,
-      })
-      expect(result).toEqual([
-        "table.name",
-        "table.description",
-        "table.amount",
-        "linkedTable.id",
-        "linkedTable.hidden",
-      ])
-    })
   })
 
   describe("view", () => {
-    let view: ViewV2
-
-    beforeEach(() => {
-      getTableMock.mockResolvedValueOnce(table)
-
-      view = {
-        version: 2,
-        id: generateViewID(table._id),
-        name: generator.word(),
-        tableId: table._id,
-      }
-    })
-
     it("extracts fields from table schema", async () => {
-      view.schema = {
-        name: { visible: false },
-        amount: { visible: true },
-      }
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
 
       const result = await buildSqlFieldList(view, {})
       expect(result).toEqual(["table.amount"])
     })
 
     it("includes all fields if there is a formula column", async () => {
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-
-      view.schema = {
-        name: { visible: false },
-        amount: { visible: true },
-        formula: { visible: true },
-      }
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
 
       const result = await buildSqlFieldList(view, {})
       expect(result).toEqual([
@@ -326,53 +332,35 @@ describe("buildSqlFieldList", () => {
     })
 
     it("does not includes all fields if the formula column is not included", async () => {
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
-
-      view.schema = {
-        name: { visible: false },
-        amount: { visible: true },
-        formula: { visible: false },
-      }
+      const table = new TableConfig("table")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
 
       const result = await buildSqlFieldList(view, {})
       expect(result).toEqual(["table.amount"])
     })
 
     it("includes relationships fields when flagged", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        primary: ["id"],
-        primaryDisplay: "name",
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
 
-      view.schema = {
-        name: { visible: true },
-        link: { visible: true },
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("amount")
+        .create()
 
       const result = await buildSqlFieldList(view, allTables, {
         relationships: true,
@@ -385,47 +373,25 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes relationships columns", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        primary: ["id"],
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: "any",
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
 
-      view.schema = {
-        name: { visible: true },
-        link: {
-          visible: true,
-          columns: {
-            name: { visible: false },
-            amount: { visible: true },
-            formula: { visible: false },
-          },
-        },
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
 
       const result = await buildSqlFieldList(view, allTables, {
         relationships: true,
@@ -438,47 +404,25 @@ describe("buildSqlFieldList", () => {
     })
 
     it("does not include relationships columns for hidden links", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        primary: ["id"],
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: "any",
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("formula", FieldType.FORMULA)
+        .withPrimary("id")
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .create()
 
-      view.schema = {
-        name: { visible: true },
-        link: {
-          visible: false,
-          columns: {
-            name: { visible: false },
-            amount: { visible: true },
-            formula: { visible: false },
-          },
-        },
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
 
       const result = await buildSqlFieldList(view, allTables, {
         relationships: true,
@@ -487,69 +431,30 @@ describe("buildSqlFieldList", () => {
     })
 
     it("includes all relationship fields if there is a formula column", async () => {
-      const otherTable: Table = {
-        ...table,
-        name: "linkedTable",
-        schema: {
-          ...table.schema,
-          id: {
-            name: "id",
-            type: FieldType.NUMBER,
-          },
-          hidden: {
-            name: "other",
-            type: FieldType.STRING,
-            visible: false,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: "any",
-          },
-          ai: {
-            name: "ai",
-            type: FieldType.AI,
-            operation: AIOperationEnum.PROMPT,
-          },
-          link: {
-            name: "link",
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            fieldName: "link",
-            tableId: "otherTableId",
-          },
-        },
-      }
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .withPrimary("id")
+        .create()
 
-      table.schema.link = {
-        name: "link",
-        type: FieldType.LINK,
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        fieldName: "link",
-        tableId: sql.utils.buildExternalTableId("ds_id", "otherTableId"),
-      }
-      table.schema.formula = {
-        name: "formula",
-        type: FieldType.FORMULA,
-        formula: "any",
-      }
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
 
-      view.schema = {
-        name: { visible: true },
-        formula: { visible: true },
-        link: {
-          visible: false,
-          columns: {
-            name: { visible: false },
-            amount: { visible: true },
-            formula: { visible: false },
-          },
-        },
-      }
-
-      const allTables: Record<string, Table> = {
-        otherTableId: otherTable,
-      }
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
 
       const result = await buildSqlFieldList(view, allTables, {
         relationships: true,

From 92e791f9a715235856c2f318965329d713c7b893 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 12:48:33 +0100
Subject: [PATCH 046/163] Add calculation tests

---
 .../row/utils/tests/sqlUtils.spec.ts          | 41 +++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index eafdd170f8..12a5edd30b 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -1,10 +1,12 @@
 import {
   AIOperationEnum,
+  CalculationType,
   FieldType,
   RelationshipType,
   SourceName,
   Table,
   ViewV2,
+  ViewV2Type,
 } from "@budibase/types"
 import { buildSqlFieldList } from "../sqlUtils"
 import { structures } from "../../../../routes/tests/utilities"
@@ -166,6 +168,21 @@ describe("buildSqlFieldList", () => {
       return this
     }
 
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
     create() {
       getTableMock.mockResolvedValueOnce(this._table)
       return cloneDeep(this._view)
@@ -471,4 +488,28 @@ describe("buildSqlFieldList", () => {
       ])
     })
   })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig("table").create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+
+        .create()
+
+      const result = await buildSqlFieldList(view, {})
+      expect(result).toEqual(["table.amount"])
+    })
+  })
 })

From 889197679e1595b78bff806292372997000cedc2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 19 Dec 2024 12:39:00 +0000
Subject: [PATCH 047/163] Comment to explain the behaviour of the junction
 document select.

---
 packages/server/src/sdk/app/rows/search/internal/sqs.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index c92d186a37..449ba477b7 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -121,6 +121,12 @@ async function buildInternalFieldList(
       if (!relatedTable) {
         continue
       }
+      // a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full
+      // structure - essentially all relationships between two tables will be inserted into a single "table"
+      // we don't use an independent junction table ID for each separate relationship between two tables. For
+      // example if we have table A and B, with two relationships between them, all the junction documents will
+      // end up in the same junction table ID. We need to retrieve the field name property of the junction documents
+      // as part of the relationship to tell us which relationship column the junction is related to.
       const relatedFields = (
         await buildInternalFieldList(relatedTable, tables)
       ).concat(

From 0ee432dd9a34e478705a0c74a0535965fd33cd37 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 13:32:47 +0100
Subject: [PATCH 048/163] Always use tableid prefix for sqs

---
 packages/server/src/sdk/app/rows/search/internal/sqs.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 449ba477b7..5b03887d36 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -142,7 +142,7 @@ async function buildInternalFieldList(
 
   if (!isView || !helpers.views.isCalculationView(source)) {
     for (const field of PROTECTED_INTERNAL_COLUMNS) {
-      fieldList.push(field)
+      fieldList.push(`${table._id}.${field}`)
     }
   }
 

From 5a9ed4ff52a9c016b65df931f542f08569820948 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 13:48:10 +0100
Subject: [PATCH 049/163] Include hidden fields for formulas

---
 .../src/sdk/app/rows/search/internal/sqs.ts   | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 5b03887d36..68d9d3420f 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -55,12 +55,16 @@ const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
 const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 const DUPLICATE_COLUMN_REGEX = new RegExp(`duplicate column name: .+`)
 
-async function buildInternalFieldList(
+export async function buildInternalFieldList(
   source: Table | ViewV2,
   tables: Table[],
-  opts?: { relationships?: RelationshipsJson[]; allowedFields?: string[] }
+  opts?: {
+    relationships?: RelationshipsJson[]
+    allowedFields?: string[]
+    includeHiddenFields?: boolean
+  }
 ) {
-  const { relationships, allowedFields } = opts || {}
+  const { relationships, allowedFields, includeHiddenFields } = opts || {}
   let schemaFields: string[] = []
 
   const isView = sdk.views.isView(source)
@@ -75,7 +79,7 @@ async function buildInternalFieldList(
     schemaFields = Object.keys(helpers.views.basicFields(source))
   } else {
     schemaFields = Object.keys(source.schema).filter(
-      key => source.schema[key].visible !== false
+      key => includeHiddenFields || source.schema[key].visible !== false
     )
   }
 
@@ -109,10 +113,16 @@ async function buildInternalFieldList(
   }
   for (let key of schemaFields) {
     const col = table.schema[key]
+
+    if ([FieldType.FORMULA, FieldType.AI].includes(col.type)) {
+      continue
+    }
+
     const isRelationship = col.type === FieldType.LINK
     if (!relationships && isRelationship) {
       continue
     }
+
     if (!isRelationship) {
       fieldList.push(`${table._id}.${mapToUserColumn(key)}`)
     } else {
@@ -121,6 +131,7 @@ async function buildInternalFieldList(
       if (!relatedTable) {
         continue
       }
+
       // a quirk of how junction documents work in Budibase, refer to the "LinkDocument" type to see the full
       // structure - essentially all relationships between two tables will be inserted into a single "table"
       // we don't use an independent junction table ID for each separate relationship between two tables. For
@@ -128,7 +139,9 @@ async function buildInternalFieldList(
       // end up in the same junction table ID. We need to retrieve the field name property of the junction documents
       // as part of the relationship to tell us which relationship column the junction is related to.
       const relatedFields = (
-        await buildInternalFieldList(relatedTable, tables)
+        await buildInternalFieldList(relatedTable, tables, {
+          includeHiddenFields: containsFormula,
+        })
       ).concat(
         getJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
       )

From 7cc03b172486c32b98144398a1940e7559dba6bf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 14:31:45 +0100
Subject: [PATCH 050/163] Create sql tests for sqs

---
 .../app/rows/search/internal/test/sqs.spec.ts | 674 ++++++++++++++++++
 1 file changed, 674 insertions(+)
 create mode 100644 packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts

diff --git a/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts
new file mode 100644
index 0000000000..31c25defad
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts
@@ -0,0 +1,674 @@
+import {
+  AIOperationEnum,
+  CalculationType,
+  FieldType,
+  RelationshipType,
+  SourceName,
+  Table,
+  ViewV2,
+  ViewV2Type,
+} from "@budibase/types"
+import { buildInternalFieldList } from "../sqs"
+import { structures } from "../../../../../../api/routes/tests/utilities"
+import { sql } from "@budibase/backend-core"
+import { generator } from "@budibase/backend-core/tests"
+import {
+  generateJunctionTableID,
+  generateViewID,
+} from "../../../../../../db/utils"
+
+import sdk from "../../../../../../sdk"
+import { cloneDeep } from "lodash"
+import { utils } from "@budibase/shared-core"
+
+jest.mock("../../../../../../sdk/app/views", () => ({
+  ...jest.requireActual("../../../../../../sdk/app/views"),
+  getTable: jest.fn(),
+}))
+const getTableMock = sdk.views.getTable as jest.MockedFunction<
+  typeof sdk.views.getTable
+>
+
+describe("buildInternalFieldList", () => {
+  let allTables: Table[]
+
+  class TableConfig {
+    private _table: Table & { _id: string }
+
+    constructor() {
+      const name = generator.word()
+      this._table = {
+        ...structures.tableForDatasource({
+          type: "datasource",
+          source: SourceName.POSTGRES,
+        }),
+        name,
+        _id: sql.utils.buildExternalTableId("ds_id", name),
+        schema: {
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          description: {
+            name: "description",
+            type: FieldType.STRING,
+          },
+          amount: {
+            name: "amount",
+            type: FieldType.NUMBER,
+          },
+        },
+      }
+
+      allTables.push(this._table)
+    }
+
+    withHiddenField(field: string) {
+      this._table.schema[field].visible = false
+      return this
+    }
+
+    withField(
+      name: string,
+      type:
+        | FieldType.STRING
+        | FieldType.NUMBER
+        | FieldType.FORMULA
+        | FieldType.AI,
+      options?: { visible: boolean }
+    ) {
+      switch (type) {
+        case FieldType.NUMBER:
+        case FieldType.STRING:
+          this._table.schema[name] = {
+            name,
+            type,
+            ...options,
+          }
+          break
+        case FieldType.FORMULA:
+          this._table.schema[name] = {
+            name,
+            type,
+            formula: "any",
+            ...options,
+          }
+          break
+        case FieldType.AI:
+          this._table.schema[name] = {
+            name,
+            type,
+            operation: AIOperationEnum.PROMPT,
+            ...options,
+          }
+          break
+        default:
+          utils.unreachable(type)
+      }
+      return this
+    }
+
+    withRelation(name: string, toTableId: string) {
+      this._table.schema[name] = {
+        name,
+        type: FieldType.LINK,
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        fieldName: "link",
+        tableId: toTableId,
+      }
+      return this
+    }
+
+    withEmptySchema() {
+      this._table.schema = {}
+      return this
+    }
+
+    create() {
+      return cloneDeep(this._table)
+    }
+  }
+
+  class ViewConfig {
+    private _table: Table
+    private _view: ViewV2
+
+    constructor(table: Table) {
+      this._table = table
+      this._view = {
+        version: 2,
+        id: generateViewID(table._id!),
+        name: generator.word(),
+        tableId: table._id!,
+      }
+    }
+
+    withVisible(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = true
+      return this
+    }
+
+    withHidden(field: string) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].visible = false
+      return this
+    }
+
+    withRelationshipColumns(
+      field: string,
+      columns: Record<string, { visible: boolean }>
+    ) {
+      this._view.schema ??= {}
+      this._view.schema[field] ??= {}
+      this._view.schema[field].columns = columns
+      return this
+    }
+
+    withCalculation(
+      name: string,
+      field: string,
+      calculationType: CalculationType
+    ) {
+      this._view.type = ViewV2Type.CALCULATION
+      this._view.schema ??= {}
+      this._view.schema[name] = {
+        field,
+        calculationType,
+        visible: true,
+      }
+      return this
+    }
+
+    create() {
+      getTableMock.mockResolvedValueOnce(this._table)
+      return cloneDeep(this._view)
+    }
+  }
+
+  beforeEach(() => {
+    jest.clearAllMocks()
+    allTables = []
+  })
+
+  describe("table", () => {
+    it("includes internal columns by default", async () => {
+      const table = new TableConfig().withEmptySchema().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const table = new TableConfig().create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("excludes hidden fields", async () => {
+      const table = new TableConfig().withHiddenField("description").create()
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("excludes non-sql fields fields", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes hidden fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const result = await buildInternalFieldList(table, [])
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields when flagged", async () => {
+      const otherTable = new TableConfig()
+        .withHiddenField("description")
+        .create()
+
+      const table = new TableConfig()
+        .withHiddenField("amount")
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withHiddenField("description")
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("never includes non-sql columns from relationships", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(table, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("view", () => {
+    it("includes internal columns by default", async () => {
+      const view = new ViewConfig(new TableConfig().create()).create()
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("extracts fields from table schema", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withVisible("amount")
+        .withHidden("name")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes all fields if there is a formula column", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withVisible("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_name`,
+        `${view.tableId}.data_description`,
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("does not includes all fields if the formula column is not included", async () => {
+      const table = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+      const view = new ViewConfig(table)
+        .withHidden("name")
+        .withVisible("amount")
+        .withHidden("formula")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([
+        `${view.tableId}.data_amount`,
+        `${view.tableId}._id`,
+        `${view.tableId}._rev`,
+        `${view.tableId}.type`,
+        `${view.tableId}.createdAt`,
+        `${view.tableId}.updatedAt`,
+        `${view.tableId}.tableId`,
+      ])
+    })
+
+    it("includes relationships fields", async () => {
+      const otherTable = new TableConfig().create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withHidden("amount")
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes relationships columns", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("does not include relationships columns for hidden links", async () => {
+      const otherTable = new TableConfig()
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+
+    it("includes all relationship fields if there is a formula column", async () => {
+      const otherTable = new TableConfig()
+        .withField("hidden", FieldType.STRING, { visible: false })
+        .withField("formula", FieldType.FORMULA)
+        .withField("ai", FieldType.AI)
+        .withRelation("link", "otherTableId")
+        .create()
+
+      const table = new TableConfig()
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withVisible("formula")
+        .withHidden("link")
+        .withRelationshipColumns("link", {
+          name: { visible: false },
+          amount: { visible: true },
+          formula: { visible: false },
+        })
+        .create()
+
+      const relationships = [{ tableName: otherTable.name, column: "link" }]
+      const result = await buildInternalFieldList(view, allTables, {
+        relationships,
+      })
+      expect(result).toEqual([
+        `${table._id}.data_name`,
+        `${table._id}.data_description`,
+        `${table._id}.data_amount`,
+        `${otherTable._id}.data_name`,
+        `${otherTable._id}.data_description`,
+        `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_hidden`,
+        `${otherTable._id}._id`,
+        `${otherTable._id}._rev`,
+        `${otherTable._id}.type`,
+        `${otherTable._id}.createdAt`,
+        `${otherTable._id}.updatedAt`,
+        `${otherTable._id}.tableId`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
+        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}._id`,
+        `${table._id}._rev`,
+        `${table._id}.type`,
+        `${table._id}.createdAt`,
+        `${table._id}.updatedAt`,
+        `${table._id}.tableId`,
+      ])
+    })
+  })
+
+  describe("calculation view", () => {
+    it("does not include calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([])
+    })
+
+    it("includes visible fields calculation fields", async () => {
+      const view = new ViewConfig(new TableConfig().create())
+        .withCalculation("average", "amount", CalculationType.AVG)
+        .withHidden("name")
+        .withVisible("amount")
+        .create()
+
+      const result = await buildInternalFieldList(view, [])
+      expect(result).toEqual([`${view.tableId}.data_amount`])
+    })
+  })
+})

From 16fb86549b7cb47148cb92f695d2e6be6449905c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 14:42:49 +0100
Subject: [PATCH 051/163] Fix tests

---
 .../src/api/controllers/row/utils/sqlUtils.ts |  4 +-
 .../row/utils/tests/sqlUtils.spec.ts          | 50 +++++++++----------
 2 files changed, 25 insertions(+), 29 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index e66a0b5bf6..1793d64b89 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -200,8 +200,8 @@ export async function buildSqlFieldList(
 
     if (
       isView &&
-      source.schema?.[field.name] &&
-      !helpers.views.isVisible(source.schema[field.name]) &&
+      (!source.schema?.[field.name] ||
+        !helpers.views.isVisible(source.schema[field.name])) &&
       !containsFormula
     ) {
       continue
diff --git a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
index 12a5edd30b..365f571fcf 100644
--- a/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
+++ b/packages/server/src/api/controllers/row/utils/tests/sqlUtils.spec.ts
@@ -362,33 +362,6 @@ describe("buildSqlFieldList", () => {
       expect(result).toEqual(["table.amount"])
     })
 
-    it("includes relationships fields when flagged", async () => {
-      const otherTable = new TableConfig("linkedTable")
-        .withField("id", FieldType.NUMBER)
-        .withPrimary("id")
-        .withDisplay("name")
-        .create()
-
-      const table = new TableConfig("table")
-        .withRelation("link", otherTable._id)
-        .withField("formula", FieldType.FORMULA)
-        .create()
-
-      const view = new ViewConfig(table)
-        .withVisible("name")
-        .withHidden("amount")
-        .create()
-
-      const result = await buildSqlFieldList(view, allTables, {
-        relationships: true,
-      })
-      expect(result).toEqual([
-        "table.name",
-        "linkedTable.id",
-        "linkedTable.name",
-      ])
-    })
-
     it("includes relationships columns", async () => {
       const otherTable = new TableConfig("linkedTable")
         .withField("id", FieldType.NUMBER)
@@ -420,6 +393,29 @@ describe("buildSqlFieldList", () => {
       ])
     })
 
+    it("excludes relationships fields when view is not included in the view", async () => {
+      const otherTable = new TableConfig("linkedTable")
+        .withField("id", FieldType.NUMBER)
+        .withPrimary("id")
+        .withDisplay("name")
+        .create()
+
+      const table = new TableConfig("table")
+        .withRelation("link", otherTable._id)
+        .withField("formula", FieldType.FORMULA)
+        .create()
+
+      const view = new ViewConfig(table)
+        .withVisible("name")
+        .withHidden("amount")
+        .create()
+
+      const result = await buildSqlFieldList(view, allTables, {
+        relationships: true,
+      })
+      expect(result).toEqual(["table.name"])
+    })
+
     it("does not include relationships columns for hidden links", async () => {
       const otherTable = new TableConfig("linkedTable")
         .withField("id", FieldType.NUMBER)

From 8905f9dd5d5cc6ad174eb6d3a4ba7420b19bad2c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 19 Dec 2024 15:27:18 +0100
Subject: [PATCH 052/163] Fix tests

---
 .../src/sdk/app/rows/search/internal/sqs.ts   |  4 --
 .../app/rows/search/internal/test/sqs.spec.ts | 70 ++-----------------
 2 files changed, 7 insertions(+), 67 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 68d9d3420f..84162a67af 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -114,10 +114,6 @@ export async function buildInternalFieldList(
   for (let key of schemaFields) {
     const col = table.schema[key]
 
-    if ([FieldType.FORMULA, FieldType.AI].includes(col.type)) {
-      continue
-    }
-
     const isRelationship = col.type === FieldType.LINK
     if (!relationships && isRelationship) {
       continue
diff --git a/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts
index 31c25defad..91674089dc 100644
--- a/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/test/sqs.spec.ts
@@ -238,27 +238,6 @@ describe("buildInternalFieldList", () => {
       ])
     })
 
-    it("excludes non-sql fields fields", async () => {
-      const table = new TableConfig()
-        .withField("formula", FieldType.FORMULA)
-        .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
-        .create()
-
-      const result = await buildInternalFieldList(table, [])
-      expect(result).toEqual([
-        `${table._id}.data_name`,
-        `${table._id}.data_description`,
-        `${table._id}.data_amount`,
-        `${table._id}._id`,
-        `${table._id}._rev`,
-        `${table._id}.type`,
-        `${table._id}.createdAt`,
-        `${table._id}.updatedAt`,
-        `${table._id}.tableId`,
-      ])
-    })
-
     it("includes hidden fields if there is a formula column", async () => {
       const table = new TableConfig()
         .withHiddenField("description")
@@ -270,6 +249,7 @@ describe("buildInternalFieldList", () => {
         `${table._id}.data_name`,
         `${table._id}.data_description`,
         `${table._id}.data_amount`,
+        `${table._id}.data_formula`,
         `${table._id}._id`,
         `${table._id}._rev`,
         `${table._id}.type`,
@@ -347,48 +327,7 @@ describe("buildInternalFieldList", () => {
         `${otherTable._id}.tableId`,
         `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
         `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
-        `${table._id}._id`,
-        `${table._id}._rev`,
-        `${table._id}.type`,
-        `${table._id}.createdAt`,
-        `${table._id}.updatedAt`,
-        `${table._id}.tableId`,
-      ])
-    })
-
-    it("never includes non-sql columns from relationships", async () => {
-      const otherTable = new TableConfig()
-        .withField("hidden", FieldType.STRING, { visible: false })
-        .withField("formula", FieldType.FORMULA)
-        .withField("ai", FieldType.AI)
-        .withRelation("link", "otherTableId")
-        .create()
-
-      const table = new TableConfig()
-        .withRelation("link", otherTable._id)
-        .withField("formula", FieldType.FORMULA)
-        .create()
-
-      const relationships = [{ tableName: otherTable.name, column: "link" }]
-      const result = await buildInternalFieldList(table, allTables, {
-        relationships,
-      })
-      expect(result).toEqual([
-        `${table._id}.data_name`,
-        `${table._id}.data_description`,
-        `${table._id}.data_amount`,
-        `${otherTable._id}.data_name`,
-        `${otherTable._id}.data_description`,
-        `${otherTable._id}.data_amount`,
-        `${otherTable._id}.data_hidden`,
-        `${otherTable._id}._id`,
-        `${otherTable._id}._rev`,
-        `${otherTable._id}.type`,
-        `${otherTable._id}.createdAt`,
-        `${otherTable._id}.updatedAt`,
-        `${otherTable._id}.tableId`,
-        `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
-        `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
         `${table._id}._id`,
         `${table._id}._rev`,
         `${table._id}.type`,
@@ -446,6 +385,7 @@ describe("buildInternalFieldList", () => {
         `${view.tableId}.data_name`,
         `${view.tableId}.data_description`,
         `${view.tableId}.data_amount`,
+        `${view.tableId}.data_formula`,
         `${view.tableId}._id`,
         `${view.tableId}._rev`,
         `${view.tableId}.type`,
@@ -545,6 +485,7 @@ describe("buildInternalFieldList", () => {
         `${otherTable._id}.data_name`,
         `${otherTable._id}.data_description`,
         `${otherTable._id}.data_amount`,
+        `${otherTable._id}.data_formula`,
         `${otherTable._id}._id`,
         `${otherTable._id}._rev`,
         `${otherTable._id}.type`,
@@ -632,6 +573,8 @@ describe("buildInternalFieldList", () => {
         `${otherTable._id}.data_description`,
         `${otherTable._id}.data_amount`,
         `${otherTable._id}.data_hidden`,
+        `${otherTable._id}.data_formula`,
+        `${otherTable._id}.data_ai`,
         `${otherTable._id}._id`,
         `${otherTable._id}._rev`,
         `${otherTable._id}.type`,
@@ -640,6 +583,7 @@ describe("buildInternalFieldList", () => {
         `${otherTable._id}.tableId`,
         `${generateJunctionTableID(table._id, otherTable._id)}.doc1.fieldName`,
         `${generateJunctionTableID(table._id, otherTable._id)}.doc2.fieldName`,
+        `${table._id}.data_formula`,
         `${table._id}._id`,
         `${table._id}._rev`,
         `${table._id}.type`,

From a7be765bf1ed06126e46c68227c2d41e0f62fe00 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 2 Jan 2025 15:26:50 +0000
Subject: [PATCH 053/163] Fix for upload parameters and api client handling of
 empty responses. AWS put returns no body on a 200 so response.json would
 mistakenly throw an error

---
 packages/client/src/components/app/forms/S3Upload.svelte | 8 ++++----
 packages/frontend-core/src/api/index.ts                  | 8 +++++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index fc4c0f8f08..5af3931492 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -88,12 +88,12 @@
     })
     loading = true
     try {
-      const res = await API.externalUpload({
+      const res = await API.externalUpload(
         datasourceId,
         bucket,
-        key: processedFileKey,
-        data,
-      })
+        processedFileKey,
+        data
+      )
       notificationStore.actions.success("File uploaded successfully")
       loading = false
       return res
diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index f7b05c338a..36f4e7ed61 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -163,7 +163,13 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
         } else if (parseResponse) {
           return await parseResponse(response)
         } else {
-          return (await response.json()) as ResponseT
+          const text = await response.text()
+          // If the response has no body at all e.g s3.put
+          if (!text) {
+            // Empty response
+            return {} as ResponseT
+          }
+          return JSON.parse(text) as ResponseT
         }
       } catch (error) {
         delete cache[url]

From 2eaf2954c145dccec05cad91588054e976edec05 Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Thu, 2 Jan 2025 17:43:32 +0000
Subject: [PATCH 054/163] PR feedback

---
 packages/client/src/components/app/forms/S3Upload.svelte | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/components/app/forms/S3Upload.svelte b/packages/client/src/components/app/forms/S3Upload.svelte
index 5af3931492..7ac0532dca 100644
--- a/packages/client/src/components/app/forms/S3Upload.svelte
+++ b/packages/client/src/components/app/forms/S3Upload.svelte
@@ -14,6 +14,8 @@
   export let validation
   export let onChange
 
+  const context = getContext("context")
+
   let fieldState
   let fieldApi
   let localFiles = []
@@ -83,9 +85,7 @@
   }
 
   const upload = async () => {
-    const processedFileKey = processStringSync(key, {
-      [$component.id]: fieldState,
-    })
+    const processedFileKey = processStringSync(key, $context)
     loading = true
     try {
       const res = await API.externalUpload(

From 52916f11a8129cd8dec61dcd73fc30c2f894f741 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 7 Jan 2025 14:06:03 +0000
Subject: [PATCH 055/163] Convert portal user store to TS

---
 .../_components/BuilderSidePanel.svelte       |   7 +-
 .../src/stores/portal/{users.js => users.ts}  | 144 ++++++++++--------
 packages/frontend-core/src/api/user.ts        |   2 +-
 packages/types/src/api/web/user.ts            |   2 +-
 4 files changed, 85 insertions(+), 70 deletions(-)
 rename packages/builder/src/stores/portal/{users.js => users.ts} (55%)

diff --git a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
index 37abd7f1eb..2260892913 100644
--- a/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
@@ -442,13 +442,11 @@
 
   const onUpdateUserInvite = async (invite, role) => {
     let updateBody = {
-      code: invite.code,
       apps: {
         ...invite.apps,
         [prodAppId]: role,
       },
     }
-
     if (role === Constants.Roles.CREATOR) {
       updateBody.builder = updateBody.builder || {}
       updateBody.builder.apps = [...(updateBody.builder.apps ?? []), prodAppId]
@@ -456,7 +454,7 @@
     } else if (role !== Constants.Roles.CREATOR && invite?.builder?.apps) {
       invite.builder.apps = []
     }
-    await users.updateInvite(updateBody)
+    await users.updateInvite(invite.code, updateBody)
     await filterInvites(query)
   }
 
@@ -470,8 +468,7 @@
     let updated = { ...invite }
     delete updated.info.apps[prodAppId]
 
-    return await users.updateInvite({
-      code: updated.code,
+    return await users.updateInvite(updated.code, {
       apps: updated.apps,
     })
   }
diff --git a/packages/builder/src/stores/portal/users.js b/packages/builder/src/stores/portal/users.ts
similarity index 55%
rename from packages/builder/src/stores/portal/users.js
rename to packages/builder/src/stores/portal/users.ts
index 99ead22317..7c0bec296e 100644
--- a/packages/builder/src/stores/portal/users.js
+++ b/packages/builder/src/stores/portal/users.ts
@@ -1,41 +1,68 @@
-import { writable } from "svelte/store"
 import { API } from "@/api"
-import { update } from "lodash"
 import { licensing } from "."
 import { sdk } from "@budibase/shared-core"
 import { Constants } from "@budibase/frontend-core"
+import {
+  DeleteInviteUsersRequest,
+  InviteUsersRequest,
+  SearchUsersRequest,
+  SearchUsersResponse,
+  UpdateInviteRequest,
+  User,
+} from "@budibase/types"
+import { BudiStore } from "../BudiStore"
 
-export function createUsersStore() {
-  const { subscribe, set } = writable({})
+type UserState = SearchUsersResponse & SearchUsersRequest
 
-  // opts can contain page and search params
-  async function search(opts = {}) {
+class UserStore extends BudiStore<UserState> {
+  constructor() {
+    super({
+      data: [],
+    })
+
+    // Update quotas after any add or remove operation
+    this.create = this.refreshUsage(this.create)
+    this.save = this.refreshUsage(this.save)
+    this.delete = this.refreshUsage(this.delete)
+    this.bulkDelete = this.refreshUsage(this.bulkDelete)
+  }
+
+  async search(opts: SearchUsersRequest = {}) {
     const paged = await API.searchUsers(opts)
-    set({
+    this.set({
       ...paged,
       ...opts,
     })
     return paged
   }
 
-  async function get(userId) {
+  async get(userId: string) {
     try {
       return await API.getUser(userId)
     } catch (err) {
       return null
     }
   }
-  const fetch = async () => {
+
+  async fetch() {
     return await API.getUsers()
   }
 
-  // One or more users.
-  async function onboard(payload) {
+  async onboard(payload: InviteUsersRequest) {
     return await API.onboardUsers(payload)
   }
 
-  async function invite(payload) {
-    const users = payload.map(user => {
+  async invite(
+    payload: {
+      admin?: boolean
+      builder?: boolean
+      creator?: boolean
+      email: string
+      apps?: any[]
+      groups?: any[]
+    }[]
+  ) {
+    const users: InviteUsersRequest = payload.map(user => {
       let builder = undefined
       if (user.admin || user.builder) {
         builder = { global: true }
@@ -55,11 +82,16 @@ export function createUsersStore() {
     return API.inviteUsers(users)
   }
 
-  async function removeInvites(payload) {
+  async removeInvites(payload: DeleteInviteUsersRequest) {
     return API.removeUserInvites(payload)
   }
 
-  async function acceptInvite(inviteCode, password, firstName, lastName) {
+  async acceptInvite(
+    inviteCode: string,
+    password: string,
+    firstName: string,
+    lastName?: string
+  ) {
     return API.acceptInvite({
       inviteCode,
       password,
@@ -68,21 +100,25 @@ export function createUsersStore() {
     })
   }
 
-  async function fetchInvite(inviteCode) {
+  async fetchInvite(inviteCode: string) {
     return API.getUserInvite(inviteCode)
   }
 
-  async function getInvites() {
+  async getInvites() {
     return API.getUserInvites()
   }
 
-  async function updateInvite(invite) {
-    return API.updateUserInvite(invite.code, invite)
+  async updateInvite(code: string, invite: UpdateInviteRequest) {
+    return API.updateUserInvite(code, invite)
   }
 
-  async function create(data) {
-    let mappedUsers = data.users.map(user => {
-      const body = {
+  async getUserCountByApp(appId: string) {
+    return await API.getUserCountByApp(appId)
+  }
+
+  async create(data: any) {
+    let mappedUsers: Omit<User, "tenantId">[] = data.users.map((user: any) => {
+      const body: Omit<User, "tenantId"> = {
         email: user.email,
         password: user.password,
         roles: {},
@@ -113,41 +149,44 @@ export function createUsersStore() {
     const response = await API.createUsers(mappedUsers, data.groups)
 
     // re-search from first page
-    await search()
+    await this.search()
     return response
   }
 
-  async function del(id) {
+  async delete(id: string) {
     await API.deleteUser(id)
-    update(users => users.filter(user => user._id !== id))
   }
 
-  async function getUserCountByApp(appId) {
-    return await API.getUserCountByApp(appId)
-  }
-
-  async function bulkDelete(users) {
+  async bulkDelete(
+    users: Array<{
+      userId: string
+      email: string
+    }>
+  ) {
     return API.deleteUsers(users)
   }
 
-  async function save(user) {
+  async save(user: User) {
     return await API.saveUser(user)
   }
 
-  async function addAppBuilder(userId, appId) {
+  async addAppBuilder(userId: string, appId: string) {
     return await API.addAppBuilder(userId, appId)
   }
 
-  async function removeAppBuilder(userId, appId) {
+  async removeAppBuilder(userId: string, appId: string) {
     return await API.removeAppBuilder(userId, appId)
   }
 
-  async function getAccountHolder() {
+  async getAccountHolder() {
     return await API.getAccountHolder()
   }
 
-  const getUserRole = user => {
-    if (user && user.email === user.tenantOwnerEmail) {
+  getUserRole(user?: User & { tenantOwnerEmail?: string }) {
+    if (!user) {
+      return Constants.BudibaseRoles.AppUser
+    }
+    if (user.email === user.tenantOwnerEmail) {
       return Constants.BudibaseRoles.Owner
     } else if (sdk.users.isAdmin(user)) {
       return Constants.BudibaseRoles.Admin
@@ -160,37 +199,16 @@ export function createUsersStore() {
     }
   }
 
-  const refreshUsage =
-    fn =>
-    async (...args) => {
+  foo = this.refreshUsage(this.create)
+  bar = this.refreshUsage(this.save)
+
+  refreshUsage<T extends any[], U>(fn: (...args: T) => Promise<U>) {
+    return async function (...args: T) {
       const response = await fn(...args)
       await licensing.setQuotaUsage()
       return response
     }
-
-  return {
-    subscribe,
-    search,
-    get,
-    getUserRole,
-    fetch,
-    invite,
-    onboard,
-    fetchInvite,
-    getInvites,
-    removeInvites,
-    updateInvite,
-    getUserCountByApp,
-    addAppBuilder,
-    removeAppBuilder,
-    // any operation that adds or deletes users
-    acceptInvite,
-    create: refreshUsage(create),
-    save: refreshUsage(save),
-    bulkDelete: refreshUsage(bulkDelete),
-    delete: refreshUsage(del),
-    getAccountHolder,
   }
 }
 
-export const users = createUsersStore()
+export const users = new UserStore()
diff --git a/packages/frontend-core/src/api/user.ts b/packages/frontend-core/src/api/user.ts
index 84ec68644d..7464b1ec4a 100644
--- a/packages/frontend-core/src/api/user.ts
+++ b/packages/frontend-core/src/api/user.ts
@@ -60,7 +60,7 @@ export interface UserEndpoints {
   getAccountHolder: () => Promise<LookupAccountHolderResponse>
   searchUsers: (data: SearchUsersRequest) => Promise<SearchUsersResponse>
   createUsers: (
-    users: User[],
+    users: Omit<User, "tenantId">[],
     groups: any[]
   ) => Promise<BulkUserCreated | undefined>
   updateUserInvite: (
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index a42449d550..8b0dfef34b 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -124,7 +124,7 @@ export interface AcceptUserInviteRequest {
   inviteCode: string
   password: string
   firstName: string
-  lastName: string
+  lastName?: string
 }
 
 export interface AcceptUserInviteResponse {

From af83f1b2e15a0268efb7bcac7abcd784575c9208 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Tue, 7 Jan 2025 17:18:36 +0000
Subject: [PATCH 056/163] Delete old feature flags.

---
 packages/builder/src/stores/portal/menu.js      | 10 +++-------
 packages/pro                                    |  2 +-
 .../src/api/controllers/row/staticFormula.ts    | 17 ++++-------------
 packages/server/src/automations/actions.ts      | 12 ++----------
 packages/server/src/automations/steps/openai.ts | 11 +++--------
 packages/types/src/sdk/featureFlag.ts           |  8 --------
 6 files changed, 13 insertions(+), 47 deletions(-)

diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index 75a9b363be..4f87a5bfde 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -1,9 +1,7 @@
 import { derived } from "svelte/store"
 import { admin } from "./admin"
 import { auth } from "./auth"
-import { isEnabled } from "@/helpers/featureFlags"
 import { sdk } from "@budibase/shared-core"
-import { FeatureFlag } from "@budibase/types"
 
 export const menu = derived([admin, auth], ([$admin, $auth]) => {
   const user = $auth?.user
@@ -63,13 +61,11 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Environment",
         href: "/builder/portal/settings/environment",
       },
-    ]
-    if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) {
-      settingsSubPages.push({
+      {
         title: "AI",
         href: "/builder/portal/settings/ai",
-      })
-    }
+      },
+    ]
 
     if (!cloud) {
       settingsSubPages.push({
diff --git a/packages/pro b/packages/pro
index 32d84f109d..23fdd50b7e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 32d84f109d4edc526145472a7446327312151442
+Subproject commit 23fdd50b7ef28cf320716ed2c46e15d63185daa7
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index b81a164807..678e824b94 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -4,15 +4,8 @@ import {
   processAIColumns,
   processFormulas,
 } from "../../../utilities/rowProcessor"
-import { context, features } from "@budibase/backend-core"
-import {
-  Table,
-  Row,
-  FeatureFlag,
-  FormulaType,
-  FieldType,
-  ViewV2,
-} from "@budibase/types"
+import { context } from "@budibase/backend-core"
+import { Table, Row, FormulaType, FieldType, ViewV2 } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import isEqual from "lodash/isEqual"
 import { cloneDeep, merge } from "lodash/fp"
@@ -163,10 +156,8 @@ export async function finaliseRow(
     contextRows: [enrichedRow],
   })
   const aiEnabled =
-    ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
-      (await pro.features.isBudibaseAIEnabled())) ||
-    ((await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
-      (await pro.features.isAICustomConfigsEnabled()))
+    (await pro.features.isBudibaseAIEnabled()) ||
+    (await pro.features.isAICustomConfigsEnabled())
   if (aiEnabled) {
     row = await processAIColumns(table, row, {
       contextRows: [enrichedRow],
diff --git a/packages/server/src/automations/actions.ts b/packages/server/src/automations/actions.ts
index 1c201d1f64..537b6befc3 100644
--- a/packages/server/src/automations/actions.ts
+++ b/packages/server/src/automations/actions.ts
@@ -27,11 +27,9 @@ import {
   Hosting,
   ActionImplementation,
   AutomationStepDefinition,
-  FeatureFlag,
 } from "@budibase/types"
 import sdk from "../sdk"
 import { getAutomationPlugin } from "../utilities/fileSystem"
-import { features } from "@budibase/backend-core"
 
 type ActionImplType = ActionImplementations<
   typeof env.SELF_HOSTED extends "true" ? Hosting.SELF : Hosting.CLOUD
@@ -78,6 +76,7 @@ export const BUILTIN_ACTION_DEFINITIONS: Record<
   LOOP: loop.definition,
   COLLECT: collect.definition,
   TRIGGER_AUTOMATION_RUN: triggerAutomationRun.definition,
+  BRANCH: branch.definition,
   // these used to be lowercase step IDs, maintain for backwards compat
   discord: discord.definition,
   slack: slack.definition,
@@ -105,14 +104,7 @@ if (env.SELF_HOSTED) {
 export async function getActionDefinitions(): Promise<
   Record<keyof typeof AutomationActionStepId, AutomationStepDefinition>
 > {
-  if (await features.isEnabled(FeatureFlag.AUTOMATION_BRANCHING)) {
-    BUILTIN_ACTION_DEFINITIONS["BRANCH"] = branch.definition
-  }
-  if (
-    env.SELF_HOSTED ||
-    (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) ||
-    (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS))
-  ) {
+  if (env.SELF_HOSTED) {
     BUILTIN_ACTION_DEFINITIONS["OPENAI"] = openai.definition
   }
 
diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts
index 19595cc0d0..53e41ceb09 100644
--- a/packages/server/src/automations/steps/openai.ts
+++ b/packages/server/src/automations/steps/openai.ts
@@ -7,9 +7,8 @@ import {
   AutomationIOType,
   OpenAIStepInputs,
   OpenAIStepOutputs,
-  FeatureFlag,
 } from "@budibase/types"
-import { env, features } from "@budibase/backend-core"
+import { env } from "@budibase/backend-core"
 import * as automationUtils from "../automationUtils"
 import * as pro from "@budibase/pro"
 
@@ -99,12 +98,8 @@ export async function run({
 
   try {
     let response
-    const customConfigsEnabled =
-      (await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) &&
-      (await pro.features.isAICustomConfigsEnabled())
-    const budibaseAIEnabled =
-      (await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
-      (await pro.features.isBudibaseAIEnabled())
+    const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled()
+    const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
 
     let llmWrapper
     if (budibaseAIEnabled || customConfigsEnabled) {
diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts
index 725ae0feb1..7b61b70772 100644
--- a/packages/types/src/sdk/featureFlag.ts
+++ b/packages/types/src/sdk/featureFlag.ts
@@ -1,16 +1,8 @@
 export enum FeatureFlag {
-  AUTOMATION_BRANCHING = "AUTOMATION_BRANCHING",
-  AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
-  DEFAULT_VALUES = "DEFAULT_VALUES",
-  BUDIBASE_AI = "BUDIBASE_AI",
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
 }
 
 export const FeatureFlagDefaults = {
-  [FeatureFlag.DEFAULT_VALUES]: true,
-  [FeatureFlag.AUTOMATION_BRANCHING]: true,
-  [FeatureFlag.AI_CUSTOM_CONFIGS]: true,
-  [FeatureFlag.BUDIBASE_AI]: true,
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
 }
 

From b9fb4416bb428acab70288fdc7966454670b738f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 13:18:40 +0000
Subject: [PATCH 057/163] Allow use of AI fields in view calculations.

---
 packages/pro                                  |  2 +-
 .../src/api/controllers/row/staticFormula.ts  |  6 ++
 .../server/src/api/routes/tests/row.spec.ts   | 40 ++++----
 .../src/api/routes/tests/viewV2.spec.ts       | 97 ++++++++++++++++++-
 .../src/tests/utilities/mocks/openai.ts       | 46 +++++++++
 .../src/utilities/rowProcessor/utils.ts       |  2 +-
 packages/types/src/documents/app/row.ts       |  1 +
 .../types/src/documents/app/table/schema.ts   |  2 +-
 8 files changed, 170 insertions(+), 26 deletions(-)
 create mode 100644 packages/server/src/tests/utilities/mocks/openai.ts

diff --git a/packages/pro b/packages/pro
index 32d84f109d..45f5b6fe9b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 32d84f109d4edc526145472a7446327312151442
+Subproject commit 45f5b6fe9bbdbdf502581740ab43b82e8153260f
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index b81a164807..afa3a1f239 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -162,6 +162,12 @@ export async function finaliseRow(
     dynamic: false,
     contextRows: [enrichedRow],
   })
+
+  const flag1 = await features.isEnabled(FeatureFlag.BUDIBASE_AI)
+  const flag2 = await pro.features.isBudibaseAIEnabled()
+  const flag3 = await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)
+  const flag4 = await pro.features.isAICustomConfigsEnabled()
+
   const aiEnabled =
     ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())) ||
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index a3012c3760..968ce9c798 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -8,7 +8,13 @@ import {
 import tk from "timekeeper"
 import emitter from "../../../../src/events"
 import { outputProcessing } from "../../../utilities/rowProcessor"
-import { context, InternalTable, tenancy, utils } from "@budibase/backend-core"
+import {
+  context,
+  setEnv,
+  InternalTable,
+  tenancy,
+  utils,
+} from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   AIOperationEnum,
@@ -42,19 +48,8 @@ import { InternalTables } from "../../../db/utils"
 import { withEnv } from "../../../environment"
 import { JsTimeoutError } from "@budibase/string-templates"
 import { isDate } from "../../../utilities"
-
-jest.mock("@budibase/pro", () => ({
-  ...jest.requireActual("@budibase/pro"),
-  ai: {
-    LargeLanguageModel: {
-      forCurrentTenant: async () => ({
-        llm: {},
-        run: jest.fn(() => `Mock LLM Response`),
-        buildPromptFromAIOperation: jest.fn(),
-      }),
-    },
-  },
-}))
+import nock from "nock"
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -99,6 +94,8 @@ if (descriptions.length) {
         const ds = await dsProvider()
         datasource = ds.datasource
         client = ds.client
+
+        mocks.licenses.useCloudFree()
       })
 
       afterAll(async () => {
@@ -172,10 +169,6 @@ if (descriptions.length) {
         )
       }
 
-      beforeEach(async () => {
-        mocks.licenses.useCloudFree()
-      })
-
       const getRowUsage = async () => {
         const { total } = await config.doInContext(undefined, () =>
           quotas.getCurrentUsageValues(
@@ -3224,10 +3217,17 @@ if (descriptions.length) {
       isInternal &&
         describe("AI fields", () => {
           let table: Table
+          let envCleanup: () => void
 
           beforeAll(async () => {
             mocks.licenses.useBudibaseAI()
             mocks.licenses.useAICustomConfigs()
+            envCleanup = setEnv({
+              OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+            })
+
+            mockChatGPTResponse("Mock LLM Response")
+
             table = await config.api.table.save(
               saveTableRequest({
                 schema: {
@@ -3251,7 +3251,9 @@ if (descriptions.length) {
           })
 
           afterAll(() => {
-            jest.unmock("@budibase/pro")
+            nock.cleanAll()
+            envCleanup()
+            mocks.licenses.useCloudFree()
           })
 
           it("should be able to save a row with an AI column", async () => {
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 6ace7e256b..57efc868e9 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -1,4 +1,5 @@
 import {
+  AIOperationEnum,
   ArrayOperator,
   BasicOperator,
   BBReferenceFieldSubType,
@@ -42,7 +43,9 @@ import {
 } from "../../../integrations/tests/utils"
 import merge from "lodash/merge"
 import { quotas } from "@budibase/pro"
-import { context, db, events, roles } from "@budibase/backend-core"
+import { context, db, events, roles, setEnv } from "@budibase/backend-core"
+import { mockChatGPTResponse } from "../../../tests/utilities/mocks/openai"
+import nock from "nock"
 
 const descriptions = datasourceDescribe({ exclude: [DatabaseName.MONGODB] })
 
@@ -100,6 +103,7 @@ if (descriptions.length) {
 
       beforeAll(async () => {
         await config.init()
+        mocks.licenses.useCloudFree()
 
         const ds = await dsProvider()
         rawDatasource = ds.rawDatasource
@@ -109,7 +113,6 @@ if (descriptions.length) {
 
       beforeEach(() => {
         jest.clearAllMocks()
-        mocks.licenses.useCloudFree()
       })
 
       describe("view crud", () => {
@@ -507,7 +510,6 @@ if (descriptions.length) {
             })
 
             it("readonly fields can be used on free license", async () => {
-              mocks.licenses.useCloudFree()
               const table = await config.api.table.save(
                 saveTableRequest({
                   schema: {
@@ -933,6 +935,94 @@ if (descriptions.length) {
                 }
               )
             })
+
+          describe("AI fields", () => {
+            let envCleanup: () => void
+            beforeAll(() => {
+              mocks.licenses.useBudibaseAI()
+              mocks.licenses.useAICustomConfigs()
+              envCleanup = setEnv({
+                OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+              })
+
+              mockChatGPTResponse(prompt => {
+                if (prompt.includes("elephant")) {
+                  return "big"
+                }
+                if (prompt.includes("mouse")) {
+                  return "small"
+                }
+                if (prompt.includes("whale")) {
+                  return "big"
+                }
+                return "unknown"
+              })
+            })
+
+            afterAll(() => {
+              nock.cleanAll()
+              envCleanup()
+              mocks.licenses.useCloudFree()
+            })
+
+            it("can use AI fields in view calculations", async () => {
+              const table = await config.api.table.save(
+                saveTableRequest({
+                  schema: {
+                    animal: {
+                      name: "animal",
+                      type: FieldType.STRING,
+                    },
+                    bigOrSmall: {
+                      name: "bigOrSmall",
+                      type: FieldType.AI,
+                      operation: AIOperationEnum.CATEGORISE_TEXT,
+                      categories: "big,small",
+                      columns: ["animal"],
+                    },
+                  },
+                })
+              )
+
+              const view = await config.api.viewV2.create({
+                tableId: table._id!,
+                name: generator.guid(),
+                type: ViewV2Type.CALCULATION,
+                schema: {
+                  bigOrSmall: {
+                    visible: true,
+                  },
+                  count: {
+                    visible: true,
+                    calculationType: CalculationType.COUNT,
+                    field: "animal",
+                  },
+                },
+              })
+
+              await config.api.row.save(table._id!, {
+                animal: "elephant",
+              })
+
+              await config.api.row.save(table._id!, {
+                animal: "mouse",
+              })
+
+              await config.api.row.save(table._id!, {
+                animal: "whale",
+              })
+
+              const { rows } = await config.api.row.search(view.id, {
+                sort: "bigOrSmall",
+                sortOrder: SortOrder.ASCENDING,
+              })
+              expect(rows).toHaveLength(2)
+              expect(rows[0].bigOrSmall).toEqual("big")
+              expect(rows[1].bigOrSmall).toEqual("small")
+              expect(rows[0].count).toEqual(2)
+              expect(rows[1].count).toEqual(1)
+            })
+          })
         })
 
         describe("update", () => {
@@ -1836,7 +1926,6 @@ if (descriptions.length) {
               },
             })
 
-            mocks.licenses.useCloudFree()
             const view = await getDelegate(res)
             expect(view.schema?.one).toEqual(
               expect.objectContaining({ visible: true, readonly: true })
diff --git a/packages/server/src/tests/utilities/mocks/openai.ts b/packages/server/src/tests/utilities/mocks/openai.ts
new file mode 100644
index 0000000000..b17491808c
--- /dev/null
+++ b/packages/server/src/tests/utilities/mocks/openai.ts
@@ -0,0 +1,46 @@
+import nock from "nock"
+
+let chatID = 1
+
+export function mockChatGPTResponse(
+  response: string | ((prompt: string) => string)
+) {
+  return nock("https://api.openai.com")
+    .post("/v1/chat/completions")
+    .reply(200, (uri, requestBody) => {
+      let content = response
+      if (typeof response === "function") {
+        const messages = (requestBody as any).messages
+        content = response(messages[0].content)
+      }
+
+      chatID++
+
+      return {
+        id: `chatcmpl-${chatID}`,
+        object: "chat.completion",
+        created: Math.floor(Date.now() / 1000),
+        model: "gpt-4o-mini",
+        system_fingerprint: `fp_${chatID}`,
+        choices: [
+          {
+            index: 0,
+            message: { role: "assistant", content },
+            logprobs: null,
+            finish_reason: "stop",
+          },
+        ],
+        usage: {
+          prompt_tokens: 0,
+          completion_tokens: 0,
+          total_tokens: 0,
+          completion_tokens_details: {
+            reasoning_tokens: 0,
+            accepted_prediction_tokens: 0,
+            rejected_prediction_tokens: 0,
+          },
+        },
+      }
+    })
+    .persist()
+}
diff --git a/packages/server/src/utilities/rowProcessor/utils.ts b/packages/server/src/utilities/rowProcessor/utils.ts
index 09d3324ded..7d2f8b49f4 100644
--- a/packages/server/src/utilities/rowProcessor/utils.ts
+++ b/packages/server/src/utilities/rowProcessor/utils.ts
@@ -160,7 +160,7 @@ export async function processAIColumns<T extends Row | Row[]>(
 
           return tracer.trace("processAIColumn", {}, async span => {
             span?.addTags({ table_id: table._id, column })
-            const llmResponse = await llmWrapper.run(prompt!)
+            const llmResponse = await llmWrapper.run(prompt)
             return {
               ...row,
               [column]: llmResponse,
diff --git a/packages/types/src/documents/app/row.ts b/packages/types/src/documents/app/row.ts
index 6b6b38a5cf..bb58933b65 100644
--- a/packages/types/src/documents/app/row.ts
+++ b/packages/types/src/documents/app/row.ts
@@ -154,6 +154,7 @@ export const GroupByTypes = [
   FieldType.BOOLEAN,
   FieldType.DATETIME,
   FieldType.BIGINT,
+  FieldType.AI,
 ]
 
 export function canGroupBy(type: FieldType) {
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 771192e2f5..f4a6d8481d 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -123,7 +123,7 @@ export interface AIFieldMetadata extends BaseFieldSchema {
   operation: AIOperationEnum
   columns?: string[]
   column?: string
-  categories?: string[]
+  categories?: string
   prompt?: string
   language?: string
 }

From 6bd4cb47c204d1c2626c0d8d93c71b2b695679a4 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 8 Jan 2025 15:54:09 +0000
Subject: [PATCH 058/163] Add new UnsavedUser type and update controllers

---
 packages/builder/src/stores/portal/users.ts   | 35 ++++++++++---------
 packages/frontend-core/src/api/user.ts        | 14 +++-----
 packages/types/src/api/web/user.ts            |  4 ++-
 .../src/api/controllers/global/users.ts       | 16 ++++++---
 4 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/packages/builder/src/stores/portal/users.ts b/packages/builder/src/stores/portal/users.ts
index 7c0bec296e..9284ad2992 100644
--- a/packages/builder/src/stores/portal/users.ts
+++ b/packages/builder/src/stores/portal/users.ts
@@ -9,9 +9,18 @@ import {
   SearchUsersResponse,
   UpdateInviteRequest,
   User,
+  UserIdentifier,
+  UnsavedUser,
 } from "@budibase/types"
 import { BudiStore } from "../BudiStore"
 
+interface UserInfo {
+  email: string
+  password: string
+  forceResetPassword?: boolean
+  role: keyof typeof Constants.BudibaseRoles
+}
+
 type UserState = SearchUsersResponse & SearchUsersRequest
 
 class UserStore extends BudiStore<UserState> {
@@ -116,9 +125,9 @@ class UserStore extends BudiStore<UserState> {
     return await API.getUserCountByApp(appId)
   }
 
-  async create(data: any) {
-    let mappedUsers: Omit<User, "tenantId">[] = data.users.map((user: any) => {
-      const body: Omit<User, "tenantId"> = {
+  async create(data: { users: UserInfo[]; groups: any[] }) {
+    let mappedUsers: UnsavedUser[] = data.users.map((user: any) => {
+      const body: UnsavedUser = {
         email: user.email,
         password: user.password,
         roles: {},
@@ -128,17 +137,17 @@ class UserStore extends BudiStore<UserState> {
       }
 
       switch (user.role) {
-        case "appUser":
+        case Constants.BudibaseRoles.AppUser:
           body.builder = { global: false }
           body.admin = { global: false }
           break
-        case "developer":
+        case Constants.BudibaseRoles.Developer:
           body.builder = { global: true }
           break
-        case "creator":
+        case Constants.BudibaseRoles.Creator:
           body.builder = { creator: true, global: false }
           break
-        case "admin":
+        case Constants.BudibaseRoles.Admin:
           body.admin = { global: true }
           body.builder = { global: true }
           break
@@ -157,12 +166,7 @@ class UserStore extends BudiStore<UserState> {
     await API.deleteUser(id)
   }
 
-  async bulkDelete(
-    users: Array<{
-      userId: string
-      email: string
-    }>
-  ) {
+  async bulkDelete(users: UserIdentifier[]) {
     return API.deleteUsers(users)
   }
 
@@ -199,9 +203,8 @@ class UserStore extends BudiStore<UserState> {
     }
   }
 
-  foo = this.refreshUsage(this.create)
-  bar = this.refreshUsage(this.save)
-
+  // Wrapper function to refresh quota usage after an operation,
+  // persisting argument and return types
   refreshUsage<T extends any[], U>(fn: (...args: T) => Promise<U>) {
     return async function (...args: T) {
       const response = await fn(...args)
diff --git a/packages/frontend-core/src/api/user.ts b/packages/frontend-core/src/api/user.ts
index 7464b1ec4a..cf66751078 100644
--- a/packages/frontend-core/src/api/user.ts
+++ b/packages/frontend-core/src/api/user.ts
@@ -21,11 +21,12 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   UpdateSelfMetadataRequest,
   UpdateSelfMetadataResponse,
-  User,
+  UserIdentifier,
 } from "@budibase/types"
 import { BaseAPIClient } from "./types"
 
@@ -38,14 +39,9 @@ export interface UserEndpoints {
   createAdminUser: (
     user: CreateAdminUserRequest
   ) => Promise<CreateAdminUserResponse>
-  saveUser: (user: User) => Promise<SaveUserResponse>
+  saveUser: (user: UnsavedUser) => Promise<SaveUserResponse>
   deleteUser: (userId: string) => Promise<DeleteUserResponse>
-  deleteUsers: (
-    users: Array<{
-      userId: string
-      email: string
-    }>
-  ) => Promise<BulkUserDeleted | undefined>
+  deleteUsers: (users: UserIdentifier[]) => Promise<BulkUserDeleted | undefined>
   onboardUsers: (data: InviteUsersRequest) => Promise<InviteUsersResponse>
   getUserInvite: (code: string) => Promise<CheckInviteResponse>
   getUserInvites: () => Promise<GetUserInvitesResponse>
@@ -60,7 +56,7 @@ export interface UserEndpoints {
   getAccountHolder: () => Promise<LookupAccountHolderResponse>
   searchUsers: (data: SearchUsersRequest) => Promise<SearchUsersResponse>
   createUsers: (
-    users: Omit<User, "tenantId">[],
+    users: UnsavedUser[],
     groups: any[]
   ) => Promise<BulkUserCreated | undefined>
   updateUserInvite: (
diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts
index 8b0dfef34b..c1f37fd3f0 100644
--- a/packages/types/src/api/web/user.ts
+++ b/packages/types/src/api/web/user.ts
@@ -22,6 +22,8 @@ export interface UserDetails {
   password?: string
 }
 
+export type UnsavedUser = Omit<User, "tenantId">
+
 export interface BulkUserRequest {
   delete?: {
     users: Array<{
@@ -31,7 +33,7 @@ export interface BulkUserRequest {
   }
   create?: {
     roles?: any[]
-    users: User[]
+    users: UnsavedUser[]
     groups: any[]
   }
 }
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index a028f4fd33..4f9135e79e 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -33,6 +33,7 @@ import {
   SaveUserResponse,
   SearchUsersRequest,
   SearchUsersResponse,
+  UnsavedUser,
   UpdateInviteRequest,
   UpdateInviteResponse,
   User,
@@ -49,6 +50,7 @@ import {
   tenancy,
   db,
   locks,
+  context,
 } from "@budibase/backend-core"
 import { checkAnyUserExists } from "../../../utilities/users"
 import { isEmailConfigured } from "../../../utilities/email"
@@ -66,10 +68,11 @@ const generatePassword = (length: number) => {
     .slice(0, length)
 }
 
-export const save = async (ctx: UserCtx<User, SaveUserResponse>) => {
+export const save = async (ctx: UserCtx<UnsavedUser, SaveUserResponse>) => {
   try {
     const currentUserId = ctx.user?._id
-    const requestUser = ctx.request.body
+    const tenantId = context.getTenantId()
+    const requestUser: User = { ...ctx.request.body, tenantId }
 
     // Do not allow the account holder role to be changed
     const accountMetadata = await users.getExistingAccounts([requestUser.email])
@@ -149,7 +152,12 @@ export const bulkUpdate = async (
   let created, deleted
   try {
     if (input.create) {
-      created = await bulkCreate(input.create.users, input.create.groups)
+      const tenantId = context.getTenantId()
+      const users: User[] = input.create.users.map(user => ({
+        ...user,
+        tenantId,
+      }))
+      created = await bulkCreate(users, input.create.groups)
     }
     if (input.delete) {
       deleted = await bulkDelete(input.delete.users, currentUserId)
@@ -441,7 +449,6 @@ export const checkInvite = async (ctx: UserCtx<void, CheckInviteResponse>) => {
   } catch (e) {
     console.warn("Error getting invite from code", e)
     ctx.throw(400, "There was a problem with the invite")
-    return
   }
   ctx.body = {
     email: invite.email,
@@ -472,7 +479,6 @@ export const updateInvite = async (
     invite = await cache.invite.getCode(code)
   } catch (e) {
     ctx.throw(400, "There was a problem with the invite")
-    return
   }
 
   let updated = {

From 7d6b822b8a76014d03ccad31c227857b660990b1 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 16:12:02 +0000
Subject: [PATCH 059/163] Improve typing around AI prompts.

---
 packages/pro                                  |  2 +-
 .../src/api/controllers/row/staticFormula.ts  |  5 --
 .../types/src/documents/app/table/schema.ts   | 57 ++++++++++++++++---
 3 files changed, 51 insertions(+), 13 deletions(-)

diff --git a/packages/pro b/packages/pro
index 45f5b6fe9b..788173a024 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 45f5b6fe9bbdbdf502581740ab43b82e8153260f
+Subproject commit 788173a024fd5ef98d3122b26dbc06d39fb51349
diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index afa3a1f239..e2743da366 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -163,11 +163,6 @@ export async function finaliseRow(
     contextRows: [enrichedRow],
   })
 
-  const flag1 = await features.isEnabled(FeatureFlag.BUDIBASE_AI)
-  const flag2 = await pro.features.isBudibaseAIEnabled()
-  const flag3 = await features.isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)
-  const flag4 = await pro.features.isAICustomConfigsEnabled()
-
   const aiEnabled =
     ((await features.isEnabled(FeatureFlag.BUDIBASE_AI)) &&
       (await pro.features.isBudibaseAIEnabled())) ||
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index f4a6d8481d..47a63e0dd9 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -118,16 +118,59 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   responseType?: FormulaResponseType
 }
 
-export interface AIFieldMetadata extends BaseFieldSchema {
+interface AITranslateFieldMetadata extends BaseFieldSchema {
   type: FieldType.AI
-  operation: AIOperationEnum
-  columns?: string[]
-  column?: string
-  categories?: string
-  prompt?: string
-  language?: string
+  operation: AIOperationEnum.TRANSLATE
+  column: string
+  language: string
 }
 
+interface AICleanDataFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.CLEAN_DATA
+  column: string
+}
+
+interface AICategoriseTextFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.CATEGORISE_TEXT
+  columns: string[]
+  categories: string
+}
+
+interface AISentimentAnalysisFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.SENTIMENT_ANALYSIS
+  column: string
+}
+
+interface AISearchWebFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.SEARCH_WEB
+  columns: string[]
+}
+
+interface AIPromptFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.PROMPT
+  prompt: string
+}
+
+interface AISummariseTextFieldMetadata extends BaseFieldSchema {
+  type: FieldType.AI
+  operation: AIOperationEnum.SUMMARISE_TEXT
+  columns: string[]
+}
+
+export type AIFieldMetadata =
+  | AITranslateFieldMetadata
+  | AICleanDataFieldMetadata
+  | AICategoriseTextFieldMetadata
+  | AISentimentAnalysisFieldMetadata
+  | AIPromptFieldMetadata
+  | AISearchWebFieldMetadata
+  | AISummariseTextFieldMetadata
+
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE

From 68c2b29e6225225589c65c41a866a64670d17715 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 17:03:40 +0000
Subject: [PATCH 060/163] Admit defeat.

---
 packages/pro                                  |  2 +-
 .../types/src/documents/app/table/schema.ts   | 57 +++----------------
 2 files changed, 8 insertions(+), 51 deletions(-)

diff --git a/packages/pro b/packages/pro
index 788173a024..71922a7d97 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 788173a024fd5ef98d3122b26dbc06d39fb51349
+Subproject commit 71922a7d979ddca6f51d9e450abcd81ba9a5b6fa
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 47a63e0dd9..f4a6d8481d 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -118,59 +118,16 @@ export interface FormulaFieldMetadata extends BaseFieldSchema {
   responseType?: FormulaResponseType
 }
 
-interface AITranslateFieldMetadata extends BaseFieldSchema {
+export interface AIFieldMetadata extends BaseFieldSchema {
   type: FieldType.AI
-  operation: AIOperationEnum.TRANSLATE
-  column: string
-  language: string
+  operation: AIOperationEnum
+  columns?: string[]
+  column?: string
+  categories?: string
+  prompt?: string
+  language?: string
 }
 
-interface AICleanDataFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.CLEAN_DATA
-  column: string
-}
-
-interface AICategoriseTextFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.CATEGORISE_TEXT
-  columns: string[]
-  categories: string
-}
-
-interface AISentimentAnalysisFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.SENTIMENT_ANALYSIS
-  column: string
-}
-
-interface AISearchWebFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.SEARCH_WEB
-  columns: string[]
-}
-
-interface AIPromptFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.PROMPT
-  prompt: string
-}
-
-interface AISummariseTextFieldMetadata extends BaseFieldSchema {
-  type: FieldType.AI
-  operation: AIOperationEnum.SUMMARISE_TEXT
-  columns: string[]
-}
-
-export type AIFieldMetadata =
-  | AITranslateFieldMetadata
-  | AICleanDataFieldMetadata
-  | AICategoriseTextFieldMetadata
-  | AISentimentAnalysisFieldMetadata
-  | AIPromptFieldMetadata
-  | AISearchWebFieldMetadata
-  | AISummariseTextFieldMetadata
-
 export interface BBReferenceFieldMetadata
   extends Omit<BaseFieldSchema, "subtype"> {
   type: FieldType.BB_REFERENCE

From 352e16f347172ece0a69f73fcc97d8456ec6c848 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 17:09:02 +0000
Subject: [PATCH 061/163] Explain the weirdness with enriched view FieldType.AI
 columns in a comment for the next person to stumble on this.

---
 packages/server/src/sdk/app/views/index.ts | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts
index 7fc78b9085..4f978253d6 100644
--- a/packages/server/src/sdk/app/views/index.ts
+++ b/packages/server/src/sdk/app/views/index.ts
@@ -432,6 +432,21 @@ export async function enrichSchema(
       ...tableSchema[key],
       ...ui,
       order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key]?.order,
+      // When this was written, the only column types in FieldSchema to have columns
+      // field were the relationship columns. We blank this out here to make sure it's
+      // not set on non-relationship columns, then below we populate it by calling
+      // populateRelSchema.
+      //
+      // For Budibase 3.0 we introduced the FieldType.AI fields. Some of these fields
+      // have `columns: string[]` and it flew under the radar here because the
+      // AIFieldMetadata type isn't a union on its subtypes, it has a collection of
+      // optional fields. So columns is `columns?: string[]` which allows undefined,
+      // and doesn't fail this type check.
+      //
+      // What this means in practice is when FieldType.AI fields get enriched, we
+      // delete their `columns`. At the time of writing, I don't believe anything in
+      // the frontend depends on this, but it is odd and will probably bite us at
+      // some point.
       columns: undefined,
     }
 

From a18256a9cfe018c993d4c53b3e68f1a6189a520b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 17:10:02 +0000
Subject: [PATCH 062/163] Limit view AI column tests to internal tables.

---
 .../src/api/routes/tests/viewV2.spec.ts       | 145 +++++++++---------
 1 file changed, 73 insertions(+), 72 deletions(-)

diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 57efc868e9..9531737d30 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -936,93 +936,94 @@ if (descriptions.length) {
               )
             })
 
-          describe("AI fields", () => {
-            let envCleanup: () => void
-            beforeAll(() => {
-              mocks.licenses.useBudibaseAI()
-              mocks.licenses.useAICustomConfigs()
-              envCleanup = setEnv({
-                OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+          isInternal &&
+            describe("AI fields", () => {
+              let envCleanup: () => void
+              beforeAll(() => {
+                mocks.licenses.useBudibaseAI()
+                mocks.licenses.useAICustomConfigs()
+                envCleanup = setEnv({
+                  OPENAI_API_KEY: "sk-abcdefghijklmnopqrstuvwxyz1234567890abcd",
+                })
+
+                mockChatGPTResponse(prompt => {
+                  if (prompt.includes("elephant")) {
+                    return "big"
+                  }
+                  if (prompt.includes("mouse")) {
+                    return "small"
+                  }
+                  if (prompt.includes("whale")) {
+                    return "big"
+                  }
+                  return "unknown"
+                })
               })
 
-              mockChatGPTResponse(prompt => {
-                if (prompt.includes("elephant")) {
-                  return "big"
-                }
-                if (prompt.includes("mouse")) {
-                  return "small"
-                }
-                if (prompt.includes("whale")) {
-                  return "big"
-                }
-                return "unknown"
+              afterAll(() => {
+                nock.cleanAll()
+                envCleanup()
+                mocks.licenses.useCloudFree()
               })
-            })
 
-            afterAll(() => {
-              nock.cleanAll()
-              envCleanup()
-              mocks.licenses.useCloudFree()
-            })
-
-            it("can use AI fields in view calculations", async () => {
-              const table = await config.api.table.save(
-                saveTableRequest({
-                  schema: {
-                    animal: {
-                      name: "animal",
-                      type: FieldType.STRING,
+              it("can use AI fields in view calculations", async () => {
+                const table = await config.api.table.save(
+                  saveTableRequest({
+                    schema: {
+                      animal: {
+                        name: "animal",
+                        type: FieldType.STRING,
+                      },
+                      bigOrSmall: {
+                        name: "bigOrSmall",
+                        type: FieldType.AI,
+                        operation: AIOperationEnum.CATEGORISE_TEXT,
+                        categories: "big,small",
+                        columns: ["animal"],
+                      },
                     },
+                  })
+                )
+
+                const view = await config.api.viewV2.create({
+                  tableId: table._id!,
+                  name: generator.guid(),
+                  type: ViewV2Type.CALCULATION,
+                  schema: {
                     bigOrSmall: {
-                      name: "bigOrSmall",
-                      type: FieldType.AI,
-                      operation: AIOperationEnum.CATEGORISE_TEXT,
-                      categories: "big,small",
-                      columns: ["animal"],
+                      visible: true,
+                    },
+                    count: {
+                      visible: true,
+                      calculationType: CalculationType.COUNT,
+                      field: "animal",
                     },
                   },
                 })
-              )
 
-              const view = await config.api.viewV2.create({
-                tableId: table._id!,
-                name: generator.guid(),
-                type: ViewV2Type.CALCULATION,
-                schema: {
-                  bigOrSmall: {
-                    visible: true,
-                  },
-                  count: {
-                    visible: true,
-                    calculationType: CalculationType.COUNT,
-                    field: "animal",
-                  },
-                },
-              })
+                await config.api.row.save(table._id!, {
+                  animal: "elephant",
+                })
 
-              await config.api.row.save(table._id!, {
-                animal: "elephant",
-              })
+                await config.api.row.save(table._id!, {
+                  animal: "mouse",
+                })
 
-              await config.api.row.save(table._id!, {
-                animal: "mouse",
-              })
+                await config.api.row.save(table._id!, {
+                  animal: "whale",
+                })
 
-              await config.api.row.save(table._id!, {
-                animal: "whale",
+                const { rows } = await config.api.row.search(view.id, {
+                  sort: "bigOrSmall",
+                  sortOrder: SortOrder.ASCENDING,
+                })
+                expect(rows).toHaveLength(2)
+                expect(rows[0].bigOrSmall).toEqual("big")
+                expect(rows[1].bigOrSmall).toEqual("small")
+                expect(rows[0].count).toEqual(2)
+                expect(rows[1].count).toEqual(1)
               })
-
-              const { rows } = await config.api.row.search(view.id, {
-                sort: "bigOrSmall",
-                sortOrder: SortOrder.ASCENDING,
-              })
-              expect(rows).toHaveLength(2)
-              expect(rows[0].bigOrSmall).toEqual("big")
-              expect(rows[1].bigOrSmall).toEqual("small")
-              expect(rows[0].count).toEqual(2)
-              expect(rows[1].count).toEqual(1)
             })
-          })
         })
 
         describe("update", () => {

From 2a9e4c307825cf6e97265a8436cfc9a58832ff07 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 8 Jan 2025 17:34:08 +0000
Subject: [PATCH 063/163] Remove double processing of AI columns.

---
 packages/server/src/api/controllers/row/staticFormula.ts | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index e2743da366..bd842ebfd0 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -185,11 +185,6 @@ export async function finaliseRow(
   enrichedRow = await processFormulas(table, enrichedRow, {
     dynamic: false,
   })
-  if (aiEnabled) {
-    enrichedRow = await processAIColumns(table, enrichedRow, {
-      contextRows: [enrichedRow],
-    })
-  }
 
   // this updates the related formulas in other rows based on the relations to this row
   if (updateFormula) {

From 45ee1570370c9e41fe503a12401ac5664081a832 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:24:37 +0100
Subject: [PATCH 064/163] Rename file

---
 packages/client/src/stores/index.js                             | 2 +-
 packages/client/src/stores/{notification.js => notification.ts} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename packages/client/src/stores/{notification.js => notification.ts} (100%)

diff --git a/packages/client/src/stores/index.js b/packages/client/src/stores/index.js
index f2b80ed732..e099434b3d 100644
--- a/packages/client/src/stores/index.js
+++ b/packages/client/src/stores/index.js
@@ -1,6 +1,6 @@
 export { authStore } from "./auth"
 export { appStore } from "./app"
-export { notificationStore } from "./notification"
+export { notificationStore } from "./notification.ts"
 export { routeStore } from "./routes"
 export { screenStore } from "./screens"
 export { builderStore } from "./builder"
diff --git a/packages/client/src/stores/notification.js b/packages/client/src/stores/notification.ts
similarity index 100%
rename from packages/client/src/stores/notification.js
rename to packages/client/src/stores/notification.ts

From 96cee21792413d638191e4a6872bf5e388d1786a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:32:10 +0100
Subject: [PATCH 065/163] Type store

---
 packages/client/src/stores/notification.ts | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/packages/client/src/stores/notification.ts b/packages/client/src/stores/notification.ts
index 054117aaba..6f5fcc9ef4 100644
--- a/packages/client/src/stores/notification.ts
+++ b/packages/client/src/stores/notification.ts
@@ -6,7 +6,7 @@ const DEFAULT_NOTIFICATION_TIMEOUT = 3000
 const createNotificationStore = () => {
   let block = false
 
-  const store = writable([])
+  const store = writable<{ id: string; message: string; count: number }[]>([])
 
   const blockNotifications = (timeout = 1000) => {
     block = true
@@ -14,17 +14,18 @@ const createNotificationStore = () => {
   }
 
   const send = (
-    message,
+    message: string,
     type = "info",
-    icon,
+    icon: string,
     autoDismiss = true,
-    duration,
+    duration: number,
     count = 1
   ) => {
     if (block) {
       return
     }
 
+    // @ts-expect-error
     if (get(routeStore).queryParams?.peek) {
       window.parent.postMessage({
         type: "notification",
@@ -66,7 +67,7 @@ const createNotificationStore = () => {
     }
   }
 
-  const dismiss = id => {
+  const dismiss = (id: string) => {
     store.update(state => {
       return state.filter(n => n.id !== id)
     })
@@ -76,13 +77,13 @@ const createNotificationStore = () => {
     subscribe: store.subscribe,
     actions: {
       send,
-      info: (msg, autoDismiss, duration) =>
+      info: (msg: string, autoDismiss: boolean, duration: number) =>
         send(msg, "info", "Info", autoDismiss ?? true, duration),
-      success: (msg, autoDismiss, duration) =>
+      success: (msg: string, autoDismiss: boolean, duration: number) =>
         send(msg, "success", "CheckmarkCircle", autoDismiss ?? true, duration),
-      warning: (msg, autoDismiss, duration) =>
+      warning: (msg: string, autoDismiss: boolean, duration: number) =>
         send(msg, "warning", "Alert", autoDismiss ?? true, duration),
-      error: (msg, autoDismiss, duration) =>
+      error: (msg: string, autoDismiss: boolean, duration: number) =>
         send(msg, "error", "Alert", autoDismiss ?? false, duration),
       blockNotifications,
       dismiss,

From f938eb3297e5dd3de249d41c36ef7c0565f1996a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:34:15 +0100
Subject: [PATCH 066/163] api to ts

---
 packages/client/src/api/{api.js => api.ts} |  6 +++++-
 packages/client/src/api/index.js           |  2 +-
 packages/client/src/stores/notification.ts | 10 +++++-----
 3 files changed, 11 insertions(+), 7 deletions(-)
 rename packages/client/src/api/{api.js => api.ts} (96%)

diff --git a/packages/client/src/api/api.js b/packages/client/src/api/api.ts
similarity index 96%
rename from packages/client/src/api/api.js
rename to packages/client/src/api/api.ts
index d4c8faa4d2..59430e4ebd 100644
--- a/packages/client/src/api/api.js
+++ b/packages/client/src/api/api.ts
@@ -1,6 +1,10 @@
 import { createAPIClient } from "@budibase/frontend-core"
 import { authStore } from "../stores/auth.js"
-import { notificationStore, devToolsEnabled, devToolsStore } from "../stores/"
+import {
+  notificationStore,
+  devToolsEnabled,
+  devToolsStore,
+} from "../stores/index.js"
 import { get } from "svelte/store"
 
 export const API = createAPIClient({
diff --git a/packages/client/src/api/index.js b/packages/client/src/api/index.js
index 5eb6b2b6f4..a63e19bfbb 100644
--- a/packages/client/src/api/index.js
+++ b/packages/client/src/api/index.js
@@ -1,4 +1,4 @@
-import { API } from "./api.js"
+import { API } from "./api.ts"
 import { patchAPI } from "./patches.js"
 
 // Certain endpoints which return rows need patched so that they transform
diff --git a/packages/client/src/stores/notification.ts b/packages/client/src/stores/notification.ts
index 6f5fcc9ef4..5e1a4774c1 100644
--- a/packages/client/src/stores/notification.ts
+++ b/packages/client/src/stores/notification.ts
@@ -18,7 +18,7 @@ const createNotificationStore = () => {
     type = "info",
     icon: string,
     autoDismiss = true,
-    duration: number,
+    duration?: number,
     count = 1
   ) => {
     if (block) {
@@ -77,13 +77,13 @@ const createNotificationStore = () => {
     subscribe: store.subscribe,
     actions: {
       send,
-      info: (msg: string, autoDismiss: boolean, duration: number) =>
+      info: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "info", "Info", autoDismiss ?? true, duration),
-      success: (msg: string, autoDismiss: boolean, duration: number) =>
+      success: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "success", "CheckmarkCircle", autoDismiss ?? true, duration),
-      warning: (msg: string, autoDismiss: boolean, duration: number) =>
+      warning: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "warning", "Alert", autoDismiss ?? true, duration),
-      error: (msg: string, autoDismiss: boolean, duration: number) =>
+      error: (msg: string, autoDismiss?: boolean, duration?: number) =>
         send(msg, "error", "Alert", autoDismiss ?? false, duration),
       blockNotifications,
       dismiss,

From a0950f15d9bb4d77dbdcc98d8f96fa93b78ef085 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:38:16 +0100
Subject: [PATCH 067/163] Add window typings

---
 packages/client/src/index.d.ts | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 packages/client/src/index.d.ts

diff --git a/packages/client/src/index.d.ts b/packages/client/src/index.d.ts
new file mode 100644
index 0000000000..7e13670b33
--- /dev/null
+++ b/packages/client/src/index.d.ts
@@ -0,0 +1,5 @@
+interface Window {
+  "##BUDIBASE_APP_ID##": string
+  "##BUDIBASE_IN_BUILDER##": string
+  MIGRATING_APP: boolean
+}

From 60c23ae021614dee53eb62a08e0ae9c4da974ff9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:42:35 +0100
Subject: [PATCH 068/163] Type auth store

---
 packages/client/src/stores/{auth.js => auth.ts} | 4 +++-
 packages/types/src/api/web/global/self.ts       | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
 rename packages/client/src/stores/{auth.js => auth.ts} (94%)

diff --git a/packages/client/src/stores/auth.js b/packages/client/src/stores/auth.ts
similarity index 94%
rename from packages/client/src/stores/auth.js
rename to packages/client/src/stores/auth.ts
index 214cc7bce2..15f44e7c32 100644
--- a/packages/client/src/stores/auth.js
+++ b/packages/client/src/stores/auth.ts
@@ -2,7 +2,9 @@ import { API } from "api"
 import { writable } from "svelte/store"
 
 const createAuthStore = () => {
-  const store = writable(null)
+  const store = writable<{
+    csrfToken?: string
+  } | null>(null)
 
   // Fetches the user object if someone is logged in and has reloaded the page
   const fetchUser = async () => {
diff --git a/packages/types/src/api/web/global/self.ts b/packages/types/src/api/web/global/self.ts
index 517559d1ca..9d99a1f1a5 100644
--- a/packages/types/src/api/web/global/self.ts
+++ b/packages/types/src/api/web/global/self.ts
@@ -15,5 +15,5 @@ export interface GetGlobalSelfResponse extends User {
   license: License
   budibaseAccess: boolean
   accountPortalAccess: boolean
-  csrfToken: boolean
+  csrfToken: string
 }

From 6e816fb6a2ae3aa8ec13f8d5fc887bc88b3c25fe Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:51:27 +0100
Subject: [PATCH 069/163] Fixes

---
 packages/client/src/api/api.ts                    | 2 +-
 packages/client/src/stores/derived/currentRole.js | 2 +-
 packages/client/src/stores/notification.ts        | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/api/api.ts b/packages/client/src/api/api.ts
index 59430e4ebd..93f59f6e9a 100644
--- a/packages/client/src/api/api.ts
+++ b/packages/client/src/api/api.ts
@@ -1,5 +1,5 @@
 import { createAPIClient } from "@budibase/frontend-core"
-import { authStore } from "../stores/auth.js"
+import { authStore } from "../stores/auth"
 import {
   notificationStore,
   devToolsEnabled,
diff --git a/packages/client/src/stores/derived/currentRole.js b/packages/client/src/stores/derived/currentRole.js
index 8bb4c5a25d..056a05f8ab 100644
--- a/packages/client/src/stores/derived/currentRole.js
+++ b/packages/client/src/stores/derived/currentRole.js
@@ -1,7 +1,7 @@
 import { derived } from "svelte/store"
 import { Constants } from "@budibase/frontend-core"
 import { devToolsStore } from "../devTools.js"
-import { authStore } from "../auth.js"
+import { authStore } from "../auth"
 import { devToolsEnabled } from "./devToolsEnabled.js"
 
 // Derive the current role of the logged-in user
diff --git a/packages/client/src/stores/notification.ts b/packages/client/src/stores/notification.ts
index 5e1a4774c1..fa28b9f40a 100644
--- a/packages/client/src/stores/notification.ts
+++ b/packages/client/src/stores/notification.ts
@@ -25,7 +25,6 @@ const createNotificationStore = () => {
       return
     }
 
-    // @ts-expect-error
     if (get(routeStore).queryParams?.peek) {
       window.parent.postMessage({
         type: "notification",

From 1b6dc51f018c80938c8f45aaf2852734f3981076 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 11:51:38 +0100
Subject: [PATCH 070/163] Type routes

---
 .../src/stores/{routes.js => routes.ts}       | 30 ++++++++++++++-----
 packages/types/src/documents/app/screen.ts    |  1 +
 2 files changed, 24 insertions(+), 7 deletions(-)
 rename packages/client/src/stores/{routes.js => routes.ts} (82%)

diff --git a/packages/client/src/stores/routes.js b/packages/client/src/stores/routes.ts
similarity index 82%
rename from packages/client/src/stores/routes.js
rename to packages/client/src/stores/routes.ts
index 8e318af2e3..3f200a9c88 100644
--- a/packages/client/src/stores/routes.js
+++ b/packages/client/src/stores/routes.ts
@@ -4,8 +4,24 @@ import { API } from "api"
 import { peekStore } from "./peek"
 import { builderStore } from "./builder"
 
+interface Route {
+  path: string
+  screenId: string
+}
+
+interface StoreType {
+  routes: Route[]
+  routeParams: {}
+  activeRoute?: Route | null
+  routeSessionId: number
+  routerLoaded: boolean
+  queryParams?: {
+    peek?: boolean
+  }
+}
+
 const createRouteStore = () => {
-  const initialState = {
+  const initialState: StoreType = {
     routes: [],
     routeParams: {},
     activeRoute: null,
@@ -22,7 +38,7 @@ const createRouteStore = () => {
     } catch (error) {
       routeConfig = null
     }
-    let routes = []
+    const routes: Route[] = []
     Object.values(routeConfig?.routes || {}).forEach(route => {
       Object.entries(route.subpaths || {}).forEach(([path, config]) => {
         routes.push({
@@ -43,13 +59,13 @@ const createRouteStore = () => {
       return state
     })
   }
-  const setRouteParams = routeParams => {
+  const setRouteParams = (routeParams: StoreType["routeParams"]) => {
     store.update(state => {
       state.routeParams = routeParams
       return state
     })
   }
-  const setQueryParams = queryParams => {
+  const setQueryParams = (queryParams: { peek?: boolean }) => {
     store.update(state => {
       state.queryParams = {
         ...queryParams,
@@ -60,13 +76,13 @@ const createRouteStore = () => {
       return state
     })
   }
-  const setActiveRoute = route => {
+  const setActiveRoute = (route: string) => {
     store.update(state => {
       state.activeRoute = state.routes.find(x => x.path === route)
       return state
     })
   }
-  const navigate = (url, peek, externalNewTab) => {
+  const navigate = (url: string, peek: boolean, externalNewTab: boolean) => {
     if (get(builderStore).inBuilder) {
       return
     }
@@ -93,7 +109,7 @@ const createRouteStore = () => {
   const setRouterLoaded = () => {
     store.update(state => ({ ...state, routerLoaded: true }))
   }
-  const createFullURL = relativeURL => {
+  const createFullURL = (relativeURL: string) => {
     if (!relativeURL?.startsWith("/")) {
       return relativeURL
     }
diff --git a/packages/types/src/documents/app/screen.ts b/packages/types/src/documents/app/screen.ts
index b2cedf31a9..e85fd5953b 100644
--- a/packages/types/src/documents/app/screen.ts
+++ b/packages/types/src/documents/app/screen.ts
@@ -36,6 +36,7 @@ export type ScreenRoutingJson = Record<
     subpaths: Record<
       string,
       {
+        screenId: string
         screens: Record<string, string>
       }
     >

From 59bdd201585b90c1e87e6cfaa7b82becf2a3ef01 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 12:12:43 +0100
Subject: [PATCH 071/163] Type onError

---
 packages/frontend-core/src/api/types.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/api/types.ts b/packages/frontend-core/src/api/types.ts
index 0db1049591..940927f499 100644
--- a/packages/frontend-core/src/api/types.ts
+++ b/packages/frontend-core/src/api/types.ts
@@ -46,7 +46,7 @@ export type Headers = Record<string, string>
 export type APIClientConfig = {
   enableCaching?: boolean
   attachHeaders?: (headers: Headers) => void
-  onError?: (error: any) => void
+  onError?: (error: APIError) => void
   onMigrationDetected?: (migration: string) => void
 }
 

From e6c45d934b70ae496a8827b5f17d75cc5c8af5b8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 12:39:44 +0100
Subject: [PATCH 072/163] Stringify api errors

---
 packages/frontend-core/src/api/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index f7b05c338a..44e85e5a68 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -74,7 +74,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
       if (json?.message) {
         message = json.message
       } else if (json?.error) {
-        message = json.error
+        message = JSON.stringify(json.error)
       }
     } catch (error) {
       // Do nothing

From 2cbf7a47c16194741d49dc654d1d0ea059c3d54d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 12:40:23 +0100
Subject: [PATCH 073/163] Type

---
 packages/frontend-core/src/api/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index 44e85e5a68..b0e1970e6f 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -226,7 +226,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
         return await handler(callConfig)
       } catch (error) {
         if (config?.onError) {
-          config.onError(error)
+          config.onError(error as APIError)
         }
         throw error
       }

From 9fade65b4a3895c7ab0609b0654d5ac3d44d455d Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 11:46:45 +0000
Subject: [PATCH 074/163] Update pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 71922a7d97..15b7f0907e 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 71922a7d979ddca6f51d9e450abcd81ba9a5b6fa
+Subproject commit 15b7f0907e66e4144338404bb071bc1ccfc98137

From 2b90d972b21325608f0f1e4f51d0d1a850be2e26 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 12:50:19 +0100
Subject: [PATCH 075/163] Clean code

---
 packages/frontend-core/src/api/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index b0e1970e6f..92dc2d9ea3 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -68,7 +68,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
   ): Promise<APIError> => {
     // Try to read a message from the error
     let message = response.statusText
-    let json: any = null
+    let json = null
     try {
       json = await response.json()
       if (json?.message) {

From 71c6480c1b3ea08cd69bc8a0cfe015aadbaa773b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 13:00:02 +0100
Subject: [PATCH 076/163] Remove unused function

---
 packages/frontend-core/src/api/index.ts | 6 +-----
 packages/frontend-core/src/api/types.ts | 3 +--
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index 92dc2d9ea3..6efc90023a 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -93,7 +93,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
   // Generates an error object from a string
   const makeError = (
     message: string,
-    url?: string,
+    url: string,
     method?: HTTPMethod
   ): APIError => {
     return {
@@ -239,13 +239,9 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
     patch: requestApiCall(HTTPMethod.PATCH),
     delete: requestApiCall(HTTPMethod.DELETE),
     put: requestApiCall(HTTPMethod.PUT),
-    error: (message: string) => {
-      throw makeError(message)
-    },
     invalidateCache: () => {
       cache = {}
     },
-
     // Generic utility to extract the current app ID. Assumes that any client
     // that exists in an app context will be attaching our app ID header.
     getAppID: (): string => {
diff --git a/packages/frontend-core/src/api/types.ts b/packages/frontend-core/src/api/types.ts
index 940927f499..4819b4cd3b 100644
--- a/packages/frontend-core/src/api/types.ts
+++ b/packages/frontend-core/src/api/types.ts
@@ -86,14 +86,13 @@ export type BaseAPIClient = {
   patch: <RequestT = null, ResponseT = void>(
     params: APICallParams<RequestT, ResponseT>
   ) => Promise<ResponseT>
-  error: (message: string) => void
   invalidateCache: () => void
   getAppID: () => string
 }
 
 export type APIError = {
   message?: string
-  url?: string
+  url: string
   method?: HTTPMethod
   json: any
   status: number

From ace5099a8cf25f6eb73878d295568cb4130ea344 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 12:05:45 +0000
Subject: [PATCH 077/163] Update pro reference.

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 23fdd50b7e..193476cdfa 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 23fdd50b7ef28cf320716ed2c46e15d63185daa7
+Subproject commit 193476cdfade6d3c613e6972f16ee0c527e01ff6

From 70a8b9e4688249211f5e9d56463e3524a97b2539 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 13:55:00 +0100
Subject: [PATCH 078/163] Remove screen id

---
 packages/types/src/documents/app/screen.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/types/src/documents/app/screen.ts b/packages/types/src/documents/app/screen.ts
index e85fd5953b..b2cedf31a9 100644
--- a/packages/types/src/documents/app/screen.ts
+++ b/packages/types/src/documents/app/screen.ts
@@ -36,7 +36,6 @@ export type ScreenRoutingJson = Record<
     subpaths: Record<
       string,
       {
-        screenId: string
         screens: Record<string, string>
       }
     >

From 88e245a5bb7a97eedfc07ea6104e44c5d2dfbd2a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:01:07 +0100
Subject: [PATCH 079/163] Fix typing

---
 packages/types/src/documents/app/screen.ts | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/types/src/documents/app/screen.ts b/packages/types/src/documents/app/screen.ts
index b2cedf31a9..a8c32118d3 100644
--- a/packages/types/src/documents/app/screen.ts
+++ b/packages/types/src/documents/app/screen.ts
@@ -33,11 +33,6 @@ export interface ScreenRoutesViewOutput extends Document {
 export type ScreenRoutingJson = Record<
   string,
   {
-    subpaths: Record<
-      string,
-      {
-        screens: Record<string, string>
-      }
-    >
+    subpaths: Record<string, any>
   }
 >

From 02b20f1102ebc05f3c292eae68df8c8a9e2b7417 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:24:59 +0100
Subject: [PATCH 080/163] Fix getSchema from forms

---
 packages/client/src/utils/schema.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index ffab142cf3..4a65d2d099 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -52,7 +52,7 @@ export const fetchDatasourceSchema = async (
   // Get the normal schema as long as we aren't wanting a form schema
   let schema
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(datasource, definition)
+    schema = instance.getSchema(definition)
   } else if (definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {

From ff0a25f0ffadf727418f757736fcc0b6091ff412 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:32:15 +0100
Subject: [PATCH 081/163] Pass datasource to instance

---
 packages/client/src/utils/schema.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index 4a65d2d099..60800afc53 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -31,7 +31,7 @@ const getDatasourceFetchInstance = datasource => {
   if (!handler) {
     return null
   }
-  return new handler({ API })
+  return new handler({ API, datasource })
 }
 
 /**

From a5ec51028099325a288ab9fc4bca5e1978d634cb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 15:58:59 +0100
Subject: [PATCH 082/163] Explicitly fetch the data from fetch

---
 packages/frontend-core/src/fetch/DataFetch.ts | 3 ---
 packages/frontend-core/src/fetch/index.ts     | 7 ++++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts
index 9312c57637..0079fec057 100644
--- a/packages/frontend-core/src/fetch/DataFetch.ts
+++ b/packages/frontend-core/src/fetch/DataFetch.ts
@@ -179,9 +179,6 @@ export default abstract class DataFetch<
       this.store.update($store => ({ ...$store, loaded: true }))
       return
     }
-
-    // Initially fetch data but don't bother waiting for the result
-    this.getInitialData()
   }
 
   /**
diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts
index 4accb0b5ec..c1f35abef2 100644
--- a/packages/frontend-core/src/fetch/index.ts
+++ b/packages/frontend-core/src/fetch/index.ts
@@ -33,7 +33,12 @@ const DataFetchMap = {
 export const fetchData = ({ API, datasource, options }: any) => {
   const Fetch =
     DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch
-  return new Fetch({ API, datasource, ...options })
+  const fetch = new Fetch({ API, datasource, ...options })
+
+  // Initially fetch data but don't bother waiting for the result
+  fetch.getInitialData()
+
+  return fetch
 }
 
 // Creates an empty fetch instance with no datasource configured, so no data

From 5c933c7f80ef4f0e631fafed7dc16d8691823079 Mon Sep 17 00:00:00 2001
From: Michael Drury <me@michaeldrury.co.uk>
Date: Thu, 9 Jan 2025 16:27:59 +0000
Subject: [PATCH 083/163] Revert "Fix issue with forms after cleanup"

---
 packages/client/src/utils/schema.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index 60800afc53..ffab142cf3 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -31,7 +31,7 @@ const getDatasourceFetchInstance = datasource => {
   if (!handler) {
     return null
   }
-  return new handler({ API, datasource })
+  return new handler({ API })
 }
 
 /**
@@ -52,7 +52,7 @@ export const fetchDatasourceSchema = async (
   // Get the normal schema as long as we aren't wanting a form schema
   let schema
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(definition)
+    schema = instance.getSchema(datasource, definition)
   } else if (definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {

From 26c0861e6899b196eed64e0396080a6a6e905d3f Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 17:05:44 +0000
Subject: [PATCH 084/163] Fix date-only searching.

---
 .../src/middleware/errorHandling.ts           |   6 +-
 packages/backend-core/src/sql/sql.ts          |  54 ++++++++-
 packages/backend-core/src/sql/utils.ts        |  12 +-
 .../src/api/routes/tests/search.spec.ts       | 106 ++++++++++++++++++
 .../src/utilities/rowProcessor/index.ts       |   9 ++
 packages/shared-core/src/filters.ts           |  22 +++-
 6 files changed, 196 insertions(+), 13 deletions(-)

diff --git a/packages/backend-core/src/middleware/errorHandling.ts b/packages/backend-core/src/middleware/errorHandling.ts
index 6ceda9cd3a..5a5a25b461 100644
--- a/packages/backend-core/src/middleware/errorHandling.ts
+++ b/packages/backend-core/src/middleware/errorHandling.ts
@@ -32,8 +32,12 @@ export async function errorHandling(ctx: any, next: any) {
     }
 
     if (environment.isTest() && ctx.headers["x-budibase-include-stacktrace"]) {
+      let rootErr = err
+      while (rootErr.cause) {
+        rootErr = rootErr.cause
+      }
       // @ts-ignore
-      error.stack = err.stack
+      error.stack = rootErr.stack
     }
 
     ctx.body = error
diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 5f462ee144..9b0c49d9f8 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -816,14 +816,29 @@ class InternalBuilder {
         filters.oneOf,
         ArrayOperator.ONE_OF,
         (q, key: string, array) => {
+          const schema = this.getFieldSchema(key)
+          const values = Array.isArray(array) ? array : [array]
           if (shouldOr) {
             q = q.or
           }
           if (this.client === SqlClient.ORACLE) {
             // @ts-ignore
             key = this.convertClobs(key)
+          } else if (
+            this.client === SqlClient.SQL_LITE &&
+            schema?.type === FieldType.DATETIME &&
+            schema.dateOnly
+          ) {
+            for (const value of values) {
+              if (value != null) {
+                q = q.or.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              } else {
+                q = q.or.whereNull(key)
+              }
+            }
+            return q
           }
-          return q.whereIn(key, Array.isArray(array) ? array : [array])
+          return q.whereIn(key, values)
         },
         (q, key: string[], array) => {
           if (shouldOr) {
@@ -882,6 +897,19 @@ class InternalBuilder {
         let high = value.high
         let low = value.low
 
+        if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (high != null) {
+            high = `${high.toISOString().slice(0, 10)}T23:59:59.999Z`
+          }
+          if (low != null) {
+            low = low.toISOString().slice(0, 10)
+          }
+        }
+
         if (this.client === SqlClient.ORACLE) {
           rawKey = this.convertClobs(key)
         } else if (
@@ -914,6 +942,7 @@ class InternalBuilder {
     }
     if (filters.equal) {
       iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -928,6 +957,16 @@ class InternalBuilder {
             // @ts-expect-error knex types are wrong, raw is fine here
             subq.whereNotNull(identifier).andWhere(identifier, value)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+          } else {
+            return q.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? = ?, FALSE)`, [
             this.rawQuotedIdentifier(key),
@@ -938,6 +977,7 @@ class InternalBuilder {
     }
     if (filters.notEqual) {
       iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => {
+        const schema = this.getFieldSchema(key)
         if (shouldOr) {
           q = q.or
         }
@@ -959,6 +999,18 @@ class InternalBuilder {
               // @ts-expect-error knex types are wrong, raw is fine here
               .or.whereNull(identifier)
           )
+        } else if (
+          this.client === SqlClient.SQL_LITE &&
+          schema?.type === FieldType.DATETIME &&
+          schema.dateOnly
+        ) {
+          if (value != null) {
+            return q.not
+              .whereLike(key, `${value.toISOString().slice(0, 10)}%`)
+              .or.whereNull(key)
+          } else {
+            return q.not.whereNull(key)
+          }
         } else {
           return q.whereRaw(`COALESCE(?? != ?, TRUE)`, [
             this.rawQuotedIdentifier(key),
diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 16b352995b..b07854b2a0 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -14,7 +14,7 @@ import environment from "../environment"
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
 const ROW_ID_REGEX = /^\[.*]$/g
 const ENCODED_SPACE = encodeURIComponent(" ")
-const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
+const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}.\d{3}Z)?$/
 const TIME_REGEX = /^(?:\d{2}:)?(?:\d{2}:)(?:\d{2})$/
 
 export function isExternalTableID(tableId: string) {
@@ -149,15 +149,7 @@ export function isInvalidISODateString(str: string) {
 }
 
 export function isValidISODateString(str: string) {
-  const trimmedValue = str.trim()
-  if (!ISO_DATE_REGEX.test(trimmedValue)) {
-    return false
-  }
-  let d = new Date(trimmedValue)
-  if (isNaN(d.getTime())) {
-    return false
-  }
-  return d.toISOString() === trimmedValue
+  return ISO_DATE_REGEX.test(str.trim())
 }
 
 export function isValidFilter(value: any) {
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index e94e567b43..dca1bf9394 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1683,6 +1683,112 @@ if (descriptions.length) {
                 })
               })
 
+            describe.only("datetime - date only", () => {
+              const JAN_1ST = "2020-01-01"
+              const JAN_10TH = "2020-01-10"
+              const JAN_30TH = "2020-01-30"
+              const UNEXISTING_DATE = "2020-01-03"
+              const NULL_DATE__ID = `null_date__id`
+
+              beforeAll(async () => {
+                tableOrViewId = await createTableOrView({
+                  dateid: { name: "dateid", type: FieldType.STRING },
+                  date: {
+                    name: "date",
+                    type: FieldType.DATETIME,
+                    dateOnly: true,
+                  },
+                })
+
+                await createRows([
+                  { dateid: NULL_DATE__ID, date: null },
+                  { date: JAN_1ST },
+                  { date: `${JAN_10TH}T00:00:00.000Z` },
+                ])
+              })
+
+              describe("equal", () => {
+                it("successfully finds a row", async () => {
+                  await expectQuery({
+                    equal: { date: JAN_1ST },
+                  }).toContainExactly([{ date: JAN_1ST }])
+                })
+
+                it("successfully finds an ISO8601 row", async () => {
+                  await expectQuery({
+                    equal: { date: JAN_10TH },
+                  }).toContainExactly([{ date: JAN_10TH }])
+                })
+
+                it("finds a row with ISO8601 timestamp", async () => {
+                  await expectQuery({
+                    equal: { date: `${JAN_1ST}T00:00:00.000Z` },
+                  }).toContainExactly([{ date: JAN_1ST }])
+                })
+
+                it("fails to find nonexistent row", async () => {
+                  await expectQuery({
+                    equal: { date: UNEXISTING_DATE },
+                  }).toFindNothing()
+                })
+              })
+
+              describe("notEqual", () => {
+                it("successfully finds a row", async () => {
+                  await expectQuery({
+                    notEqual: { date: JAN_1ST },
+                  }).toContainExactly([
+                    { date: JAN_10TH },
+                    { dateid: NULL_DATE__ID },
+                  ])
+                })
+
+                it("fails to find nonexistent row", async () => {
+                  await expectQuery({
+                    notEqual: { date: JAN_30TH },
+                  }).toContainExactly([
+                    { date: JAN_1ST },
+                    { date: JAN_10TH },
+                    { dateid: NULL_DATE__ID },
+                  ])
+                })
+              })
+
+              describe("oneOf", () => {
+                it("successfully finds a row", async () => {
+                  await expectQuery({
+                    oneOf: { date: [JAN_1ST] },
+                  }).toContainExactly([{ date: JAN_1ST }])
+                })
+
+                it("fails to find nonexistent row", async () => {
+                  await expectQuery({
+                    oneOf: { date: [UNEXISTING_DATE] },
+                  }).toFindNothing()
+                })
+              })
+
+              describe("range", () => {
+                it("successfully finds a row", async () => {
+                  await expectQuery({
+                    range: { date: { low: JAN_1ST, high: JAN_1ST } },
+                  }).toContainExactly([{ date: JAN_1ST }])
+                })
+
+                it("successfully finds multiple rows", async () => {
+                  await expectQuery({
+                    range: { date: { low: JAN_1ST, high: JAN_10TH } },
+                  }).toContainExactly([{ date: JAN_1ST }, { date: JAN_10TH }])
+                })
+
+                it("successfully finds no rows", async () => {
+                  await expectQuery({
+                    range: { date: { low: JAN_30TH, high: JAN_30TH } },
+                  }).toFindNothing()
+                })
+              })
+            })
+
             isInternal &&
               !isInMemory &&
               describe("AI Column", () => {
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 14b524fd95..8595a3483e 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -411,6 +411,15 @@ export async function coreOutputProcessing(
           row[property] = `${hours}:${minutes}:${seconds}`
         }
       }
+    } else if (column.type === FieldType.DATETIME && column.dateOnly) {
+      for (const row of rows) {
+        if (typeof row[property] === "string") {
+          row[property] = new Date(row[property])
+        }
+        if (row[property] instanceof Date) {
+          row[property] = row[property].toISOString().slice(0, 10)
+        }
+      }
     } else if (column.type === FieldType.LINK) {
       for (let row of rows) {
         // if relationship is empty - remove the array, this has been part of the API for some time
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index b711d4cb61..afe99d9565 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -699,7 +699,27 @@ export function runQuery<T extends Record<string, any>>(
       return docValue._id === testValue
     }
 
-    return docValue === testValue
+    if (docValue === testValue) {
+      return true
+    }
+
+    if (docValue == null && testValue != null) {
+      return false
+    }
+
+    if (docValue != null && testValue == null) {
+      return false
+    }
+
+    const leftDate = dayjs(docValue)
+    if (leftDate.isValid()) {
+      const rightDate = dayjs(testValue)
+      if (rightDate.isValid()) {
+        return leftDate.isSame(rightDate)
+      }
+    }
+
+    return false
   }
 
   const not =

From 7ad5879c3eb19783bd5a9d4beb36b835fe02fc3e Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 17:16:47 +0000
Subject: [PATCH 085/163] Create search cross product on whether we save/search
 with timestamp.

---
 .../src/api/routes/tests/search.spec.ts       | 219 +++++++++++-------
 1 file changed, 129 insertions(+), 90 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index dca1bf9394..69c68aad97 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1684,109 +1684,148 @@ if (descriptions.length) {
               })
 
             describe.only("datetime - date only", () => {
-              const JAN_1ST = "2020-01-01"
-              const JAN_10TH = "2020-01-10"
-              const JAN_30TH = "2020-01-30"
-              const UNEXISTING_DATE = "2020-01-03"
-              const NULL_DATE__ID = `null_date__id`
+              describe.each([true, false])(
+                "saved with timestamp: %s",
+                saveWithTimestamp => {
+                  describe.each([true, false])(
+                    "search with timestamp: %s",
+                    searchWithTimestamp => {
+                      const SAVE_SUFFIX = saveWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
+                      const SEARCH_SUFFIX = searchWithTimestamp
+                        ? "T00:00:00.000Z"
+                        : ""
 
-              beforeAll(async () => {
-                tableOrViewId = await createTableOrView({
-                  dateid: { name: "dateid", type: FieldType.STRING },
-                  date: {
-                    name: "date",
-                    type: FieldType.DATETIME,
-                    dateOnly: true,
-                  },
-                })
+                      const JAN_1ST = `2020-01-01`
+                      const JAN_10TH = `2020-01-10`
+                      const JAN_30TH = `2020-01-30`
+                      const UNEXISTING_DATE = `2020-01-03`
+                      const NULL_DATE__ID = `null_date__id`
 
-                await createRows([
-                  { dateid: NULL_DATE__ID, date: null },
-                  { date: JAN_1ST },
-                  { date: `${JAN_10TH}T00:00:00.000Z` },
-                ])
-              })
+                      beforeAll(async () => {
+                        tableOrViewId = await createTableOrView({
+                          dateid: { name: "dateid", type: FieldType.STRING },
+                          date: {
+                            name: "date",
+                            type: FieldType.DATETIME,
+                            dateOnly: true,
+                          },
+                        })
 
-              describe("equal", () => {
-                it("successfully finds a row", async () => {
-                  await expectQuery({
-                    equal: { date: JAN_1ST },
-                  }).toContainExactly([{ date: JAN_1ST }])
-                })
+                        await createRows([
+                          { dateid: NULL_DATE__ID, date: null },
+                          { date: `${JAN_1ST}${SAVE_SUFFIX}` },
+                          { date: `${JAN_10TH}${SAVE_SUFFIX}` },
+                        ])
+                      })
 
-                it("successfully finds an ISO8601 row", async () => {
-                  await expectQuery({
-                    equal: { date: JAN_10TH },
-                  }).toContainExactly([{ date: JAN_10TH }])
-                })
+                      describe("equal", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
 
-                it("finds a row with ISO8601 timestamp", async () => {
-                  await expectQuery({
-                    equal: { date: `${JAN_1ST}T00:00:00.000Z` },
-                  }).toContainExactly([{ date: JAN_1ST }])
-                })
+                        it("successfully finds an ISO8601 row", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_10TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_10TH }])
+                        })
 
-                it("fails to find nonexistent row", async () => {
-                  await expectQuery({
-                    equal: { date: UNEXISTING_DATE },
-                  }).toFindNothing()
-                })
-              })
+                        it("finds a row with ISO8601 timestamp", async () => {
+                          await expectQuery({
+                            equal: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
 
-              describe("notEqual", () => {
-                it("successfully finds a row", async () => {
-                  await expectQuery({
-                    notEqual: { date: JAN_1ST },
-                  }).toContainExactly([
-                    { date: JAN_10TH },
-                    { dateid: NULL_DATE__ID },
-                  ])
-                })
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            equal: {
+                              date: `${UNEXISTING_DATE}${SEARCH_SUFFIX}`,
+                            },
+                          }).toFindNothing()
+                        })
+                      })
 
-                it("fails to find nonexistent row", async () => {
-                  await expectQuery({
-                    notEqual: { date: JAN_30TH },
-                  }).toContainExactly([
-                    { date: JAN_1ST },
-                    { date: JAN_10TH },
-                    { dateid: NULL_DATE__ID },
-                  ])
-                })
-              })
+                      describe("notEqual", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_1ST}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
 
-              describe("oneOf", () => {
-                it("successfully finds a row", async () => {
-                  await expectQuery({
-                    oneOf: { date: [JAN_1ST] },
-                  }).toContainExactly([{ date: JAN_1ST }])
-                })
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            notEqual: { date: `${JAN_30TH}${SEARCH_SUFFIX}` },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                            { dateid: NULL_DATE__ID },
+                          ])
+                        })
+                      })
 
-                it("fails to find nonexistent row", async () => {
-                  await expectQuery({
-                    oneOf: { date: [UNEXISTING_DATE] },
-                  }).toFindNothing()
-                })
-              })
+                      describe("oneOf", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            oneOf: { date: [`${JAN_1ST}${SEARCH_SUFFIX}`] },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
 
-              describe("range", () => {
-                it("successfully finds a row", async () => {
-                  await expectQuery({
-                    range: { date: { low: JAN_1ST, high: JAN_1ST } },
-                  }).toContainExactly([{ date: JAN_1ST }])
-                })
+                        it("fails to find nonexistent row", async () => {
+                          await expectQuery({
+                            oneOf: {
+                              date: [`${UNEXISTING_DATE}${SEARCH_SUFFIX}`],
+                            },
+                          }).toFindNothing()
+                        })
+                      })
 
-                it("successfully finds multiple rows", async () => {
-                  await expectQuery({
-                    range: { date: { low: JAN_1ST, high: JAN_10TH } },
-                  }).toContainExactly([{ date: JAN_1ST }, { date: JAN_10TH }])
-                })
+                      describe("range", () => {
+                        it("successfully finds a row", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([{ date: JAN_1ST }])
+                        })
 
-                it("successfully finds no rows", async () => {
-                  await expectQuery({
-                    range: { date: { low: JAN_30TH, high: JAN_30TH } },
-                  }).toFindNothing()
-                })
-              })
+                        it("successfully finds multiple rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_1ST}${SEARCH_SUFFIX}`,
+                                high: `${JAN_10TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toContainExactly([
+                            { date: JAN_1ST },
+                            { date: JAN_10TH },
+                          ])
+                        })
+
+                        it("successfully finds no rows", async () => {
+                          await expectQuery({
+                            range: {
+                              date: {
+                                low: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                                high: `${JAN_30TH}${SEARCH_SUFFIX}`,
+                              },
+                            },
+                          }).toFindNothing()
+                        })
+                      })
+                    }
+                  )
+                }
+              )
             })
 
             isInternal &&

From b9a5a71fd8e8be5282f459280c140effa812139b Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 17:17:24 +0000
Subject: [PATCH 086/163] Remove focus test.

---
 packages/server/src/api/routes/tests/search.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 69c68aad97..4de92f21e5 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1683,7 +1683,7 @@ if (descriptions.length) {
                 })
               })
 
-            describe.only("datetime - date only", () => {
+            describe("datetime - date only", () => {
               describe.each([true, false])(
                 "saved with timestamp: %s",
                 saveWithTimestamp => {

From d15a5f330c4d8bb3c9b2dc9269b21554cf8145b6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Thu, 9 Jan 2025 17:25:31 +0000
Subject: [PATCH 087/163] Fix row.spec.ts

---
 packages/server/src/api/routes/tests/row.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 968ce9c798..e5cd54e5a5 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -2341,7 +2341,7 @@ if (descriptions.length) {
               [FieldType.ARRAY]: ["options 2", "options 4"],
               [FieldType.NUMBER]: generator.natural(),
               [FieldType.BOOLEAN]: generator.bool(),
-              [FieldType.DATETIME]: generator.date().toISOString(),
+              [FieldType.DATETIME]: generator.date().toISOString().slice(0, 10),
               [FieldType.ATTACHMENTS]: [setup.structures.basicAttachment()],
               [FieldType.ATTACHMENT_SINGLE]: setup.structures.basicAttachment(),
               [FieldType.FORMULA]: undefined, // generated field

From 40192097d819564573343d47cd85a63b00d8be39 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:32:15 +0100
Subject: [PATCH 088/163] Pass datasource to instance

---
 packages/client/src/utils/schema.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index ffab142cf3..988c2e3316 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -31,7 +31,7 @@ const getDatasourceFetchInstance = datasource => {
   if (!handler) {
     return null
   }
-  return new handler({ API })
+  return new handler({ API, datasource })
 }
 
 /**

From cbad9303d962f3b118186099aaf5c50f092b9668 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:24:59 +0100
Subject: [PATCH 089/163] Fix getSchema from forms

---
 packages/client/src/utils/schema.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index 988c2e3316..60800afc53 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -52,7 +52,7 @@ export const fetchDatasourceSchema = async (
   // Get the normal schema as long as we aren't wanting a form schema
   let schema
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(datasource, definition)
+    schema = instance.getSchema(definition)
   } else if (definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {

From aadc4da0b124cbdeaae04f85a008f41160dcb67e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:15:34 +0100
Subject: [PATCH 090/163] Remove duplicated utils

---
 .../src/components/grid/lib/utils.js          | 32 -------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 packages/frontend-core/src/components/grid/lib/utils.js

diff --git a/packages/frontend-core/src/components/grid/lib/utils.js b/packages/frontend-core/src/components/grid/lib/utils.js
deleted file mode 100644
index ee74a14bf0..0000000000
--- a/packages/frontend-core/src/components/grid/lib/utils.js
+++ /dev/null
@@ -1,32 +0,0 @@
-// TODO: remove when all stores are typed
-
-import { GeneratedIDPrefix, CellIDSeparator } from "./constants"
-import { Helpers } from "@budibase/bbui"
-
-export const parseCellID = cellId => {
-  if (!cellId) {
-    return { rowId: undefined, field: undefined }
-  }
-  const parts = cellId.split(CellIDSeparator)
-  const field = parts.pop()
-  return { rowId: parts.join(CellIDSeparator), field }
-}
-
-export const getCellID = (rowId, fieldName) => {
-  return `${rowId}${CellIDSeparator}${fieldName}`
-}
-
-export const parseEventLocation = e => {
-  return {
-    x: e.clientX ?? e.touches?.[0]?.clientX,
-    y: e.clientY ?? e.touches?.[0]?.clientY,
-  }
-}
-
-export const generateRowID = () => {
-  return `${GeneratedIDPrefix}${Helpers.uuid()}`
-}
-
-export const isGeneratedRowID = id => {
-  return id?.startsWith(GeneratedIDPrefix)
-}

From 96052679cb5bbdda0b082ee6c471b6ce7cbaff06 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:26:24 +0100
Subject: [PATCH 091/163] Convert constants.js

---
 .../src/components/grid/lib/{constants.js => constants.ts}        | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename packages/frontend-core/src/components/grid/lib/{constants.js => constants.ts} (100%)

diff --git a/packages/frontend-core/src/components/grid/lib/constants.js b/packages/frontend-core/src/components/grid/lib/constants.ts
similarity index 100%
rename from packages/frontend-core/src/components/grid/lib/constants.js
rename to packages/frontend-core/src/components/grid/lib/constants.ts

From bb1811558570b1f29d3bd064640c6249df818e81 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:26:56 +0100
Subject: [PATCH 092/163] Convert renderer

---
 .../grid/lib/{renderers.js => renderers.ts}    | 18 ++++++++++++++----
 packages/types/src/ui/stores/grid/columns.ts   |  1 +
 2 files changed, 15 insertions(+), 4 deletions(-)
 rename packages/frontend-core/src/components/grid/lib/{renderers.js => renderers.ts} (82%)

diff --git a/packages/frontend-core/src/components/grid/lib/renderers.js b/packages/frontend-core/src/components/grid/lib/renderers.ts
similarity index 82%
rename from packages/frontend-core/src/components/grid/lib/renderers.js
rename to packages/frontend-core/src/components/grid/lib/renderers.ts
index a860d01b53..b009806cc4 100644
--- a/packages/frontend-core/src/components/grid/lib/renderers.js
+++ b/packages/frontend-core/src/components/grid/lib/renderers.ts
@@ -1,4 +1,4 @@
-import { FieldType } from "@budibase/types"
+import { FieldType, UIColumn } from "@budibase/types"
 
 import OptionsCell from "../cells/OptionsCell.svelte"
 import DateCell from "../cells/DateCell.svelte"
@@ -40,13 +40,23 @@ const TypeComponentMap = {
   // Custom types for UI only
   role: RoleCell,
 }
-export const getCellRenderer = column => {
+
+function getCellRendererByType(type: FieldType | "role" | undefined) {
+  if (!type) {
+    return
+  }
+
+  return TypeComponentMap[type as keyof typeof TypeComponentMap]
+}
+
+export const getCellRenderer = (column: UIColumn) => {
   if (column.calculationType) {
     return NumberCell
   }
+
   return (
-    TypeComponentMap[column?.schema?.cellRenderType] ||
-    TypeComponentMap[column?.schema?.type] ||
+    getCellRendererByType(column.schema?.cellRenderType) ||
+    getCellRendererByType(column.schema?.type) ||
     TextCell
   )
 }
diff --git a/packages/types/src/ui/stores/grid/columns.ts b/packages/types/src/ui/stores/grid/columns.ts
index 7f20145246..2517d2a3e0 100644
--- a/packages/types/src/ui/stores/grid/columns.ts
+++ b/packages/types/src/ui/stores/grid/columns.ts
@@ -14,6 +14,7 @@ export type UIColumn = FieldSchema & {
     type: FieldType
     readonly: boolean
     autocolumn: boolean
+    cellRenderType?: FieldType | "role"
   }
   calculationType: CalculationType
   __idx: number

From e23be5acb8ccb0d0425b39d14180817355bc7e6c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:30:19 +0100
Subject: [PATCH 093/163] Websocket to ts

---
 .../components/grid/lib/{websocket.js => websocket.ts}    | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
 rename packages/frontend-core/src/components/grid/lib/{websocket.js => websocket.ts} (89%)

diff --git a/packages/frontend-core/src/components/grid/lib/websocket.js b/packages/frontend-core/src/components/grid/lib/websocket.ts
similarity index 89%
rename from packages/frontend-core/src/components/grid/lib/websocket.js
rename to packages/frontend-core/src/components/grid/lib/websocket.ts
index e7b89ff58a..437b93270d 100644
--- a/packages/frontend-core/src/components/grid/lib/websocket.js
+++ b/packages/frontend-core/src/components/grid/lib/websocket.ts
@@ -1,12 +1,14 @@
 import { get } from "svelte/store"
 import { createWebsocket } from "../../../utils"
 import { SocketEvent, GridSocketEvent } from "@budibase/shared-core"
+import { Store } from "../stores"
+import { UIDatasource } from "@budibase/types"
 
-export const createGridWebsocket = context => {
+export const createGridWebsocket = (context: Store) => {
   const { rows, datasource, users, focusedCellId, definition, API } = context
   const socket = createWebsocket("/socket/grid")
 
-  const connectToDatasource = datasource => {
+  const connectToDatasource = (datasource: UIDatasource) => {
     if (!socket.connected) {
       return
     }
@@ -65,7 +67,7 @@ export const createGridWebsocket = context => {
     GridSocketEvent.DatasourceChange,
     ({ datasource: newDatasource }) => {
       // Listen builder renames, as these aren't handled otherwise
-      if (newDatasource?.name !== get(definition).name) {
+      if (newDatasource?.name !== get(definition)?.name) {
         definition.set(newDatasource)
       }
     }

From 535f2d68031efcb84f51493644e2d6c2fa8d7de7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:45:35 +0100
Subject: [PATCH 094/163] Type

---
 packages/frontend-core/src/components/grid/lib/websocket.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/lib/websocket.ts b/packages/frontend-core/src/components/grid/lib/websocket.ts
index 437b93270d..bc41d594f4 100644
--- a/packages/frontend-core/src/components/grid/lib/websocket.ts
+++ b/packages/frontend-core/src/components/grid/lib/websocket.ts
@@ -2,7 +2,7 @@ import { get } from "svelte/store"
 import { createWebsocket } from "../../../utils"
 import { SocketEvent, GridSocketEvent } from "@budibase/shared-core"
 import { Store } from "../stores"
-import { UIDatasource } from "@budibase/types"
+import { UIDatasource, UIUser } from "@budibase/types"
 
 export const createGridWebsocket = (context: Store) => {
   const { rows, datasource, users, focusedCellId, definition, API } = context
@@ -20,7 +20,7 @@ export const createGridWebsocket = (context: Store) => {
         datasource,
         appId,
       },
-      ({ users: gridUsers }) => {
+      ({ users: gridUsers }: { users: UIUser[] }) => {
         users.set(gridUsers)
       }
     )

From 7d9debd3191668da83dac17c0f0b551a17cea0b5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:49:07 +0100
Subject: [PATCH 095/163] Export DataFetchMap

---
 packages/frontend-core/src/fetch/index.ts | 14 +++++++-------
 packages/frontend-core/src/index.ts       |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts
index c1f35abef2..e343b351bc 100644
--- a/packages/frontend-core/src/fetch/index.ts
+++ b/packages/frontend-core/src/fetch/index.ts
@@ -1,18 +1,18 @@
-import TableFetch from "./TableFetch.js"
-import ViewFetch from "./ViewFetch.js"
-import ViewV2Fetch from "./ViewV2Fetch.js"
+import TableFetch from "./TableFetch"
+import ViewFetch from "./ViewFetch"
+import ViewV2Fetch from "./ViewV2Fetch"
 import QueryFetch from "./QueryFetch"
 import RelationshipFetch from "./RelationshipFetch"
 import NestedProviderFetch from "./NestedProviderFetch"
 import FieldFetch from "./FieldFetch"
 import JSONArrayFetch from "./JSONArrayFetch"
-import UserFetch from "./UserFetch.js"
+import UserFetch from "./UserFetch"
 import GroupUserFetch from "./GroupUserFetch"
 import CustomFetch from "./CustomFetch"
-import QueryArrayFetch from "./QueryArrayFetch.js"
-import { APIClient } from "../api/types.js"
+import QueryArrayFetch from "./QueryArrayFetch"
+import { APIClient } from "../api/types"
 
-const DataFetchMap = {
+export const DataFetchMap = {
   table: TableFetch,
   view: ViewFetch,
   viewV2: ViewV2Fetch,
diff --git a/packages/frontend-core/src/index.ts b/packages/frontend-core/src/index.ts
index 37951dc776..5b24a9ac50 100644
--- a/packages/frontend-core/src/index.ts
+++ b/packages/frontend-core/src/index.ts
@@ -1,5 +1,5 @@
 export { createAPIClient } from "./api"
-export { fetchData } from "./fetch"
+export { fetchData, DataFetchMap } from "./fetch"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"

From 7df69f154a594a5250b511ef62de29a2063870ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 14:50:21 +0100
Subject: [PATCH 096/163] Basic ts conversion

---
 packages/client/src/sdk.js                    |  2 +-
 .../client/src/utils/{schema.js => schema.ts} | 24 +++----------------
 2 files changed, 4 insertions(+), 22 deletions(-)
 rename packages/client/src/utils/{schema.js => schema.ts} (73%)

diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 40d066f2ee..40f35b2ba7 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -29,7 +29,7 @@ import { ActionTypes } from "./constants"
 import {
   fetchDatasourceSchema,
   fetchDatasourceDefinition,
-} from "./utils/schema.js"
+} from "./utils/schema.ts"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"
diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.ts
similarity index 73%
rename from packages/client/src/utils/schema.js
rename to packages/client/src/utils/schema.ts
index 60800afc53..53b31b5495 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.ts
@@ -1,13 +1,5 @@
 import { API } from "api"
-import TableFetch from "@budibase/frontend-core/src/fetch/TableFetch"
-import ViewFetch from "@budibase/frontend-core/src/fetch/ViewFetch"
-import QueryFetch from "@budibase/frontend-core/src/fetch/QueryFetch"
-import RelationshipFetch from "@budibase/frontend-core/src/fetch/RelationshipFetch"
-import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProviderFetch"
-import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch"
-import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch"
-import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch"
-import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
+import { DataFetchMap } from "@budibase/frontend-core"
 
 /**
  * Constructs a fetch instance for a given datasource.
@@ -16,18 +8,8 @@ import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
  * @param datasource the datasource
  * @returns
  */
-const getDatasourceFetchInstance = datasource => {
-  const handler = {
-    table: TableFetch,
-    view: ViewFetch,
-    viewV2: ViewV2Fetch,
-    query: QueryFetch,
-    link: RelationshipFetch,
-    provider: NestedProviderFetch,
-    field: FieldFetch,
-    jsonarray: JSONArrayFetch,
-    queryarray: QueryArrayFetch,
-  }[datasource?.type]
+const getDatasourceFetchInstance = (datasource: { type: string }) => {
+  const handler = DataFetchMap[datasource?.type]
   if (!handler) {
     return null
   }

From e25f26d28d9d41bb2e46f7e0823e87b19c3c3986 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 15:04:56 +0100
Subject: [PATCH 097/163] Reuse type

---
 packages/frontend-core/src/fetch/index.ts | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts
index e343b351bc..21832e3ede 100644
--- a/packages/frontend-core/src/fetch/index.ts
+++ b/packages/frontend-core/src/fetch/index.ts
@@ -12,6 +12,8 @@ import CustomFetch from "./CustomFetch"
 import QueryArrayFetch from "./QueryArrayFetch"
 import { APIClient } from "../api/types"
 
+export type DataFetchType = keyof typeof DataFetchMap
+
 export const DataFetchMap = {
   table: TableFetch,
   view: ViewFetch,
@@ -31,8 +33,7 @@ export const DataFetchMap = {
 
 // Constructs a new fetch model for a certain datasource
 export const fetchData = ({ API, datasource, options }: any) => {
-  const Fetch =
-    DataFetchMap[datasource?.type as keyof typeof DataFetchMap] || TableFetch
+  const Fetch = DataFetchMap[datasource?.type as DataFetchType] || TableFetch
   const fetch = new Fetch({ API, datasource, ...options })
 
   // Initially fetch data but don't bother waiting for the result
@@ -43,18 +44,14 @@ export const fetchData = ({ API, datasource, options }: any) => {
 
 // Creates an empty fetch instance with no datasource configured, so no data
 // will initially be loaded
-const createEmptyFetchInstance = <
-  TDatasource extends {
-    type: keyof typeof DataFetchMap
-  }
->({
+const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
   API,
   datasource,
 }: {
   API: APIClient
   datasource: TDatasource
 }) => {
-  const handler = DataFetchMap[datasource?.type as keyof typeof DataFetchMap]
+  const handler = DataFetchMap[datasource?.type as DataFetchType]
   if (!handler) {
     return null
   }
@@ -63,9 +60,7 @@ const createEmptyFetchInstance = <
 
 // Fetches the definition of any type of datasource
 export const getDatasourceDefinition = async <
-  TDatasource extends {
-    type: keyof typeof DataFetchMap
-  }
+  TDatasource extends { type: DataFetchType }
 >({
   API,
   datasource,
@@ -79,9 +74,7 @@ export const getDatasourceDefinition = async <
 
 // Fetches the schema of any type of datasource
 export const getDatasourceSchema = <
-  TDatasource extends {
-    type: keyof typeof DataFetchMap
-  }
+  TDatasource extends { type: DataFetchType }
 >({
   API,
   datasource,

From 4f06592685522e7d1d58af1a43e7549b24e9002a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 15:23:20 +0100
Subject: [PATCH 098/163] Use trimmed and typed datasources

---
 packages/frontend-core/src/fetch/DataFetch.ts        |  3 ++-
 packages/frontend-core/src/fetch/FieldFetch.ts       |  9 ++++++---
 packages/frontend-core/src/fetch/GroupUserFetch.ts   |  2 ++
 packages/frontend-core/src/fetch/JSONArrayFetch.ts   |  2 +-
 .../frontend-core/src/fetch/NestedProviderFetch.ts   |  1 +
 packages/frontend-core/src/fetch/QueryArrayFetch.ts  |  2 +-
 packages/frontend-core/src/fetch/QueryFetch.ts       |  1 +
 .../frontend-core/src/fetch/RelationshipFetch.ts     |  1 +
 packages/frontend-core/src/fetch/TableFetch.ts       |  9 +++++++--
 packages/frontend-core/src/fetch/UserFetch.ts        |  8 ++++++--
 packages/frontend-core/src/fetch/ViewFetch.ts        | 11 +++++++++--
 packages/frontend-core/src/fetch/ViewV2Fetch.ts      | 12 ++++++++++--
 packages/frontend-core/src/fetch/index.ts            |  8 ++++++--
 packages/frontend-core/src/index.ts                  |  1 +
 14 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts
index 0079fec057..8f475339b4 100644
--- a/packages/frontend-core/src/fetch/DataFetch.ts
+++ b/packages/frontend-core/src/fetch/DataFetch.ts
@@ -13,6 +13,7 @@ import {
   UISearchFilter,
 } from "@budibase/types"
 import { APIClient } from "../api/types"
+import { DataFetchType } from "."
 
 const { buildQuery, limit: queryLimit, runQuery, sort } = QueryUtils
 
@@ -59,7 +60,7 @@ export interface DataFetchParams<
  * For other types of datasource, this class is overridden and extended.
  */
 export default abstract class DataFetch<
-  TDatasource extends {},
+  TDatasource extends { type: DataFetchType },
   TDefinition extends {
     schema?: Record<string, any> | null
     primaryDisplay?: string
diff --git a/packages/frontend-core/src/fetch/FieldFetch.ts b/packages/frontend-core/src/fetch/FieldFetch.ts
index ac1e683c51..694443a5dc 100644
--- a/packages/frontend-core/src/fetch/FieldFetch.ts
+++ b/packages/frontend-core/src/fetch/FieldFetch.ts
@@ -1,7 +1,10 @@
 import { Row } from "@budibase/types"
 import DataFetch from "./DataFetch"
 
-export interface FieldDatasource {
+type Types = "field" | "queryarray" | "jsonarray"
+
+export interface FieldDatasource<TType extends Types> {
+  type: TType
   tableId: string
   fieldType: "attachment" | "array"
   value: string[] | Row[]
@@ -15,8 +18,8 @@ function isArrayOfStrings(value: string[] | Row[]): value is string[] {
   return Array.isArray(value) && !!value[0] && typeof value[0] !== "object"
 }
 
-export default class FieldFetch extends DataFetch<
-  FieldDatasource,
+export default class FieldFetch<TType extends Types> extends DataFetch<
+  FieldDatasource<TType>,
   FieldDefinition
 > {
   async getDefinition(): Promise<FieldDefinition | null> {
diff --git a/packages/frontend-core/src/fetch/GroupUserFetch.ts b/packages/frontend-core/src/fetch/GroupUserFetch.ts
index a14623bfb0..e07e5331d4 100644
--- a/packages/frontend-core/src/fetch/GroupUserFetch.ts
+++ b/packages/frontend-core/src/fetch/GroupUserFetch.ts
@@ -8,6 +8,7 @@ interface GroupUserQuery {
 }
 
 interface GroupUserDatasource {
+  type: "groupUser"
   tableId: TableNames.USERS
 }
 
@@ -20,6 +21,7 @@ export default class GroupUserFetch extends DataFetch<
     super({
       ...opts,
       datasource: {
+        type: "groupUser",
         tableId: TableNames.USERS,
       },
     })
diff --git a/packages/frontend-core/src/fetch/JSONArrayFetch.ts b/packages/frontend-core/src/fetch/JSONArrayFetch.ts
index cae9a1e521..d746c923f8 100644
--- a/packages/frontend-core/src/fetch/JSONArrayFetch.ts
+++ b/packages/frontend-core/src/fetch/JSONArrayFetch.ts
@@ -1,7 +1,7 @@
 import FieldFetch from "./FieldFetch"
 import { getJSONArrayDatasourceSchema } from "../utils/json"
 
-export default class JSONArrayFetch extends FieldFetch {
+export default class JSONArrayFetch extends FieldFetch<"jsonarray"> {
   async getDefinition() {
     const { datasource } = this.options
 
diff --git a/packages/frontend-core/src/fetch/NestedProviderFetch.ts b/packages/frontend-core/src/fetch/NestedProviderFetch.ts
index 666340610f..af121fcef8 100644
--- a/packages/frontend-core/src/fetch/NestedProviderFetch.ts
+++ b/packages/frontend-core/src/fetch/NestedProviderFetch.ts
@@ -2,6 +2,7 @@ import { Row, TableSchema } from "@budibase/types"
 import DataFetch from "./DataFetch"
 
 interface NestedProviderDatasource {
+  type: "provider"
   value?: {
     schema: TableSchema
     primaryDisplay: string
diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.ts b/packages/frontend-core/src/fetch/QueryArrayFetch.ts
index 9142000fe6..7f4d34aaa6 100644
--- a/packages/frontend-core/src/fetch/QueryArrayFetch.ts
+++ b/packages/frontend-core/src/fetch/QueryArrayFetch.ts
@@ -4,7 +4,7 @@ import {
   generateQueryArraySchemas,
 } from "../utils/json"
 
-export default class QueryArrayFetch extends FieldFetch {
+export default class QueryArrayFetch extends FieldFetch<"queryarray"> {
   async getDefinition() {
     const { datasource } = this.options
 
diff --git a/packages/frontend-core/src/fetch/QueryFetch.ts b/packages/frontend-core/src/fetch/QueryFetch.ts
index 0754edd267..09dde86cbd 100644
--- a/packages/frontend-core/src/fetch/QueryFetch.ts
+++ b/packages/frontend-core/src/fetch/QueryFetch.ts
@@ -4,6 +4,7 @@ import { ExecuteQueryRequest, Query } from "@budibase/types"
 import { get } from "svelte/store"
 
 interface QueryDatasource {
+  type: "query"
   _id: string
   fields: Record<string, any> & {
     pagination?: {
diff --git a/packages/frontend-core/src/fetch/RelationshipFetch.ts b/packages/frontend-core/src/fetch/RelationshipFetch.ts
index f853a753cd..89a85ab0e4 100644
--- a/packages/frontend-core/src/fetch/RelationshipFetch.ts
+++ b/packages/frontend-core/src/fetch/RelationshipFetch.ts
@@ -2,6 +2,7 @@ import { Table } from "@budibase/types"
 import DataFetch from "./DataFetch"
 
 interface RelationshipDatasource {
+  type: "link"
   tableId: string
   rowId: string
   rowTableId: string
diff --git a/packages/frontend-core/src/fetch/TableFetch.ts b/packages/frontend-core/src/fetch/TableFetch.ts
index f5927262cb..67cac6b6a7 100644
--- a/packages/frontend-core/src/fetch/TableFetch.ts
+++ b/packages/frontend-core/src/fetch/TableFetch.ts
@@ -1,8 +1,13 @@
 import { get } from "svelte/store"
 import DataFetch from "./DataFetch"
-import { SortOrder, Table, UITable } from "@budibase/types"
+import { SortOrder, Table } from "@budibase/types"
 
-export default class TableFetch extends DataFetch<UITable, Table> {
+interface TableDatasource {
+  type: "table"
+  tableId: string
+}
+
+export default class TableFetch extends DataFetch<TableDatasource, Table> {
   async determineFeatureFlags() {
     return {
       supportsSearch: true,
diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts
index 656cd840fe..54147fbccf 100644
--- a/packages/frontend-core/src/fetch/UserFetch.ts
+++ b/packages/frontend-core/src/fetch/UserFetch.ts
@@ -14,18 +14,22 @@ interface UserFetchQuery {
 }
 
 interface UserDatasource {
-  tableId: string
+  type: "user"
+  tableId: TableNames.USERS
 }
 
+interface UserDefinition {}
+
 export default class UserFetch extends DataFetch<
   UserDatasource,
-  {},
+  UserDefinition,
   UserFetchQuery
 > {
   constructor(opts: DataFetchParams<UserDatasource, UserFetchQuery>) {
     super({
       ...opts,
       datasource: {
+        type: "user",
         tableId: TableNames.USERS,
       },
     })
diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts
index b6830e7118..c075d80ce0 100644
--- a/packages/frontend-core/src/fetch/ViewFetch.ts
+++ b/packages/frontend-core/src/fetch/ViewFetch.ts
@@ -1,7 +1,14 @@
-import { Table, View } from "@budibase/types"
+import { Table } from "@budibase/types"
 import DataFetch from "./DataFetch"
 
-type ViewV1 = View & { name: string }
+type ViewV1 = {
+  type: "view"
+  name: string
+  tableId: string
+  calculation: string
+  field: string
+  groupBy: string
+}
 
 export default class ViewFetch extends DataFetch<ViewV1, Table> {
   async getDefinition() {
diff --git a/packages/frontend-core/src/fetch/ViewV2Fetch.ts b/packages/frontend-core/src/fetch/ViewV2Fetch.ts
index cdd3bab6ed..aa5fbd60a2 100644
--- a/packages/frontend-core/src/fetch/ViewV2Fetch.ts
+++ b/packages/frontend-core/src/fetch/ViewV2Fetch.ts
@@ -1,9 +1,17 @@
-import { SortOrder, UIView, ViewV2, ViewV2Type } from "@budibase/types"
+import { SortOrder, ViewV2Enriched, ViewV2Type } from "@budibase/types"
 import DataFetch from "./DataFetch"
 import { get } from "svelte/store"
 import { helpers } from "@budibase/shared-core"
 
-export default class ViewV2Fetch extends DataFetch<UIView, ViewV2> {
+interface ViewDatasource {
+  type: "viewV2"
+  id: string
+}
+
+export default class ViewV2Fetch extends DataFetch<
+  ViewDatasource,
+  ViewV2Enriched
+> {
   async determineFeatureFlags() {
     return {
       supportsSearch: true,
diff --git a/packages/frontend-core/src/fetch/index.ts b/packages/frontend-core/src/fetch/index.ts
index 21832e3ede..d80aa10df6 100644
--- a/packages/frontend-core/src/fetch/index.ts
+++ b/packages/frontend-core/src/fetch/index.ts
@@ -26,7 +26,7 @@ export const DataFetchMap = {
 
   // Client specific datasource types
   provider: NestedProviderFetch,
-  field: FieldFetch,
+  field: FieldFetch<"field">,
   jsonarray: JSONArrayFetch,
   queryarray: QueryArrayFetch,
 }
@@ -55,7 +55,11 @@ const createEmptyFetchInstance = <TDatasource extends { type: DataFetchType }>({
   if (!handler) {
     return null
   }
-  return new handler({ API, datasource: null as any, query: null as any })
+  return new handler({
+    API,
+    datasource: null as never,
+    query: null as any,
+  })
 }
 
 // Fetches the definition of any type of datasource
diff --git a/packages/frontend-core/src/index.ts b/packages/frontend-core/src/index.ts
index 5b24a9ac50..c0baa63ab6 100644
--- a/packages/frontend-core/src/index.ts
+++ b/packages/frontend-core/src/index.ts
@@ -1,5 +1,6 @@
 export { createAPIClient } from "./api"
 export { fetchData, DataFetchMap } from "./fetch"
+export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"
 export * from "./stores"
 export * from "./utils"

From eb73370460f9b3fb98303f086bf21e29a1ae9bc7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 9 Jan 2025 15:35:16 +0100
Subject: [PATCH 099/163] Fix types

---
 packages/client/src/utils/schema.ts           | 56 +++++++++++++------
 packages/frontend-core/src/fetch/DataFetch.ts |  2 +-
 2 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/packages/client/src/utils/schema.ts b/packages/client/src/utils/schema.ts
index 53b31b5495..5400d62087 100644
--- a/packages/client/src/utils/schema.ts
+++ b/packages/client/src/utils/schema.ts
@@ -1,5 +1,5 @@
 import { API } from "api"
-import { DataFetchMap } from "@budibase/frontend-core"
+import { DataFetchMap, DataFetchType } from "@budibase/frontend-core"
 
 /**
  * Constructs a fetch instance for a given datasource.
@@ -8,12 +8,20 @@ import { DataFetchMap } from "@budibase/frontend-core"
  * @param datasource the datasource
  * @returns
  */
-const getDatasourceFetchInstance = (datasource: { type: string }) => {
+const getDatasourceFetchInstance = <
+  TDatasource extends { type: DataFetchType }
+>(
+  datasource: TDatasource
+) => {
   const handler = DataFetchMap[datasource?.type]
   if (!handler) {
     return null
   }
-  return new handler({ API, datasource })
+  return new handler({
+    API,
+    datasource: datasource as never,
+    query: null as any,
+  })
 }
 
 /**
@@ -21,21 +29,23 @@ const getDatasourceFetchInstance = (datasource: { type: string }) => {
  * @param datasource the datasource to fetch the schema for
  * @param options options for enriching the schema
  */
-export const fetchDatasourceSchema = async (
-  datasource,
+export const fetchDatasourceSchema = async <
+  TDatasource extends { type: DataFetchType }
+>(
+  datasource: TDatasource,
   options = { enrichRelationships: false, formSchema: false }
 ) => {
   const instance = getDatasourceFetchInstance(datasource)
-  const definition = await instance?.getDefinition(datasource)
-  if (!definition) {
+  const definition = await instance?.getDefinition()
+  if (!instance || !definition) {
     return null
   }
 
   // Get the normal schema as long as we aren't wanting a form schema
-  let schema
+  let schema: any
   if (datasource?.type !== "query" || !options?.formSchema) {
-    schema = instance.getSchema(definition)
-  } else if (definition.parameters?.length) {
+    schema = instance.getSchema(definition as any)
+  } else if ("parameters" in definition && definition.parameters?.length) {
     schema = {}
     definition.parameters.forEach(param => {
       schema[param.name] = { ...param, type: "string" }
@@ -55,7 +65,12 @@ export const fetchDatasourceSchema = async (
   }
 
   // Enrich schema with relationships if required
-  if (definition?.sql && options?.enrichRelationships) {
+  if (
+    definition &&
+    "sql" in definition &&
+    definition.sql &&
+    options?.enrichRelationships
+  ) {
     const relationshipAdditions = await getRelationshipSchemaAdditions(schema)
     schema = {
       ...schema,
@@ -71,20 +86,26 @@ export const fetchDatasourceSchema = async (
  * Fetches the definition of any kind of datasource.
  * @param datasource the datasource to fetch the schema for
  */
-export const fetchDatasourceDefinition = async datasource => {
+export const fetchDatasourceDefinition = async <
+  TDatasource extends { type: DataFetchType }
+>(
+  datasource: TDatasource
+) => {
   const instance = getDatasourceFetchInstance(datasource)
-  return await instance?.getDefinition(datasource)
+  return await instance?.getDefinition()
 }
 
 /**
  * Fetches the schema of relationship fields for a SQL table schema
  * @param schema the schema to enrich
  */
-export const getRelationshipSchemaAdditions = async schema => {
+export const getRelationshipSchemaAdditions = async (
+  schema: Record<string, any>
+) => {
   if (!schema) {
     return null
   }
-  let relationshipAdditions = {}
+  let relationshipAdditions: Record<string, any> = {}
   for (let fieldKey of Object.keys(schema)) {
     const fieldSchema = schema[fieldKey]
     if (fieldSchema?.type === "link") {
@@ -92,7 +113,10 @@ export const getRelationshipSchemaAdditions = async schema => {
         type: "table",
         tableId: fieldSchema?.tableId,
       })
-      Object.keys(linkSchema || {}).forEach(linkKey => {
+      if (!linkSchema) {
+        continue
+      }
+      Object.keys(linkSchema).forEach(linkKey => {
         relationshipAdditions[`${fieldKey}.${linkKey}`] = {
           type: linkSchema[linkKey].type,
           externalType: linkSchema[linkKey].externalType,
diff --git a/packages/frontend-core/src/fetch/DataFetch.ts b/packages/frontend-core/src/fetch/DataFetch.ts
index 8f475339b4..b10a8b0a69 100644
--- a/packages/frontend-core/src/fetch/DataFetch.ts
+++ b/packages/frontend-core/src/fetch/DataFetch.ts
@@ -369,7 +369,7 @@ export default abstract class DataFetch<
    * @param schema the datasource schema
    * @return {object} the enriched datasource schema
    */
-  private enrichSchema(schema: TableSchema): TableSchema {
+  enrichSchema(schema: TableSchema): TableSchema {
     // Check for any JSON fields so we can add any top level properties
     let jsonAdditions: Record<string, { type: string; nestedJSON: true }> = {}
     for (const fieldKey of Object.keys(schema)) {

From be41a2ae6c2c087deea46c2d8b1861c7448afec2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 10:07:45 +0100
Subject: [PATCH 100/163] Add type to customDatasource

---
 packages/frontend-core/src/fetch/CustomFetch.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/frontend-core/src/fetch/CustomFetch.ts b/packages/frontend-core/src/fetch/CustomFetch.ts
index afd3d18ba9..dfd29c4a02 100644
--- a/packages/frontend-core/src/fetch/CustomFetch.ts
+++ b/packages/frontend-core/src/fetch/CustomFetch.ts
@@ -1,6 +1,7 @@
 import DataFetch from "./DataFetch"
 
 interface CustomDatasource {
+  type: "custom"
   data: any
 }
 

From 8520ad2aeb9cc0424301950cebd0bc8a597bb2e4 Mon Sep 17 00:00:00 2001
From: Peter Clement <peter@budibase.com>
Date: Fri, 10 Jan 2025 11:31:14 +0000
Subject: [PATCH 101/163] convert queries store to typescript

---
 .../builder/src/stores/builder/queries.js     | 130 ---------------
 .../builder/src/stores/builder/queries.ts     | 156 ++++++++++++++++++
 2 files changed, 156 insertions(+), 130 deletions(-)
 delete mode 100644 packages/builder/src/stores/builder/queries.js
 create mode 100644 packages/builder/src/stores/builder/queries.ts

diff --git a/packages/builder/src/stores/builder/queries.js b/packages/builder/src/stores/builder/queries.js
deleted file mode 100644
index 7aeb9ff8fd..0000000000
--- a/packages/builder/src/stores/builder/queries.js
+++ /dev/null
@@ -1,130 +0,0 @@
-import { writable, get, derived } from "svelte/store"
-import { datasources } from "./datasources"
-import { integrations } from "./integrations"
-import { API } from "@/api"
-import { duplicateName } from "@/helpers/duplicate"
-
-const sortQueries = queryList => {
-  queryList.sort((q1, q2) => {
-    return q1.name.localeCompare(q2.name)
-  })
-}
-
-export function createQueriesStore() {
-  const store = writable({
-    list: [],
-    selectedQueryId: null,
-  })
-  const derivedStore = derived(store, $store => ({
-    ...$store,
-    selected: $store.list?.find(q => q._id === $store.selectedQueryId),
-  }))
-
-  const fetch = async () => {
-    const queries = await API.getQueries()
-    sortQueries(queries)
-    store.update(state => ({
-      ...state,
-      list: queries,
-    }))
-  }
-
-  const save = async (datasourceId, query) => {
-    const _integrations = get(integrations)
-    const dataSource = get(datasources).list.filter(
-      ds => ds._id === datasourceId
-    )
-    // Check if readable attribute is found
-    if (dataSource.length !== 0) {
-      const integration = _integrations[dataSource[0].source]
-      const readable = integration.query[query.queryVerb].readable
-      if (readable) {
-        query.readable = readable
-      }
-    }
-    query.datasourceId = datasourceId
-    const savedQuery = await API.saveQuery(query)
-    store.update(state => {
-      const idx = state.list.findIndex(query => query._id === savedQuery._id)
-      const queries = state.list
-      if (idx >= 0) {
-        queries.splice(idx, 1, savedQuery)
-      } else {
-        queries.push(savedQuery)
-      }
-      sortQueries(queries)
-      return {
-        list: queries,
-        selectedQueryId: savedQuery._id,
-      }
-    })
-    return savedQuery
-  }
-
-  const importQueries = async ({ data, datasourceId }) => {
-    return await API.importQueries(datasourceId, data)
-  }
-
-  const select = id => {
-    store.update(state => ({
-      ...state,
-      selectedQueryId: id,
-    }))
-  }
-
-  const preview = async query => {
-    const result = await API.previewQuery(query)
-    // Assume all the fields are strings and create a basic schema from the
-    // unique fields returned by the server
-    const schema = {}
-    for (let [field, metadata] of Object.entries(result.schema)) {
-      schema[field] = metadata || { type: "string" }
-    }
-    return { ...result, schema, rows: result.rows || [] }
-  }
-
-  const deleteQuery = async query => {
-    await API.deleteQuery(query._id, query._rev)
-    store.update(state => {
-      state.list = state.list.filter(existing => existing._id !== query._id)
-      return state
-    })
-  }
-
-  const duplicate = async query => {
-    let list = get(store).list
-    const newQuery = { ...query }
-    const datasourceId = query.datasourceId
-
-    delete newQuery._id
-    delete newQuery._rev
-    newQuery.name = duplicateName(
-      query.name,
-      list.map(q => q.name)
-    )
-
-    return await save(datasourceId, newQuery)
-  }
-
-  const removeDatasourceQueries = datasourceId => {
-    store.update(state => ({
-      ...state,
-      list: state.list.filter(table => table.datasourceId !== datasourceId),
-    }))
-  }
-
-  return {
-    subscribe: derivedStore.subscribe,
-    fetch,
-    init: fetch,
-    select,
-    save,
-    import: importQueries,
-    delete: deleteQuery,
-    preview,
-    duplicate,
-    removeDatasourceQueries,
-  }
-}
-
-export const queries = createQueriesStore()
diff --git a/packages/builder/src/stores/builder/queries.ts b/packages/builder/src/stores/builder/queries.ts
new file mode 100644
index 0000000000..c6511dc346
--- /dev/null
+++ b/packages/builder/src/stores/builder/queries.ts
@@ -0,0 +1,156 @@
+import { derived, get, Writable } from "svelte/store"
+import { datasources } from "./datasources"
+import { integrations } from "./integrations"
+import { API } from "@/api"
+import { duplicateName } from "@/helpers/duplicate"
+import { DerivedBudiStore } from "@/stores/BudiStore"
+import {
+  Query,
+  QueryPreview,
+  PreviewQueryResponse,
+  SaveQueryRequest,
+  ImportRestQueryRequest,
+  QuerySchema,
+} from "@budibase/types"
+
+const sortQueries = (queryList: Query[]) => {
+  queryList.sort((q1, q2) => {
+    return q1.name.localeCompare(q2.name)
+  })
+}
+
+interface BuilderQueryStore {
+  list: Query[]
+  selectedQueryId: string | null
+}
+
+interface DerivedQueryStore extends BuilderQueryStore {
+  selected?: Query
+}
+
+export class QueryStore extends DerivedBudiStore<
+  BuilderQueryStore,
+  DerivedQueryStore
+> {
+  constructor() {
+    const makeDerivedStore = (store: Writable<BuilderQueryStore>) => {
+      return derived(store, ($store): DerivedQueryStore => {
+        return {
+          list: $store.list,
+          selectedQueryId: $store.selectedQueryId,
+          selected: $store.list?.find(q => q._id === $store.selectedQueryId),
+        }
+      })
+    }
+
+    super(
+      {
+        list: [],
+        selectedQueryId: null,
+      },
+      makeDerivedStore
+    )
+
+    this.select = this.select.bind(this)
+  }
+
+  async fetch() {
+    const queries = await API.getQueries()
+    sortQueries(queries)
+    this.store.update(state => ({
+      ...state,
+      list: queries,
+    }))
+  }
+
+  async save(datasourceId: string, query: SaveQueryRequest) {
+    const _integrations = get(integrations)
+    const dataSource = get(datasources).list.filter(
+      ds => ds._id === datasourceId
+    )
+    // Check if readable attribute is found
+    if (dataSource.length !== 0) {
+      const integration = _integrations[dataSource[0].source]
+      const readable = integration.query[query.queryVerb].readable
+      if (readable) {
+        query.readable = readable
+      }
+    }
+    query.datasourceId = datasourceId
+    const savedQuery = await API.saveQuery(query)
+    this.store.update(state => {
+      const idx = state.list.findIndex(query => query._id === savedQuery._id)
+      const queries = state.list
+      if (idx >= 0) {
+        queries.splice(idx, 1, savedQuery)
+      } else {
+        queries.push(savedQuery)
+      }
+      sortQueries(queries)
+      return {
+        list: queries,
+        selectedQueryId: savedQuery._id || null,
+      }
+    })
+    return savedQuery
+  }
+
+  async importQueries(data: ImportRestQueryRequest) {
+    return await API.importQueries(data)
+  }
+
+  select(id: string | null) {
+    this.store.update(state => ({
+      ...state,
+      selectedQueryId: id,
+    }))
+  }
+
+  async preview(query: QueryPreview): Promise<PreviewQueryResponse> {
+    const result = await API.previewQuery(query)
+    // Assume all the fields are strings and create a basic schema from the
+    // unique fields returned by the server
+    const schema: Record<string, QuerySchema> = {}
+    for (let [field, metadata] of Object.entries(result.schema)) {
+      schema[field] = (metadata as QuerySchema) || { type: "string" }
+    }
+    return { ...result, schema, rows: result.rows || [] }
+  }
+
+  async delete(query: Query) {
+    if (!query._id || !query._rev) {
+      throw new Error("Query ID or Revision is missing")
+    }
+    await API.deleteQuery(query._id, query._rev)
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(existing => existing._id !== query._id),
+    }))
+  }
+
+  async duplicate(query: Query) {
+    let list = get(this.store).list
+    const newQuery = { ...query }
+    const datasourceId = query.datasourceId
+
+    delete newQuery._id
+    delete newQuery._rev
+    newQuery.name = duplicateName(
+      query.name,
+      list.map(q => q.name)
+    )
+
+    return await this.save(datasourceId, newQuery)
+  }
+
+  removeDatasourceQueries(datasourceId: string) {
+    this.store.update(state => ({
+      ...state,
+      list: state.list.filter(table => table.datasourceId !== datasourceId),
+    }))
+  }
+
+  init = this.fetch
+}
+
+export const queries = new QueryStore()

From 616e89716c8ea89bcd1b34b8e16a72463b089924 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 12:47:49 +0100
Subject: [PATCH 102/163] Remove unneeded extension

---
 packages/client/src/sdk.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 40f35b2ba7..68d75d2806 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -29,7 +29,7 @@ import { ActionTypes } from "./constants"
 import {
   fetchDatasourceSchema,
   fetchDatasourceDefinition,
-} from "./utils/schema.ts"
+} from "./utils/schema"
 import { getAPIKey } from "./utils/api.js"
 import { enrichButtonActions } from "./utils/buttonActions.js"
 import { processStringSync, makePropSafe } from "@budibase/string-templates"

From b54eb876d81007c37b918e33f61ded9db8c4ef1f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 12:50:08 +0100
Subject: [PATCH 103/163] Renames

---
 packages/frontend-core/src/fetch/ViewFetch.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/frontend-core/src/fetch/ViewFetch.ts b/packages/frontend-core/src/fetch/ViewFetch.ts
index c075d80ce0..df00b9bbfc 100644
--- a/packages/frontend-core/src/fetch/ViewFetch.ts
+++ b/packages/frontend-core/src/fetch/ViewFetch.ts
@@ -1,7 +1,7 @@
 import { Table } from "@budibase/types"
 import DataFetch from "./DataFetch"
 
-type ViewV1 = {
+type ViewV1Datasource = {
   type: "view"
   name: string
   tableId: string
@@ -10,7 +10,7 @@ type ViewV1 = {
   groupBy: string
 }
 
-export default class ViewFetch extends DataFetch<ViewV1, Table> {
+export default class ViewFetch extends DataFetch<ViewV1Datasource, Table> {
   async getDefinition() {
     const { datasource } = this.options
 

From ed6adbae08cb76f4278084b6d84fc9dcf0f65d45 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 Jan 2025 11:52:48 +0000
Subject: [PATCH 104/163] Bump version to 3.2.38

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 647c9f202d..ff69a18459 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.37",
+  "version": "3.2.38",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

From 6ecb01ae830d7ea94fe14b4e2696cfbb29f82cb1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 14:47:09 +0100
Subject: [PATCH 105/163] Fix close modal on grid+modal generation

---
 packages/frontend-core/src/utils/utils.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/frontend-core/src/utils/utils.js b/packages/frontend-core/src/utils/utils.js
index c424aea5b2..55603b0129 100644
--- a/packages/frontend-core/src/utils/utils.js
+++ b/packages/frontend-core/src/utils/utils.js
@@ -209,6 +209,9 @@ export const buildFormBlockButtonConfig = props => {
     {
       "##eventHandlerType": "Close Side Panel",
     },
+    {
+      "##eventHandlerType": "Close Modal",
+    },
 
     ...(actionUrl
       ? [

From 23025f46396b828e27c356ae21c7bddb473c90c9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 15:19:55 +0100
Subject: [PATCH 106/163] Clean

---
 packages/client/src/api/api.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/api/api.ts b/packages/client/src/api/api.ts
index 93f59f6e9a..b944f7bd7c 100644
--- a/packages/client/src/api/api.ts
+++ b/packages/client/src/api/api.ts
@@ -4,7 +4,7 @@ import {
   notificationStore,
   devToolsEnabled,
   devToolsStore,
-} from "../stores/index.js"
+} from "../stores/index"
 import { get } from "svelte/store"
 
 export const API = createAPIClient({

From 9d0e3a17222f303e1f3a952e7c5e298d7187f070 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 15:22:34 +0100
Subject: [PATCH 107/163] Remove unnecessary extensions

---
 packages/client/src/api/index.js    | 4 ++--
 packages/client/src/stores/index.js | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/api/index.js b/packages/client/src/api/index.js
index a63e19bfbb..3c53045cfd 100644
--- a/packages/client/src/api/index.js
+++ b/packages/client/src/api/index.js
@@ -1,5 +1,5 @@
-import { API } from "./api.ts"
-import { patchAPI } from "./patches.js"
+import { API } from "./api"
+import { patchAPI } from "./patches"
 
 // Certain endpoints which return rows need patched so that they transform
 // and enrich the row docs, so that they can be correctly handled by the
diff --git a/packages/client/src/stores/index.js b/packages/client/src/stores/index.js
index e099434b3d..f2b80ed732 100644
--- a/packages/client/src/stores/index.js
+++ b/packages/client/src/stores/index.js
@@ -1,6 +1,6 @@
 export { authStore } from "./auth"
 export { appStore } from "./app"
-export { notificationStore } from "./notification.ts"
+export { notificationStore } from "./notification"
 export { routeStore } from "./routes"
 export { screenStore } from "./screens"
 export { builderStore } from "./builder"

From 9e2915ff0fc0a7279b95b1b05f9dd8e8baec1cbc Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 16:24:55 +0100
Subject: [PATCH 108/163] Fix wrong conversion

---
 packages/frontend-core/src/fetch/UserFetch.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts
index 656cd840fe..58aa4f5a96 100644
--- a/packages/frontend-core/src/fetch/UserFetch.ts
+++ b/packages/frontend-core/src/fetch/UserFetch.ts
@@ -52,7 +52,7 @@ export default class UserFetch extends DataFetch<
 
     const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
       ? rest
-      : { [BasicOperator.EMPTY]: { email: null } }
+      : { [BasicOperator.STRING]: { email: null as any } }
 
     try {
       const opts: SearchUsersRequest = {

From f54d917f3a1b375948491757298bc71fdb968192 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Fri, 10 Jan 2025 15:27:50 +0000
Subject: [PATCH 109/163] account-portal login v2 feature flag (#15343)

---
 packages/types/src/sdk/featureFlag.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts
index 7b61b70772..996d3bba8d 100644
--- a/packages/types/src/sdk/featureFlag.ts
+++ b/packages/types/src/sdk/featureFlag.ts
@@ -1,9 +1,15 @@
 export enum FeatureFlag {
   USE_ZOD_VALIDATOR = "USE_ZOD_VALIDATOR",
+
+  // Account-portal
+  DIRECT_LOGIN_TO_ACCOUNT_PORTAL = "DIRECT_LOGIN_TO_ACCOUNT_PORTAL",
 }
 
 export const FeatureFlagDefaults = {
   [FeatureFlag.USE_ZOD_VALIDATOR]: false,
+
+  // Account-portal
+  [FeatureFlag.DIRECT_LOGIN_TO_ACCOUNT_PORTAL]: false,
 }
 
 export type FeatureFlags = typeof FeatureFlagDefaults

From e7400f982cc6bab2bada801e94a76b167332b84e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 16:29:42 +0100
Subject: [PATCH 110/163] Add comment

---
 packages/frontend-core/src/fetch/UserFetch.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts
index e8e7dc8721..1fe9c0a383 100644
--- a/packages/frontend-core/src/fetch/UserFetch.ts
+++ b/packages/frontend-core/src/fetch/UserFetch.ts
@@ -56,7 +56,7 @@ export default class UserFetch extends DataFetch<
 
     const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
       ? rest
-      : { [BasicOperator.STRING]: { email: null as any } }
+      : { [BasicOperator.STRING]: { email: null as any } } // TODO: chech. Left as any to not change the behaviour it had when it was js
 
     try {
       const opts: SearchUsersRequest = {

From 337339d6e5366b36d437dc384f29963506cee6df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 16:39:55 +0100
Subject: [PATCH 111/163] Remove confusing empty filter

---
 packages/frontend-core/src/fetch/UserFetch.ts | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/fetch/UserFetch.ts b/packages/frontend-core/src/fetch/UserFetch.ts
index 1fe9c0a383..36aebac506 100644
--- a/packages/frontend-core/src/fetch/UserFetch.ts
+++ b/packages/frontend-core/src/fetch/UserFetch.ts
@@ -2,11 +2,7 @@ import { get } from "svelte/store"
 import DataFetch, { DataFetchParams } from "./DataFetch"
 import { TableNames } from "../constants"
 import { utils } from "@budibase/shared-core"
-import {
-  BasicOperator,
-  SearchFilters,
-  SearchUsersRequest,
-} from "@budibase/types"
+import { SearchFilters, SearchUsersRequest } from "@budibase/types"
 
 interface UserFetchQuery {
   appId: string
@@ -56,7 +52,7 @@ export default class UserFetch extends DataFetch<
 
     const finalQuery: SearchFilters = utils.isSupportedUserSearch(rest)
       ? rest
-      : { [BasicOperator.STRING]: { email: null as any } } // TODO: chech. Left as any to not change the behaviour it had when it was js
+      : {}
 
     try {
       const opts: SearchUsersRequest = {

From 6ab389edde98138f90b85f216cea44b6af804875 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 10 Jan 2025 15:51:18 +0000
Subject: [PATCH 112/163] Bump version to 3.2.39

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index ff69a18459..0dc09b27be 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.38",
+  "version": "3.2.39",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

From 640dd930def662fecec33f3f65afd059e6b2bdad Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 10 Jan 2025 17:45:51 +0000
Subject: [PATCH 113/163] Updating query row to correctly decode the input
 table Id as it will be encoded. The CTX expects this to be passed in its
 decoded state as Koa would do to encoded URLs.

---
 .../server/src/automations/steps/queryRows.ts |  3 +-
 .../src/automations/tests/queryRows.spec.ts   | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/automations/steps/queryRows.ts b/packages/server/src/automations/steps/queryRows.ts
index 93ca8a3f33..4a02e80686 100644
--- a/packages/server/src/automations/steps/queryRows.ts
+++ b/packages/server/src/automations/steps/queryRows.ts
@@ -122,9 +122,10 @@ export async function run({
     sortType =
       fieldType === FieldType.NUMBER ? FieldType.NUMBER : FieldType.STRING
   }
+  // when passing the tableId in the Ctx it needs to be decoded
   const ctx = buildCtx(appId, null, {
     params: {
-      tableId,
+      tableId: decodeURIComponent(tableId),
     },
     body: {
       sortType,
diff --git a/packages/server/src/automations/tests/queryRows.spec.ts b/packages/server/src/automations/tests/queryRows.spec.ts
index 21e28673f3..6ec7b7abfb 100644
--- a/packages/server/src/automations/tests/queryRows.spec.ts
+++ b/packages/server/src/automations/tests/queryRows.spec.ts
@@ -2,6 +2,7 @@ import { EmptyFilterOption, SortOrder, Table } from "@budibase/types"
 import * as setup from "./utilities"
 import { createAutomationBuilder } from "./utilities/AutomationTestBuilder"
 import * as automation from "../index"
+import { basicTable } from "../../tests/utilities/structures"
 
 const NAME = "Test"
 
@@ -13,6 +14,7 @@ describe("Test a query step automation", () => {
     await automation.init()
     await config.init()
     table = await config.createTable()
+
     const row = {
       name: NAME,
       description: "original description",
@@ -153,4 +155,32 @@ describe("Test a query step automation", () => {
     expect(result.steps[0].outputs.rows).toBeDefined()
     expect(result.steps[0].outputs.rows.length).toBe(2)
   })
+
+  it("return rows when querying a table with a space in the name", async () => {
+    const tableWithSpaces = await config.createTable({
+      ...basicTable(),
+      name: "table with spaces",
+    })
+    await config.createRow({
+      name: NAME,
+      tableId: tableWithSpaces._id,
+    })
+    const result = await createAutomationBuilder({
+      name: "Return All Test",
+      config,
+    })
+      .appAction({ fields: {} })
+      .queryRows(
+        {
+          tableId: tableWithSpaces._id!,
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+          filters: {},
+        },
+        { stepName: "Query table with spaces" }
+      )
+      .run()
+    expect(result.steps[0].outputs.success).toBe(true)
+    expect(result.steps[0].outputs.rows).toBeDefined()
+    expect(result.steps[0].outputs.rows.length).toBe(1)
+  })
 })

From fcb3a3a1986eae17208ecf44cfa0d9b3e513d09c Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:24:23 +0000
Subject: [PATCH 114/163] Fix this reference

---
 packages/builder/src/stores/portal/users.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/builder/src/stores/portal/users.ts b/packages/builder/src/stores/portal/users.ts
index 9284ad2992..1beb73a6c0 100644
--- a/packages/builder/src/stores/portal/users.ts
+++ b/packages/builder/src/stores/portal/users.ts
@@ -29,6 +29,8 @@ class UserStore extends BudiStore<UserState> {
       data: [],
     })
 
+    this.search = this.search.bind(this)
+
     // Update quotas after any add or remove operation
     this.create = this.refreshUsage(this.create)
     this.save = this.refreshUsage(this.save)

From a6ac76eb05f37c2402cd86c693334b36c6bee757 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:24:37 +0000
Subject: [PATCH 115/163] Fix group actions error

---
 .../builder/portal/users/groups/_components/GroupUsers.svelte   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte
index 8d99d406fd..71fd4c0be3 100644
--- a/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte
+++ b/packages/builder/src/pages/builder/portal/users/groups/_components/GroupUsers.svelte
@@ -52,7 +52,7 @@
   ]
 
   const removeUser = async id => {
-    await groups.actions.removeUser(groupId, id)
+    await groups.removeUser(groupId, id)
     fetchGroupUsers.refresh()
   }
 

From d8abff1a7569e31204e173a8e02d66f14a0c892a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 11:26:32 +0100
Subject: [PATCH 116/163] js to ts

---
 packages/client/src/api/{index.js => index.ts}                  | 0
 packages/client/src/api/{patches.js => patches.ts}              | 0
 packages/client/src/components/Block.svelte                     | 2 +-
 packages/client/src/components/app/blocks/CardsBlock.svelte     | 2 +-
 packages/client/src/components/app/deprecated/TableBlock.svelte | 2 +-
 packages/client/src/utils/blocks.js                             | 2 +-
 6 files changed, 4 insertions(+), 4 deletions(-)
 rename packages/client/src/api/{index.js => index.ts} (100%)
 rename packages/client/src/api/{patches.js => patches.ts} (100%)

diff --git a/packages/client/src/api/index.js b/packages/client/src/api/index.ts
similarity index 100%
rename from packages/client/src/api/index.js
rename to packages/client/src/api/index.ts
diff --git a/packages/client/src/api/patches.js b/packages/client/src/api/patches.ts
similarity index 100%
rename from packages/client/src/api/patches.js
rename to packages/client/src/api/patches.ts
diff --git a/packages/client/src/components/Block.svelte b/packages/client/src/components/Block.svelte
index a739065015..48c11f152a 100644
--- a/packages/client/src/components/Block.svelte
+++ b/packages/client/src/components/Block.svelte
@@ -1,7 +1,7 @@
 <script>
   import { getContext, onDestroy, onMount, setContext } from "svelte"
   import { builderStore } from "stores/builder.js"
-  import { blockStore } from "stores/blocks.js"
+  import { blockStore } from "stores/blocks"
 
   const component = getContext("component")
   const { styleable } = getContext("sdk")
diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte
index 0368a27e93..34707c394d 100644
--- a/packages/client/src/components/app/blocks/CardsBlock.svelte
+++ b/packages/client/src/components/app/blocks/CardsBlock.svelte
@@ -3,7 +3,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { get } from "svelte/store"
 
   export let title
diff --git a/packages/client/src/components/app/deprecated/TableBlock.svelte b/packages/client/src/components/app/deprecated/TableBlock.svelte
index e67dc5b061..cdb45cfed7 100644
--- a/packages/client/src/components/app/deprecated/TableBlock.svelte
+++ b/packages/client/src/components/app/deprecated/TableBlock.svelte
@@ -5,7 +5,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { Utils } from "@budibase/frontend-core"
 
   export let title
diff --git a/packages/client/src/utils/blocks.js b/packages/client/src/utils/blocks.js
index 88cf4b095b..2522f3442c 100644
--- a/packages/client/src/utils/blocks.js
+++ b/packages/client/src/utils/blocks.js
@@ -1,5 +1,5 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
-import { API } from "../api/index.js"
+import { API } from "../api"
 import { UILogicalOperator } from "@budibase/types"
 import { Constants } from "@budibase/frontend-core"
 

From 20a226a0a5b0c174d3e7734de374d4880d1560b0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 11:30:04 +0100
Subject: [PATCH 117/163] Initial typings

---
 packages/client/src/api/patches.ts      | 9 +++++----
 packages/frontend-core/src/api/index.ts | 2 ++
 packages/frontend-core/src/index.ts     | 1 +
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/client/src/api/patches.ts b/packages/client/src/api/patches.ts
index 722167e16d..77061968d7 100644
--- a/packages/client/src/api/patches.ts
+++ b/packages/client/src/api/patches.ts
@@ -1,19 +1,20 @@
-import { Constants } from "@budibase/frontend-core"
+import { Constants, APIClient } from "@budibase/frontend-core"
 import { FieldTypes } from "../constants"
+import { Row, Table } from "@budibase/types"
 
-export const patchAPI = API => {
+export const patchAPI = (API: APIClient) => {
   /**
    * Enriches rows which contain certain field types so that they can
    * be properly displayed.
    * The ability to create these bindings has been removed, but they will still
    * exist in client apps to support backwards compatibility.
    */
-  const enrichRows = async (rows, tableId) => {
+  const enrichRows = async (rows: Row[], tableId: string) => {
     if (!Array.isArray(rows)) {
       return []
     }
     if (rows.length) {
-      const tables = {}
+      const tables: Record<string, Table> = {}
       for (let row of rows) {
         // Fall back to passed in tableId if row doesn't have it specified
         let rowTableId = row.tableId || tableId
diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index 6efc90023a..485e4d8b5a 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -46,6 +46,8 @@ import { buildLogsEndpoints } from "./logs"
 import { buildMigrationEndpoints } from "./migrations"
 import { buildRowActionEndpoints } from "./rowActions"
 
+export type { APIClient } from "./types"
+
 /**
  * Random identifier to uniquely identify a session in a tab. This is
  * used to determine the originator of calls to the API, which is in
diff --git a/packages/frontend-core/src/index.ts b/packages/frontend-core/src/index.ts
index c0baa63ab6..fd5595e04d 100644
--- a/packages/frontend-core/src/index.ts
+++ b/packages/frontend-core/src/index.ts
@@ -1,4 +1,5 @@
 export { createAPIClient } from "./api"
+export type { APIClient } from "./api"
 export { fetchData, DataFetchMap } from "./fetch"
 export type { DataFetchType } from "./fetch"
 export * as Constants from "./constants"

From 1ec4b4c6b7fc3f0a7f91b52797669612c061761e Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:38:00 +0000
Subject: [PATCH 118/163] Fix this reference

---
 .../src/pages/builder/portal/users/users/index.svelte  |  1 +
 packages/builder/src/stores/portal/users.ts            | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index 97120c55d4..c77e40c964 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -251,6 +251,7 @@
       passwordModal.show()
       await fetch.refresh()
     } catch (error) {
+      console.error(error)
       notifications.error("Error creating user")
     }
   }
diff --git a/packages/builder/src/stores/portal/users.ts b/packages/builder/src/stores/portal/users.ts
index 1beb73a6c0..6503fc9280 100644
--- a/packages/builder/src/stores/portal/users.ts
+++ b/packages/builder/src/stores/portal/users.ts
@@ -29,13 +29,11 @@ class UserStore extends BudiStore<UserState> {
       data: [],
     })
 
-    this.search = this.search.bind(this)
-
     // Update quotas after any add or remove operation
-    this.create = this.refreshUsage(this.create)
-    this.save = this.refreshUsage(this.save)
-    this.delete = this.refreshUsage(this.delete)
-    this.bulkDelete = this.refreshUsage(this.bulkDelete)
+    this.create = this.refreshUsage(this.create.bind(this))
+    this.save = this.refreshUsage(this.save.bind(this))
+    this.delete = this.refreshUsage(this.delete.bind(this))
+    this.bulkDelete = this.refreshUsage(this.bulkDelete.bind(this))
   }
 
   async search(opts: SearchUsersRequest = {}) {

From bd378f0bd44f24bdd22c7fa55657ff96f4051d62 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:50:23 +0000
Subject: [PATCH 119/163] Simplify usage quota refreshing when doing user CRUD

---
 packages/builder/src/stores/portal/users.ts | 26 +++++++--------------
 1 file changed, 8 insertions(+), 18 deletions(-)

diff --git a/packages/builder/src/stores/portal/users.ts b/packages/builder/src/stores/portal/users.ts
index 6503fc9280..605f8612aa 100644
--- a/packages/builder/src/stores/portal/users.ts
+++ b/packages/builder/src/stores/portal/users.ts
@@ -28,12 +28,6 @@ class UserStore extends BudiStore<UserState> {
     super({
       data: [],
     })
-
-    // Update quotas after any add or remove operation
-    this.create = this.refreshUsage(this.create.bind(this))
-    this.save = this.refreshUsage(this.save.bind(this))
-    this.delete = this.refreshUsage(this.delete.bind(this))
-    this.bulkDelete = this.refreshUsage(this.bulkDelete.bind(this))
   }
 
   async search(opts: SearchUsersRequest = {}) {
@@ -156,6 +150,7 @@ class UserStore extends BudiStore<UserState> {
       return body
     })
     const response = await API.createUsers(mappedUsers, data.groups)
+    licensing.setQuotaUsage()
 
     // re-search from first page
     await this.search()
@@ -164,14 +159,19 @@ class UserStore extends BudiStore<UserState> {
 
   async delete(id: string) {
     await API.deleteUser(id)
+    licensing.setQuotaUsage()
   }
 
   async bulkDelete(users: UserIdentifier[]) {
-    return API.deleteUsers(users)
+    const res = API.deleteUsers(users)
+    licensing.setQuotaUsage()
+    return res
   }
 
   async save(user: User) {
-    return await API.saveUser(user)
+    const res = await API.saveUser(user)
+    licensing.setQuotaUsage()
+    return res
   }
 
   async addAppBuilder(userId: string, appId: string) {
@@ -202,16 +202,6 @@ class UserStore extends BudiStore<UserState> {
       return Constants.BudibaseRoles.AppUser
     }
   }
-
-  // Wrapper function to refresh quota usage after an operation,
-  // persisting argument and return types
-  refreshUsage<T extends any[], U>(fn: (...args: T) => Promise<U>) {
-    return async function (...args: T) {
-      const response = await fn(...args)
-      await licensing.setQuotaUsage()
-      return response
-    }
-  }
 }
 
 export const users = new UserStore()

From 6b780e686505e57b0276cde4a36c11932dca0bc3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 11:52:15 +0100
Subject: [PATCH 120/163] Add comment

---
 packages/client/src/api/patches.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/client/src/api/patches.ts b/packages/client/src/api/patches.ts
index 77061968d7..8f09eb5442 100644
--- a/packages/client/src/api/patches.ts
+++ b/packages/client/src/api/patches.ts
@@ -91,13 +91,14 @@ export const patchAPI = (API: APIClient) => {
     return await enrichRows(rows, tableId)
   }
 
-  // Wipe any HBS formulae from table definitions, as these interfere with
+  // Wipe any HBS formulas from table definitions, as these interfere with
   // handlebars enrichment
   const fetchTableDefinition = API.fetchTableDefinition
   API.fetchTableDefinition = async tableId => {
     const definition = await fetchTableDefinition(tableId)
     Object.keys(definition?.schema || {}).forEach(field => {
       if (definition.schema[field]?.type === "formula") {
+        // @ts-expect-error TODO check what use case removing that would break
         delete definition.schema[field].formula
       }
     })

From 53b12075bb1b35bc7839f46384c6b7e907a92d15 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:57:10 +0000
Subject: [PATCH 121/163] Fix unrelated automation log error message

---
 .../src/pages/builder/portal/apps/index.svelte       | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index f94bad2147..5c3ee674e9 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -176,6 +176,8 @@
       notifications.error("Error getting init info")
     }
   })
+
+  $: console.log(automationErrors)
 </script>
 
 <Page>
@@ -191,8 +193,14 @@
           ? "View errors"
           : "View error"}
         on:dismiss={async () => {
-          await automationStore.actions.clearLogErrors({ appId })
-          await appsStore.load()
+          const automationId = Object.keys(automationErrors[appId] || {})[0]
+          if (automationId) {
+            await automationStore.actions.clearLogErrors({
+              appId,
+              automationId,
+            })
+            await appsStore.load()
+          }
         }}
         message={automationErrorMessage(appId)}
       />

From 79a74ff709706ed37aa3a3a9084f0ba7f4fcb133 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Mon, 13 Jan 2025 10:58:31 +0000
Subject: [PATCH 122/163] Remove log

---
 packages/builder/src/pages/builder/portal/apps/index.svelte | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/apps/index.svelte b/packages/builder/src/pages/builder/portal/apps/index.svelte
index 5c3ee674e9..bcd59cd948 100644
--- a/packages/builder/src/pages/builder/portal/apps/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/index.svelte
@@ -176,8 +176,6 @@
       notifications.error("Error getting init info")
     }
   })
-
-  $: console.log(automationErrors)
 </script>
 
 <Page>

From 432f80f1a6876026e1bf27b98141e9e148e35aca Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 11:59:30 +0100
Subject: [PATCH 123/163] Fix types

---
 packages/client/src/api/patches.ts     | 2 +-
 packages/frontend-core/src/api/self.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/client/src/api/patches.ts b/packages/client/src/api/patches.ts
index 8f09eb5442..9482644fa7 100644
--- a/packages/client/src/api/patches.ts
+++ b/packages/client/src/api/patches.ts
@@ -55,7 +55,7 @@ export const patchAPI = (API: APIClient) => {
   const fetchSelf = API.fetchSelf
   API.fetchSelf = async () => {
     const user = await fetchSelf()
-    if (user && user._id) {
+    if (user && "_id" in user && user._id) {
       if (user.roleId === "PUBLIC") {
         // Don't try to enrich a public user as it will 403
         return user
diff --git a/packages/frontend-core/src/api/self.ts b/packages/frontend-core/src/api/self.ts
index db5b1226eb..0f4ff1f985 100644
--- a/packages/frontend-core/src/api/self.ts
+++ b/packages/frontend-core/src/api/self.ts
@@ -13,7 +13,7 @@ export interface SelfEndpoints {
   generateAPIKey: () => Promise<string | undefined>
   fetchDeveloperInfo: () => Promise<FetchAPIKeyResponse>
   fetchBuilderSelf: () => Promise<GetGlobalSelfResponse>
-  fetchSelf: () => Promise<AppSelfResponse>
+  fetchSelf: () => Promise<AppSelfResponse | null>
 }
 
 export const buildSelfEndpoints = (API: BaseAPIClient): SelfEndpoints => ({

From da4014e5aaa7b046638c781f8d12d90c7707de3e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 12:19:44 +0100
Subject: [PATCH 124/163] Remove outdated TODO

---
 packages/backend-core/src/sql/sql.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index cc5f226d56..fa9ef9a4d6 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -313,8 +313,6 @@ class InternalBuilder {
         // Time gets returned as timestamp from mssql, not matching the expected
         // HH:mm format
 
-        // TODO: figure out how to express this safely without string
-        // interpolation.
         return this.knex.raw(`CONVERT(varchar, ??, 108) as ??`, [
           this.rawQuotedIdentifier(field),
           this.knex.raw(this.quote(field)),

From 831df35c5eacdeb076f46cf58c5ba82195067498 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 11:40:19 +0100
Subject: [PATCH 125/163] Initial blocks conversion

---
 packages/client/src/components/app/blocks/CardsBlock.svelte     | 2 +-
 packages/client/src/components/app/deprecated/TableBlock.svelte | 2 +-
 packages/client/src/utils/{blocks.js => blocks.ts}              | 0
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename packages/client/src/utils/{blocks.js => blocks.ts} (100%)

diff --git a/packages/client/src/components/app/blocks/CardsBlock.svelte b/packages/client/src/components/app/blocks/CardsBlock.svelte
index 0368a27e93..34707c394d 100644
--- a/packages/client/src/components/app/blocks/CardsBlock.svelte
+++ b/packages/client/src/components/app/blocks/CardsBlock.svelte
@@ -3,7 +3,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { get } from "svelte/store"
 
   export let title
diff --git a/packages/client/src/components/app/deprecated/TableBlock.svelte b/packages/client/src/components/app/deprecated/TableBlock.svelte
index e67dc5b061..cdb45cfed7 100644
--- a/packages/client/src/components/app/deprecated/TableBlock.svelte
+++ b/packages/client/src/components/app/deprecated/TableBlock.svelte
@@ -5,7 +5,7 @@
   import Block from "components/Block.svelte"
   import BlockComponent from "components/BlockComponent.svelte"
   import { makePropSafe as safe } from "@budibase/string-templates"
-  import { enrichSearchColumns, enrichFilter } from "utils/blocks.js"
+  import { enrichSearchColumns, enrichFilter } from "utils/blocks"
   import { Utils } from "@budibase/frontend-core"
 
   export let title
diff --git a/packages/client/src/utils/blocks.js b/packages/client/src/utils/blocks.ts
similarity index 100%
rename from packages/client/src/utils/blocks.js
rename to packages/client/src/utils/blocks.ts

From c4ba33cf5bdc1fdb1f71ee53df1ed26fe2ec9cf0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 11:43:00 +0100
Subject: [PATCH 126/163] Fix types

---
 packages/client/src/utils/blocks.ts | 31 +++++++++++++++++++++++------
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/packages/client/src/utils/blocks.ts b/packages/client/src/utils/blocks.ts
index 88cf4b095b..58d79aa1aa 100644
--- a/packages/client/src/utils/blocks.ts
+++ b/packages/client/src/utils/blocks.ts
@@ -1,10 +1,16 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
 import { API } from "../api/index.js"
-import { UILogicalOperator } from "@budibase/types"
+import {
+  BasicOperator,
+  LegacyFilter,
+  UIColumn,
+  UILogicalOperator,
+  UISearchFilter,
+} from "@budibase/types"
 import { Constants } from "@budibase/frontend-core"
 
 // Map of data types to component types for search fields inside blocks
-const schemaComponentMap = {
+const schemaComponentMap: Record<string, string> = {
   string: "stringfield",
   options: "optionsfield",
   number: "numberfield",
@@ -19,7 +25,16 @@ const schemaComponentMap = {
  * @param searchColumns the search columns to use
  * @param schema the datasource schema
  */
-export const enrichSearchColumns = async (searchColumns, schema) => {
+export const enrichSearchColumns = async (
+  searchColumns: string[],
+  schema: Record<
+    string,
+    {
+      tableId: string
+      type: string
+    }
+  >
+) => {
   if (!searchColumns?.length || !schema) {
     return []
   }
@@ -61,12 +76,16 @@ export const enrichSearchColumns = async (searchColumns, schema) => {
  * @param columns the enriched search column structure
  * @param formId the ID of the form containing the search fields
  */
-export const enrichFilter = (filter, columns, formId) => {
+export const enrichFilter = (
+  filter: UISearchFilter,
+  columns: UIColumn[],
+  formId: string
+) => {
   if (!columns?.length) {
     return filter
   }
 
-  let newFilters = []
+  const newFilters: LegacyFilter[] = []
   columns?.forEach(column => {
     const safePath = column.name.split(".").map(safe).join(".")
     const stringType = column.type === "string" || column.type === "formula"
@@ -99,7 +118,7 @@ export const enrichFilter = (filter, columns, formId) => {
       newFilters.push({
         field: column.name,
         type: column.type,
-        operator: stringType ? "string" : "equal",
+        operator: stringType ? BasicOperator.STRING : BasicOperator.EQUAL,
         valueType: "Binding",
         value: `{{ ${binding} }}`,
       })

From 558b7c88a7fe013454614ab0dba916dc5b82edf8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 11:46:58 +0100
Subject: [PATCH 127/163] Basic grid conversion

---
 .../client/src/utils/{grid.js => grid.ts}     | 28 ++++++++++---------
 packages/client/tsconfig.json                 |  1 +
 2 files changed, 16 insertions(+), 13 deletions(-)
 rename packages/client/src/utils/{grid.js => grid.ts} (89%)

diff --git a/packages/client/src/utils/grid.js b/packages/client/src/utils/grid.ts
similarity index 89%
rename from packages/client/src/utils/grid.js
rename to packages/client/src/utils/grid.ts
index 142a7ed55a..eec2c9c173 100644
--- a/packages/client/src/utils/grid.js
+++ b/packages/client/src/utils/grid.ts
@@ -1,4 +1,4 @@
-import { GridSpacing, GridRowHeight } from "constants"
+import { GridSpacing, GridRowHeight } from "@/constants"
 import { builderStore } from "stores"
 import { buildStyleString } from "utils/styleable.js"
 
@@ -44,10 +44,11 @@ export const GridDragModes = {
 }
 
 // Builds a CSS variable name for a certain piece of grid metadata
-export const getGridVar = (device, param) => `--grid-${device}-${param}`
+export const getGridVar = (device: string, param: string) =>
+  `--grid-${device}-${param}`
 
 // Determines whether a JS event originated from immediately within a grid
-export const isGridEvent = e => {
+export const isGridEvent = (e: any) => {
   return (
     e.target.dataset?.indicator === "true" ||
     e.target
@@ -59,11 +60,11 @@ export const isGridEvent = e => {
 
 // Svelte action to apply required class names and styles to our component
 // wrappers
-export const gridLayout = (node, metadata) => {
-  let selectComponent
+export const gridLayout = (node: HTMLDivElement, metadata: any) => {
+  let selectComponent: any
 
   // Applies the required listeners, CSS and classes to a component DOM node
-  const applyMetadata = metadata => {
+  const applyMetadata = (metadata: any) => {
     const {
       id,
       styles,
@@ -86,7 +87,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Callback to select the component when clicking on the wrapper
-    selectComponent = e => {
+    selectComponent = (e: Event) => {
       e.stopPropagation()
       builderStore.actions.selectComponent(id)
     }
@@ -100,7 +101,7 @@ export const gridLayout = (node, metadata) => {
     }
     width += 2 * GridSpacing
     height += 2 * GridSpacing
-    let vars = {
+    let vars: any = {
       "--default-width": width,
       "--default-height": height,
     }
@@ -135,7 +136,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Apply some metadata to data attributes to speed up lookups
-    const addDataTag = (tagName, device, param) => {
+    const addDataTag = (tagName: string, device: string, param: string) => {
       const val = `${vars[getGridVar(device, param)]}`
       if (node.dataset[tagName] !== val) {
         node.dataset[tagName] = val
@@ -147,11 +148,12 @@ export const gridLayout = (node, metadata) => {
     addDataTag("gridMobileHAlign", Devices.Mobile, GridParams.HAlign)
     addDataTag("gridDesktopVAlign", Devices.Desktop, GridParams.VAlign)
     addDataTag("gridMobileVAlign", Devices.Mobile, GridParams.VAlign)
-    if (node.dataset.insideGrid !== true) {
-      node.dataset.insideGrid = true
+    if (node.dataset.insideGrid !== "true") {
+      node.dataset.insideGrid = "true"
     }
 
     // Apply all CSS variables to the wrapper
+    // @ts-expect-error TODO
     node.style = buildStyleString(vars)
 
     // Add a listener to select this node on click
@@ -160,7 +162,7 @@ export const gridLayout = (node, metadata) => {
     }
 
     // Add draggable attribute
-    node.setAttribute("draggable", !!draggable)
+    node.setAttribute("draggable", (!!draggable).toString())
   }
 
   // Removes the previously set up listeners
@@ -176,7 +178,7 @@ export const gridLayout = (node, metadata) => {
   applyMetadata(metadata)
 
   return {
-    update(newMetadata) {
+    update(newMetadata: any) {
       removeListeners()
       applyMetadata(newMetadata)
     },
diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json
index 740b129738..ab07343525 100644
--- a/packages/client/tsconfig.json
+++ b/packages/client/tsconfig.json
@@ -14,6 +14,7 @@
         "../*",
         "../../node_modules/@budibase/*"
       ],
+      "@/*": ["./src/*"],
       "*": ["./src/*"]
     }
   }

From ec0de61cf71d7924b68b2cc8d7308cd2b89cfcc7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 11:57:06 +0100
Subject: [PATCH 128/163] Types

---
 packages/client/src/components/Component.svelte | 2 +-
 packages/client/src/utils/grid.ts               | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/client/src/components/Component.svelte b/packages/client/src/components/Component.svelte
index 7a1341c67c..79b4ca6f68 100644
--- a/packages/client/src/components/Component.svelte
+++ b/packages/client/src/components/Component.svelte
@@ -39,7 +39,7 @@
     getActionContextKey,
     getActionDependentContextKeys,
   } from "../utils/buttonActions.js"
-  import { gridLayout } from "utils/grid.js"
+  import { gridLayout } from "utils/grid"
 
   export let instance = {}
   export let parent = null
diff --git a/packages/client/src/utils/grid.ts b/packages/client/src/utils/grid.ts
index eec2c9c173..519f47f342 100644
--- a/packages/client/src/utils/grid.ts
+++ b/packages/client/src/utils/grid.ts
@@ -48,11 +48,13 @@ export const getGridVar = (device: string, param: string) =>
   `--grid-${device}-${param}`
 
 // Determines whether a JS event originated from immediately within a grid
-export const isGridEvent = (e: any) => {
+export const isGridEvent = (e: Event & { target: HTMLElement }): boolean => {
   return (
     e.target.dataset?.indicator === "true" ||
+    // @ts-expect-error: api is not properly typed
     e.target
       .closest?.(".component")
+      // @ts-expect-error
       ?.parentNode.closest(".component")
       ?.childNodes[0]?.classList?.contains("grid")
   )
@@ -101,7 +103,7 @@ export const gridLayout = (node: HTMLDivElement, metadata: any) => {
     }
     width += 2 * GridSpacing
     height += 2 * GridSpacing
-    let vars: any = {
+    const vars: Record<string, string> = {
       "--default-width": width,
       "--default-height": height,
     }

From d88ba716bba1bf355f88cf907833bbb0473f9812 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 12:43:32 +0100
Subject: [PATCH 129/163] Type grid

---
 packages/client/src/utils/grid.ts | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/packages/client/src/utils/grid.ts b/packages/client/src/utils/grid.ts
index 519f47f342..119d008375 100644
--- a/packages/client/src/utils/grid.ts
+++ b/packages/client/src/utils/grid.ts
@@ -2,6 +2,26 @@ import { GridSpacing, GridRowHeight } from "@/constants"
 import { builderStore } from "stores"
 import { buildStyleString } from "utils/styleable.js"
 
+interface GridMetadata {
+  id: string
+  styles: Record<string, string | number> & {
+    "--default-width"?: number
+    "--default-height"?: number
+  }
+  interactive: boolean
+  errored: boolean
+  definition?: {
+    size?: {
+      width: number
+      height: number
+    }
+    grid?: { hAlign: string; vAlign: string }
+  }
+  draggable: boolean
+  insideGrid: boolean
+  ignoresLayout: boolean
+}
+
 /**
  * We use CSS variables on components to control positioning and layout of
  * components inside grids.
@@ -62,11 +82,11 @@ export const isGridEvent = (e: Event & { target: HTMLElement }): boolean => {
 
 // Svelte action to apply required class names and styles to our component
 // wrappers
-export const gridLayout = (node: HTMLDivElement, metadata: any) => {
+export const gridLayout = (node: HTMLDivElement, metadata: GridMetadata) => {
   let selectComponent: any
 
   // Applies the required listeners, CSS and classes to a component DOM node
-  const applyMetadata = (metadata: any) => {
+  const applyMetadata = (metadata: GridMetadata) => {
     const {
       id,
       styles,
@@ -103,7 +123,7 @@ export const gridLayout = (node: HTMLDivElement, metadata: any) => {
     }
     width += 2 * GridSpacing
     height += 2 * GridSpacing
-    const vars: Record<string, string> = {
+    const vars: Record<string, string | number> = {
       "--default-width": width,
       "--default-height": height,
     }
@@ -180,7 +200,7 @@ export const gridLayout = (node: HTMLDivElement, metadata: any) => {
   applyMetadata(metadata)
 
   return {
-    update(newMetadata: any) {
+    update(newMetadata: GridMetadata) {
       removeListeners()
       applyMetadata(newMetadata)
     },

From a566b49ec38fddb2ea1da590056798048fa58540 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 12:45:02 +0100
Subject: [PATCH 130/163] Fix @/constants usages

---
 packages/client/vite.config.mjs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/client/vite.config.mjs b/packages/client/vite.config.mjs
index 22f451fadd..7c163181ae 100644
--- a/packages/client/vite.config.mjs
+++ b/packages/client/vite.config.mjs
@@ -67,6 +67,10 @@ export default defineConfig(({ mode }) => {
           find: "constants",
           replacement: path.resolve("./src/constants"),
         },
+        {
+          find: "@/constants",
+          replacement: path.resolve("./src/constants"),
+        },
         {
           find: "sdk",
           replacement: path.resolve("./src/sdk"),

From c38d694727a873b44d825b489dce0f3a1c3fd374 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 10 Jan 2025 12:46:20 +0100
Subject: [PATCH 131/163] Convert linkable

---
 packages/client/src/utils/{linkable.js => linkable.ts} | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename packages/client/src/utils/{linkable.js => linkable.ts} (61%)

diff --git a/packages/client/src/utils/linkable.js b/packages/client/src/utils/linkable.ts
similarity index 61%
rename from packages/client/src/utils/linkable.js
rename to packages/client/src/utils/linkable.ts
index d82a53474b..2b6cd0ebfd 100644
--- a/packages/client/src/utils/linkable.js
+++ b/packages/client/src/utils/linkable.ts
@@ -1,8 +1,8 @@
 import { get } from "svelte/store"
-import { link } from "svelte-spa-router"
+import { link, LinkActionOpts } from "svelte-spa-router"
 import { builderStore } from "stores"
 
-export const linkable = (node, href) => {
+export const linkable = (node: HTMLElement, href?: LinkActionOpts) => {
   if (get(builderStore).inBuilder) {
     node.onclick = e => {
       e.preventDefault()

From 92a1b0b4acb9ab8bba9243d09419923db9318394 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 12:17:03 +0100
Subject: [PATCH 132/163] Type

---
 packages/client/src/utils/grid.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/client/src/utils/grid.ts b/packages/client/src/utils/grid.ts
index 119d008375..69797e9120 100644
--- a/packages/client/src/utils/grid.ts
+++ b/packages/client/src/utils/grid.ts
@@ -83,7 +83,7 @@ export const isGridEvent = (e: Event & { target: HTMLElement }): boolean => {
 // Svelte action to apply required class names and styles to our component
 // wrappers
 export const gridLayout = (node: HTMLDivElement, metadata: GridMetadata) => {
-  let selectComponent: any
+  let selectComponent: ((e: Event) => void) | null
 
   // Applies the required listeners, CSS and classes to a component DOM node
   const applyMetadata = (metadata: GridMetadata) => {

From 3f6dcb569ee406af0fc56cfe2038ca77c0e7e93a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 12:56:24 +0100
Subject: [PATCH 133/163] Add comment

---
 packages/client/src/sdk.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/client/src/sdk.js b/packages/client/src/sdk.js
index 68d75d2806..c641d339d5 100644
--- a/packages/client/src/sdk.js
+++ b/packages/client/src/sdk.js
@@ -74,6 +74,7 @@ export default {
   fetchData,
   QueryUtils,
   ContextScopes: Constants.ContextScopes,
+  // This is not used internally but exposed to users to be used in plugins
   getAPIKey,
   enrichButtonActions,
   processStringSync,

From 0d6474a38af3a42ee7e933d78b7e660ca37755a0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 13 Jan 2025 12:30:19 +0000
Subject: [PATCH 134/163] Other steps.

---
 packages/server/src/automations/steps/createRow.ts | 2 +-
 packages/server/src/automations/steps/deleteRow.ts | 2 +-
 packages/server/src/automations/steps/updateRow.ts | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/automations/steps/createRow.ts b/packages/server/src/automations/steps/createRow.ts
index e71b1e4556..cd16be2cf3 100644
--- a/packages/server/src/automations/steps/createRow.ts
+++ b/packages/server/src/automations/steps/createRow.ts
@@ -97,7 +97,7 @@ export async function run({
   const ctx: any = buildCtx(appId, emitter, {
     body: inputs.row,
     params: {
-      tableId: inputs.row.tableId,
+      tableId: decodeURIComponent(inputs.row.tableId),
     },
   })
   try {
diff --git a/packages/server/src/automations/steps/deleteRow.ts b/packages/server/src/automations/steps/deleteRow.ts
index 8c61d256d1..6b8a68eeb8 100644
--- a/packages/server/src/automations/steps/deleteRow.ts
+++ b/packages/server/src/automations/steps/deleteRow.ts
@@ -85,7 +85,7 @@ export async function run({
       _rev: inputs.revision,
     },
     params: {
-      tableId: inputs.tableId,
+      tableId: decodeURIComponent(inputs.tableId),
     },
   })
 
diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts
index ecdbdc68c8..552d846f18 100644
--- a/packages/server/src/automations/steps/updateRow.ts
+++ b/packages/server/src/automations/steps/updateRow.ts
@@ -90,6 +90,8 @@ export async function run({
     }
   }
   const tableId = inputs.row.tableId
+    ? decodeURIComponent(inputs.row.tableId)
+    : inputs.row.tableId
 
   // Base update
   let rowUpdate: Record<string, any>
@@ -157,7 +159,7 @@ export async function run({
       },
       params: {
         rowId: inputs.rowId,
-        tableId,
+        tableId: tableId,
       },
     })
     await rowController.patch(ctx)

From 411924b49790171ffbbb531db52dc6b39c7aeafb Mon Sep 17 00:00:00 2001
From: Dean <deanhannigan@gmail.com>
Date: Mon, 13 Jan 2025 12:42:31 +0000
Subject: [PATCH 135/163] PR feedback. Limit API changes only to the S3 PR

---
 packages/frontend-core/src/api/attachments.ts | 1 +
 packages/frontend-core/src/api/index.ts       | 8 +-------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/frontend-core/src/api/attachments.ts b/packages/frontend-core/src/api/attachments.ts
index 0cedfb2cf1..e680218c8f 100644
--- a/packages/frontend-core/src/api/attachments.ts
+++ b/packages/frontend-core/src/api/attachments.ts
@@ -100,6 +100,7 @@ export const buildAttachmentEndpoints = (
         body: data,
         json: false,
         external: true,
+        parseResponse: response => response as any,
       })
       return { publicUrl }
     },
diff --git a/packages/frontend-core/src/api/index.ts b/packages/frontend-core/src/api/index.ts
index d3209e0a54..6efc90023a 100644
--- a/packages/frontend-core/src/api/index.ts
+++ b/packages/frontend-core/src/api/index.ts
@@ -163,13 +163,7 @@ export const createAPIClient = (config: APIClientConfig = {}): APIClient => {
         } else if (parseResponse) {
           return await parseResponse(response)
         } else {
-          const text = await response.text()
-          // If the response has no body at all e.g s3.put
-          if (!text) {
-            // Empty response
-            return {} as ResponseT
-          }
-          return JSON.parse(text) as ResponseT
+          return (await response.json()) as ResponseT
         }
       } catch (error) {
         delete cache[url]

From 6a4d86434eea32546e1770744a2d81f4bc056aa3 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 13 Jan 2025 14:38:00 +0000
Subject: [PATCH 136/163] Bump version to 3.2.40

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 0dc09b27be..f1edc49261 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.39",
+  "version": "3.2.40",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

From 378e611222c569d6656865574faa615b380adba6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:23:12 +0100
Subject: [PATCH 137/163] Remove DEFAULT_VALUES flag check

---
 .../DataTable/modals/CreateEditColumn.svelte  | 91 +++++++++----------
 packages/builder/src/helpers/utils.js         | 17 ++--
 2 files changed, 51 insertions(+), 57 deletions(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index db5b8a7d49..60243b1dab 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -168,7 +168,6 @@
   // used to select what different options can be displayed for column type
   $: canBeDisplay =
     canBeDisplayColumn(editableColumn) && !editableColumn.autocolumn
-  $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES")
   $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type)
   $: canBeRequired =
     editableColumn?.type !== FieldType.LINK &&
@@ -300,7 +299,7 @@
     }
 
     // Ensure we don't have a default value if we can't have one
-    if (!canHaveDefault || !defaultValuesEnabled) {
+    if (!canHaveDefault) {
       delete saveColumn.default
     }
 
@@ -848,51 +847,49 @@
     </div>
   {/if}
 
-  {#if defaultValuesEnabled}
-    {#if editableColumn.type === FieldType.OPTIONS}
-      <Select
-        disabled={!canHaveDefault}
-        options={editableColumn.constraints?.inclusion || []}
-        label="Default value"
-        value={editableColumn.default}
-        on:change={e => (editableColumn.default = e.detail)}
-        placeholder="None"
-      />
-    {:else if editableColumn.type === FieldType.ARRAY}
-      <Multiselect
-        disabled={!canHaveDefault}
-        options={editableColumn.constraints?.inclusion || []}
-        label="Default value"
-        value={editableColumn.default}
-        on:change={e =>
-          (editableColumn.default = e.detail?.length ? e.detail : undefined)}
-        placeholder="None"
-      />
-    {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
-      {@const defaultValue =
-        editableColumn.type === FieldType.BB_REFERENCE_SINGLE
-          ? SingleUserDefault
-          : MultiUserDefault}
-      <Toggle
-        disabled={!canHaveDefault}
-        text="Default to current user"
-        value={editableColumn.default === defaultValue}
-        on:change={e =>
-          (editableColumn.default = e.detail ? defaultValue : undefined)}
-      />
-    {:else}
-      <ModalBindableInput
-        disabled={!canHaveDefault}
-        panel={ServerBindingPanel}
-        title="Default value"
-        label="Default value"
-        placeholder="None"
-        value={editableColumn.default}
-        on:change={e => (editableColumn.default = e.detail)}
-        bindings={defaultValueBindings}
-        allowJS
-      />
-    {/if}
+  {#if editableColumn.type === FieldType.OPTIONS}
+    <Select
+      disabled={!canHaveDefault}
+      options={editableColumn.constraints?.inclusion || []}
+      label="Default value"
+      value={editableColumn.default}
+      on:change={e => (editableColumn.default = e.detail)}
+      placeholder="None"
+    />
+  {:else if editableColumn.type === FieldType.ARRAY}
+    <Multiselect
+      disabled={!canHaveDefault}
+      options={editableColumn.constraints?.inclusion || []}
+      label="Default value"
+      value={editableColumn.default}
+      on:change={e =>
+        (editableColumn.default = e.detail?.length ? e.detail : undefined)}
+      placeholder="None"
+    />
+  {:else if editableColumn.subtype === BBReferenceFieldSubType.USER}
+    {@const defaultValue =
+      editableColumn.type === FieldType.BB_REFERENCE_SINGLE
+        ? SingleUserDefault
+        : MultiUserDefault}
+    <Toggle
+      disabled={!canHaveDefault}
+      text="Default to current user"
+      value={editableColumn.default === defaultValue}
+      on:change={e =>
+        (editableColumn.default = e.detail ? defaultValue : undefined)}
+    />
+  {:else}
+    <ModalBindableInput
+      disabled={!canHaveDefault}
+      panel={ServerBindingPanel}
+      title="Default value"
+      label="Default value"
+      placeholder="None"
+      value={editableColumn.default}
+      on:change={e => (editableColumn.default = e.detail)}
+      bindings={defaultValueBindings}
+      allowJS
+    />
   {/if}
 </Layout>
 
diff --git a/packages/builder/src/helpers/utils.js b/packages/builder/src/helpers/utils.js
index 27f459d6f2..7af7d4ad22 100644
--- a/packages/builder/src/helpers/utils.js
+++ b/packages/builder/src/helpers/utils.js
@@ -7,22 +7,19 @@ import {
   FIELDS,
   isAutoColumnUserRelationship,
 } from "@/constants/backend"
-import { isEnabled } from "@/helpers/featureFlags"
 
 export function getAutoColumnInformation(enabled = true) {
   let info = {}
   for (const [key, subtype] of Object.entries(AUTO_COLUMN_SUB_TYPES)) {
     // Because it's possible to replicate the functionality of CREATED_AT and
-    // CREATED_BY columns, we disable their creation when the DEFAULT_VALUES
-    // feature flag is enabled.
-    if (isEnabled("DEFAULT_VALUES")) {
-      if (
-        subtype === AUTO_COLUMN_SUB_TYPES.CREATED_AT ||
-        subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY
-      ) {
-        continue
-      }
+    // CREATED_BY columns with user column default values, we disable their creation
+    if (
+      subtype === AUTO_COLUMN_SUB_TYPES.CREATED_AT ||
+      subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY
+    ) {
+      continue
     }
+
     info[subtype] = { enabled, name: AUTO_COLUMN_DISPLAY_NAMES[key] }
   }
   return info

From 29dcdba7c58f887942d3ca734e3fa84aad676759 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:29:06 +0100
Subject: [PATCH 138/163] Lint

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 60243b1dab..55ac8474a6 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -49,7 +49,6 @@
   import { RowUtils, canBeDisplayColumn } from "@budibase/frontend-core"
   import ServerBindingPanel from "@/components/common/bindings/ServerBindingPanel.svelte"
   import OptionsEditor from "./OptionsEditor.svelte"
-  import { isEnabled } from "@/helpers/featureFlags"
   import { getUserBindings } from "@/dataBinding"
 
   export let field

From 1fb530dd8e23a32383731c0bfd09617a2a204804 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 14 Jan 2025 09:01:25 +0000
Subject: [PATCH 139/163] Bump version to 3.2.41

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index f1edc49261..e8cd581c98 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.40",
+  "version": "3.2.41",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

From d9eb7eaa552523a46d09c38f6fc379dcec347a5c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 13:34:09 +0100
Subject: [PATCH 140/163] Type constants

---
 .../src/constants/backend/{automations.js => automations.ts}   | 0
 .../builder/src/constants/backend/{backups.js => backups.ts}   | 0
 packages/builder/src/constants/backend/{index.js => index.ts}  | 3 ++-
 3 files changed, 2 insertions(+), 1 deletion(-)
 rename packages/builder/src/constants/backend/{automations.js => automations.ts} (100%)
 rename packages/builder/src/constants/backend/{backups.js => backups.ts} (100%)
 rename packages/builder/src/constants/backend/{index.js => index.ts} (98%)

diff --git a/packages/builder/src/constants/backend/automations.js b/packages/builder/src/constants/backend/automations.ts
similarity index 100%
rename from packages/builder/src/constants/backend/automations.js
rename to packages/builder/src/constants/backend/automations.ts
diff --git a/packages/builder/src/constants/backend/backups.js b/packages/builder/src/constants/backend/backups.ts
similarity index 100%
rename from packages/builder/src/constants/backend/backups.js
rename to packages/builder/src/constants/backend/backups.ts
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.ts
similarity index 98%
rename from packages/builder/src/constants/backend/index.js
rename to packages/builder/src/constants/backend/index.ts
index 6ddf4c2138..c28416ac3e 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.ts
@@ -4,6 +4,7 @@ import {
   INTERNAL_TABLE_SOURCE_ID,
   AutoFieldSubType,
   Hosting,
+  FieldSubType,
 } from "@budibase/types"
 import { Constants } from "@budibase/frontend-core"
 
@@ -209,7 +210,7 @@ export const Roles = {
   BUILDER: "BUILDER",
 }
 
-export function isAutoColumnUserRelationship(subtype) {
+export function isAutoColumnUserRelationship(subtype: FieldSubType) {
   return (
     subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY ||
     subtype === AUTO_COLUMN_SUB_TYPES.UPDATED_BY

From 1bf99d31de180ac35479b1b3866eb781c836207c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 13:34:46 +0100
Subject: [PATCH 141/163] Type constants

---
 packages/builder/src/constants/{index.js => index.ts} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename packages/builder/src/constants/{index.js => index.ts} (100%)

diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.ts
similarity index 100%
rename from packages/builder/src/constants/index.js
rename to packages/builder/src/constants/index.ts

From 2fd120c656e6d605a69b186b174b2e50d66156c8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 13:36:25 +0100
Subject: [PATCH 142/163] Convert utils

---
 packages/builder/src/helpers/keyUtils.js            | 7 -------
 packages/builder/src/helpers/keyUtils.ts            | 7 +++++++
 packages/builder/src/helpers/{utils.js => utils.ts} | 0
 3 files changed, 7 insertions(+), 7 deletions(-)
 delete mode 100644 packages/builder/src/helpers/keyUtils.js
 create mode 100644 packages/builder/src/helpers/keyUtils.ts
 rename packages/builder/src/helpers/{utils.js => utils.ts} (100%)

diff --git a/packages/builder/src/helpers/keyUtils.js b/packages/builder/src/helpers/keyUtils.js
deleted file mode 100644
index 8d6dfb06dc..0000000000
--- a/packages/builder/src/helpers/keyUtils.js
+++ /dev/null
@@ -1,7 +0,0 @@
-function handleEnter(fnc) {
-  return e => e.key === "Enter" && fnc()
-}
-
-export const keyUtils = {
-  handleEnter,
-}
diff --git a/packages/builder/src/helpers/keyUtils.ts b/packages/builder/src/helpers/keyUtils.ts
new file mode 100644
index 0000000000..f78b05ec2b
--- /dev/null
+++ b/packages/builder/src/helpers/keyUtils.ts
@@ -0,0 +1,7 @@
+function handleEnter(fnc: () => void) {
+  return (e: KeyboardEvent) => e.key === "Enter" && fnc()
+}
+
+export const keyUtils = {
+  handleEnter,
+}
diff --git a/packages/builder/src/helpers/utils.js b/packages/builder/src/helpers/utils.ts
similarity index 100%
rename from packages/builder/src/helpers/utils.js
rename to packages/builder/src/helpers/utils.ts

From 819f6f52e3cbb71a3e90ff0f07a4699ca14ff7dd Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 13:48:20 +0100
Subject: [PATCH 143/163] Typings

---
 .../builder/src/constants/backend/index.ts    |  5 +++-
 packages/builder/src/helpers/utils.ts         | 27 ++++++++++++++-----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/packages/builder/src/constants/backend/index.ts b/packages/builder/src/constants/backend/index.ts
index c28416ac3e..910823a553 100644
--- a/packages/builder/src/constants/backend/index.ts
+++ b/packages/builder/src/constants/backend/index.ts
@@ -17,7 +17,10 @@ export {
 
 export const AUTO_COLUMN_SUB_TYPES = AutoFieldSubType
 
-export const AUTO_COLUMN_DISPLAY_NAMES = {
+export const AUTO_COLUMN_DISPLAY_NAMES: Record<
+  keyof typeof AUTO_COLUMN_SUB_TYPES,
+  string
+> = {
   AUTO_ID: "Auto ID",
   CREATED_BY: "Created By",
   CREATED_AT: "Created At",
diff --git a/packages/builder/src/helpers/utils.ts b/packages/builder/src/helpers/utils.ts
index 7af7d4ad22..988f4cbb99 100644
--- a/packages/builder/src/helpers/utils.ts
+++ b/packages/builder/src/helpers/utils.ts
@@ -1,4 +1,4 @@
-import { FieldType } from "@budibase/types"
+import { AutoFieldSubType, Automation, FieldType } from "@budibase/types"
 import { ActionStepID } from "@/constants/backend/automations"
 import { TableNames } from "@/constants"
 import {
@@ -8,8 +8,14 @@ import {
   isAutoColumnUserRelationship,
 } from "@/constants/backend"
 
-export function getAutoColumnInformation(enabled = true) {
-  let info = {}
+type AutoColumnInformation = Partial<
+  Record<AutoFieldSubType, { enabled: boolean; name: string }>
+>
+
+export function getAutoColumnInformation(
+  enabled = true
+): AutoColumnInformation {
+  const info: AutoColumnInformation = {}
   for (const [key, subtype] of Object.entries(AUTO_COLUMN_SUB_TYPES)) {
     // Because it's possible to replicate the functionality of CREATED_AT and
     // CREATED_BY columns with user column default values, we disable their creation
@@ -19,13 +25,20 @@ export function getAutoColumnInformation(enabled = true) {
     ) {
       continue
     }
-
-    info[subtype] = { enabled, name: AUTO_COLUMN_DISPLAY_NAMES[key] }
+    const typedKey = key as keyof typeof AUTO_COLUMN_SUB_TYPES
+    info[subtype] = {
+      enabled,
+      name: AUTO_COLUMN_DISPLAY_NAMES[typedKey],
+    }
   }
   return info
 }
 
-export function buildAutoColumn(tableName, name, subtype) {
+export function buildAutoColumn(
+  tableName: string,
+  name: string,
+  subtype: AutoFieldSubType
+) {
   let type, constraints
   switch (subtype) {
     case AUTO_COLUMN_SUB_TYPES.UPDATED_BY:
@@ -65,7 +78,7 @@ export function buildAutoColumn(tableName, name, subtype) {
   return base
 }
 
-export function checkForCollectStep(automation) {
+export function checkForCollectStep(automation: Automation) {
   return automation.definition.steps.some(
     step => step.stepId === ActionStepID.COLLECT
   )

From d97c492d36bac6004b92ce29f0eaa59e5b16345e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 14:06:24 +0100
Subject: [PATCH 144/163] Typing utils

---
 .../builder/src/constants/backend/index.ts    |  8 --
 packages/builder/src/helpers/utils.ts         | 75 +++++++++++--------
 .../types/src/documents/app/table/schema.ts   |  1 +
 3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/packages/builder/src/constants/backend/index.ts b/packages/builder/src/constants/backend/index.ts
index 910823a553..b7d3f584be 100644
--- a/packages/builder/src/constants/backend/index.ts
+++ b/packages/builder/src/constants/backend/index.ts
@@ -4,7 +4,6 @@ import {
   INTERNAL_TABLE_SOURCE_ID,
   AutoFieldSubType,
   Hosting,
-  FieldSubType,
 } from "@budibase/types"
 import { Constants } from "@budibase/frontend-core"
 
@@ -213,13 +212,6 @@ export const Roles = {
   BUILDER: "BUILDER",
 }
 
-export function isAutoColumnUserRelationship(subtype: FieldSubType) {
-  return (
-    subtype === AUTO_COLUMN_SUB_TYPES.CREATED_BY ||
-    subtype === AUTO_COLUMN_SUB_TYPES.UPDATED_BY
-  )
-}
-
 export const PrettyRelationshipDefinitions = {
   MANY: "Many rows",
   ONE: "One row",
diff --git a/packages/builder/src/helpers/utils.ts b/packages/builder/src/helpers/utils.ts
index 988f4cbb99..140daaef3b 100644
--- a/packages/builder/src/helpers/utils.ts
+++ b/packages/builder/src/helpers/utils.ts
@@ -1,12 +1,20 @@
-import { AutoFieldSubType, Automation, FieldType } from "@budibase/types"
+import {
+  AutoFieldSubType,
+  Automation,
+  DateFieldMetadata,
+  FieldType,
+  NumberFieldMetadata,
+  RelationshipFieldMetadata,
+  RelationshipType,
+} from "@budibase/types"
 import { ActionStepID } from "@/constants/backend/automations"
 import { TableNames } from "@/constants"
 import {
   AUTO_COLUMN_DISPLAY_NAMES,
   AUTO_COLUMN_SUB_TYPES,
   FIELDS,
-  isAutoColumnUserRelationship,
 } from "@/constants/backend"
+import { utils } from "@budibase/shared-core"
 
 type AutoColumnInformation = Partial<
   Record<AutoFieldSubType, { enabled: boolean; name: string }>
@@ -38,44 +46,47 @@ export function buildAutoColumn(
   tableName: string,
   name: string,
   subtype: AutoFieldSubType
-) {
-  let type, constraints
+): RelationshipFieldMetadata | NumberFieldMetadata | DateFieldMetadata {
+  const base = {
+    name,
+    icon: "ri-magic-line",
+    autocolumn: true,
+  }
+
   switch (subtype) {
     case AUTO_COLUMN_SUB_TYPES.UPDATED_BY:
     case AUTO_COLUMN_SUB_TYPES.CREATED_BY:
-      type = FieldType.LINK
-      constraints = FIELDS.LINK.constraints
-      break
+      return {
+        ...base,
+        type: FieldType.LINK,
+        subtype,
+        constraints: FIELDS.LINK.constraints,
+        tableId: TableNames.USERS,
+        fieldName: `${tableName}-${name}`,
+        relationshipType: RelationshipType.MANY_TO_ONE,
+      }
+
     case AUTO_COLUMN_SUB_TYPES.AUTO_ID:
-      type = FieldType.NUMBER
-      constraints = FIELDS.NUMBER.constraints
-      break
+      return {
+        ...base,
+        type: FieldType.NUMBER,
+        subtype,
+        constraints: FIELDS.NUMBER.constraints,
+      }
     case AUTO_COLUMN_SUB_TYPES.UPDATED_AT:
     case AUTO_COLUMN_SUB_TYPES.CREATED_AT:
-      type = FieldType.DATETIME
-      constraints = FIELDS.DATETIME.constraints
-      break
+      return {
+        ...base,
+        type: FieldType.DATETIME,
+        subtype,
+        constraints: FIELDS.DATETIME.constraints,
+      }
+
     default:
-      type = FieldType.STRING
-      constraints = FIELDS.STRING.constraints
-      break
+      throw utils.unreachable(subtype, {
+        message: "Cannot build auto column with supplied subtype",
+      })
   }
-  if (Object.values(AUTO_COLUMN_SUB_TYPES).indexOf(subtype) === -1) {
-    throw "Cannot build auto column with supplied subtype"
-  }
-  const base = {
-    name,
-    type,
-    subtype,
-    icon: "ri-magic-line",
-    autocolumn: true,
-    constraints,
-  }
-  if (isAutoColumnUserRelationship(subtype)) {
-    base.tableId = TableNames.USERS
-    base.fieldName = `${tableName}-${name}`
-  }
-  return base
 }
 
 export function checkForCollectStep(automation: Automation) {
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 551b1f16a8..13eecec3a4 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -109,6 +109,7 @@ export interface LongFormFieldMetadata extends BaseFieldSchema {
 export interface StringFieldMetadata extends BaseFieldSchema {
   type: FieldType.STRING
   default?: string
+  subtype?: never
 }
 
 export interface FormulaFieldMetadata extends BaseFieldSchema {

From bdd0d10d0d5af482ab7de2a5b1cf2fdc28c761c8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 14:07:54 +0100
Subject: [PATCH 145/163] Extra typings

---
 packages/builder/src/helpers/{planTitle.js => planTitle.ts}   | 4 ++--
 .../builder/src/helpers/{sanitizeUrl.js => sanitizeUrl.ts}    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename packages/builder/src/helpers/{planTitle.js => planTitle.ts} (86%)
 rename packages/builder/src/helpers/{sanitizeUrl.js => sanitizeUrl.ts} (88%)

diff --git a/packages/builder/src/helpers/planTitle.js b/packages/builder/src/helpers/planTitle.ts
similarity index 86%
rename from packages/builder/src/helpers/planTitle.js
rename to packages/builder/src/helpers/planTitle.ts
index c08b8bf3fe..ab342b4d93 100644
--- a/packages/builder/src/helpers/planTitle.js
+++ b/packages/builder/src/helpers/planTitle.ts
@@ -1,6 +1,6 @@
 import { PlanType } from "@budibase/types"
 
-export function getFormattedPlanName(userPlanType) {
+export function getFormattedPlanName(userPlanType: PlanType) {
   let planName
   switch (userPlanType) {
     case PlanType.PRO:
@@ -29,6 +29,6 @@ export function getFormattedPlanName(userPlanType) {
   return `${planName} Plan`
 }
 
-export function isPremiumOrAbove(userPlanType) {
+export function isPremiumOrAbove(userPlanType: PlanType) {
   return ![PlanType.PRO, PlanType.TEAM, PlanType.FREE].includes(userPlanType)
 }
diff --git a/packages/builder/src/helpers/sanitizeUrl.js b/packages/builder/src/helpers/sanitizeUrl.ts
similarity index 88%
rename from packages/builder/src/helpers/sanitizeUrl.js
rename to packages/builder/src/helpers/sanitizeUrl.ts
index 4d00c503fb..5b76930037 100644
--- a/packages/builder/src/helpers/sanitizeUrl.js
+++ b/packages/builder/src/helpers/sanitizeUrl.ts
@@ -1,4 +1,4 @@
-export default function (url) {
+export default function (url: string) {
   return url
     .split("/")
     .map(part => {

From b3d18b16cf5841b24452d3a5d0d8b97d412b59de Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 14:11:39 +0100
Subject: [PATCH 146/163] Type pagination

---
 .../src/helpers/{pagination.js => pagination.ts} | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)
 rename packages/builder/src/helpers/{pagination.js => pagination.ts} (76%)

diff --git a/packages/builder/src/helpers/pagination.js b/packages/builder/src/helpers/pagination.ts
similarity index 76%
rename from packages/builder/src/helpers/pagination.js
rename to packages/builder/src/helpers/pagination.ts
index 122973f1a1..0280cd1052 100644
--- a/packages/builder/src/helpers/pagination.js
+++ b/packages/builder/src/helpers/pagination.ts
@@ -1,6 +1,16 @@
 import { writable } from "svelte/store"
 
-function defaultValue() {
+interface PaginationStore {
+  nextPage: string | null | undefined
+  page: string | null | undefined
+  hasPrevPage: boolean
+  hasNextPage: boolean
+  loading: boolean
+  pageNumber: number
+  pages: string[]
+}
+
+function defaultValue(): PaginationStore {
   return {
     nextPage: null,
     page: undefined,
@@ -29,13 +39,13 @@ export function createPaginationStore() {
     update(state => {
       state.pageNumber++
       state.page = state.nextPage
-      state.pages.push(state.page)
+      state.pages.push(state.page!)
       state.hasPrevPage = state.pageNumber > 1
       return state
     })
   }
 
-  function fetched(hasNextPage, nextPage) {
+  function fetched(hasNextPage: boolean, nextPage: string) {
     update(state => {
       state.hasNextPage = hasNextPage
       state.nextPage = nextPage

From 8f282d54d2830e268d80ae27a12b49da60d64553 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 14:13:20 +0100
Subject: [PATCH 147/163] Type duplicates

---
 .../builder/src/helpers/{duplicate.js => duplicate.ts} | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
 rename packages/builder/src/helpers/{duplicate.js => duplicate.ts} (92%)

diff --git a/packages/builder/src/helpers/duplicate.js b/packages/builder/src/helpers/duplicate.ts
similarity index 92%
rename from packages/builder/src/helpers/duplicate.js
rename to packages/builder/src/helpers/duplicate.ts
index 361e1faa25..4816bcfd7b 100644
--- a/packages/builder/src/helpers/duplicate.js
+++ b/packages/builder/src/helpers/duplicate.ts
@@ -10,13 +10,13 @@
  *
  * Repl
  */
-export const duplicateName = (name, allNames) => {
+export const duplicateName = (name: string, allNames: string[]) => {
   const duplicatePattern = new RegExp(`\\s(\\d+)$`)
   const baseName = name.split(duplicatePattern)[0]
   const isDuplicate = new RegExp(`${baseName}\\s(\\d+)$`)
 
   // get the sequence from matched names
-  const sequence = []
+  const sequence: number[] = []
   allNames.filter(n => {
     if (n === baseName) {
       return true
@@ -71,9 +71,9 @@ export const duplicateName = (name, allNames) => {
  *  flat array of strings
  */
 export const getSequentialName = (
-  items,
-  prefix,
-  { getName = x => x, numberFirstItem = false } = {}
+  items: string[],
+  prefix: string,
+  { getName = (x: string) => x, numberFirstItem = false } = {}
 ) => {
   if (!prefix?.length || !getName) {
     return null

From 2a853d3992c67e59ba00fbc3c083051080e4b2c1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 14:20:10 +0100
Subject: [PATCH 148/163] Type helpers

---
 packages/builder/src/helpers/helpers.js       | 46 -----------------
 packages/builder/src/helpers/helpers.ts       | 50 +++++++++++++++++++
 .../src/helpers/{index.js => index.ts}        |  0
 3 files changed, 50 insertions(+), 46 deletions(-)
 delete mode 100644 packages/builder/src/helpers/helpers.js
 create mode 100644 packages/builder/src/helpers/helpers.ts
 rename packages/builder/src/helpers/{index.js => index.ts} (100%)

diff --git a/packages/builder/src/helpers/helpers.js b/packages/builder/src/helpers/helpers.js
deleted file mode 100644
index 99483d40e2..0000000000
--- a/packages/builder/src/helpers/helpers.js
+++ /dev/null
@@ -1,46 +0,0 @@
-import { last, flow } from "lodash/fp"
-
-export const buildStyle = styles => {
-  let str = ""
-  for (let s in styles) {
-    if (styles[s]) {
-      let key = convertCamel(s)
-      str += `${key}: ${styles[s]}; `
-    }
-  }
-  return str
-}
-
-export const convertCamel = str => {
-  return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`)
-}
-
-export const pipe = (arg, funcs) => flow(funcs)(arg)
-
-export const capitalise = s => {
-  if (!s) {
-    return s
-  }
-  return s.substring(0, 1).toUpperCase() + s.substring(1)
-}
-
-export const lowercase = s => s.substring(0, 1).toLowerCase() + s.substring(1)
-
-export const lowercaseExceptFirst = s =>
-  s.charAt(0) + s.substring(1).toLowerCase()
-
-export const get_name = s => (!s ? "" : last(s.split("/")))
-
-export const get_capitalised_name = name => pipe(name, [get_name, capitalise])
-
-export const isBuilderInputFocused = e => {
-  const activeTag = document.activeElement?.tagName.toLowerCase()
-  const inCodeEditor = document.activeElement?.classList?.contains("cm-content")
-  if (
-    (inCodeEditor || ["input", "textarea"].indexOf(activeTag) !== -1) &&
-    e.key !== "Escape"
-  ) {
-    return true
-  }
-  return false
-}
diff --git a/packages/builder/src/helpers/helpers.ts b/packages/builder/src/helpers/helpers.ts
new file mode 100644
index 0000000000..f767e6f59f
--- /dev/null
+++ b/packages/builder/src/helpers/helpers.ts
@@ -0,0 +1,50 @@
+import type { Many } from "lodash"
+import { last, flow } from "lodash/fp"
+
+export const buildStyle = (styles: Record<string, any>) => {
+  let str = ""
+  for (let s in styles) {
+    if (styles[s]) {
+      let key = convertCamel(s)
+      str += `${key}: ${styles[s]}; `
+    }
+  }
+  return str
+}
+
+export const convertCamel = (str: string) => {
+  return str.replace(/[A-Z]/g, match => `-${match.toLowerCase()}`)
+}
+
+export const pipe = (arg: string, funcs: Many<(...args: any[]) => any>) =>
+  flow(funcs)(arg)
+
+export const capitalise = (s: string) => {
+  if (!s) {
+    return s
+  }
+  return s.substring(0, 1).toUpperCase() + s.substring(1)
+}
+
+export const lowercase = (s: string) =>
+  s.substring(0, 1).toLowerCase() + s.substring(1)
+
+export const lowercaseExceptFirst = (s: string) =>
+  s.charAt(0) + s.substring(1).toLowerCase()
+
+export const get_name = (s: string) => (!s ? "" : last(s.split("/")))
+
+export const get_capitalised_name = (name: string) =>
+  pipe(name, [get_name, capitalise])
+
+export const isBuilderInputFocused = (e: KeyboardEvent) => {
+  const activeTag = document.activeElement?.tagName.toLowerCase()
+  const inCodeEditor = document.activeElement?.classList?.contains("cm-content")
+  if (
+    (inCodeEditor || ["input", "textarea"].indexOf(activeTag!) !== -1) &&
+    e.key !== "Escape"
+  ) {
+    return true
+  }
+  return false
+}
diff --git a/packages/builder/src/helpers/index.js b/packages/builder/src/helpers/index.ts
similarity index 100%
rename from packages/builder/src/helpers/index.js
rename to packages/builder/src/helpers/index.ts

From a8d7a97bd678d73621d65e2fd9c69082d7ea52a7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:08:37 +0100
Subject: [PATCH 149/163] Type warnings

---
 packages/builder/src/helpers/{warnings.js => warnings.ts} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename packages/builder/src/helpers/{warnings.js => warnings.ts} (85%)

diff --git a/packages/builder/src/helpers/warnings.js b/packages/builder/src/helpers/warnings.ts
similarity index 85%
rename from packages/builder/src/helpers/warnings.js
rename to packages/builder/src/helpers/warnings.ts
index ad943a8578..ae6c65666c 100644
--- a/packages/builder/src/helpers/warnings.js
+++ b/packages/builder/src/helpers/warnings.ts
@@ -1,4 +1,4 @@
-export const suppressWarnings = warnings => {
+export const suppressWarnings = (warnings: string[]) => {
   if (!warnings?.length) {
     return
   }

From d912e95849295e03e4b363423e792cc9b42eb580 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:10:40 +0100
Subject: [PATCH 150/163] Type fetchData

---
 .../src/helpers/{fetchData.js => fetchData.ts}     | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)
 rename packages/builder/src/helpers/{fetchData.js => fetchData.ts} (57%)

diff --git a/packages/builder/src/helpers/fetchData.js b/packages/builder/src/helpers/fetchData.ts
similarity index 57%
rename from packages/builder/src/helpers/fetchData.js
rename to packages/builder/src/helpers/fetchData.ts
index 085d0ae5b4..22b82c070e 100644
--- a/packages/builder/src/helpers/fetchData.js
+++ b/packages/builder/src/helpers/fetchData.ts
@@ -1,13 +1,21 @@
 import { writable } from "svelte/store"
 import { API } from "@/api"
 
-export default function (url) {
-  const store = writable({ status: "LOADING", data: {}, error: {} })
+export default function (url: string) {
+  const store = writable<{
+    status: "LOADING" | "SUCCESS" | "ERROR"
+    data: object
+    error?: unknown
+  }>({
+    status: "LOADING",
+    data: {},
+    error: {},
+  })
 
   async function get() {
     store.update(u => ({ ...u, status: "LOADING" }))
     try {
-      const data = await API.get({ url })
+      const data = await API.get<any>({ url })
       store.set({ data, status: "SUCCESS" })
     } catch (e) {
       store.set({ data: {}, error: e, status: "ERROR" })

From 2558bd3e6ad2e71bb355ac1c97c2dbc25287a6c2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:12:20 +0100
Subject: [PATCH 151/163] Type feature flags

---
 .../builder/src/helpers/{featureFlags.js => featureFlags.ts}   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
 rename packages/builder/src/helpers/{featureFlags.js => featureFlags.ts} (58%)

diff --git a/packages/builder/src/helpers/featureFlags.js b/packages/builder/src/helpers/featureFlags.ts
similarity index 58%
rename from packages/builder/src/helpers/featureFlags.js
rename to packages/builder/src/helpers/featureFlags.ts
index e9054e8a9c..625468f06b 100644
--- a/packages/builder/src/helpers/featureFlags.js
+++ b/packages/builder/src/helpers/featureFlags.ts
@@ -1,7 +1,8 @@
+import { FeatureFlag } from "@budibase/types"
 import { auth } from "../stores/portal"
 import { get } from "svelte/store"
 
-export const isEnabled = featureFlag => {
+export const isEnabled = (featureFlag: FeatureFlag) => {
   const user = get(auth).user
   return !!user?.flags?.[featureFlag]
 }

From 8281ec4e3ee5f47293e3cba7c49c01a2178b0d5c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 13 Jan 2025 17:13:50 +0100
Subject: [PATCH 152/163] Allow feature flags as strings

---
 packages/builder/src/helpers/featureFlags.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/helpers/featureFlags.ts b/packages/builder/src/helpers/featureFlags.ts
index 625468f06b..faa57892e8 100644
--- a/packages/builder/src/helpers/featureFlags.ts
+++ b/packages/builder/src/helpers/featureFlags.ts
@@ -2,7 +2,7 @@ import { FeatureFlag } from "@budibase/types"
 import { auth } from "../stores/portal"
 import { get } from "svelte/store"
 
-export const isEnabled = (featureFlag: FeatureFlag) => {
+export const isEnabled = (featureFlag: FeatureFlag | `${FeatureFlag}`) => {
   const user = get(auth).user
   return !!user?.flags?.[featureFlag]
 }

From ca1015baf6b6bc5942c7a5cbd0caf4cd499665ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 10:07:50 +0100
Subject: [PATCH 153/163] Remove unexpected never field

---
 packages/types/src/documents/app/table/schema.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 13eecec3a4..551b1f16a8 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -109,7 +109,6 @@ export interface LongFormFieldMetadata extends BaseFieldSchema {
 export interface StringFieldMetadata extends BaseFieldSchema {
   type: FieldType.STRING
   default?: string
-  subtype?: never
 }
 
 export interface FormulaFieldMetadata extends BaseFieldSchema {

From 41c8ab976fd4c30661b4e75927c5bdbe4ec7a74e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 10:09:10 +0100
Subject: [PATCH 154/163] Type

---
 packages/builder/src/helpers/fetchData.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/helpers/fetchData.ts b/packages/builder/src/helpers/fetchData.ts
index 22b82c070e..cffb8f4d7d 100644
--- a/packages/builder/src/helpers/fetchData.ts
+++ b/packages/builder/src/helpers/fetchData.ts
@@ -15,7 +15,7 @@ export default function (url: string) {
   async function get() {
     store.update(u => ({ ...u, status: "LOADING" }))
     try {
-      const data = await API.get<any>({ url })
+      const data = await API.get<object>({ url })
       store.set({ data, status: "SUCCESS" })
     } catch (e) {
       store.set({ data: {}, error: e, status: "ERROR" })

From 8624997f4c371fb853159605a5fc81a4289baa99 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 11:14:30 +0100
Subject: [PATCH 155/163] Fix types

---
 packages/builder/src/helpers/duplicate.ts         | 12 +++++++++---
 packages/builder/src/stores/builder/rowActions.ts |  2 +-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/packages/builder/src/helpers/duplicate.ts b/packages/builder/src/helpers/duplicate.ts
index 4816bcfd7b..35beb4df38 100644
--- a/packages/builder/src/helpers/duplicate.ts
+++ b/packages/builder/src/helpers/duplicate.ts
@@ -70,10 +70,16 @@ export const duplicateName = (name: string, allNames: string[]) => {
  * @param getName optional function to extract the name for an item, if not a
  *  flat array of strings
  */
-export const getSequentialName = (
-  items: string[],
+export const getSequentialName = <T extends string | object>(
+  items: T[],
   prefix: string,
-  { getName = (x: string) => x, numberFirstItem = false } = {}
+  {
+    getName,
+    numberFirstItem,
+  }: {
+    getName: (item: T) => string
+    numberFirstItem?: boolean
+  }
 ) => {
   if (!prefix?.length || !getName) {
     return null
diff --git a/packages/builder/src/stores/builder/rowActions.ts b/packages/builder/src/stores/builder/rowActions.ts
index 9576eccd1b..2b3077926e 100644
--- a/packages/builder/src/stores/builder/rowActions.ts
+++ b/packages/builder/src/stores/builder/rowActions.ts
@@ -62,7 +62,7 @@ export class RowActionStore extends BudiStore<RowActionState> {
       const existingRowActions = get(this)[tableId] || []
       name = getSequentialName(existingRowActions, "New row action ", {
         getName: x => x.name,
-      })
+      })!
     }
 
     if (!name) {

From 06013929e44cd822cc19c146f944303fa2a36e90 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 11:14:34 +0100
Subject: [PATCH 156/163] Fix usage

---
 .../src/components/common/bindings/SnippetDrawer.svelte       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/common/bindings/SnippetDrawer.svelte b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte
index 90ca6ffd9f..f10c44f81f 100644
--- a/packages/builder/src/components/common/bindings/SnippetDrawer.svelte
+++ b/packages/builder/src/components/common/bindings/SnippetDrawer.svelte
@@ -28,7 +28,9 @@
   let loading = false
   let deleteConfirmationDialog
 
-  $: defaultName = getSequentialName($snippets, "MySnippet", x => x.name)
+  $: defaultName = getSequentialName($snippets, "MySnippet", {
+    getName: x => x.name,
+  })
   $: key = snippet?.name
   $: name = snippet?.name || defaultName
   $: code = snippet?.code ? encodeJSBinding(snippet.code) : ""

From 35d5e51898a94054284239b5e061557dfdc8c9b9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 11:35:43 +0100
Subject: [PATCH 157/163] Convert and fix tests

---
 packages/builder/src/helpers/duplicate.ts          | 14 +++++++-------
 .../tests/{duplicate.test.js => duplicate.test.ts} |  0
 2 files changed, 7 insertions(+), 7 deletions(-)
 rename packages/builder/src/helpers/tests/{duplicate.test.js => duplicate.test.ts} (100%)

diff --git a/packages/builder/src/helpers/duplicate.ts b/packages/builder/src/helpers/duplicate.ts
index 35beb4df38..b4740b3e52 100644
--- a/packages/builder/src/helpers/duplicate.ts
+++ b/packages/builder/src/helpers/duplicate.ts
@@ -70,18 +70,18 @@ export const duplicateName = (name: string, allNames: string[]) => {
  * @param getName optional function to extract the name for an item, if not a
  *  flat array of strings
  */
-export const getSequentialName = <T extends string | object>(
-  items: T[],
-  prefix: string,
+export const getSequentialName = <T extends any>(
+  items: T[] | null,
+  prefix: string | null,
   {
     getName,
     numberFirstItem,
   }: {
-    getName: (item: T) => string
+    getName?: (item: T) => string
     numberFirstItem?: boolean
-  }
+  } = {}
 ) => {
-  if (!prefix?.length || !getName) {
+  if (!prefix?.length) {
     return null
   }
   const trimmedPrefix = prefix.trim()
@@ -91,7 +91,7 @@ export const getSequentialName = <T extends string | object>(
   }
   let max = 0
   items.forEach(item => {
-    const name = getName(item)
+    const name = getName?.(item) ?? item
     if (typeof name !== "string" || !name.startsWith(trimmedPrefix)) {
       return
     }
diff --git a/packages/builder/src/helpers/tests/duplicate.test.js b/packages/builder/src/helpers/tests/duplicate.test.ts
similarity index 100%
rename from packages/builder/src/helpers/tests/duplicate.test.js
rename to packages/builder/src/helpers/tests/duplicate.test.ts

From 0665ff1307a95d6e5b5dd0c50a4b3ab13f206771 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 11:37:14 +0100
Subject: [PATCH 158/163] Convert test

---
 .../helpers/tests/{nameHelpers.spec.js => nameHelpers.spec.ts}    | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename packages/builder/src/helpers/tests/{nameHelpers.spec.js => nameHelpers.spec.ts} (100%)

diff --git a/packages/builder/src/helpers/tests/nameHelpers.spec.js b/packages/builder/src/helpers/tests/nameHelpers.spec.ts
similarity index 100%
rename from packages/builder/src/helpers/tests/nameHelpers.spec.js
rename to packages/builder/src/helpers/tests/nameHelpers.spec.ts

From a3bcaddabbca7d02f5ba306b9b98ebf34e61dfa9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 16:05:14 +0100
Subject: [PATCH 159/163] Re-add tests files in backend-core/dist

---
 packages/backend-core/package.json       | 2 +-
 packages/backend-core/tsconfig.test.json | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 packages/backend-core/tsconfig.test.json

diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json
index 3e1b5f324b..1ab05cd14b 100644
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@@ -21,7 +21,7 @@
   "scripts": {
     "prebuild": "rimraf dist/",
     "prepack": "cp package.json dist",
-    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null",
+    "build": "node ./scripts/build.js && tsc -p tsconfig.build.json --emitDeclarationOnly --paths null && tsc -p tsconfig.test.json --paths null",
     "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
     "build:oss": "node ./scripts/build.js",
     "check:types": "tsc -p tsconfig.json --noEmit --paths null",
diff --git a/packages/backend-core/tsconfig.test.json b/packages/backend-core/tsconfig.test.json
new file mode 100644
index 0000000000..48322b7a94
--- /dev/null
+++ b/packages/backend-core/tsconfig.test.json
@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.build.json",
+  "compilerOptions": {
+    "outDir": "dist"
+  },
+  "include": ["tests/**/*.js", "tests/**/*.ts"],
+  "exclude": ["node_modules", "dist"]
+}

From bc1e95b012aa357fa646ccecffd1bb65cc0624fb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 16:15:24 +0100
Subject: [PATCH 160/163] Add sourcemap

---
 packages/backend-core/tsconfig.test.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/tsconfig.test.json b/packages/backend-core/tsconfig.test.json
index 48322b7a94..0e3e7a79b2 100644
--- a/packages/backend-core/tsconfig.test.json
+++ b/packages/backend-core/tsconfig.test.json
@@ -1,7 +1,8 @@
 {
   "extends": "../../tsconfig.build.json",
   "compilerOptions": {
-    "outDir": "dist"
+    "outDir": "dist",
+    "sourceMap": true
   },
   "include": ["tests/**/*.js", "tests/**/*.ts"],
   "exclude": ["node_modules", "dist"]

From 3715bd0bed58432a13657126e69b222df6762a86 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 16:51:31 +0100
Subject: [PATCH 161/163] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 193476cdfa..19295c7e4d 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 193476cdfade6d3c613e6972f16ee0c527e01ff6
+Subproject commit 19295c7e4d223bec493a108a905c110d5f024258

From 2acbf29e625f60452adb3a8f2f696b05641f1939 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 14 Jan 2025 16:20:54 +0000
Subject: [PATCH 162/163] Bump version to 3.2.42

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index e8cd581c98..d0f0bd23c5 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.2.41",
+  "version": "3.2.42",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

From ccf3e787011ff11838f48fa9b569d50345b2d5cf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 14 Jan 2025 23:35:18 +0100
Subject: [PATCH 163/163] Update pro submodule

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 19295c7e4d..a4f63b2267 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 19295c7e4d223bec493a108a905c110d5f024258
+Subproject commit a4f63b22675e16dcdcaa4d9e83b298eee6466a07