From 1b9211ee6c9684fa325871716643e0f6d3e6e786 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 26 Jun 2024 16:40:20 +0100
Subject: [PATCH 01/11] Initial implementation of solving SQS issue with case
 insensitivity.

---
 .../server/src/sdk/app/rows/search/sqs.ts     | 49 ++++++++++++++++++-
 .../server/src/sdk/app/tables/internal/sqs.ts | 15 +++++-
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 174ecc0e38..e33d41c99f 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -18,6 +18,7 @@ import {
   buildInternalRelationships,
   sqlOutputProcessing,
 } from "../../../../api/controllers/row/utils"
+import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs"
 import sdk from "../../../index"
 import {
   context,
@@ -59,7 +60,7 @@ function buildInternalFieldList(
         buildInternalFieldList(relatedTable, tables, { relationships: false })
       )
     } else {
-      fieldList.push(`${table._id}.${col.name}`)
+      fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
     }
   }
   return fieldList
@@ -90,6 +91,28 @@ function cleanupFilters(
       )
   )
 
+  // generate a map of all possible column names (these can be duplicated across tables
+  // the map of them will always be the same
+  const userColumnMap: Record<string, string> = {}
+  allTables.forEach(table =>
+    Object.keys(table.schema).forEach(
+      key => (userColumnMap[key] = mapToUserColumn(key))
+    )
+  )
+  // update the keys of filters to manage user columns
+  for (let filter of Object.values(filters)) {
+    for (let key of Object.keys(filter)) {
+      const found = Object.keys(userColumnMap).find(possibleColumn =>
+        key.endsWith(possibleColumn)
+      )
+      if (found) {
+        const newKey = key.replace(found, userColumnMap[found])
+        filter[newKey] = filter[key]
+        delete filter[key]
+      }
+    }
+  }
+
   return filters
 }
 
@@ -106,6 +129,26 @@ function buildTableMap(tables: Table[]) {
   return tableMap
 }
 
+function reverseUserColumnMapping(rows: Row[]) {
+  const prefixLength = USER_COLUMN_PREFIX.length
+  return rows.map(row => {
+    const finalRow: Row = {}
+    for (let key of Object.keys(row)) {
+      // it should be the last prefix
+      const lastIndex = key.lastIndexOf(USER_COLUMN_PREFIX)
+      if (lastIndex !== -1 && lastIndex < key.length - prefixLength) {
+        // cut out the prefix
+        const newKey =
+          key.slice(0, lastIndex) + key.slice(lastIndex + prefixLength)
+        finalRow[newKey] = row[key]
+      } else {
+        finalRow[key] = row[key]
+      }
+    }
+    return finalRow
+  })
+}
+
 function runSqlQuery(json: QueryJson, tables: Table[]): Promise<Row[]>
 function runSqlQuery(
   json: QueryJson,
@@ -147,6 +190,8 @@ async function runSqlQuery(
   const response = await alias.queryWithAliasing(json, processSQLQuery)
   if (opts?.countTotalRows) {
     return processRowCountResponse(response)
+  } else if (Array.isArray(response)) {
+    return reverseUserColumnMapping(response)
   } else {
     return response
   }
@@ -197,7 +242,7 @@ export async function search(
     const sortType =
       sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING
     request.sort = {
-      [sortField.name]: {
+      [mapToUserColumn(sortField.name)]: {
         direction: params.sortOrder || SortOrder.ASCENDING,
         type: sortType as SortType,
       },
diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts
index 4819b9d8d5..f892a9c6c8 100644
--- a/packages/server/src/sdk/app/tables/internal/sqs.ts
+++ b/packages/server/src/sdk/app/tables/internal/sqs.ts
@@ -62,10 +62,18 @@ function buildRelationshipDefinitions(
   }
 }
 
+export const USER_COLUMN_PREFIX = "data_"
+
+// utility function to denote that columns in SQLite are mapped to avoid overlap issues
+// the overlaps can occur due to case insensitivity and some of the columns which Budibase requires
+export function mapToUserColumn(key: string) {
+  return `${USER_COLUMN_PREFIX}${key}`
+}
+
 // this can generate relationship tables as part of the mapping
 function mapTable(table: Table): SQLiteTables {
   const tables: SQLiteTables = {}
-  const fields: Record<string, SQLiteType> = {}
+  const fields: Record<string, { field: string; type: SQLiteType }> = {}
   for (let [key, column] of Object.entries(table.schema)) {
     // relationships should be handled differently
     if (column.type === FieldType.LINK) {
@@ -78,7 +86,10 @@ function mapTable(table: Table): SQLiteTables {
     if (!FieldTypeMap[column.type]) {
       throw new Error(`Unable to map type "${column.type}" to SQLite type`)
     }
-    fields[key] = FieldTypeMap[column.type]
+    fields[mapToUserColumn(key)] = {
+      field: key,
+      type: FieldTypeMap[column.type],
+    }
   }
   // there are some extra columns to map - add these in
   const constantMap: Record<string, SQLiteType> = {}

From b0e6d3e72c826ef0a7c6cc242e5761e90b3ef5b1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 27 Jun 2024 17:55:09 +0100
Subject: [PATCH 02/11] Correctly handling overlapping column names.

---
 packages/server/src/sdk/app/rows/search/sqs.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index e33d41c99f..8c376f726e 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -99,10 +99,15 @@ function cleanupFilters(
       key => (userColumnMap[key] = mapToUserColumn(key))
     )
   )
+
+  // sort longest first, don't find substrings
+  const userColumnList = Object.keys(userColumnMap).sort(
+    (a, b) => b.length - a.length
+  )
   // update the keys of filters to manage user columns
   for (let filter of Object.values(filters)) {
     for (let key of Object.keys(filter)) {
-      const found = Object.keys(userColumnMap).find(possibleColumn =>
+      const found = userColumnList.find(possibleColumn =>
         key.endsWith(possibleColumn)
       )
       if (found) {

From 3394e974d3db216204041fa9f8c1fc77f54eae20 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 27 Jun 2024 18:18:41 +0100
Subject: [PATCH 03/11] Getting casting working again, have to pass down prefix
 for checking against fields.

---
 packages/backend-core/src/sql/sql.ts           | 15 +++++++++++++--
 packages/backend-core/src/sql/sqlStatements.ts | 12 ++++++++++--
 packages/server/src/sdk/app/rows/search/sqs.ts |  1 +
 packages/types/src/sdk/search.ts               |  2 ++
 4 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 34b950bf2c..615753efc3 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -184,7 +184,11 @@ class InternalBuilder {
     query: Knex.QueryBuilder,
     filters: SearchFilters | undefined,
     table: Table,
-    opts: { aliases?: Record<string, string>; relationship?: boolean }
+    opts: {
+      aliases?: Record<string, string>
+      relationship?: boolean
+      columnPrefix?: string
+    }
   ): Knex.QueryBuilder {
     if (!filters) {
       return query
@@ -192,7 +196,10 @@ class InternalBuilder {
     filters = parseFilters(filters)
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
-    const sqlStatements = new SqlStatements(this.client, table, { allOr })
+    const sqlStatements = new SqlStatements(this.client, table, {
+      allOr,
+      columnPrefix: opts.columnPrefix,
+    })
     const tableName =
       this.client === SqlClient.SQL_LITE ? table._id! : table.name
 
@@ -663,6 +670,7 @@ class InternalBuilder {
     }
     // add filters to the query (where)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
 
@@ -698,6 +706,7 @@ class InternalBuilder {
     }
 
     return this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       relationship: true,
       aliases: tableAliases,
     })
@@ -708,6 +717,7 @@ class InternalBuilder {
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
     // mysql can't use returning
@@ -722,6 +732,7 @@ class InternalBuilder {
     const { endpoint, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     query = this.addFilters(query, filters, json.meta.table, {
+      columnPrefix: json.meta.columnPrefix,
       aliases: tableAliases,
     })
     // mysql can't use returning
diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts
index a80defd8b8..311f7c7d49 100644
--- a/packages/backend-core/src/sql/sqlStatements.ts
+++ b/packages/backend-core/src/sql/sqlStatements.ts
@@ -5,19 +5,27 @@ export class SqlStatements {
   client: string
   table: Table
   allOr: boolean | undefined
+  columnPrefix: string | undefined
+
   constructor(
     client: string,
     table: Table,
-    { allOr }: { allOr?: boolean } = {}
+    { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {}
   ) {
     this.client = client
     this.table = table
     this.allOr = allOr
+    this.columnPrefix = columnPrefix
   }
 
   getField(key: string): FieldSchema | undefined {
     const fieldName = key.split(".")[1]
-    return this.table.schema[fieldName]
+    let found = this.table.schema[fieldName]
+    if (!found && this.columnPrefix) {
+      const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "")
+      found = this.table.schema[prefixRemovedFieldName]
+    }
+    return found
   }
 
   between(
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 8c376f726e..2e7a478ffb 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -235,6 +235,7 @@ export async function search(
     meta: {
       table,
       tables: allTablesMap,
+      columnPrefix: USER_COLUMN_PREFIX,
     },
     resource: {
       fields: buildInternalFieldList(table, allTables),
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index e5cbccf5c1..c40f1c3b84 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -122,6 +122,8 @@ export interface QueryJson {
     table: Table
     tables?: Record<string, Table>
     renamed?: RenameColumn
+    // can specify something that columns could be prefixed with
+    columnPrefix?: string
   }
   extra?: {
     idFilter?: SearchFilters

From 6b4852bf3e93e0b22e9442cd9ef6663bd66d0f2b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 27 Jun 2024 18:30:22 +0100
Subject: [PATCH 04/11] Updating migration test case.

---
 .../tests/20240604153647_initial_sqs.spec.ts        | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
index 86e50a5812..572e694855 100644
--- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
+++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts
@@ -15,6 +15,7 @@ import {
 import { processMigrations } from "../../migrationsProcessor"
 import migration from "../20240604153647_initial_sqs"
 import { AppMigration } from "src/appMigrations"
+import sdk from "../../../sdk"
 
 const MIGRATIONS: AppMigration[] = [
   {
@@ -27,6 +28,8 @@ const MIGRATIONS: AppMigration[] = [
 const config = setup.getConfig()
 let tableId: string
 
+const prefix = sdk.tables.sqs.mapToUserColumn
+
 function oldLinkDocInfo() {
   const tableId1 = `${DocumentType.TABLE}_a`,
     tableId2 = `${DocumentType.TABLE}_b`
@@ -102,8 +105,14 @@ describe("SQS migration", () => {
       expect(designDoc.sql.tables).toBeDefined()
       const mainTableDef = designDoc.sql.tables[tableId]
       expect(mainTableDef).toBeDefined()
-      expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT)
-      expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT)
+      expect(mainTableDef.fields[prefix("name")]).toEqual({
+        field: "name",
+        type: SQLiteType.TEXT,
+      })
+      expect(mainTableDef.fields[prefix("description")]).toEqual({
+        field: "description",
+        type: SQLiteType.TEXT,
+      })
 
       const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo()
       const linkDoc = await db.get<LinkDocument>(oldLinkDocID())

From 048d15956b11e9dec36a5c0e99f68016b1408918 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 12:47:57 +0100
Subject: [PATCH 05/11] Updating environments which need to re-sync
 definitions.

---
 packages/server/src/sdk/app/rows/search/sqs.ts | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 2e7a478ffb..937dbbb1ed 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -38,6 +38,7 @@ import {
 } from "./filters"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
+const NO_SUCH_COLUMN_REGEX = new RegExp(`no such colum.+${USER_COLUMN_PREFIX}`)
 
 function buildInternalFieldList(
   table: Table,
@@ -329,7 +330,10 @@ export async function search(
     return response
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    if (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) {
+    const syncAndRepeat =
+      (err.status === 400 && msg?.match(NO_SUCH_COLUMN_REGEX)) ||
+      (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID))
+    if (syncAndRepeat) {
       await sdk.tables.sqs.syncDefinition()
       return search(options, table)
     }

From 2d31e327e79e99f135baa1eec3fef05c4f63706a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 13:49:52 +0100
Subject: [PATCH 06/11] Resolving PR comments.

---
 packages/server/src/sdk/app/rows/search/sqs.ts | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 937dbbb1ed..dbd4231664 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -140,12 +140,11 @@ function reverseUserColumnMapping(rows: Row[]) {
   return rows.map(row => {
     const finalRow: Row = {}
     for (let key of Object.keys(row)) {
-      // it should be the last prefix
-      const lastIndex = key.lastIndexOf(USER_COLUMN_PREFIX)
-      if (lastIndex !== -1 && lastIndex < key.length - prefixLength) {
+      // it should be the first prefix
+      const index = key.indexOf(USER_COLUMN_PREFIX)
+      if (index !== -1) {
         // cut out the prefix
-        const newKey =
-          key.slice(0, lastIndex) + key.slice(lastIndex + prefixLength)
+        const newKey = key.slice(0, index) + key.slice(index + prefixLength)
         finalRow[newKey] = row[key]
       } else {
         finalRow[key] = row[key]
@@ -198,9 +197,8 @@ async function runSqlQuery(
     return processRowCountResponse(response)
   } else if (Array.isArray(response)) {
     return reverseUserColumnMapping(response)
-  } else {
-    return response
   }
+  return response
 }
 
 export async function search(

From 510baf4f6eeecd1e62f3d3e5f072e51f5312aa1f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 14:19:11 +0100
Subject: [PATCH 07/11] Adding test case and updating how prefix updates in
 filters work.

---
 .../src/api/routes/tests/search.spec.ts       | 22 +++++++++++++++++++
 .../server/src/sdk/app/rows/search/sqs.ts     | 20 +++++++----------
 packages/shared-core/src/filters.ts           | 15 ++++++++++---
 3 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 589f129f31..872759ac13 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2108,4 +2108,26 @@ describe.each([
         }).toNotHaveProperty(["totalRows"])
       })
     })
+
+  describe("special data_ case", () => {
+    beforeAll(async () => {
+      table = await createTable({
+        name_data_test: {
+          name: "name_data_test",
+          type: FieldType.STRING,
+        },
+      })
+      await createRows([{ name_data_test: "a" }, { name_data_test: "b" }])
+    })
+
+    it("should be able to query a column with data_ in it", async () => {
+      await expectSearch({
+        query: {
+          equal: {
+            ["1:name_data_test"]: "a",
+          },
+        },
+      }).toContainExactly([{ name_data_test: "a" }])
+    })
+  })
 })
diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index dbd4231664..644864efb0 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -36,6 +36,7 @@ import {
   getRelationshipColumns,
   getTableIDList,
 } from "./filters"
+import { dataFilters } from "@budibase/shared-core"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const NO_SUCH_COLUMN_REGEX = new RegExp(`no such colum.+${USER_COLUMN_PREFIX}`)
@@ -101,19 +102,14 @@ function cleanupFilters(
     )
   )
 
-  // sort longest first, don't find substrings
-  const userColumnList = Object.keys(userColumnMap).sort(
-    (a, b) => b.length - a.length
-  )
   // update the keys of filters to manage user columns
-  for (let filter of Object.values(filters)) {
-    for (let key of Object.keys(filter)) {
-      const found = userColumnList.find(possibleColumn =>
-        key.endsWith(possibleColumn)
-      )
-      if (found) {
-        const newKey = key.replace(found, userColumnMap[found])
-        filter[newKey] = filter[key]
+  const keyInAnyTable = (key: string): boolean =>
+    allTables.some(table => table.schema[key])
+  for (const filter of Object.values(filters)) {
+    for (const key of Object.keys(filter)) {
+      const { prefix, key: rawKey } = dataFilters.getKeyNumbering(key)
+      if (keyInAnyTable(rawKey)) {
+        filter[`${prefix || ""}${mapToUserColumn(rawKey)}`] = filter[key]
         delete filter[key]
       }
     }
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 443dbbce8d..07c21f0481 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -131,13 +131,22 @@ const cleanupQuery = (query: SearchFilters) => {
  * Removes a numeric prefix on field names designed to give fields uniqueness
  */
 export const removeKeyNumbering = (key: string): string => {
+  return getKeyNumbering(key).key
+}
+
+/**
+ * Gets the part of the keys, returning the numeric prefix and the field name
+ */
+export const getKeyNumbering = (
+  key: string
+): { prefix?: string; key: string } => {
   if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) {
     const parts = key.split(":")
     // remove the number
-    parts.shift()
-    return parts.join(":")
+    const number = parts.shift()
+    return { prefix: `${number}:`, key: parts.join(":") }
   } else {
-    return key
+    return { key }
   }
 }
 

From 13ac273c8397aaba4f6d14e58a84fa6783c0f43d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 14:59:00 +0100
Subject: [PATCH 08/11] Updating how search filters are converted, needs full
 column metadata to work it out this way.

---
 .../server/src/sdk/app/rows/search/sqs.ts     | 12 +++--
 packages/shared-core/src/filters.ts           | 49 +++++++++++++++++++
 yarn.lock                                     | 12 ++---
 3 files changed, 64 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts
index 644864efb0..0720600a15 100644
--- a/packages/server/src/sdk/app/rows/search/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/sqs.ts
@@ -105,11 +105,17 @@ function cleanupFilters(
   // update the keys of filters to manage user columns
   const keyInAnyTable = (key: string): boolean =>
     allTables.some(table => table.schema[key])
+
+  const splitter = new dataFilters.ColumnSplitter(allTables)
   for (const filter of Object.values(filters)) {
     for (const key of Object.keys(filter)) {
-      const { prefix, key: rawKey } = dataFilters.getKeyNumbering(key)
-      if (keyInAnyTable(rawKey)) {
-        filter[`${prefix || ""}${mapToUserColumn(rawKey)}`] = filter[key]
+      const { numberPrefix, relationshipPrefix, column } = splitter.run(key)
+      if (keyInAnyTable(column)) {
+        filter[
+          `${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn(
+            column
+          )}`
+        ] = filter[key]
         delete filter[key]
       }
     }
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 07c21f0481..032439f9bf 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -13,6 +13,7 @@ import {
   RowSearchParams,
   EmptyFilterOption,
   SearchResponse,
+  Table,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
@@ -150,6 +151,54 @@ export const getKeyNumbering = (
   }
 }
 
+/**
+ * Generates a splitter which can be used to split columns from a context into
+ * their components (number prefix, relationship column/table, column name)
+ */
+export class ColumnSplitter {
+  tableNames: string[]
+  tableIds: string[]
+  relationshipColumnNames: string[]
+  relationships: string[]
+
+  constructor(tables: Table[]) {
+    this.tableNames = tables.map(table => table.name)
+    this.tableIds = tables.map(table => table._id!)
+    this.relationshipColumnNames = tables.flatMap(table =>
+      Object.keys(table.schema).filter(
+        columnName => table.schema[columnName].type === FieldType.LINK
+      )
+    )
+    this.relationships = this.tableNames
+      .concat(this.tableIds)
+      .concat(this.relationshipColumnNames)
+  }
+
+  run(key: string): {
+    numberPrefix?: string
+    relationshipPrefix?: string
+    column: string
+  } {
+    let { prefix, key: splitKey } = getKeyNumbering(key)
+    let relationship: string | undefined
+    for (let possibleRelationship of this.relationships) {
+      const withDot = `${possibleRelationship}.`
+      if (splitKey.startsWith(withDot)) {
+        const finalKeyParts = splitKey.split(withDot)
+        finalKeyParts.shift()
+        relationship = withDot
+        splitKey = finalKeyParts.join(".")
+        break
+      }
+    }
+    return {
+      numberPrefix: prefix,
+      relationshipPrefix: relationship,
+      column: splitKey,
+    }
+  }
+}
+
 /**
  * Builds a JSON query from the filter structure generated in the builder
  * @param filter the builder filter structure
diff --git a/yarn.lock b/yarn.lock
index 4cc63badcb..c060cc9da3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -16917,7 +16917,7 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
-openai@4.52.1:
+openai@^4.52.1:
   version "4.52.1"
   resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
   integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
@@ -21786,16 +21786,16 @@ typescript-eslint@^7.3.1:
     "@typescript-eslint/eslint-plugin" "7.3.1"
     "@typescript-eslint/parser" "7.3.1"
 
+typescript@5.2.2, "typescript@>=3 < 6":
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
+  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
+
 typescript@5.5.2:
   version "5.5.2"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507"
   integrity sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==
 
-"typescript@>=3 < 6":
-  version "5.2.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
-  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
-
 typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7:
   version "3.9.10"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8"

From b3d07aa22812ce8f2c5e081572d4e3aeeab6d4e7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 15:21:39 +0100
Subject: [PATCH 09/11] Adding parameterised tests.

---
 .../src/api/routes/tests/search.spec.ts       | 43 ++++++++++---------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 872759ac13..2b2542a3b2 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2109,25 +2109,28 @@ describe.each([
       })
     })
 
-  describe("special data_ case", () => {
-    beforeAll(async () => {
-      table = await createTable({
-        name_data_test: {
-          name: "name_data_test",
-          type: FieldType.STRING,
-        },
-      })
-      await createRows([{ name_data_test: "a" }, { name_data_test: "b" }])
-    })
-
-    it("should be able to query a column with data_ in it", async () => {
-      await expectSearch({
-        query: {
-          equal: {
-            ["1:name_data_test"]: "a",
+  describe.each(["data_name_test", "name_data_test", "name_test_data_"])(
+    "special (%s) case",
+    column => {
+      beforeAll(async () => {
+        table = await createTable({
+          [column]: {
+            name: column,
+            type: FieldType.STRING,
           },
-        },
-      }).toContainExactly([{ name_data_test: "a" }])
-    })
-  })
+        })
+        await createRows([{ [column]: "a" }, { [column]: "b" }])
+      })
+
+      it("should be able to query a column with data_ in it", async () => {
+        await expectSearch({
+          query: {
+            equal: {
+              [`1:${column}`]: "a",
+            },
+          },
+        }).toContainExactly([{ [column]: "a" }])
+      })
+    }
+  )
 })

From d7f6c494ed4d31654dbc549b26f25ef158fab4d4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 28 Jun 2024 15:22:34 +0100
Subject: [PATCH 10/11] Quick change to splitting.

---
 packages/shared-core/src/filters.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 032439f9bf..3c6901e195 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -172,6 +172,8 @@ export class ColumnSplitter {
     this.relationships = this.tableNames
       .concat(this.tableIds)
       .concat(this.relationshipColumnNames)
+      // sort by length - makes sure there's no mis-matches due to similarities (sub column names)
+      .sort((a, b) => b.length - a.length)
   }
 
   run(key: string): {

From de333a87fd99bbe13d46bbd7f589a26158775e06 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Mon, 1 Jul 2024 15:01:04 +0100
Subject: [PATCH 11/11] Add verify_sso_login flow (#14060)

---
 packages/shared-core/src/constants/index.ts |  3 +++
 scripts/deploy-camunda.sh                   |  1 +
 yarn.lock                                   | 12 ++++++------
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts
index c9d1a8fc8f..0713b5d2f8 100644
--- a/packages/shared-core/src/constants/index.ts
+++ b/packages/shared-core/src/constants/index.ts
@@ -164,14 +164,17 @@ export const InvalidFileExtensions = [
 
 export enum BpmCorrelationKey {
   ONBOARDING = "budibase:onboarding:correlationkey",
+  VERIFY_SSO_LOGIN = "budibase:verify_sso_login:correlationkey",
 }
 
 export enum BpmInstanceKey {
   ONBOARDING = "budibase:onboarding:instancekey",
+  VERIFY_SSO_LOGIN = "budibase:verify_sso_login:instancekey",
 }
 
 export enum BpmStatusKey {
   ONBOARDING = "budibase:onboarding:status",
+  VERIFY_SSO_LOGIN = "budibase:verify_sso_login:status",
 }
 
 export enum BpmStatusValue {
diff --git a/scripts/deploy-camunda.sh b/scripts/deploy-camunda.sh
index 7059b6f072..90400a0449 100755
--- a/scripts/deploy-camunda.sh
+++ b/scripts/deploy-camunda.sh
@@ -23,6 +23,7 @@ echo "deploy processes..."
 zbctl deploy resource offboarding.bpmn --insecure
 zbctl deploy resource onboarding.bpmn --insecure
 zbctl deploy resource free_trial.bpmn --insecure
+zbctl deploy resource verify_sso_login.bpmn --insecure
 
 cd ../../../../../budibase/packages/account-portal/packages/server 
 
diff --git a/yarn.lock b/yarn.lock
index 4cc63badcb..c060cc9da3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -16917,7 +16917,7 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
-openai@4.52.1:
+openai@^4.52.1:
   version "4.52.1"
   resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65"
   integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg==
@@ -21786,16 +21786,16 @@ typescript-eslint@^7.3.1:
     "@typescript-eslint/eslint-plugin" "7.3.1"
     "@typescript-eslint/parser" "7.3.1"
 
+typescript@5.2.2, "typescript@>=3 < 6":
+  version "5.2.2"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
+  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
+
 typescript@5.5.2:
   version "5.5.2"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507"
   integrity sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==
 
-"typescript@>=3 < 6":
-  version "5.2.2"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78"
-  integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==
-
 typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7:
   version "3.9.10"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8"