From ab5f50d2b8db2be4be57cf54eb64d77473e9d2e8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 23 Aug 2024 18:00:52 +0100
Subject: [PATCH 01/69] Implementing a JSON aggregate method of selecting
 relationships.

---
 packages/backend-core/src/sql/sql.ts | 130 +++++++++++++++++++--------
 1 file changed, 95 insertions(+), 35 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 6a8518ea7b..1c87abf5ae 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -126,12 +126,20 @@ class InternalBuilder {
   }
 
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { resource, meta } = this.query
+    const { endpoint, resource, meta, tableAliases } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
     }
 
+    // no relationships - select everything in SQLite
+    if (this.client === SqlClient.SQL_LITE) {
+      const alias = tableAliases?.[endpoint.entityId]
+        ? tableAliases?.[endpoint.entityId]
+        : endpoint.entityId
+      return [this.knex.raw(`${this.quote(alias)}.*`)]
+    }
+
     const schema = meta.table.schema
     return resource.fields.map(field => {
       const parts = field.split(/\./g)
@@ -745,16 +753,83 @@ class InternalBuilder {
     return withSchema
   }
 
+  addJsonRelationships(
+    query: Knex.QueryBuilder,
+    fromTable: string,
+    relationships: RelationshipsJson[]
+  ): Knex.QueryBuilder {
+    const { resource, tableAliases: aliases, endpoint } = this.query
+    const fields = resource?.fields || []
+    const jsonField = (field: string) => {
+      const unAliased = field.split(".").slice(1).join(".")
+      return `'${unAliased}',${field}`
+    }
+    for (let relationship of relationships) {
+      const {
+        tableName: toTable,
+        through: throughTable,
+        to: toKey,
+        from: fromKey,
+        fromPrimary,
+        toPrimary,
+      } = relationship
+      // skip invalid relationships
+      if (!toTable || !fromTable || !fromPrimary || !toPrimary) {
+        continue
+      }
+      if (!throughTable) {
+        throw new Error("Only many-to-many implemented for JSON relationships")
+      }
+      const toAlias = aliases?.[toTable] || toTable,
+        throughAlias = aliases?.[throughTable] || throughTable,
+        fromAlias = aliases?.[fromTable] || fromTable
+      let toTableWithSchema = this.tableNameWithSchema(toTable, {
+        alias: toAlias,
+        schema: endpoint.schema,
+      })
+      let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
+        alias: throughAlias,
+        schema: endpoint.schema,
+      })
+      const relationshipFields = fields.filter(
+        field => field.split(".")[0] === toAlias
+      )
+      const fieldList: string = relationshipFields
+        .map(field => jsonField(field))
+        .join(",")
+      let rawJsonArray: Knex.Raw
+      switch (this.client) {
+        case SqlClient.SQL_LITE:
+          rawJsonArray = this.knex.raw(
+            `json_group_array(json_object(${fieldList}))`
+          )
+          break
+        default:
+          throw new Error(`JSON relationships not implement for ${this.client}`)
+      }
+      const subQuery = this.knex
+        .select(rawJsonArray)
+        .from(toTableWithSchema)
+        .join(throughTableWithSchema, function () {
+          this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
+        })
+        .where(
+          `${throughAlias}.${fromKey}`,
+          "=",
+          this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
+        )
+      query = query.select({ [relationship.column]: subQuery })
+    }
+    return query
+  }
+
   addRelationships(
     query: Knex.QueryBuilder,
     fromTable: string,
-    relationships: RelationshipsJson[] | undefined,
-    schema: string | undefined,
+    relationships: RelationshipsJson[],
+    schema?: string,
     aliases?: Record<string, string>
   ): Knex.QueryBuilder {
-    if (!relationships) {
-      return query
-    }
     const tableSets: Record<string, [RelationshipsJson]> = {}
     // aggregate into table sets (all the same to tables)
     for (let relationship of relationships) {
@@ -957,42 +1032,27 @@ class InternalBuilder {
       if (foundOffset != null) {
         query = query.offset(foundOffset)
       }
-      // add sorting to pre-query
-      // no point in sorting when counting
-      query = this.addSorting(query)
     }
-    // add filters to the query (where)
-    query = this.addFilters(query, filters)
 
-    const alias = tableAliases?.[tableName] || tableName
-    let preQuery: Knex.QueryBuilder = this.knex({
-      // the typescript definition for the knex constructor doesn't support this
-      // syntax, but it is the only way to alias a pre-query result as part of
-      // a query - there is an alias dictionary type, but it assumes it can only
-      // be a table name, not a pre-query
-      [alias]: query as any,
-    })
     // if counting, use distinct count, else select
-    preQuery = !counting
-      ? preQuery.select(this.generateSelectStatement())
-      : this.addDistinctCount(preQuery)
+    query = !counting
+      ? query.select(this.generateSelectStatement())
+      : this.addDistinctCount(query)
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL && !counting) {
-      preQuery = this.addSorting(preQuery)
+      query = this.addSorting(query)
     }
     // handle joins
-    query = this.addRelationships(
-      preQuery,
-      tableName,
-      relationships,
-      endpoint.schema,
-      tableAliases
-    )
-
-    // add a base limit over the whole query
-    // if counting we can't set this limit
-    if (limits?.base) {
-      query = query.limit(limits.base)
+    if (relationships && this.client === SqlClient.SQL_LITE) {
+      query = this.addJsonRelationships(query, tableName, relationships)
+    } else if (relationships) {
+      query = this.addRelationships(
+        query,
+        tableName,
+        relationships,
+        endpoint.schema,
+        tableAliases
+      )
     }
 
     return this.addFilters(query, filters, { relationship: true })

From 80f3e5954bf5ecb84a8559789fcc50f5b9f5e474 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 23 Aug 2024 18:30:29 +0100
Subject: [PATCH 02/69] Getting processing of SQS relationships working.

---
 .../src/api/controllers/row/utils/basic.ts    | 11 +++++++---
 .../src/api/controllers/row/utils/utils.ts    | 20 ++++++++++---------
 .../src/sdk/app/rows/search/internal/sqs.ts   | 13 +++++++++++-
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index f28f650422..8f3607bc73 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -1,5 +1,5 @@
 // need to handle table name + field or just field, depending on if relationships used
-import { FieldType, Row, Table } from "@budibase/types"
+import { FieldSchema, FieldType, Row, Table } from "@budibase/types"
 import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
 import { generateRowIdField } from "../../../../integrations/utils"
 
@@ -82,7 +82,7 @@ export function basicProcessing({
       value = value.toString()
     }
     // all responses include "select col as table.col" so that overlaps are handled
-    if (value != null) {
+    else if (value != null) {
       thisRow[fieldName] = value
     }
   }
@@ -93,12 +93,17 @@ export function basicProcessing({
   } else {
     const columns = Object.keys(table.schema)
     for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
-      thisRow[internalColumn] = extractFieldValue({
+      const schema: FieldSchema | undefined = table.schema[internalColumn]
+      let value = extractFieldValue({
         row,
         tableName: table._id!,
         fieldName: internalColumn,
         isLinked,
       })
+      if (sqs && schema?.type === FieldType.LINK && typeof value === "string") {
+        value = JSON.parse(value)
+      }
+      thisRow[internalColumn] = value
     }
   }
   return thisRow
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index 911cfe8d5b..fdf05baf3c 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -147,7 +147,7 @@ export async function sqlOutputProcessing(
       row._id = rowId
     }
     // this is a relationship of some sort
-    if (finalRows[rowId]) {
+    if (!opts?.sqs && finalRows[rowId]) {
       finalRows = await updateRelationshipColumns(
         table,
         tables,
@@ -174,14 +174,16 @@ export async function sqlOutputProcessing(
     finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
 
     // do this at end once its been added to the final rows
-    finalRows = await updateRelationshipColumns(
-      table,
-      tables,
-      row,
-      finalRows,
-      relationships,
-      opts
-    )
+    if (!opts?.sqs) {
+      finalRows = await updateRelationshipColumns(
+        table,
+        tables,
+        row,
+        finalRows,
+        relationships,
+        opts
+      )
+    }
   }
 
   // make sure all related rows are correct
diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index 6736ff6abf..fb140e3c14 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -37,9 +37,9 @@ import { outputProcessing } from "../../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
 import { processRowCountResponse } from "../../utils"
 import {
-  updateFilterKeys,
   getRelationshipColumns,
   getTableIDList,
+  updateFilterKeys,
 } from "../filters"
 import {
   dataFilters,
@@ -368,6 +368,17 @@ export async function search(
       })
     )
 
+    // make sure relationships have columns reversed correctly
+    for (let columnName of Object.keys(table.schema)) {
+      if (table.schema[columnName].type !== FieldType.LINK) {
+        continue
+      }
+      // process the relationships (JSON generated by SQS)
+      for (let row of processed) {
+        row[columnName] = reverseUserColumnMapping(row[columnName])
+      }
+    }
+
     // check for pagination final row
     let nextRow: boolean = false
     if (paginate && params.limit && rows.length > params.limit) {

From 5d53e64360f9832cfee92521d6305b94aff42299 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 23 Aug 2024 18:45:13 +0100
Subject: [PATCH 03/69] Getting fields from all relationships loading
 correctly.

---
 packages/backend-core/src/sql/sql.ts | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 1c87abf5ae..74678a79b5 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -761,8 +761,17 @@ class InternalBuilder {
     const { resource, tableAliases: aliases, endpoint } = this.query
     const fields = resource?.fields || []
     const jsonField = (field: string) => {
-      const unAliased = field.split(".").slice(1).join(".")
-      return `'${unAliased}',${field}`
+      const parts = field.split(".")
+      let tableField: string, unaliased: string
+      if (parts.length > 1) {
+        const alias = parts.shift()!
+        unaliased = parts.join(".")
+        tableField = `${this.quote(alias)}.${this.quote(unaliased)}`
+      } else {
+        unaliased = parts.join(".")
+        tableField = this.quote(unaliased)
+      }
+      return `'${unaliased}',${tableField}`
     }
     for (let relationship of relationships) {
       const {

From b11ee56a38ca8b056f74794c97b5a3366409d3c4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 23 Aug 2024 18:54:46 +0100
Subject: [PATCH 04/69] Adding limit in for wide tables to be related
 correctly.

---
 packages/backend-core/src/sql/sql.ts | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 74678a79b5..5f8a8e1a9b 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -39,6 +39,7 @@ import { dataFilters, helpers } from "@budibase/shared-core"
 import { cloneDeep } from "lodash"
 
 type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
+const MAX_SQS_RELATIONSHIP_FIELDS = 63
 
 function getBaseLimit() {
   const envLimit = environment.SQL_MAX_ROWS
@@ -800,9 +801,15 @@ class InternalBuilder {
         alias: throughAlias,
         schema: endpoint.schema,
       })
-      const relationshipFields = fields.filter(
+      let relationshipFields = fields.filter(
         field => field.split(".")[0] === toAlias
       )
+      if (this.client === SqlClient.SQL_LITE) {
+        relationshipFields = relationshipFields.slice(
+          0,
+          MAX_SQS_RELATIONSHIP_FIELDS
+        )
+      }
       const fieldList: string = relationshipFields
         .map(field => jsonField(field))
         .join(",")

From 0c604b7310a96aba482dab0b25296b5f428fabe3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Aug 2024 18:34:05 +0100
Subject: [PATCH 05/69] Moving things around, making join logic more
 accessible.

---
 packages/backend-core/src/sql/sql.ts | 169 ++++++++++++++++++---------
 1 file changed, 112 insertions(+), 57 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 5f8a8e1a9b..9fbb02a69e 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -337,6 +337,40 @@ class InternalBuilder {
     return filters
   }
 
+  addRelationshipForFilter(
+    query: Knex.QueryBuilder,
+    filterKey: string,
+    whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
+  ): Knex.QueryBuilder {
+    const mainKnex = this.knex
+    const { relationships, endpoint } = this.query
+    const tableName = endpoint.entityId
+    if (!relationships) {
+      return query
+    }
+    for (const relationship of relationships) {
+      // this is the relationship which is being filtered
+      if (filterKey.startsWith(relationship.column) && relationship.to) {
+        const subQuery = query.whereExists(function () {
+          this.select(mainKnex.raw(1)).from(relationship.tableName!)
+        })
+        query = whereCb(
+          this.addJoin(
+            subQuery,
+            {
+              from: relationship.tableName,
+              to: tableName,
+              through: relationship.through,
+            },
+            [relationship]
+          )
+        )
+        break
+      }
+    }
+    return query
+  }
+
   // right now we only do filters on the specific table being queried
   addFilters(
     query: Knex.QueryBuilder,
@@ -387,6 +421,7 @@ class InternalBuilder {
           fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
         }
         if (opts?.relationship && isRelationshipField) {
+          // TODO: need to update fn to take the query
           const [filterTableName, property] = updatedKey.split(".")
           const alias = getTableAlias(filterTableName)
           fn(alias ? `${alias}.${property}` : property, value)
@@ -839,12 +874,75 @@ class InternalBuilder {
     return query
   }
 
+  addJoin(
+    query: Knex.QueryBuilder,
+    tables: { from: string; to: string; through?: string },
+    columns: {
+      from?: string
+      to?: string
+      fromPrimary?: string
+      toPrimary?: string
+    }[]
+  ): Knex.QueryBuilder {
+    const { tableAliases: aliases, endpoint } = this.query
+    const schema = endpoint.schema
+    const toTable = tables.to,
+      fromTable = tables.from,
+      throughTable = tables.through
+    const toAlias = aliases?.[toTable] || toTable,
+      throughAlias = (throughTable && aliases?.[throughTable]) || throughTable,
+      fromAlias = aliases?.[fromTable] || fromTable
+    let toTableWithSchema = this.tableNameWithSchema(toTable, {
+      alias: toAlias,
+      schema,
+    })
+    let throughTableWithSchema = throughTable
+      ? this.tableNameWithSchema(throughTable, {
+          alias: throughAlias,
+          schema,
+        })
+      : undefined
+    if (!throughTable) {
+      // @ts-ignore
+      query = query.leftJoin(toTableWithSchema, function () {
+        for (let relationship of columns) {
+          const from = relationship.from,
+            to = relationship.to
+          // @ts-ignore
+          this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
+        }
+      })
+    } else {
+      query = query
+        // @ts-ignore
+        .leftJoin(throughTableWithSchema, function () {
+          for (let relationship of columns) {
+            const fromPrimary = relationship.fromPrimary
+            const from = relationship.from
+            // @ts-ignore
+            this.orOn(
+              `${fromAlias}.${fromPrimary}`,
+              "=",
+              `${throughAlias}.${from}`
+            )
+          }
+        })
+        .leftJoin(toTableWithSchema, function () {
+          for (let relationship of columns) {
+            const toPrimary = relationship.toPrimary
+            const to = relationship.to
+            // @ts-ignore
+            this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
+          }
+        })
+    }
+    return query
+  }
+
   addRelationships(
     query: Knex.QueryBuilder,
     fromTable: string,
-    relationships: RelationshipsJson[],
-    schema?: string,
-    aliases?: Record<string, string>
+    relationships: RelationshipsJson[]
   ): Knex.QueryBuilder {
     const tableSets: Record<string, [RelationshipsJson]> = {}
     // aggregate into table sets (all the same to tables)
@@ -865,51 +963,15 @@ class InternalBuilder {
     }
     for (let [key, relationships] of Object.entries(tableSets)) {
       const { toTable, throughTable } = JSON.parse(key)
-      const toAlias = aliases?.[toTable] || toTable,
-        throughAlias = aliases?.[throughTable] || throughTable,
-        fromAlias = aliases?.[fromTable] || fromTable
-      let toTableWithSchema = this.tableNameWithSchema(toTable, {
-        alias: toAlias,
-        schema,
-      })
-      let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
-        alias: throughAlias,
-        schema,
-      })
-      if (!throughTable) {
-        // @ts-ignore
-        query = query.leftJoin(toTableWithSchema, function () {
-          for (let relationship of relationships) {
-            const from = relationship.from,
-              to = relationship.to
-            // @ts-ignore
-            this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
-          }
-        })
-      } else {
-        query = query
-          // @ts-ignore
-          .leftJoin(throughTableWithSchema, function () {
-            for (let relationship of relationships) {
-              const fromPrimary = relationship.fromPrimary
-              const from = relationship.from
-              // @ts-ignore
-              this.orOn(
-                `${fromAlias}.${fromPrimary}`,
-                "=",
-                `${throughAlias}.${from}`
-              )
-            }
-          })
-          .leftJoin(toTableWithSchema, function () {
-            for (let relationship of relationships) {
-              const toPrimary = relationship.toPrimary
-              const to = relationship.to
-              // @ts-ignore
-              this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
-            }
-          })
-      }
+      query = this.addJoin(
+        query,
+        {
+          from: fromTable,
+          to: toTable,
+          through: throughTable,
+        },
+        relationships
+      )
     }
     return query
   }
@@ -1015,8 +1077,7 @@ class InternalBuilder {
       limits?: { base: number; query: number }
     } = {}
   ): Knex.QueryBuilder {
-    let { endpoint, filters, paginate, relationships, tableAliases } =
-      this.query
+    let { endpoint, filters, paginate, relationships } = this.query
     const { limits } = opts
     const counting = endpoint.operation === Operation.COUNT
 
@@ -1062,13 +1123,7 @@ class InternalBuilder {
     if (relationships && this.client === SqlClient.SQL_LITE) {
       query = this.addJsonRelationships(query, tableName, relationships)
     } else if (relationships) {
-      query = this.addRelationships(
-        query,
-        tableName,
-        relationships,
-        endpoint.schema,
-        tableAliases
-      )
+      query = this.addRelationships(query, tableName, relationships)
     }
 
     return this.addFilters(query, filters, { relationship: true })

From 49c1f34b5d5fe78fbfd21c5bb80a934149c8e81a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Aug 2024 18:41:02 +0100
Subject: [PATCH 06/69] Saving at this point - got exists working.

---
 packages/backend-core/src/sql/sql.ts | 173 ++++++++++++++++-----------
 1 file changed, 101 insertions(+), 72 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 9fbb02a69e..1243eaea8a 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -343,18 +343,24 @@ class InternalBuilder {
     whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder
   ): Knex.QueryBuilder {
     const mainKnex = this.knex
-    const { relationships, endpoint } = this.query
+    const { relationships, endpoint, tableAliases: aliases } = this.query
     const tableName = endpoint.entityId
     if (!relationships) {
       return query
     }
     for (const relationship of relationships) {
       // this is the relationship which is being filtered
-      if (filterKey.startsWith(relationship.column) && relationship.to) {
-        const subQuery = query.whereExists(function () {
-          this.select(mainKnex.raw(1)).from(relationship.tableName!)
-        })
-        query = whereCb(
+      if (
+        filterKey.startsWith(`${relationship.tableName}.`) &&
+        relationship.to &&
+        relationship.tableName
+      ) {
+        const relatedTableName = relationship.tableName
+        const alias = aliases?.[relatedTableName] || relatedTableName
+        let subQuery = mainKnex
+          .select(mainKnex.raw(1))
+          .from({ [alias]: relatedTableName })
+        subQuery = whereCb(
           this.addJoin(
             subQuery,
             {
@@ -365,6 +371,7 @@ class InternalBuilder {
             [relationship]
           )
         )
+        query = query.whereExists(subQuery)
         break
       }
     }
@@ -382,12 +389,13 @@ class InternalBuilder {
     if (!filters) {
       return query
     }
+    const builder = this
     filters = this.parseFilters({ ...filters })
     const aliases = this.query.tableAliases
     // if all or specified in filters, then everything is an or
     const allOr = filters.allOr
-    const tableName =
-      this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name
+    const isSqlite = this.client === SqlClient.SQL_LITE
+    const tableName = isSqlite ? this.table._id! : this.table.name
 
     function getTableAlias(name: string) {
       const alias = aliases?.[name]
@@ -395,13 +403,33 @@ class InternalBuilder {
     }
     function iterate(
       structure: AnySearchFilter,
-      fn: (key: string, value: any) => void,
-      complexKeyFn?: (key: string[], value: any) => void
+      fn: (
+        query: Knex.QueryBuilder,
+        key: string,
+        value: any
+      ) => Knex.QueryBuilder,
+      complexKeyFn?: (
+        query: Knex.QueryBuilder,
+        key: string[],
+        value: any
+      ) => Knex.QueryBuilder
     ) {
+      const handleRelationship = (
+        q: Knex.QueryBuilder,
+        key: string,
+        value: any
+      ) => {
+        const [filterTableName, ...otherProperties] = key.split(".")
+        const property = otherProperties.join(".")
+        const alias = getTableAlias(filterTableName)
+        return fn(q, alias ? `${alias}.${property}` : property, value)
+      }
       for (const key in structure) {
         const value = structure[key]
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
+        const shouldProcessRelationship =
+          opts?.relationship && isRelationshipField
 
         let castedTypeValue
         if (
@@ -410,7 +438,8 @@ class InternalBuilder {
           complexKeyFn
         ) {
           const alias = getTableAlias(tableName)
-          complexKeyFn(
+          query = complexKeyFn(
+            query,
             castedTypeValue.id.map((x: string) =>
               alias ? `${alias}.${x}` : x
             ),
@@ -418,27 +447,31 @@ class InternalBuilder {
           )
         } else if (!isRelationshipField) {
           const alias = getTableAlias(tableName)
-          fn(alias ? `${alias}.${updatedKey}` : updatedKey, value)
-        }
-        if (opts?.relationship && isRelationshipField) {
-          // TODO: need to update fn to take the query
-          const [filterTableName, property] = updatedKey.split(".")
-          const alias = getTableAlias(filterTableName)
-          fn(alias ? `${alias}.${property}` : property, value)
+          query = fn(
+            query,
+            alias ? `${alias}.${updatedKey}` : updatedKey,
+            value
+          )
+        } else if (isSqlite && shouldProcessRelationship) {
+          query = builder.addRelationshipForFilter(query, updatedKey, q => {
+            return handleRelationship(q, updatedKey, value)
+          })
+        } else if (shouldProcessRelationship) {
+          query = handleRelationship(query, updatedKey, value)
         }
       }
     }
 
-    const like = (key: string, value: any) => {
+    const like = (q: Knex.QueryBuilder, key: string, value: any) => {
       const fuzzyOr = filters?.fuzzyOr
       const fnc = fuzzyOr || allOr ? "orWhere" : "where"
       // postgres supports ilike, nothing else does
       if (this.client === SqlClient.POSTGRES) {
-        query = query[fnc](key, "ilike", `%${value}%`)
+        return q[fnc](key, "ilike", `%${value}%`)
       } else {
         const rawFnc = `${fnc}Raw`
         // @ts-ignore
-        query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+        return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
           `%${value.toLowerCase()}%`,
         ])
       }
@@ -456,13 +489,13 @@ class InternalBuilder {
         return `[${value.join(",")}]`
       }
       if (this.client === SqlClient.POSTGRES) {
-        iterate(mode, (key, value) => {
+        iterate(mode, (q, key, value) => {
           const wrap = any ? "" : "'"
           const op = any ? "\\?| array" : "@>"
           const fieldNames = key.split(/\./g)
           const table = fieldNames[0]
           const col = fieldNames[1]
-          query = query[rawFnc](
+          return q[rawFnc](
             `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray(
               value,
               any ? "'" : '"'
@@ -471,8 +504,8 @@ class InternalBuilder {
         })
       } else if (this.client === SqlClient.MY_SQL) {
         const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS"
-        iterate(mode, (key, value) => {
-          query = query[rawFnc](
+        iterate(mode, (q, key, value) => {
+          return q[rawFnc](
             `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray(
               value
             )}'), FALSE)`
@@ -480,7 +513,7 @@ class InternalBuilder {
         })
       } else {
         const andOr = mode === filters?.containsAny ? " OR " : " AND "
-        iterate(mode, (key, value) => {
+        iterate(mode, (q, key, value) => {
           let statement = ""
           const identifier = this.quotedIdentifier(key)
           for (let i in value) {
@@ -495,16 +528,16 @@ class InternalBuilder {
           }
 
           if (statement === "") {
-            return
+            return q
           }
 
           if (not) {
-            query = query[rawFnc](
+            return q[rawFnc](
               `(NOT (${statement}) OR ${identifier} IS NULL)`,
               value
             )
           } else {
-            query = query[rawFnc](statement, value)
+            return q[rawFnc](statement, value)
           }
         })
       }
@@ -534,39 +567,39 @@ class InternalBuilder {
       const fnc = allOr ? "orWhereIn" : "whereIn"
       iterate(
         filters.oneOf,
-        (key: string, array) => {
+        (q, key: string, array) => {
           if (this.client === SqlClient.ORACLE) {
             key = this.convertClobs(key)
             array = Array.isArray(array) ? array : [array]
             const binding = new Array(array.length).fill("?").join(",")
-            query = query.whereRaw(`${key} IN (${binding})`, array)
+            return q.whereRaw(`${key} IN (${binding})`, array)
           } else {
-            query = query[fnc](key, Array.isArray(array) ? array : [array])
+            return q[fnc](key, Array.isArray(array) ? array : [array])
           }
         },
-        (key: string[], array) => {
+        (q, key: string[], array) => {
           if (this.client === SqlClient.ORACLE) {
             const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})`
             const binding = `(${array
               .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`)
               .join(",")})`
-            query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat())
+            return q.whereRaw(`${keyStr} IN ${binding}`, array.flat())
           } else {
-            query = query[fnc](key, Array.isArray(array) ? array : [array])
+            return q[fnc](key, Array.isArray(array) ? array : [array])
           }
         }
       )
     }
     if (filters.string) {
-      iterate(filters.string, (key, value) => {
+      iterate(filters.string, (q, key, value) => {
         const fnc = allOr ? "orWhere" : "where"
         // postgres supports ilike, nothing else does
         if (this.client === SqlClient.POSTGRES) {
-          query = query[fnc](key, "ilike", `${value}%`)
+          return q[fnc](key, "ilike", `${value}%`)
         } else {
           const rawFnc = `${fnc}Raw`
           // @ts-ignore
-          query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
+          return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [
             `${value.toLowerCase()}%`,
           ])
         }
@@ -576,7 +609,7 @@ class InternalBuilder {
       iterate(filters.fuzzy, like)
     }
     if (filters.range) {
-      iterate(filters.range, (key, value) => {
+      iterate(filters.range, (q, key, value) => {
         const isEmptyObject = (val: any) => {
           return (
             val &&
@@ -605,97 +638,93 @@ class InternalBuilder {
             schema?.type === FieldType.BIGINT &&
             this.client === SqlClient.SQL_LITE
           ) {
-            query = query.whereRaw(
+            return q.whereRaw(
               `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
               [value.low, value.high]
             )
           } else {
             const fnc = allOr ? "orWhereBetween" : "whereBetween"
-            query = query[fnc](key, [value.low, value.high])
+            return q[fnc](key, [value.low, value.high])
           }
         } else if (lowValid) {
           if (
             schema?.type === FieldType.BIGINT &&
             this.client === SqlClient.SQL_LITE
           ) {
-            query = query.whereRaw(
-              `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`,
-              [value.low]
-            )
+            return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
+              value.low,
+            ])
           } else {
             const fnc = allOr ? "orWhere" : "where"
-            query = query[fnc](key, ">=", value.low)
+            return q[fnc](key, ">=", value.low)
           }
         } else if (highValid) {
           if (
             schema?.type === FieldType.BIGINT &&
             this.client === SqlClient.SQL_LITE
           ) {
-            query = query.whereRaw(
-              `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`,
-              [value.high]
-            )
+            return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
+              value.high,
+            ])
           } else {
             const fnc = allOr ? "orWhere" : "where"
-            query = query[fnc](key, "<=", value.high)
+            return q[fnc](key, "<=", value.high)
           }
         }
+        return q
       })
     }
     if (filters.equal) {
-      iterate(filters.equal, (key, value) => {
+      iterate(filters.equal, (q, key, value) => {
         const fnc = allOr ? "orWhereRaw" : "whereRaw"
         if (this.client === SqlClient.MS_SQL) {
-          query = query[fnc](
+          return q[fnc](
             `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`,
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
           const identifier = this.convertClobs(key)
-          query = query[fnc](
-            `(${identifier} IS NOT NULL AND ${identifier} = ?)`,
-            [value]
-          )
+          return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [
+            value,
+          ])
         } else {
-          query = query[fnc](
-            `COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`,
-            [value]
-          )
+          return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [
+            value,
+          ])
         }
       })
     }
     if (filters.notEqual) {
-      iterate(filters.notEqual, (key, value) => {
+      iterate(filters.notEqual, (q, key, value) => {
         const fnc = allOr ? "orWhereRaw" : "whereRaw"
         if (this.client === SqlClient.MS_SQL) {
-          query = query[fnc](
+          return q[fnc](
             `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`,
             [value]
           )
         } else if (this.client === SqlClient.ORACLE) {
           const identifier = this.convertClobs(key)
-          query = query[fnc](
+          return q[fnc](
             `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`,
             [value]
           )
         } else {
-          query = query[fnc](
-            `COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`,
-            [value]
-          )
+          return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [
+            value,
+          ])
         }
       })
     }
     if (filters.empty) {
-      iterate(filters.empty, key => {
+      iterate(filters.empty, (q, key) => {
         const fnc = allOr ? "orWhereNull" : "whereNull"
-        query = query[fnc](key)
+        return q[fnc](key)
       })
     }
     if (filters.notEmpty) {
-      iterate(filters.notEmpty, key => {
+      iterate(filters.notEmpty, (q, key) => {
         const fnc = allOr ? "orWhereNotNull" : "whereNotNull"
-        query = query[fnc](key)
+        return q[fnc](key)
       })
     }
     if (filters.contains) {

From 628964364a3ce5e7c6a988656b8f0fa535d15c07 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Aug 2024 18:55:15 +0100
Subject: [PATCH 07/69] Getting through join working as expected.

---
 packages/backend-core/src/sql/sql.ts | 40 +++++++++++++++++++---------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 1243eaea8a..84231288a2 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -345,6 +345,7 @@ class InternalBuilder {
     const mainKnex = this.knex
     const { relationships, endpoint, tableAliases: aliases } = this.query
     const tableName = endpoint.entityId
+    const fromAlias = aliases?.[tableName] || tableName
     if (!relationships) {
       return query
     }
@@ -356,22 +357,37 @@ class InternalBuilder {
         relationship.tableName
       ) {
         const relatedTableName = relationship.tableName
-        const alias = aliases?.[relatedTableName] || relatedTableName
+        const toAlias = aliases?.[relatedTableName] || relatedTableName
         let subQuery = mainKnex
           .select(mainKnex.raw(1))
-          .from({ [alias]: relatedTableName })
-        subQuery = whereCb(
-          this.addJoin(
-            subQuery,
-            {
-              from: relationship.tableName,
-              to: tableName,
-              through: relationship.through,
-            },
-            [relationship]
+          .from({ [toAlias]: relatedTableName })
+        let mainTableRelatesTo = toAlias
+        if (relationship.through) {
+          const throughAlias =
+            aliases?.[relationship.through] || relationship.through
+          let throughTable = this.tableNameWithSchema(relationship.through, {
+            alias: throughAlias,
+            schema: endpoint.schema,
+          })
+          subQuery = subQuery.innerJoin(throughTable, function () {
+            // @ts-ignore
+            this.orOn(
+              `${toAlias}.${relationship.toPrimary}`,
+              "=",
+              `${throughAlias}.${relationship.to}`
+            )
+          })
+          mainTableRelatesTo = throughAlias
+        }
+        // "join" to the main table, making sure the ID matches that of the main
+        subQuery = subQuery.where(
+          `${mainTableRelatesTo}.${relationship.from}`,
+          "=",
+          mainKnex.raw(
+            this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`)
           )
         )
-        query = query.whereExists(subQuery)
+        query = query.whereExists(whereCb(subQuery))
         break
       }
     }

From 3e51dde6d2be09351faaed3e6c79f214977d65ff Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 29 Aug 2024 17:58:11 +0100
Subject: [PATCH 08/69] Check for alias as well when deciding whether filter
 requires relationship addition.

---
 packages/backend-core/src/sql/sql.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 84231288a2..9b6819d433 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -346,21 +346,25 @@ class InternalBuilder {
     const { relationships, endpoint, tableAliases: aliases } = this.query
     const tableName = endpoint.entityId
     const fromAlias = aliases?.[tableName] || tableName
+    const matches = (possibleTable: string) =>
+      filterKey.startsWith(`${possibleTable}`)
     if (!relationships) {
       return query
     }
     for (const relationship of relationships) {
+      const relatedTableName = relationship.tableName
+      const toAlias = aliases?.[relatedTableName] || relatedTableName
       // this is the relationship which is being filtered
       if (
-        filterKey.startsWith(`${relationship.tableName}.`) &&
+        (matches(relatedTableName) || matches(toAlias)) &&
         relationship.to &&
         relationship.tableName
       ) {
-        const relatedTableName = relationship.tableName
-        const toAlias = aliases?.[relatedTableName] || relatedTableName
         let subQuery = mainKnex
           .select(mainKnex.raw(1))
           .from({ [toAlias]: relatedTableName })
+          // relationships should never have more than the base limit
+          .limit(getBaseLimit())
         let mainTableRelatesTo = toAlias
         if (relationship.through) {
           const throughAlias =

From a9b1a2240332e7ea337ed0e2c6625b97228ab2ad Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 29 Aug 2024 18:56:14 +0100
Subject: [PATCH 09/69] Some improvements to get SQS tests passing.

---
 packages/backend-core/src/sql/sql.ts          | 20 +++++++++++++++----
 .../src/api/routes/tests/search.spec.ts       |  3 ++-
 .../src/utilities/rowProcessor/index.ts       |  7 +++++++
 3 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 9b6819d433..e6738d4b36 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -337,6 +337,11 @@ class InternalBuilder {
     return filters
   }
 
+  addJoinFieldCheck(query: Knex.QueryBuilder, relationship: RelationshipsJson) {
+    const document = relationship.from?.split(".")[0] || ""
+    return query.andWhere(`${document}.fieldName`, "=", relationship.column)
+  }
+
   addRelationshipForFilter(
     query: Knex.QueryBuilder,
     filterKey: string,
@@ -363,8 +368,6 @@ class InternalBuilder {
         let subQuery = mainKnex
           .select(mainKnex.raw(1))
           .from({ [toAlias]: relatedTableName })
-          // relationships should never have more than the base limit
-          .limit(getBaseLimit())
         let mainTableRelatesTo = toAlias
         if (relationship.through) {
           const throughAlias =
@@ -375,12 +378,15 @@ class InternalBuilder {
           })
           subQuery = subQuery.innerJoin(throughTable, function () {
             // @ts-ignore
-            this.orOn(
+            this.on(
               `${toAlias}.${relationship.toPrimary}`,
               "=",
               `${throughAlias}.${relationship.to}`
             )
           })
+          if (this.client === SqlClient.SQL_LITE) {
+            subQuery = this.addJoinFieldCheck(subQuery, relationship)
+          }
           mainTableRelatesTo = throughAlias
         }
         // "join" to the main table, making sure the ID matches that of the main
@@ -907,7 +913,7 @@ class InternalBuilder {
         default:
           throw new Error(`JSON relationships not implement for ${this.client}`)
       }
-      const subQuery = this.knex
+      let subQuery = this.knex
         .select(rawJsonArray)
         .from(toTableWithSchema)
         .join(throughTableWithSchema, function () {
@@ -918,6 +924,12 @@ class InternalBuilder {
           "=",
           this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
         )
+        // relationships should never have more than the base limit
+        .limit(getBaseLimit())
+      // need to check the junction table document is to the right column
+      if (this.client === SqlClient.SQL_LITE) {
+        subQuery = this.addJoinFieldCheck(subQuery, relationship)
+      }
       query = query.select({ [relationship.column]: subQuery })
     }
     return query
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index bac9b6f774..44fe18d0ef 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2687,7 +2687,8 @@ describe.each([
       })
     })
 
-  isSql &&
+  // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem
+  !isInternal &&
     describe("pagination edge case with relationships", () => {
       let mainRows: Row[] = []
 
diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts
index 795f6970ab..6028b2d248 100644
--- a/packages/server/src/utilities/rowProcessor/index.ts
+++ b/packages/server/src/utilities/rowProcessor/index.ts
@@ -336,6 +336,13 @@ export async function outputProcessing<T extends Row[] | Row>(
           row[property] = `${hours}:${minutes}:${seconds}`
         }
       }
+    } else if (column.type === FieldType.LINK) {
+      for (let row of enriched) {
+        // if relationship is empty - remove the array, this has been part of the API for some time
+        if (Array.isArray(row[property]) && row[property].length === 0) {
+          delete row[property]
+        }
+      }
     }
   }
 

From 67301054913024ecab6e0bc5650ee464c73cd34d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 30 Aug 2024 13:44:23 +0100
Subject: [PATCH 10/69] Adding the option to disable user sync, always
 importing large apps which are problematic.

---
 packages/server/src/environment.ts               |  1 +
 packages/server/src/sdk/app/applications/sync.ts | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 05ebbe3151..585eb6a7f2 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -112,6 +112,7 @@ const environment = {
     parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) ||
     DEFAULTS.JS_RUNNER_MEMORY_LIMIT,
   LOG_JS_ERRORS: process.env.LOG_JS_ERRORS,
+  DISABLE_USER_SYNC: process.env.DISABLE_USER_SYNC,
   // old
   CLIENT_ID: process.env.CLIENT_ID,
   _set(key: string, value: any) {
diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts
index 44e44a5aaa..37450acf1d 100644
--- a/packages/server/src/sdk/app/applications/sync.ts
+++ b/packages/server/src/sdk/app/applications/sync.ts
@@ -8,6 +8,10 @@ import { generateUserMetadataID, InternalTables } from "../../../db/utils"
 
 type DeletedUser = { _id: string; deleted: boolean }
 
+function userSyncEnabled() {
+  return !env.DISABLE_USER_SYNC
+}
+
 async function syncUsersToApp(
   appId: string,
   users: (User | DeletedUser)[],
@@ -56,7 +60,7 @@ async function syncUsersToApp(
 
       // the user doesn't exist, or doesn't have a role anymore
       // get rid of their metadata
-      if (deletedUser || !roleId) {
+      if (userSyncEnabled() && (deletedUser || !roleId)) {
         await db.remove(metadata)
         continue
       }
@@ -149,7 +153,9 @@ export async function syncApp(
   }
 
   // sync the users - kept for safe keeping
-  await sdk.users.syncGlobalUsers()
+  if (userSyncEnabled()) {
+    await sdk.users.syncGlobalUsers()
+  }
 
   if (error) {
     throw error

From 1a57e37d38502e12b88ac625d475e656388f74de Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 2 Sep 2024 15:22:33 +0100
Subject: [PATCH 11/69] initial framework for platform AI

---
 .../portal/settings/ai/ConfigModal.svelte     | 81 +++++++++++++++++++
 .../builder/portal/settings/ai/index.svelte   | 74 +++++++++++++++++
 packages/builder/src/stores/portal/menu.js    |  4 +
 3 files changed, 159 insertions(+)
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/index.svelte

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
new file mode 100644
index 0000000000..408e71bd1f
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -0,0 +1,81 @@
+<script>
+  import {
+    ModalContent,
+    Label,
+    Input,
+    Select,
+    Toggle,
+    Dropzone,
+    Body,
+    notifications,
+  } from "@budibase/bbui"
+  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
+  import { plugins } from "stores/portal"
+  import { PluginSource } from "constants"
+
+  const options = [
+    "OpenAI",
+    "Anthropic",
+    "Together AI",
+    "Azure Open AI",
+    "Custom"
+  ]
+
+  let source = PluginSource.GITHUB
+  let dynamicValues = {}
+
+  let name
+  let baseUrl
+  let apiKey
+  let active
+  let isDefault
+
+  let validation
+</script>
+
+<ModalContent
+    confirmText={"Save"}
+    onConfirm={() => {}}
+    disabled={!validation}
+    size="M"
+    title="Custom AI Configuration"
+>
+  <div class="form-row">
+    <Label size="M">Provider</Label>
+    <Select
+        placeholder={null}
+        bind:value={source}
+        options={options}
+    />
+  </div>
+  <div class="form-row">
+    <Label size="M">Model</Label>
+    <Select
+        placeholder={null}
+        bind:value={source}
+        options={Object.values(PluginSource)}
+    />
+  </div>
+  <div class="form-row">
+    <Label size="M">Name</Label>
+    <Input placeholder={"Test 1"} bind:value={name} />
+  </div>
+  <div class="form-row">
+    <Label size="M">Base URL</Label>
+    <Input placeholder={"www.google.com"} bind:value={baseUrl} />
+  </div>
+  <div class="form-row">
+    <Label size="M">API Key</Label>
+    <Input bind:value={apiKey} />
+  </div>
+  <Toggle text="Active" bind:value={active} />
+  <Toggle text="Set as default" bind:value={isDefault} />
+</ModalContent>
+
+<style>
+    .form-row {
+        display: grid;
+        grid-gap: var(--spacing-s);
+        align-items: center;
+    }
+</style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
new file mode 100644
index 0000000000..e40a6cf4ff
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -0,0 +1,74 @@
+<script>
+  import {
+    Button,
+    Layout,
+    Heading,
+    Body,
+    Helpers,
+    Divider,
+    notifications,
+    Modal,
+    ModalContent,
+  } from "@budibase/bbui"
+  import { auth, admin } from "stores/portal"
+  import { redirect } from "@roxi/routify"
+  import { API } from "api"
+  import AIConfigModal from "./ConfigModal.svelte"
+  import { onMount } from "svelte"
+  import { sdk } from "@budibase/shared-core"
+
+  let diagnosticInfo = ""
+  let modal
+
+  // async function fetchSystemDebugInfo() {
+  //   const diagnostics = await API.fetchSystemDebugInfo()
+  //   diagnosticInfo = {
+  //     browser: {
+  //       language: navigator.language || navigator.userLanguage,
+  //       userAgent: navigator.userAgent,
+  //       platform: navigator.platform,
+  //       vendor: navigator.vendor,
+  //     },
+  //     server: diagnostics,
+  //   }
+  // }
+  //
+  // const copyToClipboard = async () => {
+  //   await Helpers.copyToClipboard(JSON.stringify(diagnosticInfo, undefined, 2))
+  //   notifications.success("Copied")
+  // }
+
+  onMount(async () => {
+    // await fetchSystemDebugInfo()
+  })
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+
+<Modal bind:this={modal}>
+  <AIConfigModal />
+</Modal>
+<Layout noPadding>
+  <Layout gap="XS" noPadding>
+    <Heading size="M">AI</Heading>
+    <Body>Configure your AI settings within this section:</Body>
+  </Layout>
+  <Divider />
+  <Layout noPadding>
+    <div class="config-heading">
+      <Heading size="S">AI Configurations</Heading>
+      <Button size="S" cta on:click={modal.show}>Add configuration</Button>
+    </div>
+    <Body size="S">Use the following interface to select your preferred AI configuration.</Body>
+    <Body size="S">Select your AI Model:</Body>
+  </Layout>
+</Layout>
+
+<style>
+  .config-heading {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+  }
+</style>
diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index c7c71d89bc..ec40f6d4ee 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -45,6 +45,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Auth",
         href: "/builder/portal/settings/auth",
       },
+      {
+        title: "AI",
+        href: "/builder/portal/settings/ai",
+      },
       {
         title: "Email",
         href: "/builder/portal/settings/email",

From 99035ad8aea269ef93c6e26cd892c167503797a9 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 2 Sep 2024 20:16:37 +0100
Subject: [PATCH 12/69] rough pass on AI platform CRUD

---
 packages/backend-core/src/configs/configs.ts  |  41 +++++++
 .../portal/settings/ai/ConfigModal.svelte     | 106 +++++++++++++-----
 .../builder/portal/settings/ai/ai.spec.ts     |   0
 .../builder/portal/settings/ai/constants.ts   |   8 ++
 .../builder/portal/settings/ai/index.svelte   |  69 +++++++-----
 .../builder/src/stores/portal/licensing.js    |  12 ++
 .../functions/backfill/global/configs.ts      |   7 +-
 packages/types/src/documents/global/config.ts |  18 +++
 packages/types/src/sdk/licensing/feature.ts   |   2 +
 .../src/api/controllers/global/configs.ts     |  14 +++
 10 files changed, 220 insertions(+), 57 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/constants.ts

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 0d189e3f7d..6062d3624f 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -1,4 +1,5 @@
 import {
+  AIConfig, AIInnerConfig,
   Config,
   ConfigType,
   GoogleConfig,
@@ -17,6 +18,7 @@ import { DocumentType, SEPARATOR } from "../constants"
 import { CacheKey, TTL, withCache } from "../cache"
 import * as context from "../context"
 import env from "../environment"
+import { getConfigParams } from "@budibase/server/src/migrations/functions/backfill/global/configs"
 
 // UTILS
 
@@ -254,3 +256,42 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
   const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
   return config?.config
 }
+
+
+// AI
+
+// TODO: Can we assume that you are licensed when you hit this endpoint?
+export async function getAIConfig(): Promise<AIConfig | undefined> {
+  if (!env.SELF_HOSTED) {
+    // always use the env vars in cloud
+    // TODO: Licensing stuff - make this right
+    if (env.OPENAI_API_KEY) {
+      return getDefaultBudibaseAIConfig()
+    }
+  }
+
+  // prefer the config in self-host
+  let config = await getConfig<AIConfig>(ConfigType.AI)
+
+  // fallback to env vars
+  if (!config || !config.activated) {
+    config = getDefaultBudibaseAIConfig()
+  }
+
+  return config
+}
+
+export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined {
+  if (env.OPENAI_API_KEY) {
+    return {
+      provider: "",
+      isDefault: true,
+      name: "Budibase AI",
+      active: true,
+      baseUrl: "",
+      apiKey: env.OPENAI_API_KEY,
+      // TODO: should be enum
+      defaultModel: ""
+    }
+  }
+}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 408e71bd1f..30830cff1a 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -5,15 +5,14 @@
     Input,
     Select,
     Toggle,
-    Dropzone,
     Body,
     notifications,
   } from "@budibase/bbui"
-  import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte"
-  import { plugins } from "stores/portal"
-  import { PluginSource } from "constants"
+  import { ConfigMap } from "./constants"
+  import { API } from "api"
 
-  const options = [
+  // TODO: Update these
+  const providers = [
     "OpenAI",
     "Anthropic",
     "Together AI",
@@ -21,21 +20,69 @@
     "Custom"
   ]
 
-  let source = PluginSource.GITHUB
-  let dynamicValues = {}
+  const models = [
+    "gpt4o-mini",
+  ]
 
-  let name
-  let baseUrl
-  let apiKey
-  let active
-  let isDefault
+  let formValues = {}
 
   let validation
+
+  $: {
+    const { provider, model, name, apiKey } = formValues
+    validation = provider && model && name && apiKey
+  }
+
+  function prefillConfig(evt) {
+    const provider = evt.detail
+    // grab the preset config from the constants for that provider and fill it in
+    if (ConfigMap[provider]) {
+      formValues = {
+        ...formValues,
+        ...ConfigMap[provider]
+      }
+    } else {
+      formValues = {
+        provider
+      }
+    }
+  }
+
+  async function saveConfig() {
+    try {
+      const savedConfig = await API.saveConfig(formValues)
+      formValues._rev = savedConfig._rev
+      formValues._id = savedConfig._id
+      notifications.success(`Configuration saved`)
+    } catch (error) {
+      notifications.error(
+        `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
+      )
+    }
+  }
+
+  async function deleteConfig() {
+    // Delete a configuration
+    try {
+      // await API.deleteConfig({
+      //   id: smtpConfig._id,
+      //   rev: smtpConfig._rev,
+      // })
+      notifications.success(`Deleted config`)
+    } catch (error) {
+      notifications.error(
+        `Failed to clear email settings, reason: ${error?.message || "Unknown"}`
+      )
+    }
+  }
+
 </script>
 
 <ModalContent
     confirmText={"Save"}
-    onConfirm={() => {}}
+    cancelText={"Delete"}
+    onConfirm={saveConfig}
+    onCancel={deleteConfig}
     disabled={!validation}
     size="M"
     title="Custom AI Configuration"
@@ -43,39 +90,40 @@
   <div class="form-row">
     <Label size="M">Provider</Label>
     <Select
-        placeholder={null}
-        bind:value={source}
-        options={options}
+      placeholder={null}
+      bind:value={formValues.provider}
+      options={providers}
+      on:change={prefillConfig}
     />
   </div>
   <div class="form-row">
     <Label size="M">Model</Label>
     <Select
-        placeholder={null}
-        bind:value={source}
-        options={Object.values(PluginSource)}
+      placeholder={null}
+      bind:value={formValues.model}
+      options={models}
     />
   </div>
   <div class="form-row">
     <Label size="M">Name</Label>
-    <Input placeholder={"Test 1"} bind:value={name} />
+    <Input placeholder={"Test 1"} bind:value={formValues.name}/>
   </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
-    <Input placeholder={"www.google.com"} bind:value={baseUrl} />
+    <Input placeholder={"www.google.com"} bind:value={formValues.baseUrl}/>
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
-    <Input bind:value={apiKey} />
+    <Input bind:value={formValues.apiKey}/>
   </div>
-  <Toggle text="Active" bind:value={active} />
-  <Toggle text="Set as default" bind:value={isDefault} />
+  <Toggle text="Active" bind:value={formValues.active}/>
+  <Toggle text="Set as default" bind:value={formValues.isDefault}/>
 </ModalContent>
 
 <style>
-    .form-row {
-        display: grid;
-        grid-gap: var(--spacing-s);
-        align-items: center;
-    }
+  .form-row {
+    display: grid;
+    grid-gap: var(--spacing-s);
+    align-items: center;
+  }
 </style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts b/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
new file mode 100644
index 0000000000..df5508109d
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -0,0 +1,8 @@
+export const ConfigMap = {
+  OpenAI: {
+    baseUrl: "https://api.openai.com"
+  },
+  Anthropic: {
+    baseUrl: "https://api.together.xyz/v1"
+  },
+}
\ No newline at end of file
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index e40a6cf4ff..427b09a7f2 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -8,57 +8,76 @@
     Divider,
     notifications,
     Modal,
-    ModalContent,
+    Tags,
+    Tag
   } from "@budibase/bbui"
-  import { auth, admin } from "stores/portal"
-  import { redirect } from "@roxi/routify"
+  import { admin, licensing } from "stores/portal"
   import { API } from "api"
   import AIConfigModal from "./ConfigModal.svelte"
   import { onMount } from "svelte"
   import { sdk } from "@budibase/shared-core"
 
-  let diagnosticInfo = ""
-  let modal
+  const ConfigTypes = {
+    AI: "ai",
+  }
 
-  // async function fetchSystemDebugInfo() {
-  //   const diagnostics = await API.fetchSystemDebugInfo()
-  //   diagnosticInfo = {
-  //     browser: {
-  //       language: navigator.language || navigator.userLanguage,
-  //       userAgent: navigator.userAgent,
-  //       platform: navigator.platform,
-  //       vendor: navigator.vendor,
-  //     },
-  //     server: diagnostics,
-  //   }
-  // }
-  //
-  // const copyToClipboard = async () => {
-  //   await Helpers.copyToClipboard(JSON.stringify(diagnosticInfo, undefined, 2))
-  //   notifications.success("Copied")
-  // }
+  let modal
+  let aiConfig
+  let loading = false
+
+  $: isCloud = $admin.cloud
+  $: budibaseAIEnabled = $licensing.budibaseAIEnabled
+  $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled
+
+  async function fetchAIConfig() {
+    loading = true
+    try {
+      // Fetch the configs for smtp
+      const aiDoc = await API.getConfig(ConfigTypes.AI)
+      if (aiDoc._id) {
+        aiConfig = aiDoc
+      }
+      loading = false
+    } catch (error) {
+      notifications.error("Error fetching AI config")
+    }
+  }
 
   onMount(async () => {
-    // await fetchSystemDebugInfo()
+    await fetchAIConfig()
   })
 </script>
 
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
-
 <Modal bind:this={modal}>
   <AIConfigModal />
 </Modal>
 <Layout noPadding>
   <Layout gap="XS" noPadding>
     <Heading size="M">AI</Heading>
+    {#if isCloud && !budibaseAIEnabled}
+      <Tags>
+        <Tag icon="LockClosed">Premium</Tag>
+      </Tags>
+    {/if}
     <Body>Configure your AI settings within this section:</Body>
   </Layout>
   <Divider />
   <Layout noPadding>
     <div class="config-heading">
       <Heading size="S">AI Configurations</Heading>
-      <Button size="S" cta on:click={modal.show}>Add configuration</Button>
+      {#if !isCloud && !customAIConfigsEnabled}
+        <Tags>
+          <Tag icon="LockClosed">Premium</Tag>
+        </Tags>
+      {:else if isCloud && !customAIConfigsEnabled}
+        <Tags>
+          <Tag icon="LockClosed">Enterprise</Tag>
+        </Tags>
+      {:else}
+        <Button size="S" cta on:click={modal.show}>Add configuration</Button>
+      {/if}
     </div>
     <Body size="S">Use the following interface to select your preferred AI configuration.</Body>
     <Body size="S">Select your AI Model:</Body>
diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js
index 0e44650479..d48377207c 100644
--- a/packages/builder/src/stores/portal/licensing.js
+++ b/packages/builder/src/stores/portal/licensing.js
@@ -22,6 +22,8 @@ export const createLicensingStore = () => {
     backupsEnabled: false,
     brandingEnabled: false,
     scimEnabled: false,
+    budibaseAIEnabled: false,
+    customAIConfigsEnabled: false,
     // the currently used quotas from the db
     quotaUsage: undefined,
     // derived quota metrics for percentages used
@@ -142,6 +144,14 @@ export const createLicensingStore = () => {
         Constants.Features.VIEW_READONLY_COLUMNS
       )
 
+      const budibaseAIEnabled = license.features.includes(
+        Constants.Features.BUDIBASE_AI
+      )
+
+      const customAIConfigsEnabled = license.features.includes(
+        Constants.Features.AI_CUSTOM_CONFIGS
+      )
+
       store.update(state => {
         return {
           ...state,
@@ -153,6 +163,8 @@ export const createLicensingStore = () => {
           groupsEnabled,
           backupsEnabled,
           brandingEnabled,
+          budibaseAIEnabled,
+          customAIConfigsEnabled,
           scimEnabled,
           environmentVariablesEnabled,
           auditLogsEnabled,
diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts
index 04eb9caff2..bb0781eba3 100644
--- a/packages/server/src/migrations/functions/backfill/global/configs.ts
+++ b/packages/server/src/migrations/functions/backfill/global/configs.ts
@@ -15,11 +15,12 @@ import {
 } from "@budibase/types"
 import env from "./../../../../environment"
 
-export function getConfigParams(): DatabaseQueryOpts {
+export function getConfigParams(type?: ConfigType): DatabaseQueryOpts {
+  const configType = type || ""
   return {
     include_docs: true,
-    startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
-    endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
+    startkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}`,
+    endkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}${UNICODE_MAX}`,
   }
 }
 
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 3fd352aada..246874d0f1 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -111,6 +111,20 @@ export interface SCIMInnerConfig {
 
 export interface SCIMConfig extends Config<SCIMInnerConfig> {}
 
+export interface AIInnerConfig {
+  // TODO: should be enum
+  provider: string
+  isDefault: boolean
+  name: string
+  active: boolean
+  baseUrl: string
+  apiKey: string
+  // TODO: should be enum
+  defaultModel: string
+}
+
+export interface AIConfig extends Config<AIInnerConfig> {}
+
 export const isSettingsConfig = (config: Config): config is SettingsConfig =>
   config.type === ConfigType.SETTINGS
 
@@ -126,6 +140,9 @@ export const isOIDCConfig = (config: Config): config is OIDCConfig =>
 export const isSCIMConfig = (config: Config): config is SCIMConfig =>
   config.type === ConfigType.SCIM
 
+export const isAIConfig = (config: Config): config is AIConfig =>
+  config.type === ConfigType.AI
+
 export enum ConfigType {
   SETTINGS = "settings",
   ACCOUNT = "account",
@@ -134,4 +151,5 @@ export enum ConfigType {
   OIDC = "oidc",
   OIDC_LOGOS = "logos_oidc",
   SCIM = "scim",
+  AI = "ai"
 }
diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts
index 0d8db8258e..bf2bcb72b6 100644
--- a/packages/types/src/sdk/licensing/feature.ts
+++ b/packages/types/src/sdk/licensing/feature.ts
@@ -15,6 +15,8 @@ export enum Feature {
   EXPANDED_PUBLIC_API = "expandedPublicApi",
   VIEW_PERMISSIONS = "viewPermissions",
   VIEW_READONLY_COLUMNS = "viewReadonlyColumns",
+  BUDIBASE_AI = "budibaseAI",
+  AI_CUSTOM_CONFIGS = "customAIConfigs"
 }
 
 export type PlanFeatures = { [key in PlanType]: Feature[] | undefined }
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index a44b173869..43aac19b18 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -197,6 +197,12 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
 }
 
+async function verifyAIConfig(config: OIDCConfigs) {
+  // await verifySSOConfig(ConfigType.OIDC, config.configs[0])
+  // Shape should be `config_ai`
+  return true
+}
+
 export async function save(ctx: UserCtx<Config>) {
   const body = ctx.request.body
   const type = body.type
@@ -224,6 +230,9 @@ export async function save(ctx: UserCtx<Config>) {
       case ConfigType.OIDC:
         await verifyOIDCConfig(config)
         break
+      case ConfigType.AI:
+        await verifyAIConfig(config)
+        break
     }
   } catch (err: any) {
     ctx.throw(400, err)
@@ -314,6 +323,11 @@ export async function find(ctx: UserCtx) {
       if (type === ConfigType.OIDC_LOGOS) {
         enrichOIDCLogos(scopedConfig)
       }
+
+      if (type === ConfigType.AI) {
+        // TODO: strip the keys from the configs here
+        // TODO: do the licensing checks here and return the right things based on the license
+      }
       ctx.body = scopedConfig
     } else {
       // don't throw an error, there simply is nothing to return

From 67d5219f754ad15ed313a4c88a2ea2621e61b15f Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 2 Sep 2024 20:20:58 +0100
Subject: [PATCH 13/69] basic crud working

---
 .../src/pages/builder/portal/settings/ai/ConfigModal.svelte      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 30830cff1a..e9ec5159f9 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -49,6 +49,7 @@
   }
 
   async function saveConfig() {
+    formValues.type = "ai"
     try {
       const savedConfig = await API.saveConfig(formValues)
       formValues._rev = savedConfig._rev

From f328ae4bf9dff216e8a23471cc60b074f0e41e0d Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 2 Sep 2024 20:45:32 +0100
Subject: [PATCH 14/69] account for arrays

---
 packages/backend-core/src/configs/configs.ts  |  3 +-
 .../portal/settings/ai/ConfigModal.svelte     | 49 ++++++++++++-------
 .../worker/src/api/routes/global/configs.ts   | 20 +++++++-
 3 files changed, 51 insertions(+), 21 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 6062d3624f..1771716f57 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -1,5 +1,6 @@
 import {
-  AIConfig, AIInnerConfig,
+  AIConfig,
+  AIInnerConfig,
   Config,
   ConfigType,
   GoogleConfig,
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index e9ec5159f9..749cebeb2c 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -23,37 +23,48 @@
   const models = [
     "gpt4o-mini",
   ]
+  
+  const defaultConfig = {
+    active: false,
+    isDefault: false,
+  }
 
-  let formValues = {}
-
+  let aiConfig = defaultConfig
   let validation
 
   $: {
-    const { provider, model, name, apiKey } = formValues
-    validation = provider && model && name && apiKey
+    const { provider, defaultModel, name, apiKey } = aiConfig
+    validation = provider && defaultModel && name && apiKey
   }
 
   function prefillConfig(evt) {
     const provider = evt.detail
     // grab the preset config from the constants for that provider and fill it in
     if (ConfigMap[provider]) {
-      formValues = {
-        ...formValues,
+      aiConfig = {
+        ...aiConfig,
         ...ConfigMap[provider]
       }
     } else {
-      formValues = {
+      aiConfig = {
+        ...aiConfig,
         provider
       }
     }
   }
 
   async function saveConfig() {
-    formValues.type = "ai"
+    const config = {
+      type: "ai",
+      config: [
+        // TODO: include the ones that are already there, or just handle this in the backend
+        aiConfig,
+      ]
+    }
     try {
-      const savedConfig = await API.saveConfig(formValues)
-      formValues._rev = savedConfig._rev
-      formValues._id = savedConfig._id
+      const savedConfig = await API.saveConfig(config)
+      aiConfig._rev = savedConfig._rev
+      aiConfig._id = savedConfig._id
       notifications.success(`Configuration saved`)
     } catch (error) {
       notifications.error(
@@ -92,33 +103,33 @@
     <Label size="M">Provider</Label>
     <Select
       placeholder={null}
-      bind:value={formValues.provider}
+      bind:value={aiConfig.provider}
       options={providers}
       on:change={prefillConfig}
     />
   </div>
   <div class="form-row">
-    <Label size="M">Model</Label>
+    <Label size="M">Default Model</Label>
     <Select
       placeholder={null}
-      bind:value={formValues.model}
+      bind:value={aiConfig.defaultModel}
       options={models}
     />
   </div>
   <div class="form-row">
     <Label size="M">Name</Label>
-    <Input placeholder={"Test 1"} bind:value={formValues.name}/>
+    <Input placeholder={"Test 1"} bind:value={aiConfig.name}/>
   </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
-    <Input placeholder={"www.google.com"} bind:value={formValues.baseUrl}/>
+    <Input placeholder={"www.google.com"} bind:value={aiConfig.baseUrl}/>
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
-    <Input bind:value={formValues.apiKey}/>
+    <Input bind:value={aiConfig.apiKey}/>
   </div>
-  <Toggle text="Active" bind:value={formValues.active}/>
-  <Toggle text="Set as default" bind:value={formValues.isDefault}/>
+  <Toggle text="Active" bind:value={aiConfig.active}/>
+  <Toggle text="Set as default" bind:value={aiConfig.isDefault}/>
 </ModalContent>
 
 <style>
diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts
index fcab2b58e0..b857647652 100644
--- a/packages/worker/src/api/routes/global/configs.ts
+++ b/packages/worker/src/api/routes/global/configs.ts
@@ -65,6 +65,23 @@ function scimValidation() {
   }).unknown(true)
 }
 
+function aiValidation() {
+  // prettier-ignore
+  return Joi.array().items(
+    Joi.object({
+      provider: Joi.string().required(),
+      isDefault: Joi.boolean().required(),
+      name: Joi.string().required(),
+      active: Joi.boolean().required(),
+      baseUrl: Joi.string().optional(),
+      apiKey: Joi.string().required(),
+      // TODO: should be enum
+      defaultModel: Joi.string().optional(),
+
+    })
+  ).required()
+}
+
 function buildConfigSaveValidation() {
   // prettier-ignore
   return auth.joiValidator.body(Joi.object({
@@ -82,7 +99,8 @@ function buildConfigSaveValidation() {
           { is: ConfigType.ACCOUNT, then: Joi.object().unknown(true) },
           { is: ConfigType.GOOGLE, then: googleValidation() },
           { is: ConfigType.OIDC, then: oidcValidation() },
-          { is: ConfigType.SCIM, then: scimValidation() }
+          { is: ConfigType.SCIM, then: scimValidation() },
+          { is: ConfigType.AI, then: aiValidation() }
         ],
       }),
   }).required().unknown(true),

From ac7838f80dd45735a4808afe888da236f7c7272e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 3 Sep 2024 12:09:33 +0100
Subject: [PATCH 15/69] Fixing an issue with inconsistent relationship order.

---
 packages/backend-core/src/sql/sql.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index e6738d4b36..626ab3bf8e 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -926,6 +926,8 @@ class InternalBuilder {
         )
         // relationships should never have more than the base limit
         .limit(getBaseLimit())
+        // add sorting to get consistent order
+        .orderBy(`${toAlias}.${toPrimary}`)
       // need to check the junction table document is to the right column
       if (this.client === SqlClient.SQL_LITE) {
         subQuery = this.addJoinFieldCheck(subQuery, relationship)

From ca4b17bc9bea58f33704bb7481adf0e895abf6a8 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 15:11:52 +0100
Subject: [PATCH 16/69] icons, styling of AI configs and further simplification

---
 packages/backend-core/src/configs/configs.ts  | 57 ++++++------
 .../portal/settings/ai/AIConfigTile.svelte    | 90 +++++++++++++++++++
 .../portal/settings/ai/ConfigModal.svelte     | 79 ++++------------
 .../builder/portal/settings/ai/constants.ts   | 39 ++++++++
 .../builder/portal/settings/ai/index.svelte   | 78 ++++++++++++++--
 .../portal/settings/ai/logos/Anthropic.svelte | 15 ++++
 .../portal/settings/ai/logos/OpenAI.svelte    |  5 ++
 packages/types/src/documents/global/config.ts |  2 +-
 .../src/api/controllers/global/configs.ts     | 11 ++-
 .../worker/src/api/routes/global/configs.ts   |  2 +-
 10 files changed, 276 insertions(+), 102 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 1771716f57..17f4abfe24 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -19,7 +19,6 @@ import { DocumentType, SEPARATOR } from "../constants"
 import { CacheKey, TTL, withCache } from "../cache"
 import * as context from "../context"
 import env from "../environment"
-import { getConfigParams } from "@budibase/server/src/migrations/functions/backfill/global/configs"
 
 // UTILS
 
@@ -262,37 +261,37 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
 // AI
 
 // TODO: Can we assume that you are licensed when you hit this endpoint?
-export async function getAIConfig(): Promise<AIConfig | undefined> {
-  if (!env.SELF_HOSTED) {
-    // always use the env vars in cloud
-    // TODO: Licensing stuff - make this right
-    if (env.OPENAI_API_KEY) {
-      return getDefaultBudibaseAIConfig()
-    }
-  }
+// export async function getAIConfig(): Promise<AIConfig | undefined> {
+  // if (!env.SELF_HOSTED) {
+  //   // always use the env vars in cloud
+  //   // TODO: Licensing stuff - make this right
+  //   if (env.OPENAI_API_KEY) {
+  //     return getDefaultBudibaseAIConfig()
+  //   }
+  // }
 
   // prefer the config in self-host
-  let config = await getConfig<AIConfig>(ConfigType.AI)
+  // let config = await getConfig<AIConfig>(ConfigType.AI)
 
   // fallback to env vars
-  if (!config || !config.activated) {
-    config = getDefaultBudibaseAIConfig()
-  }
+  // if (!config || !config.activated) {
+  //   config = getDefaultBudibaseAIConfig()
+  // }
 
-  return config
-}
+  // return config
+// }
 
-export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined {
-  if (env.OPENAI_API_KEY) {
-    return {
-      provider: "",
-      isDefault: true,
-      name: "Budibase AI",
-      active: true,
-      baseUrl: "",
-      apiKey: env.OPENAI_API_KEY,
-      // TODO: should be enum
-      defaultModel: ""
-    }
-  }
-}
+// export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined {
+//   if (env.OPENAI_API_KEY) {
+//     return {
+//       provider: "",
+//       isDefault: true,
+//       name: "Budibase AI",
+//       active: true,
+//       baseUrl: "",
+//       apiKey: env.OPENAI_API_KEY,
+//       // TODO: should be enum
+//       defaultModel: ""
+//     }
+//   }
+// }
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
new file mode 100644
index 0000000000..12cfa1d174
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -0,0 +1,90 @@
+<script>
+  import {Body, Label, Icon, Tag} from "@budibase/bbui"
+  import OpenAILogo from "./logos/OpenAI.svelte"
+  import AnthropicLogo from "./logos/Anthropic.svelte"
+  import { Providers } from "./constants"
+
+  export let config
+  export let disabled
+
+  export let editHandler
+  export let deleteHandler
+</script>
+
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<div on:click class:disabled class="option">
+  <div class="icon">
+    {#if config.provider === Providers.OpenAI.name}
+      <OpenAILogo height="30" width="30"/>
+    {:else if config.provider === Providers.Anthropic.name}
+      <AnthropicLogo height="30" width="30"/>
+    {/if}
+  </div>
+  <div class="header">
+    <Body>{config.name}</Body>
+    <Label>{config.name}</Label>
+  </div>
+  <div class="controls">
+    <Icon
+      on:click={editHandler}
+      size="S"
+      hoverable
+      name="Edit"
+    />
+    <Icon
+      on:click={deleteHandler}
+      size="S"
+      hoverable
+      name="Delete"
+    />
+    <div>Activated</div>
+  </div>
+</div>
+
+<style>
+  .option {
+    background-color: var(--background);
+    border: 1px solid var(--grey-4);
+    padding: 10px 16px 14px;
+    border-radius: 4px;
+    cursor: pointer;
+    display: grid;
+    grid-template-columns: 6% 1fr 20%;
+    grid-gap: 20px;
+    align-items: center;
+  }
+
+  .option :global(label) {
+    cursor: pointer;
+  }
+
+  .option:hover {
+    background-color: var(--background-alt);
+  }
+
+  .header {
+    align-items: center;
+  }
+
+  .icon {
+    background-color: white;
+    height: 38px;
+    width: 38px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    border-radius: 5px;
+  }
+
+  .disabled {
+    pointer-events: none;
+  }
+
+  .controls {
+    display: grid;
+    grid-auto-flow: column;
+    grid-gap: 10px;
+    align-items: center;
+  }
+</style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 749cebeb2c..70ddba5407 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -8,27 +8,17 @@
     Body,
     notifications,
   } from "@budibase/bbui"
-  import { ConfigMap } from "./constants"
+  import { ConfigMap, Providers } from "./constants"
   import { API } from "api"
 
-  // TODO: Update these
-  const providers = [
-    "OpenAI",
-    "Anthropic",
-    "Together AI",
-    "Azure Open AI",
-    "Custom"
-  ]
-
-  const models = [
-    "gpt4o-mini",
-  ]
-  
-  const defaultConfig = {
+  export let defaultConfig = {
     active: false,
     isDefault: false,
   }
 
+  export let saveHandler
+  export let deleteHandler
+
   let aiConfig = defaultConfig
   let validation
 
@@ -43,48 +33,15 @@
     if (ConfigMap[provider]) {
       aiConfig = {
         ...aiConfig,
-        ...ConfigMap[provider]
-      }
-    } else {
-      aiConfig = {
-        ...aiConfig,
+        ...ConfigMap[provider],
         provider
       }
-    }
-  }
-
-  async function saveConfig() {
-    const config = {
-      type: "ai",
-      config: [
-        // TODO: include the ones that are already there, or just handle this in the backend
-        aiConfig,
-      ]
-    }
-    try {
-      const savedConfig = await API.saveConfig(config)
-      aiConfig._rev = savedConfig._rev
-      aiConfig._id = savedConfig._id
-      notifications.success(`Configuration saved`)
-    } catch (error) {
-      notifications.error(
-        `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
-      )
-    }
-  }
-
-  async function deleteConfig() {
-    // Delete a configuration
-    try {
-      // await API.deleteConfig({
-      //   id: smtpConfig._id,
-      //   rev: smtpConfig._rev,
-      // })
-      notifications.success(`Deleted config`)
-    } catch (error) {
-      notifications.error(
-        `Failed to clear email settings, reason: ${error?.message || "Unknown"}`
-      )
+    } else {
+      aiConfig.provider = provider
+      // aiConfig = {
+      //   ...aiConfig,
+      //   provider
+      // }
     }
   }
 
@@ -93,8 +50,8 @@
 <ModalContent
     confirmText={"Save"}
     cancelText={"Delete"}
-    onConfirm={saveConfig}
-    onCancel={deleteConfig}
+    onConfirm={saveHandler}
+    onCancel={deleteHandler}
     disabled={!validation}
     size="M"
     title="Custom AI Configuration"
@@ -104,16 +61,16 @@
     <Select
       placeholder={null}
       bind:value={aiConfig.provider}
-      options={providers}
+      options={Object.keys(Providers)}
       on:change={prefillConfig}
     />
   </div>
   <div class="form-row">
     <Label size="M">Default Model</Label>
     <Select
-      placeholder={null}
+      placeholder={aiConfig.provider ? "Choose an option" : "Select a provider first"}
       bind:value={aiConfig.defaultModel}
-      options={models}
+      options={aiConfig.provider ? Providers[aiConfig.provider].models : []}
     />
   </div>
   <div class="form-row">
@@ -126,7 +83,7 @@
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
-    <Input bind:value={aiConfig.apiKey}/>
+    <Input type="password" bind:value={aiConfig.apiKey}/>
   </div>
   <Toggle text="Active" bind:value={aiConfig.active}/>
   <Toggle text="Set as default" bind:value={aiConfig.isDefault}/>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index df5508109d..07e0ea1cb7 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -1,8 +1,47 @@
+export const Providers = {
+  OpenAI: {
+    name: "OpenAI",
+    models: [
+      "gpt-4o-mini",
+      "gpt-4o",
+      "gpt-3.5-turbo",
+      "chatgpt-4o-latest",
+      "gpt-4-turbo",
+      "gpt-4",
+    ]
+  },
+  Anthropic: {
+    name: "Anthropic",
+    models: [
+      "claude-3-5-sonnet-20240620",
+      "claude-3-sonnet-20240229",
+      "claude-3-opus-20240229",
+      "claude-3-haiku-20240307"
+    ]
+  },
+  TogetherAI: {
+    name: "Together AI",
+    // TODO: too many - probably need to use an autocomplete for this
+    models: [""]
+  },
+  Custom: {
+    name: "Custom",
+    // TODO: too many - probably need to use an autocomplete for this
+    models: [""]
+  },
+}
+
 export const ConfigMap = {
   OpenAI: {
     baseUrl: "https://api.openai.com"
   },
   Anthropic: {
+    baseUrl: ""
+  },
+  TogetherAI: {
     baseUrl: "https://api.together.xyz/v1"
   },
+  Custom: {
+    baseUrl: ""
+  }
 }
\ No newline at end of file
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 427b09a7f2..65cd137833 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -1,4 +1,5 @@
 <script>
+  import { onMount } from "svelte"
   import {
     Button,
     Layout,
@@ -14,8 +15,7 @@
   import { admin, licensing } from "stores/portal"
   import { API } from "api"
   import AIConfigModal from "./ConfigModal.svelte"
-  import { onMount } from "svelte"
-  import { sdk } from "@budibase/shared-core"
+  import AIConfigTile from "./AIConfigTile.svelte"
 
   const ConfigTypes = {
     AI: "ai",
@@ -23,26 +23,75 @@
 
   let modal
   let aiConfig
-  let loading = false
+  let currentlyEditingConfig
 
   $: isCloud = $admin.cloud
   $: budibaseAIEnabled = $licensing.budibaseAIEnabled
   $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled
 
   async function fetchAIConfig() {
-    loading = true
     try {
-      // Fetch the configs for smtp
+      // Fetch the AI configs
       const aiDoc = await API.getConfig(ConfigTypes.AI)
       if (aiDoc._id) {
         aiConfig = aiDoc
       }
-      loading = false
     } catch (error) {
       notifications.error("Error fetching AI config")
     }
   }
 
+  async function saveConfig() {
+    // Update the config that was changed
+    const updateConfigs = aiConfig.config
+
+    const config = {
+      type: ConfigTypes.AI,
+      config: [
+        // TODO: include the ones that are already there, or just handle this in the backend
+        aiConfig,
+      ]
+    }
+    try {
+      const isNew = !!aiConfig._rev
+      const savedConfig = await API.saveConfig(config)
+      aiConfig._rev = savedConfig._rev
+      aiConfig._id = savedConfig._id
+      notifications.success(`Successfully saved and activated ${isNew ? "new" : ""} AI Configuration`)
+    } catch (error) {
+      notifications.error(
+        `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
+      )
+    }
+  }
+
+  async function deleteConfig(name) {
+    // Delete a configuration
+    const idx = aiConfig.config.findIndex(config => config.name === currentlyEditingConfig?.name || name)
+    aiConfig.config.splice(idx, 1)
+
+    try {
+      const savedConfig = await API.saveConfig(aiConfig)
+      aiConfig._rev = savedConfig._rev
+      aiConfig._id = savedConfig._id
+      notifications.success(`Deleted config`)
+    } catch (error) {
+      notifications.error(
+        `Failed to delete config, reason: ${error?.message || "Unknown"}`
+      )
+    }
+  }
+
+  function editConfig(config) {
+    currentlyEditingConfig = config
+    modal.show()
+  }
+
+  function newConfig() {
+    currentlyEditingConfig = undefined
+    modal.show()
+  }
+
   onMount(async () => {
     await fetchAIConfig()
   })
@@ -51,7 +100,11 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <Modal bind:this={modal}>
-  <AIConfigModal />
+  <AIConfigModal
+    saveHandler={saveConfig}
+    deleteHandler={deleteConfig}
+    defaultConfig={currentlyEditingConfig}
+  />
 </Modal>
 <Layout noPadding>
   <Layout gap="XS" noPadding>
@@ -76,11 +129,20 @@
           <Tag icon="LockClosed">Enterprise</Tag>
         </Tags>
       {:else}
-        <Button size="S" cta on:click={modal.show}>Add configuration</Button>
+        <Button size="S" cta on:click={newConfig}>Add configuration</Button>
       {/if}
     </div>
     <Body size="S">Use the following interface to select your preferred AI configuration.</Body>
     <Body size="S">Select your AI Model:</Body>
+    {#if aiConfig}
+      {#each aiConfig.config as config}
+        <AIConfigTile
+          {config}
+          editHandler={() => editConfig(config)}
+          deleteHandler={modal.show}
+        />
+      {/each}
+    {/if}
   </Layout>
 </Layout>
 
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
new file mode 100644
index 0000000000..4c598872a2
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
@@ -0,0 +1,15 @@
+<script>
+  export let height
+  export let width
+</script>
+<svg id="katman_1" {height} {width} data-name="katman 1" xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 841.89 595.28">
+  <defs>
+    <style>
+      .cls-1 {
+        fill: #000;
+        stroke-width: 0px;
+      }
+    </style>
+  </defs>
+  <path class="cls-1" d="M552.13,91.34h-90.99l162.93,412.61h88.87l-160.81-412.61ZM289.76,91.34l-160.81,412.61h90.99l35.97-86.75h169.28l33.86,84.64h90.99L384.97,91.34h-95.22ZM281.29,341.02l55.01-146,57.13,146h-112.14Z"/>
+</svg>
\ No newline at end of file
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
new file mode 100644
index 0000000000..a7a2927dbc
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
@@ -0,0 +1,5 @@
+<script>
+ export let height
+ export let width
+</script>
+<svg fill="#000000" {width} {height} viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>OpenAI icon</title><path d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"/></svg>
\ No newline at end of file
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 246874d0f1..249893520c 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -123,7 +123,7 @@ export interface AIInnerConfig {
   defaultModel: string
 }
 
-export interface AIConfig extends Config<AIInnerConfig> {}
+export interface AIConfig extends Config<AIInnerConfig[]> {}
 
 export const isSettingsConfig = (config: Config): config is SettingsConfig =>
   config.type === ConfigType.SETTINGS
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 43aac19b18..e5c142f562 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -28,7 +28,7 @@ import {
   SSOConfig,
   SSOConfigType,
   UserCtx,
-  OIDCLogosConfig,
+  OIDCLogosConfig, AIConfig,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
 
@@ -313,6 +313,13 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
   )
 }
 
+function sanitizeAIConfig(aiConfig: AIConfig) {
+  for (let providerConfig of aiConfig.config) {
+    delete providerConfig.apiKey
+  }
+  return aiConfig
+}
+
 export async function find(ctx: UserCtx) {
   try {
     // Find the config with the most granular scope based on context
@@ -325,8 +332,8 @@ export async function find(ctx: UserCtx) {
       }
 
       if (type === ConfigType.AI) {
-        // TODO: strip the keys from the configs here
         // TODO: do the licensing checks here and return the right things based on the license
+        sanitizeAIConfig(scopedConfig)
       }
       ctx.body = scopedConfig
     } else {
diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts
index b857647652..417498d829 100644
--- a/packages/worker/src/api/routes/global/configs.ts
+++ b/packages/worker/src/api/routes/global/configs.ts
@@ -73,7 +73,7 @@ function aiValidation() {
       isDefault: Joi.boolean().required(),
       name: Joi.string().required(),
       active: Joi.boolean().required(),
-      baseUrl: Joi.string().optional(),
+      baseUrl: Joi.string().optional().allow("", null),
       apiKey: Joi.string().required(),
       // TODO: should be enum
       defaultModel: Joi.string().optional(),

From 43135e42746032672cd59299e86ec9e80f1cf3ac Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 17:18:46 +0100
Subject: [PATCH 17/69] AI Config CRUD complete

---
 .../portal/settings/ai/AIConfigTile.svelte    |  2 +-
 .../portal/settings/ai/ConfigModal.svelte     | 61 +++++++---------
 .../builder/portal/settings/ai/index.svelte   | 69 ++++++++++---------
 .../src/api/controllers/global/configs.ts     | 20 ++++--
 .../worker/src/api/routes/global/configs.ts   | 26 +++----
 5 files changed, 91 insertions(+), 87 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 12cfa1d174..4b9b4cc506 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -1,5 +1,5 @@
 <script>
-  import {Body, Label, Icon, Tag} from "@budibase/bbui"
+  import { Body, Label, Icon } from "@budibase/bbui"
   import OpenAILogo from "./logos/OpenAI.svelte"
   import AnthropicLogo from "./logos/Anthropic.svelte"
   import { Providers } from "./constants"
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 70ddba5407..703f1e8129 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -5,13 +5,10 @@
     Input,
     Select,
     Toggle,
-    Body,
-    notifications,
   } from "@budibase/bbui"
-  import { ConfigMap, Providers } from "./constants"
-  import { API } from "api"
+  import {ConfigMap, Providers} from "./constants"
 
-  export let defaultConfig = {
+  export let config = {
     active: false,
     isDefault: false,
   }
@@ -19,11 +16,10 @@
   export let saveHandler
   export let deleteHandler
 
-  let aiConfig = defaultConfig
   let validation
 
   $: {
-    const { provider, defaultModel, name, apiKey } = aiConfig
+    const {provider, defaultModel, name, apiKey} = config
     validation = provider && defaultModel && name && apiKey
   }
 
@@ -31,36 +27,35 @@
     const provider = evt.detail
     // grab the preset config from the constants for that provider and fill it in
     if (ConfigMap[provider]) {
-      aiConfig = {
-        ...aiConfig,
+      config = {
+        ...config,
         ...ConfigMap[provider],
-        provider
+        provider,
       }
     } else {
-      aiConfig.provider = provider
-      // aiConfig = {
-      //   ...aiConfig,
-      //   provider
-      // }
+      config.provider = provider
     }
   }
-
 </script>
 
 <ModalContent
-    confirmText={"Save"}
-    cancelText={"Delete"}
-    onConfirm={saveHandler}
-    onCancel={deleteHandler}
-    disabled={!validation}
-    size="M"
-    title="Custom AI Configuration"
+  confirmText={"Save"}
+  cancelText={"Delete"}
+  onConfirm={saveHandler}
+  onCancel={deleteHandler}
+  disabled={!validation}
+  size="M"
+  title="Custom AI Configuration"
 >
   <div class="form-row">
+    <div class="form-row">
+      <Label size="M">Name</Label>
+      <Input placeholder={"Test 1"} bind:value={config.name}/>
+    </div>
     <Label size="M">Provider</Label>
     <Select
       placeholder={null}
-      bind:value={aiConfig.provider}
+      bind:value={config.provider}
       options={Object.keys(Providers)}
       on:change={prefillConfig}
     />
@@ -68,25 +63,21 @@
   <div class="form-row">
     <Label size="M">Default Model</Label>
     <Select
-      placeholder={aiConfig.provider ? "Choose an option" : "Select a provider first"}
-      bind:value={aiConfig.defaultModel}
-      options={aiConfig.provider ? Providers[aiConfig.provider].models : []}
+      placeholder={config.provider ? "Choose an option" : "Select a provider first"}
+      bind:value={config.defaultModel}
+      options={config.provider ? Providers[config.provider].models : []}
     />
   </div>
-  <div class="form-row">
-    <Label size="M">Name</Label>
-    <Input placeholder={"Test 1"} bind:value={aiConfig.name}/>
-  </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
-    <Input placeholder={"www.google.com"} bind:value={aiConfig.baseUrl}/>
+    <Input placeholder={"www.google.com"} bind:value={config.baseUrl}/>
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
-    <Input type="password" bind:value={aiConfig.apiKey}/>
+    <Input type="password" bind:value={config.apiKey}/>
   </div>
-  <Toggle text="Active" bind:value={aiConfig.active}/>
-  <Toggle text="Set as default" bind:value={aiConfig.isDefault}/>
+  <Toggle text="Active" bind:value={config.active}/>
+  <Toggle text="Set as default" bind:value={config.isDefault}/>
 </ModalContent>
 
 <style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 65cd137833..d0ced7041a 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -22,8 +22,9 @@
   }
 
   let modal
-  let aiConfig
-  let currentlyEditingConfig
+  let fullAIConfig
+  let editingAIConfig = {}
+  let editingUuid
 
   $: isCloud = $admin.cloud
   $: budibaseAIEnabled = $licensing.budibaseAIEnabled
@@ -34,7 +35,7 @@
       // Fetch the AI configs
       const aiDoc = await API.getConfig(ConfigTypes.AI)
       if (aiDoc._id) {
-        aiConfig = aiDoc
+        fullAIConfig = aiDoc
       }
     } catch (error) {
       notifications.error("Error fetching AI config")
@@ -42,22 +43,27 @@
   }
 
   async function saveConfig() {
-    // Update the config that was changed
-    const updateConfigs = aiConfig.config
+    // Use existing key or generate new one
+    const id = editingUuid || Helpers.uuid()
 
-    const config = {
-      type: ConfigTypes.AI,
-      config: [
-        // TODO: include the ones that are already there, or just handle this in the backend
-        aiConfig,
-      ]
+    // Creating first custom AI Config
+    if (!fullAIConfig) {
+      fullAIConfig = {
+        type: ConfigTypes.AI,
+        config: {
+          [id]: editingAIConfig
+        }
+      }
+    } else {
+      // Add new or update existing custom AI Config
+      fullAIConfig.config[id] = editingAIConfig
     }
+
     try {
-      const isNew = !!aiConfig._rev
-      const savedConfig = await API.saveConfig(config)
-      aiConfig._rev = savedConfig._rev
-      aiConfig._id = savedConfig._id
-      notifications.success(`Successfully saved and activated ${isNew ? "new" : ""} AI Configuration`)
+      const savedConfig = await API.saveConfig(fullAIConfig)
+      fullAIConfig._rev = savedConfig._rev
+      fullAIConfig._id = savedConfig._id
+      notifications.success(`Successfully saved and activated AI Configuration`)
     } catch (error) {
       notifications.error(
         `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
@@ -65,15 +71,14 @@
     }
   }
 
-  async function deleteConfig(name) {
+  async function deleteConfig(key) {
     // Delete a configuration
-    const idx = aiConfig.config.findIndex(config => config.name === currentlyEditingConfig?.name || name)
-    aiConfig.config.splice(idx, 1)
+    delete fullAIConfig.config[key]
 
     try {
-      const savedConfig = await API.saveConfig(aiConfig)
-      aiConfig._rev = savedConfig._rev
-      aiConfig._id = savedConfig._id
+      const savedConfig = await API.saveConfig(fullAIConfig)
+      fullAIConfig._rev = savedConfig._rev
+      fullAIConfig._id = savedConfig._id
       notifications.success(`Deleted config`)
     } catch (error) {
       notifications.error(
@@ -82,13 +87,15 @@
     }
   }
 
-  function editConfig(config) {
-    currentlyEditingConfig = config
+  function editConfig(uuid) {
+    editingUuid = uuid
+    editingAIConfig = fullAIConfig?.config[editingUuid]
     modal.show()
   }
 
   function newConfig() {
-    currentlyEditingConfig = undefined
+    editingUuid = undefined
+    editingAIConfig = undefined
     modal.show()
   }
 
@@ -103,7 +110,7 @@
   <AIConfigModal
     saveHandler={saveConfig}
     deleteHandler={deleteConfig}
-    defaultConfig={currentlyEditingConfig}
+    bind:config={editingAIConfig}
   />
 </Modal>
 <Layout noPadding>
@@ -134,12 +141,12 @@
     </div>
     <Body size="S">Use the following interface to select your preferred AI configuration.</Body>
     <Body size="S">Select your AI Model:</Body>
-    {#if aiConfig}
-      {#each aiConfig.config as config}
+    {#if fullAIConfig?.config}
+      {#each Object.keys(fullAIConfig.config) as key}
         <AIConfigTile
-          {config}
-          editHandler={() => editConfig(config)}
-          deleteHandler={modal.show}
+          config={fullAIConfig.config[key]}
+          editHandler={() => editConfig(key)}
+          deleteHandler={() => deleteConfig(key)}
         />
       {/each}
     {/if}
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index e5c142f562..0b8e525eb8 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -28,7 +28,9 @@ import {
   SSOConfig,
   SSOConfigType,
   UserCtx,
-  OIDCLogosConfig, AIConfig,
+  OIDCLogosConfig,
+  AIConfig,
+  PASSWORD_REPLACEMENT,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
 
@@ -197,9 +199,11 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
 }
 
-async function verifyAIConfig(config: OIDCConfigs) {
-  // await verifySSOConfig(ConfigType.OIDC, config.configs[0])
-  // Shape should be `config_ai`
+async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) {
+  // ensure that the redacted API keys are not overwritten in the DB
+  // for (let aiConfig of existingConfig) {
+  //
+  // }
   return true
 }
 
@@ -231,7 +235,7 @@ export async function save(ctx: UserCtx<Config>) {
         await verifyOIDCConfig(config)
         break
       case ConfigType.AI:
-        await verifyAIConfig(config)
+        await verifyAIConfig(config, existingConfig)
         break
     }
   } catch (err: any) {
@@ -314,8 +318,10 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
 }
 
 function sanitizeAIConfig(aiConfig: AIConfig) {
-  for (let providerConfig of aiConfig.config) {
-    delete providerConfig.apiKey
+  for (let key in aiConfig.config) {
+    if (aiConfig.config[key].apiKey) {
+      aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
+    }
   }
   return aiConfig
 }
diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts
index 417498d829..409e0a7797 100644
--- a/packages/worker/src/api/routes/global/configs.ts
+++ b/packages/worker/src/api/routes/global/configs.ts
@@ -67,19 +67,19 @@ function scimValidation() {
 
 function aiValidation() {
   // prettier-ignore
-  return Joi.array().items(
-    Joi.object({
-      provider: Joi.string().required(),
-      isDefault: Joi.boolean().required(),
-      name: Joi.string().required(),
-      active: Joi.boolean().required(),
-      baseUrl: Joi.string().optional().allow("", null),
-      apiKey: Joi.string().required(),
-      // TODO: should be enum
-      defaultModel: Joi.string().optional(),
-
-    })
-  ).required()
+  return Joi.object().pattern(
+     Joi.string(),
+     Joi.object({
+       provider: Joi.string().required(),
+       isDefault: Joi.boolean().required(),
+       name: Joi.string().required(),
+       active: Joi.boolean().required(),
+       baseUrl: Joi.string().optional().allow("", null),
+       apiKey: Joi.string().required(),
+       // TODO: should be enum
+       defaultModel: Joi.string().optional(),
+     }).required()
+  )
 }
 
 function buildConfigSaveValidation() {

From b29a4e2b37020683139c19524c489a5a8b65f14b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 3 Sep 2024 18:24:50 +0100
Subject: [PATCH 18/69] Work to support all SQL DBs across the board using the
 aggregation method.

---
 packages/backend-core/src/sql/sql.ts          | 301 ++++++++++--------
 packages/backend-core/src/sql/utils.ts        |  30 +-
 .../src/api/controllers/row/utils/basic.ts    |  61 +++-
 .../src/api/controllers/row/utils/sqlUtils.ts |  67 ----
 .../src/api/controllers/row/utils/utils.ts    |  27 +-
 .../src/api/routes/tests/search.spec.ts       | 287 ++++++-----------
 packages/types/src/sdk/search.ts              |  11 +
 7 files changed, 375 insertions(+), 409 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 626ab3bf8e..6181d18046 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -7,6 +7,7 @@ import {
   isValidFilter,
   isValidISODateString,
   sqlLog,
+  validateManyToMany,
 } from "./utils"
 import SqlTableQueryBuilder from "./sqlTable"
 import {
@@ -133,80 +134,78 @@ class InternalBuilder {
       return "*"
     }
 
-    // no relationships - select everything in SQLite
-    if (this.client === SqlClient.SQL_LITE) {
-      const alias = tableAliases?.[endpoint.entityId]
-        ? tableAliases?.[endpoint.entityId]
-        : endpoint.entityId
-      return [this.knex.raw(`${this.quote(alias)}.*`)]
-    }
-
-    const schema = meta.table.schema
-    return resource.fields.map(field => {
-      const parts = field.split(/\./g)
-      let table: string | undefined = undefined
-      let column: string | undefined = undefined
-
-      // Just a column name, e.g.: "column"
-      if (parts.length === 1) {
-        column = parts[0]
-      }
-
-      // A table name and a column name, e.g.: "table.column"
-      if (parts.length === 2) {
-        table = parts[0]
-        column = parts[1]
-      }
-
-      // A link doc, e.g.: "table.doc1.fieldName"
-      if (parts.length > 2) {
-        table = parts[0]
-        column = parts.slice(1).join(".")
-      }
-
-      if (!column) {
-        throw new Error(`Invalid field name: ${field}`)
-      }
-
-      const columnSchema = schema[column]
-
-      if (
-        this.client === SqlClient.POSTGRES &&
-        columnSchema?.externalType?.includes("money")
-      ) {
-        return this.knex.raw(
-          `${this.quotedIdentifier(
-            [table, column].join(".")
-          )}::money::numeric as ${this.quote(field)}`
-        )
-      }
-
-      if (
-        this.client === SqlClient.MS_SQL &&
-        columnSchema?.type === FieldType.DATETIME &&
-        columnSchema.timeOnly
-      ) {
-        // Time gets returned as timestamp from mssql, not matching the expected
-        // HH:mm format
-        return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
-      }
-
-      // There's at least two edge cases being handled in the expression below.
-      //  1. The column name could start/end with a space, and in that case we
-      //     want to preseve that space.
-      //  2. Almost all column names are specified in the form table.column, except
-      //     in the case of relationships, where it's table.doc1.column. In that
-      //     case, we want to split it into `table`.`doc1.column` for reasons that
-      //     aren't actually clear to me, but `table`.`doc1` breaks things with the
-      //     sample data tests.
-      if (table) {
-        return this.knex.raw(
-          `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
-        )
-      } else {
-        return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
-      }
-    })
+    const alias = tableAliases?.[endpoint.entityId]
+      ? tableAliases?.[endpoint.entityId]
+      : endpoint.entityId
+    return [this.knex.raw(`${this.quote(alias)}.*`)]
+    //
+    //
+    // const schema = meta.table.schema
+    // return resource.fields.map(field => {
+    //   const parts = field.split(/\./g)
+    //   let table: string | undefined = undefined
+    //   let column: string | undefined = undefined
+    //
+    //   // Just a column name, e.g.: "column"
+    //   if (parts.length === 1) {
+    //     column = parts[0]
+    //   }
+    //
+    //   // A table name and a column name, e.g.: "table.column"
+    //   if (parts.length === 2) {
+    //     table = parts[0]
+    //     column = parts[1]
+    //   }
+    //
+    //   // A link doc, e.g.: "table.doc1.fieldName"
+    //   if (parts.length > 2) {
+    //     table = parts[0]
+    //     column = parts.slice(1).join(".")
+    //   }
+    //
+    //   if (!column) {
+    //     throw new Error(`Invalid field name: ${field}`)
+    //   }
+    //
+    //   const columnSchema = schema[column]
+    //
+    //   if (
+    //     this.client === SqlClient.POSTGRES &&
+    //     columnSchema?.externalType?.includes("money")
+    //   ) {
+    //     return this.knex.raw(
+    //       `${this.quotedIdentifier(
+    //         [table, column].join(".")
+    //       )}::money::numeric as ${this.quote(field)}`
+    //     )
+    //   }
+    //
+    //   if (
+    //     this.client === SqlClient.MS_SQL &&
+    //     columnSchema?.type === FieldType.DATETIME &&
+    //     columnSchema.timeOnly
+    //   ) {
+    //     // Time gets returned as timestamp from mssql, not matching the expected
+    //     // HH:mm format
+    //     return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+    //   }
+    //
+    //   // There's at least two edge cases being handled in the expression below.
+    //   //  1. The column name could start/end with a space, and in that case we
+    //   //     want to preseve that space.
+    //   //  2. Almost all column names are specified in the form table.column, except
+    //   //     in the case of relationships, where it's table.doc1.column. In that
+    //   //     case, we want to split it into `table`.`doc1.column` for reasons that
+    //   //     aren't actually clear to me, but `table`.`doc1` breaks things with the
+    //   //     sample data tests.
+    //   if (table) {
+    //     return this.knex.raw(
+    //       `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
+    //     )
+    //   } else {
+    //     return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
+    //   }
+    // })
   }
 
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,
@@ -368,35 +367,47 @@ class InternalBuilder {
         let subQuery = mainKnex
           .select(mainKnex.raw(1))
           .from({ [toAlias]: relatedTableName })
-        let mainTableRelatesTo = toAlias
-        if (relationship.through) {
+        const manyToMany = validateManyToMany(relationship)
+        if (manyToMany) {
           const throughAlias =
-            aliases?.[relationship.through] || relationship.through
-          let throughTable = this.tableNameWithSchema(relationship.through, {
+            aliases?.[manyToMany.through] || relationship.through
+          let throughTable = this.tableNameWithSchema(manyToMany.through, {
             alias: throughAlias,
             schema: endpoint.schema,
           })
-          subQuery = subQuery.innerJoin(throughTable, function () {
-            // @ts-ignore
-            this.on(
-              `${toAlias}.${relationship.toPrimary}`,
+          subQuery = subQuery
+            // add a join through the junction table
+            .innerJoin(throughTable, function () {
+              // @ts-ignore
+              this.on(
+                `${toAlias}.${manyToMany.toPrimary}`,
+                "=",
+                `${throughAlias}.${manyToMany.to}`
+              )
+            })
+            // check the document in the junction table points to the main table
+            .where(
+              `${throughAlias}.${manyToMany.from}`,
               "=",
-              `${throughAlias}.${relationship.to}`
+              mainKnex.raw(
+                this.quotedIdentifier(`${fromAlias}.${manyToMany.fromPrimary}`)
+              )
             )
-          })
+          // in SQS the same junction table is used for different many-to-many relationships between the
+          // two same tables, this is needed to avoid rows ending up in all columns
           if (this.client === SqlClient.SQL_LITE) {
-            subQuery = this.addJoinFieldCheck(subQuery, relationship)
+            subQuery = this.addJoinFieldCheck(subQuery, manyToMany)
           }
-          mainTableRelatesTo = throughAlias
-        }
-        // "join" to the main table, making sure the ID matches that of the main
-        subQuery = subQuery.where(
-          `${mainTableRelatesTo}.${relationship.from}`,
-          "=",
-          mainKnex.raw(
-            this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`)
+        } else {
+          // "join" to the main table, making sure the ID matches that of the main
+          subQuery = subQuery.where(
+            `${toAlias}.${relationship.to}`,
+            "=",
+            mainKnex.raw(
+              this.quotedIdentifier(`${fromAlias}.${relationship.from}`)
+            )
           )
-        )
+        }
         query = query.whereExists(whereCb(subQuery))
         break
       }
@@ -478,12 +489,10 @@ class InternalBuilder {
             alias ? `${alias}.${updatedKey}` : updatedKey,
             value
           )
-        } else if (isSqlite && shouldProcessRelationship) {
+        } else if (shouldProcessRelationship) {
           query = builder.addRelationshipForFilter(query, updatedKey, q => {
             return handleRelationship(q, updatedKey, value)
           })
-        } else if (shouldProcessRelationship) {
-          query = handleRelationship(query, updatedKey, value)
         }
       }
     }
@@ -849,6 +858,7 @@ class InternalBuilder {
     fromTable: string,
     relationships: RelationshipsJson[]
   ): Knex.QueryBuilder {
+    const sqlClient = this.client
     const { resource, tableAliases: aliases, endpoint } = this.query
     const fields = resource?.fields || []
     const jsonField = (field: string) => {
@@ -862,7 +872,15 @@ class InternalBuilder {
         unaliased = parts.join(".")
         tableField = this.quote(unaliased)
       }
-      return `'${unaliased}',${tableField}`
+      let separator = ","
+      switch (sqlClient) {
+        case SqlClient.ORACLE:
+          separator = " VALUE "
+          break
+        case SqlClient.MS_SQL:
+          separator = ":"
+      }
+      return `'${unaliased}'${separator}${tableField}`
     }
     for (let relationship of relationships) {
       const {
@@ -874,23 +892,15 @@ class InternalBuilder {
         toPrimary,
       } = relationship
       // skip invalid relationships
-      if (!toTable || !fromTable || !fromPrimary || !toPrimary) {
+      if (!toTable || !fromTable) {
         continue
       }
-      if (!throughTable) {
-        throw new Error("Only many-to-many implemented for JSON relationships")
-      }
       const toAlias = aliases?.[toTable] || toTable,
-        throughAlias = aliases?.[throughTable] || throughTable,
         fromAlias = aliases?.[fromTable] || fromTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
         schema: endpoint.schema,
       })
-      let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
-        alias: throughAlias,
-        schema: endpoint.schema,
-      })
       let relationshipFields = fields.filter(
         field => field.split(".")[0] === toAlias
       )
@@ -903,32 +913,75 @@ class InternalBuilder {
       const fieldList: string = relationshipFields
         .map(field => jsonField(field))
         .join(",")
-      let rawJsonArray: Knex.Raw
-      switch (this.client) {
+      let rawJsonArray: Knex.Raw, limit: number
+      switch (sqlClient) {
         case SqlClient.SQL_LITE:
           rawJsonArray = this.knex.raw(
             `json_group_array(json_object(${fieldList}))`
           )
+          limit = getBaseLimit()
+          break
+        case SqlClient.POSTGRES:
+          rawJsonArray = this.knex.raw(
+            `json_agg(json_build_object(${fieldList}))`
+          )
+          limit = 1
+          break
+        case SqlClient.MY_SQL:
+        case SqlClient.ORACLE:
+          rawJsonArray = this.knex.raw(
+            `json_arrayagg(json_object(${fieldList}))`
+          )
+          limit = getBaseLimit()
+          break
+        case SqlClient.MS_SQL:
+          rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`)
+          limit = 1
           break
         default:
           throw new Error(`JSON relationships not implement for ${this.client}`)
       }
+      // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
+      // it reduces the result set rather than limiting how much data it filters over
+      const primaryKey = `${toAlias}.${toPrimary || toKey}`
       let subQuery = this.knex
         .select(rawJsonArray)
         .from(toTableWithSchema)
-        .join(throughTableWithSchema, function () {
-          this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
-        })
-        .where(
-          `${throughAlias}.${fromKey}`,
-          "=",
-          this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
-        )
-        // relationships should never have more than the base limit
-        .limit(getBaseLimit())
+        .limit(limit)
         // add sorting to get consistent order
-        .orderBy(`${toAlias}.${toPrimary}`)
-      // need to check the junction table document is to the right column
+        .orderBy(primaryKey)
+
+      if (sqlClient === SqlClient.POSTGRES) {
+        subQuery = subQuery.groupBy(primaryKey)
+      }
+
+      // many-to-many relationship with junction table
+      if (throughTable && toPrimary && fromPrimary) {
+        const throughAlias = aliases?.[throughTable] || throughTable
+        let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
+          alias: throughAlias,
+          schema: endpoint.schema,
+        })
+        subQuery = subQuery
+          .join(throughTableWithSchema, function () {
+            this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`)
+          })
+          .where(
+            `${throughAlias}.${fromKey}`,
+            "=",
+            this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
+          )
+      }
+      // one-to-many relationship with foreign key
+      else {
+        subQuery = subQuery.where(
+          `${toAlias}.${toKey}`,
+          "=",
+          this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
+        )
+      }
+
+      // need to check the junction table document is to the right column, this is just for SQS
       if (this.client === SqlClient.SQL_LITE) {
         subQuery = this.addJoinFieldCheck(subQuery, relationship)
       }
@@ -1179,14 +1232,12 @@ class InternalBuilder {
       ? query.select(this.generateSelectStatement())
       : this.addDistinctCount(query)
     // have to add after as well (this breaks MS-SQL)
-    if (this.client !== SqlClient.MS_SQL && !counting) {
+    if (!counting) {
       query = this.addSorting(query)
     }
     // handle joins
-    if (relationships && this.client === SqlClient.SQL_LITE) {
+    if (relationships) {
       query = this.addJsonRelationships(query, tableName, relationships)
-    } else if (relationships) {
-      query = this.addRelationships(query, tableName, relationships)
     }
 
     return this.addFilters(query, filters, { relationship: true })
diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts
index 1b32cc6da7..1b80ff337d 100644
--- a/packages/backend-core/src/sql/utils.ts
+++ b/packages/backend-core/src/sql/utils.ts
@@ -1,4 +1,11 @@
-import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
+import {
+  DocumentType,
+  ManyToManyRelationshipJson,
+  RelationshipsJson,
+  SqlQuery,
+  Table,
+  TableSourceType,
+} from "@budibase/types"
 import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
 import { Knex } from "knex"
 import { SEPARATOR } from "../db"
@@ -163,3 +170,24 @@ export function sqlLog(client: string, query: string, values?: any[]) {
   }
   console.log(string)
 }
+
+function isValidManyToManyRelationship(
+  relationship: RelationshipsJson
+): relationship is ManyToManyRelationshipJson {
+  return (
+    !!relationship.through &&
+    !!relationship.fromPrimary &&
+    !!relationship.from &&
+    !!relationship.toPrimary &&
+    !!relationship.to
+  )
+}
+
+export function validateManyToMany(
+  relationship: RelationshipsJson
+): ManyToManyRelationshipJson | undefined {
+  if (isValidManyToManyRelationship(relationship)) {
+    return relationship
+  }
+  return undefined
+}
diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index 8f3607bc73..9d5a315628 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -1,6 +1,10 @@
 // need to handle table name + field or just field, depending on if relationships used
 import { FieldSchema, FieldType, Row, Table } from "@budibase/types"
-import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core"
+import {
+  helpers,
+  PROTECTED_EXTERNAL_COLUMNS,
+  PROTECTED_INTERNAL_COLUMNS,
+} from "@budibase/shared-core"
 import { generateRowIdField } from "../../../../integrations/utils"
 
 function extractFieldValue({
@@ -61,11 +65,13 @@ export function generateIdForRow(
 export function basicProcessing({
   row,
   table,
+  tables,
   isLinked,
   sqs,
 }: {
   row: Row
   table: Table
+  tables: Table[]
   isLinked: boolean
   sqs?: boolean
 }): Row {
@@ -86,24 +92,65 @@ export function basicProcessing({
       thisRow[fieldName] = value
     }
   }
+  let columns: string[] = Object.keys(table.schema)
   if (!sqs) {
     thisRow._id = generateIdForRow(row, table, isLinked)
     thisRow.tableId = table._id
     thisRow._rev = "rev"
+    columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
   } else {
-    const columns = Object.keys(table.schema)
+    columns = columns.concat(PROTECTED_EXTERNAL_COLUMNS)
     for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) {
-      const schema: FieldSchema | undefined = table.schema[internalColumn]
-      let value = extractFieldValue({
+      thisRow[internalColumn] = extractFieldValue({
         row,
         tableName: table._id!,
         fieldName: internalColumn,
         isLinked,
       })
-      if (sqs && schema?.type === FieldType.LINK && typeof value === "string") {
-        value = JSON.parse(value)
+    }
+  }
+  for (let col of columns) {
+    const schema: FieldSchema | undefined = table.schema[col]
+    if (schema?.type !== FieldType.LINK) {
+      continue
+    }
+    const relatedTable = tables.find(tbl => tbl._id === schema.tableId)
+    if (!relatedTable) {
+      continue
+    }
+    const value = extractFieldValue({
+      row,
+      tableName: table._id!,
+      fieldName: col,
+      isLinked,
+    })
+    const array: Row[] = Array.isArray(value)
+      ? value
+      : typeof value === "string"
+      ? JSON.parse(value)
+      : undefined
+    if (array) {
+      thisRow[col] = array
+      // make sure all of them have an _id
+      if (Array.isArray(thisRow[col])) {
+        const sortField =
+          relatedTable.primaryDisplay || relatedTable.primary![0]!
+        thisRow[col] = (thisRow[col] as Row[])
+          .map(relatedRow => {
+            relatedRow._id = relatedRow._id
+              ? relatedRow._id
+              : generateIdForRow(relatedRow, relatedTable)
+            return relatedRow
+          })
+          .sort((a, b) => {
+            if (!a?.[sortField]) {
+              return 1
+            } else if (!b?.[sortField]) {
+              return -1
+            }
+            return a[sortField].localeCompare(b[sortField])
+          })
       }
-      thisRow[internalColumn] = value
     }
   }
   return thisRow
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index a24ec17c26..d6e5c3e8f1 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -36,73 +36,6 @@ function isCorrectRelationship(
   return !!possibleColumns.find(col => row[col] === relationship.column)
 }
 
-/**
- * This iterates through the returned rows and works out what elements of the rows
- * actually match up to another row (based on primary keys) - this is pretty specific
- * to SQL and the way that SQL relationships are returned based on joins.
- * This is complicated, but the idea is that when a SQL query returns all the relations
- * will be separate rows, with all of the data in each row. We have to decipher what comes
- * from where (which tables) and how to convert that into budibase columns.
- */
-export async function updateRelationshipColumns(
-  table: Table,
-  tables: TableMap,
-  row: Row,
-  rows: { [key: string]: Row },
-  relationships: RelationshipsJson[],
-  opts?: { sqs?: boolean }
-) {
-  const columns: { [key: string]: any } = {}
-  for (let relationship of relationships) {
-    const linkedTable = tables[relationship.tableName]
-    if (!linkedTable) {
-      continue
-    }
-    const fromColumn = `${table.name}.${relationship.from}`
-    const toColumn = `${linkedTable.name}.${relationship.to}`
-    // this is important when working with multiple relationships
-    // between the same tables, don't want to overlap/multiply the relations
-    if (
-      !relationship.through &&
-      row[fromColumn]?.toString() !== row[toColumn]?.toString()
-    ) {
-      continue
-    }
-
-    let linked = basicProcessing({
-      row,
-      table: linkedTable,
-      isLinked: true,
-      sqs: opts?.sqs,
-    })
-    if (!linked._id) {
-      continue
-    }
-    if (
-      !opts?.sqs ||
-      isCorrectRelationship(relationship, table, linkedTable, row)
-    ) {
-      columns[relationship.column] = linked
-    }
-  }
-  for (let [column, related] of Object.entries(columns)) {
-    if (!row._id) {
-      continue
-    }
-    const rowId: string = row._id
-    if (!Array.isArray(rows[rowId][column])) {
-      rows[rowId][column] = []
-    }
-    // make sure relationship hasn't been found already
-    if (
-      !rows[rowId][column].find((relation: Row) => relation._id === related._id)
-    ) {
-      rows[rowId][column].push(related)
-    }
-  }
-  return rows
-}
-
 /**
  * Gets the list of relationship JSON structures based on the columns in the table,
  * this will be used by the underlying library to build whatever relationship mechanism
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index e52ee63e42..45f0cef085 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -13,7 +13,7 @@ import {
   processDates,
   processFormulas,
 } from "../../../../utilities/rowProcessor"
-import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils"
+import { isKnexEmptyReadResponse } from "./sqlUtils"
 import {
   basicProcessing,
   generateIdForRow,
@@ -149,22 +149,11 @@ export async function sqlOutputProcessing(
       rowId = generateIdForRow(row, table)
       row._id = rowId
     }
-    // this is a relationship of some sort
-    if (!opts?.sqs && finalRows[rowId]) {
-      finalRows = await updateRelationshipColumns(
-        table,
-        tables,
-        row,
-        finalRows,
-        relationships,
-        opts
-      )
-      continue
-    }
     const thisRow = fixArrayTypes(
       basicProcessing({
         row,
         table,
+        tables: Object.values(tables),
         isLinked: false,
         sqs: opts?.sqs,
       }),
@@ -175,18 +164,6 @@ export async function sqlOutputProcessing(
     }
 
     finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table })
-
-    // do this at end once its been added to the final rows
-    if (!opts?.sqs) {
-      finalRows = await updateRelationshipColumns(
-        table,
-        tables,
-        row,
-        finalRows,
-        relationships,
-        opts
-      )
-    }
   }
 
   // make sure all related rows are correct
diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index de7fac4f1b..86d86b1161 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2126,81 +2126,76 @@ describe.each([
     })
   })
 
-  // This will never work for Lucene.
-  !isLucene &&
-    // It also can't work for in-memory searching because the related table name
-    // isn't available.
-    !isInMemory &&
-    describe("relations", () => {
-      let productCategoryTable: Table, productCatRows: Row[]
+  describe("relations", () => {
+    let productCategoryTable: Table, productCatRows: Row[]
 
-      beforeAll(async () => {
-        productCategoryTable = await createTable(
-          {
-            name: { name: "name", type: FieldType.STRING },
-          },
-          "productCategory"
-        )
-        table = await createTable(
-          {
-            name: { name: "name", type: FieldType.STRING },
-            productCat: {
-              type: FieldType.LINK,
-              relationshipType: RelationshipType.ONE_TO_MANY,
-              name: "productCat",
-              fieldName: "product",
-              tableId: productCategoryTable._id!,
-              constraints: {
-                type: "array",
-              },
+    beforeAll(async () => {
+      productCategoryTable = await createTable(
+        {
+          name: { name: "name", type: FieldType.STRING },
+        },
+        "productCategory"
+      )
+      table = await createTable(
+        {
+          name: { name: "name", type: FieldType.STRING },
+          productCat: {
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            name: "productCat",
+            fieldName: "product",
+            tableId: productCategoryTable._id!,
+            constraints: {
+              type: "array",
             },
           },
-          "product"
-        )
+        },
+        "product"
+      )
 
-        productCatRows = await Promise.all([
-          config.api.row.save(productCategoryTable._id!, { name: "foo" }),
-          config.api.row.save(productCategoryTable._id!, { name: "bar" }),
-        ])
+      productCatRows = await Promise.all([
+        config.api.row.save(productCategoryTable._id!, { name: "foo" }),
+        config.api.row.save(productCategoryTable._id!, { name: "bar" }),
+      ])
 
-        await Promise.all([
-          config.api.row.save(table._id!, {
-            name: "foo",
-            productCat: [productCatRows[0]._id],
-          }),
-          config.api.row.save(table._id!, {
-            name: "bar",
-            productCat: [productCatRows[1]._id],
-          }),
-          config.api.row.save(table._id!, {
-            name: "baz",
-            productCat: [],
-          }),
-        ])
-      })
-
-      it("should be able to filter by relationship using column name", async () => {
-        await expectQuery({
-          equal: { ["productCat.name"]: "foo" },
-        }).toContainExactly([
-          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-        ])
-      })
-
-      it("should be able to filter by relationship using table name", async () => {
-        await expectQuery({
-          equal: { ["productCategory.name"]: "foo" },
-        }).toContainExactly([
-          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-        ])
-      })
-
-      it("shouldn't return any relationship for last row", async () => {
-        await expectQuery({
-          equal: { ["name"]: "baz" },
-        }).toContainExactly([{ name: "baz", productCat: undefined }])
-      })
+      await Promise.all([
+        config.api.row.save(table._id!, {
+          name: "foo",
+          productCat: [productCatRows[0]._id],
+        }),
+        config.api.row.save(table._id!, {
+          name: "bar",
+          productCat: [productCatRows[1]._id],
+        }),
+        config.api.row.save(table._id!, {
+          name: "baz",
+          productCat: [],
+        }),
+      ])
     })
+
+    it("should be able to filter by relationship using column name", async () => {
+      await expectQuery({
+        equal: { ["productCat.name"]: "foo" },
+      }).toContainExactly([
+        { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+      ])
+    })
+
+    it("should be able to filter by relationship using table name", async () => {
+      await expectQuery({
+        equal: { ["productCategory.name"]: "foo" },
+      }).toContainExactly([
+        { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+      ])
+    })
+
+    it("shouldn't return any relationship for last row", async () => {
+      await expectQuery({
+        equal: { ["name"]: "baz" },
+      }).toContainExactly([{ name: "baz", productCat: undefined }])
+    })
+  })
   ;(isSqs || isLucene) &&
     describe("relations to same table", () => {
       let relatedTable: Table, relatedRows: Row[]
@@ -2610,50 +2605,50 @@ describe.each([
       })
     })
 
-  !isInMemory &&
-    describe("search by _id", () => {
-      let row: Row
+  // !isInMemory &&
+  describe("search by _id", () => {
+    let row: Row
 
-      beforeAll(async () => {
-        const toRelateTable = await createTable({
-          name: {
-            name: "name",
-            type: FieldType.STRING,
-          },
-        })
-        table = await createTable({
-          name: {
-            name: "name",
-            type: FieldType.STRING,
-          },
-          rel: {
-            name: "rel",
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.MANY_TO_MANY,
-            tableId: toRelateTable._id!,
-            fieldName: "rel",
-          },
-        })
-        const [row1, row2] = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
-        ])
-        row = await config.api.row.save(table._id!, {
-          name: "product 1",
-          rel: [row1._id, row2._id],
-        })
+    beforeAll(async () => {
+      const toRelateTable = await createTable({
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
       })
-
-      it("can filter by the row ID with limit 1", async () => {
-        await expectSearch({
-          query: {
-            equal: { _id: row._id },
-          },
-          limit: 1,
-        }).toContainExactly([row])
+      table = await createTable({
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+        },
+        rel: {
+          name: "rel",
+          type: FieldType.LINK,
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          tableId: toRelateTable._id!,
+          fieldName: "rel",
+        },
+      })
+      const [row1, row2] = await Promise.all([
+        config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
+        config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
+      ])
+      row = await config.api.row.save(table._id!, {
+        name: "product 1",
+        rel: [row1._id, row2._id],
       })
     })
 
+    it("can filter by the row ID with limit 1", async () => {
+      await expectSearch({
+        query: {
+          equal: { _id: row._id },
+        },
+        limit: 1,
+      }).toContainExactly([row])
+    })
+  })
+
   !isInternal &&
     describe("search by composite key", () => {
       beforeAll(async () => {
@@ -2690,82 +2685,6 @@ describe.each([
       })
     })
 
-  // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem
-  !isInternal &&
-    describe("pagination edge case with relationships", () => {
-      let mainRows: Row[] = []
-
-      beforeAll(async () => {
-        const toRelateTable = await createTable({
-          name: {
-            name: "name",
-            type: FieldType.STRING,
-          },
-        })
-        table = await createTable({
-          name: {
-            name: "name",
-            type: FieldType.STRING,
-          },
-          rel: {
-            name: "rel",
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.MANY_TO_ONE,
-            tableId: toRelateTable._id!,
-            fieldName: "rel",
-          },
-        })
-        const relatedRows = await Promise.all([
-          config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 3" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 4" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 5" }),
-          config.api.row.save(toRelateTable._id!, { name: "tag 6" }),
-        ])
-        mainRows = await Promise.all([
-          config.api.row.save(table._id!, {
-            name: "product 1",
-            rel: relatedRows.map(row => row._id),
-          }),
-          config.api.row.save(table._id!, {
-            name: "product 2",
-            rel: [],
-          }),
-          config.api.row.save(table._id!, {
-            name: "product 3",
-            rel: [],
-          }),
-        ])
-      })
-
-      it("can still page when the hard limit is hit", async () => {
-        await withCoreEnv(
-          {
-            SQL_MAX_ROWS: "6",
-          },
-          async () => {
-            const params: Omit<RowSearchParams, "tableId"> = {
-              query: {},
-              paginate: true,
-              limit: 3,
-              sort: "name",
-              sortType: SortType.STRING,
-              sortOrder: SortOrder.ASCENDING,
-            }
-            const page1 = await expectSearch(params).toContain([mainRows[0]])
-            expect(page1.hasNextPage).toBe(true)
-            expect(page1.bookmark).toBeDefined()
-            const page2 = await expectSearch({
-              ...params,
-              bookmark: page1.bookmark,
-            }).toContain([mainRows[1], mainRows[2]])
-            expect(page2.hasNextPage).toBe(false)
-          }
-        )
-      })
-    })
-
   isSql &&
     describe("primaryDisplay", () => {
       beforeAll(async () => {
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 6feea40766..7d61aebdfb 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -134,6 +134,17 @@ export interface RelationshipsJson {
   column: string
 }
 
+// TODO - this can be combined with the above type
+export interface ManyToManyRelationshipJson {
+  through: string
+  from: string
+  to: string
+  fromPrimary: string
+  toPrimary: string
+  tableName: string
+  column: string
+}
+
 export interface QueryJson {
   endpoint: {
     datasourceId: string

From 2a24a3dda683cafbbb65b0893c102240ea8c7b1e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 3 Sep 2024 18:40:20 +0100
Subject: [PATCH 19/69] Correcting test cases.

---
 .../src/api/routes/tests/search.spec.ts       | 211 +++++++++---------
 1 file changed, 108 insertions(+), 103 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 86d86b1161..f5c5ade2f8 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -2126,76 +2126,81 @@ describe.each([
     })
   })
 
-  describe("relations", () => {
-    let productCategoryTable: Table, productCatRows: Row[]
+  // This will never work for Lucene.
+  !isLucene &&
+    // It also can't work for in-memory searching because the related table name
+    // isn't available.
+    !isInMemory &&
+    describe("relations", () => {
+      let productCategoryTable: Table, productCatRows: Row[]
 
-    beforeAll(async () => {
-      productCategoryTable = await createTable(
-        {
-          name: { name: "name", type: FieldType.STRING },
-        },
-        "productCategory"
-      )
-      table = await createTable(
-        {
-          name: { name: "name", type: FieldType.STRING },
-          productCat: {
-            type: FieldType.LINK,
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            name: "productCat",
-            fieldName: "product",
-            tableId: productCategoryTable._id!,
-            constraints: {
-              type: "array",
+      beforeAll(async () => {
+        productCategoryTable = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+          },
+          "productCategory"
+        )
+        table = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+            productCat: {
+              type: FieldType.LINK,
+              relationshipType: RelationshipType.ONE_TO_MANY,
+              name: "productCat",
+              fieldName: "product",
+              tableId: productCategoryTable._id!,
+              constraints: {
+                type: "array",
+              },
             },
           },
-        },
-        "product"
-      )
+          "product"
+        )
 
-      productCatRows = await Promise.all([
-        config.api.row.save(productCategoryTable._id!, { name: "foo" }),
-        config.api.row.save(productCategoryTable._id!, { name: "bar" }),
-      ])
+        productCatRows = await Promise.all([
+          config.api.row.save(productCategoryTable._id!, { name: "foo" }),
+          config.api.row.save(productCategoryTable._id!, { name: "bar" }),
+        ])
 
-      await Promise.all([
-        config.api.row.save(table._id!, {
-          name: "foo",
-          productCat: [productCatRows[0]._id],
-        }),
-        config.api.row.save(table._id!, {
-          name: "bar",
-          productCat: [productCatRows[1]._id],
-        }),
-        config.api.row.save(table._id!, {
-          name: "baz",
-          productCat: [],
-        }),
-      ])
+        await Promise.all([
+          config.api.row.save(table._id!, {
+            name: "foo",
+            productCat: [productCatRows[0]._id],
+          }),
+          config.api.row.save(table._id!, {
+            name: "bar",
+            productCat: [productCatRows[1]._id],
+          }),
+          config.api.row.save(table._id!, {
+            name: "baz",
+            productCat: [],
+          }),
+        ])
+      })
+
+      it("should be able to filter by relationship using column name", async () => {
+        await expectQuery({
+          equal: { ["productCat.name"]: "foo" },
+        }).toContainExactly([
+          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+        ])
+      })
+
+      it("should be able to filter by relationship using table name", async () => {
+        await expectQuery({
+          equal: { ["productCategory.name"]: "foo" },
+        }).toContainExactly([
+          { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
+        ])
+      })
+
+      it("shouldn't return any relationship for last row", async () => {
+        await expectQuery({
+          equal: { ["name"]: "baz" },
+        }).toContainExactly([{ name: "baz", productCat: undefined }])
+      })
     })
-
-    it("should be able to filter by relationship using column name", async () => {
-      await expectQuery({
-        equal: { ["productCat.name"]: "foo" },
-      }).toContainExactly([
-        { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-      ])
-    })
-
-    it("should be able to filter by relationship using table name", async () => {
-      await expectQuery({
-        equal: { ["productCategory.name"]: "foo" },
-      }).toContainExactly([
-        { name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
-      ])
-    })
-
-    it("shouldn't return any relationship for last row", async () => {
-      await expectQuery({
-        equal: { ["name"]: "baz" },
-      }).toContainExactly([{ name: "baz", productCat: undefined }])
-    })
-  })
   ;(isSqs || isLucene) &&
     describe("relations to same table", () => {
       let relatedTable: Table, relatedRows: Row[]
@@ -2605,50 +2610,50 @@ describe.each([
       })
     })
 
-  // !isInMemory &&
-  describe("search by _id", () => {
-    let row: Row
+  !isInMemory &&
+    describe("search by _id", () => {
+      let row: Row
 
-    beforeAll(async () => {
-      const toRelateTable = await createTable({
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-        },
+      beforeAll(async () => {
+        const toRelateTable = await createTable({
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+        })
+        table = await createTable({
+          name: {
+            name: "name",
+            type: FieldType.STRING,
+          },
+          rel: {
+            name: "rel",
+            type: FieldType.LINK,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+            tableId: toRelateTable._id!,
+            fieldName: "rel",
+          },
+        })
+        const [row1, row2] = await Promise.all([
+          config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
+          config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
+        ])
+        row = await config.api.row.save(table._id!, {
+          name: "product 1",
+          rel: [row1._id, row2._id],
+        })
       })
-      table = await createTable({
-        name: {
-          name: "name",
-          type: FieldType.STRING,
-        },
-        rel: {
-          name: "rel",
-          type: FieldType.LINK,
-          relationshipType: RelationshipType.MANY_TO_MANY,
-          tableId: toRelateTable._id!,
-          fieldName: "rel",
-        },
-      })
-      const [row1, row2] = await Promise.all([
-        config.api.row.save(toRelateTable._id!, { name: "tag 1" }),
-        config.api.row.save(toRelateTable._id!, { name: "tag 2" }),
-      ])
-      row = await config.api.row.save(table._id!, {
-        name: "product 1",
-        rel: [row1._id, row2._id],
+
+      it("can filter by the row ID with limit 1", async () => {
+        await expectSearch({
+          query: {
+            equal: { _id: row._id },
+          },
+          limit: 1,
+        }).toContainExactly([row])
       })
     })
 
-    it("can filter by the row ID with limit 1", async () => {
-      await expectSearch({
-        query: {
-          equal: { _id: row._id },
-        },
-        limit: 1,
-      }).toContainExactly([row])
-    })
-  })
-
   !isInternal &&
     describe("search by composite key", () => {
       beforeAll(async () => {

From 2d6a8d9ff51c224177d7dcee2df44b4a920edfd5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 3 Sep 2024 18:50:01 +0100
Subject: [PATCH 20/69] Fix for sorting, didn't account for some primitive
 types.

---
 packages/server/src/api/controllers/row/utils/basic.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index 9d5a315628..b754e288ed 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -143,12 +143,16 @@ export function basicProcessing({
             return relatedRow
           })
           .sort((a, b) => {
-            if (!a?.[sortField]) {
+            const aField = a?.[sortField],
+              bField = b?.[sortField]
+            if (!aField) {
               return 1
-            } else if (!b?.[sortField]) {
+            } else if (!bField) {
               return -1
             }
-            return a[sortField].localeCompare(b[sortField])
+            return aField.localeCompare
+              ? aField.localeCompare(bField)
+              : aField - bField
           })
       }
     }

From 50d64de72acb9ee97a7735aeb65470d51195f522 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 18:51:45 +0100
Subject: [PATCH 21/69] styling updates

---
 .../portal/settings/ai/AIConfigTile.svelte    | 33 +++++++++++++++++--
 .../builder/portal/settings/ai/constants.ts   |  2 +-
 .../portal/settings/ai/logos/Budibase.svelte  | 17 ++++++++++
 .../settings/ai/logos/TogetherAI.svelte       | 17 ++++++++++
 4 files changed, 66 insertions(+), 3 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 4b9b4cc506..d02c2c62c3 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -2,6 +2,7 @@
   import { Body, Label, Icon } from "@budibase/bbui"
   import OpenAILogo from "./logos/OpenAI.svelte"
   import AnthropicLogo from "./logos/Anthropic.svelte"
+  import TogetherAILogo from "./logos/TogetherAI.svelte"
   import { Providers } from "./constants"
 
   export let config
@@ -19,6 +20,8 @@
       <OpenAILogo height="30" width="30"/>
     {:else if config.provider === Providers.Anthropic.name}
       <AnthropicLogo height="30" width="30"/>
+    {:else if config.provider === Providers.TogetherAI.name}
+      <TogetherAILogo height="30" width="30"/>
     {/if}
   </div>
   <div class="header">
@@ -28,17 +31,23 @@
   <div class="controls">
     <Icon
       on:click={editHandler}
+      color="var(--grey-6)"
       size="S"
       hoverable
       name="Edit"
     />
     <Icon
       on:click={deleteHandler}
+      color="var(--grey-6)"
       size="S"
       hoverable
       name="Delete"
     />
-    <div>Activated</div>
+    {#if config.active}
+      <div class="tag active">Activated</div>
+    {:else if !config.active}
+      <div class="tag disabled">Disabled</div>
+    {/if}
   </div>
 </div>
 
@@ -46,7 +55,7 @@
   .option {
     background-color: var(--background);
     border: 1px solid var(--grey-4);
-    padding: 10px 16px 14px;
+    padding: 16px;
     border-radius: 4px;
     cursor: pointer;
     display: grid;
@@ -87,4 +96,24 @@
     grid-gap: 10px;
     align-items: center;
   }
+
+  .tag {
+    display: flex;
+    color: var(--spectrum-body-m-text-color);
+    padding: 4px 8px;
+    justify-content: center;
+    align-items: center;
+    gap: 8px;
+    font-size: 12px;
+  }
+
+  .active {
+    border-radius: 5px;
+    background: var(--spectrum-global-color-green-600);
+  }
+
+  .disabled {
+    border-radius: 5px;
+    background: var(--spectrum-global-color-red-600);
+  }
 </style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index 07e0ea1cb7..9c82f714a4 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -22,7 +22,7 @@ export const Providers = {
   TogetherAI: {
     name: "Together AI",
     // TODO: too many - probably need to use an autocomplete for this
-    models: [""]
+    models: ["whatever"]
   },
   Custom: {
     name: "Custom",
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
new file mode 100644
index 0000000000..f69c7d8281
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
@@ -0,0 +1,17 @@
+<script>
+  export let width
+  export let height
+</script>
+<svg {width} {height} viewBox="0 0 265 265" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <g clip-path="url(#clip0_1_1799)">
+    <path d="M158.2 8.6V116.6C158.2 121.3 162 125.2 166.8 125.2H213.8C218 125.2 222 123.2 224.6 119.8L262.9 68.9C265.7 65.2 265.7 60.1 262.9 56.4L224.6 5.4C222 2 218 0 213.8 0H166.8C162 0 158.2 3.8 158.2 8.6Z" fill="#FF4E4E"/>
+    <path d="M158.2 148.4V256.4C158.2 261.1 162 265 166.8 265H213.8C218 265 222 263 224.6 259.6L262.9 208.7C265.7 205 265.7 199.9 262.9 196.2L224.6 145.3C222.1 141.9 218.1 139.9 213.8 139.9H166.8C162 139.8 158.2 143.7 158.2 148.4Z" fill="#6E56FF"/>
+    <path d="M0 8.6V116.6C0 121.3 3.8 125.2 8.6 125.2H109.6C113.8 125.2 117.8 123.2 120.4 119.8L155.9 72.5C160.3 66.6 160.3 58.5 155.9 52.6L120.3 5.4C117.8 2 113.8 0 109.5 0H8.6C3.8 0 0 3.8 0 8.6Z" fill="#F97777"/>
+    <path d="M0 148.4V256.4C0 261.1 3.8 265 8.6 265H109.6C113.8 265 117.8 263 120.4 259.6L155.9 212.3C160.3 206.4 160.3 198.3 155.9 192.4L120.4 145.1C117.9 141.7 113.9 139.7 109.6 139.7H8.6C3.8 139.8 0 143.7 0 148.4Z" fill="#9F8FFF"/>
+  </g>
+  <defs>
+    <clipPath id="clip0_1_1799">
+      <rect width="265" height="265" fill="white"/>
+    </clipPath>
+  </defs>
+</svg>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte
new file mode 100644
index 0000000000..1afd5626b1
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte
@@ -0,0 +1,17 @@
+<script>
+  export let height
+  export let width
+</script>
+<svg {width} {height} viewBox="0 0 107 21" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path d="M3.0727 6.13665H0.99707V4.44456H3.0727V1.2183H4.92272V4.44456H7.83312V6.13665H4.92272V12.9276C4.92272 13.4089 5.01297 13.7548 5.19346 13.9654C5.38899 14.1609 5.71988 14.2587 6.18615 14.2587H8.1941V15.9508H6.07334C4.9904 15.9508 4.2158 15.7101 3.74954 15.2288C3.29831 14.7475 3.0727 13.9879 3.0727 12.9501V6.13665Z" fill="currentColor"></path>
+  <path d="M14.1547 16.0861C13.0417 16.0861 12.049 15.838 11.1766 15.3416C10.3193 14.8453 9.64995 14.1534 9.16864 13.266C8.68734 12.3786 8.44669 11.3558 8.44669 10.1977C8.44669 9.03952 8.68734 8.01675 9.16864 7.12934C9.64995 6.24193 10.3193 5.55006 11.1766 5.05371C12.049 4.55736 13.0417 4.30919 14.1547 4.30919C15.2677 4.30919 16.2529 4.55736 17.1102 5.05371C17.9826 5.55006 18.6594 6.24193 19.1407 7.12934C19.622 8.01675 19.8627 9.03952 19.8627 10.1977C19.8627 11.3558 19.622 12.3786 19.1407 13.266C18.6594 14.1534 17.9826 14.8453 17.1102 15.3416C16.2529 15.838 15.2677 16.0861 14.1547 16.0861ZM14.1547 14.4392C14.9067 14.4392 15.5685 14.2587 16.1401 13.8977C16.7266 13.5367 17.1779 13.0329 17.4937 12.3861C17.8096 11.7393 17.9675 11.0099 17.9675 10.1977C17.9675 9.38546 17.8096 8.65598 17.4937 8.00923C17.1779 7.36247 16.7266 6.85861 16.1401 6.49763C15.5685 6.13665 14.9067 5.95616 14.1547 5.95616C13.4026 5.95616 12.7333 6.13665 12.1467 6.49763C11.5752 6.85861 11.1315 7.36247 10.8156 8.00923C10.4998 8.65598 10.3418 9.38546 10.3418 10.1977C10.3418 11.0099 10.4998 11.7393 10.8156 12.3861C11.1315 13.0329 11.5752 13.5367 12.1467 13.8977C12.7333 14.2587 13.4026 14.4392 14.1547 14.4392Z" fill="currentColor"></path>
+  <path d="M32.5327 4.44456V15.3867C32.5327 18.8612 30.6301 20.5984 26.8247 20.5984C25.3357 20.5984 24.1174 20.26 23.1698 19.5831C22.2373 18.9063 21.6958 17.9437 21.5454 16.6953H23.4406C23.591 17.4172 23.9745 17.9738 24.5912 18.3648C25.2079 18.7559 25.9975 18.9514 26.9601 18.9514C29.4418 18.9514 30.6827 17.7406 30.6827 15.3191V13.9654C29.8555 15.3792 28.5394 16.0861 26.7345 16.0861C25.6666 16.0861 24.7115 15.853 23.8692 15.3867C23.042 14.9205 22.3877 14.2436 21.9064 13.3562C21.4401 12.4688 21.207 11.416 21.207 10.1977C21.207 9.03952 21.4401 8.01675 21.9064 7.12934C22.3877 6.24193 23.0495 5.55006 23.8918 5.05371C24.7341 4.55736 25.6816 4.30919 26.7345 4.30919C27.667 4.30919 28.4642 4.4972 29.126 4.87322C29.7878 5.2342 30.3067 5.75311 30.6827 6.42994L30.9309 4.44456H32.5327ZM26.915 14.4392C27.652 14.4392 28.3063 14.2662 28.8778 13.9203C29.4644 13.5593 29.9156 13.0629 30.2315 12.4312C30.5473 11.7845 30.7053 11.055 30.7053 10.2428C30.7053 9.41554 30.5473 8.67854 30.2315 8.03179C29.9156 7.36999 29.4644 6.85861 28.8778 6.49763C28.3063 6.13665 27.652 5.95616 26.915 5.95616C25.7719 5.95616 24.8469 6.35474 24.14 7.1519C23.4481 7.93402 23.1021 8.94928 23.1021 10.1977C23.1021 11.4461 23.4481 12.4688 24.14 13.266C24.8469 14.0481 25.7719 14.4392 26.915 14.4392Z" fill="currentColor"></path>
+  <path d="M34.2758 10.1977C34.2758 9.02448 34.5014 7.99419 34.9526 7.10678C35.4189 6.21937 36.0656 5.53502 36.8929 5.05371C37.7201 4.55736 38.6677 4.30919 39.7356 4.30919C40.7884 4.30919 41.721 4.52728 42.5332 4.96346C43.3454 5.39965 43.9846 6.03136 44.4509 6.85861C44.9171 7.68585 45.1653 8.65598 45.1954 9.769C45.1954 9.93445 45.1803 10.1977 45.1503 10.5586H36.216V10.7166C36.2461 11.8446 36.592 12.7471 37.2538 13.4239C37.9156 14.1008 38.7805 14.4392 39.8484 14.4392C40.6756 14.4392 41.375 14.2361 41.9466 13.83C42.5332 13.4089 42.9242 12.8298 43.1198 12.0928H44.9923C44.7667 13.266 44.2027 14.2286 43.3002 14.9806C42.3978 15.7176 41.2923 16.0861 39.9837 16.0861C38.8406 16.0861 37.8404 15.8455 36.9831 15.3642C36.1258 14.8678 35.4565 14.176 34.9752 13.2885C34.5089 12.3861 34.2758 11.3558 34.2758 10.1977ZM43.2777 9.02448C43.1874 8.04683 42.8189 7.28727 42.1722 6.7458C41.5405 6.20433 40.7358 5.9336 39.7581 5.9336C38.8858 5.9336 38.1187 6.21937 37.4569 6.79092C36.7951 7.36247 36.4191 8.10699 36.3288 9.02448H43.2777Z" fill="currentColor"></path>
+  <path d="M47.5703 6.13665H45.4947V4.44456H47.5703V1.2183H49.4203V4.44456H52.3307V6.13665H49.4203V12.9276C49.4203 13.4089 49.5106 13.7548 49.6911 13.9654C49.8866 14.1609 50.2175 14.2587 50.6838 14.2587H52.6917V15.9508H50.571C49.488 15.9508 48.7134 15.7101 48.2472 15.2288C47.7959 14.7475 47.5703 13.9879 47.5703 12.9501V6.13665Z" fill="currentColor"></path>
+  <path d="M59.2771 4.30919C60.6759 4.30919 61.804 4.73033 62.6613 5.57262C63.5186 6.4149 63.9473 7.73849 63.9473 9.54339V15.9508H62.0973V9.6562C62.0973 8.46797 61.8265 7.56552 61.2851 6.94885C60.7436 6.31714 59.969 6.00128 58.9613 6.00128C57.8934 6.00128 57.0436 6.3773 56.4119 7.12934C55.7952 7.88138 55.4868 8.90415 55.4868 10.1977V15.9508H53.6368V0.157928H55.4868V6.3397C55.8629 5.73807 56.3592 5.24924 56.9759 4.87322C57.6076 4.4972 58.3747 4.30919 59.2771 4.30919Z" fill="currentColor"></path>
+  <path d="M65.5284 10.1977C65.5284 9.02448 65.754 7.99419 66.2052 7.10678C66.6715 6.21937 67.3182 5.53502 68.1455 5.05371C68.9727 4.55736 69.9203 4.30919 70.9882 4.30919C72.041 4.30919 72.9736 4.52728 73.7858 4.96346C74.598 5.39965 75.2372 6.03136 75.7035 6.85861C76.1697 7.68585 76.4179 8.65598 76.448 9.769C76.448 9.93445 76.4329 10.1977 76.4029 10.5586H67.4686V10.7166C67.4987 11.8446 67.8446 12.7471 68.5064 13.4239C69.1682 14.1008 70.0331 14.4392 71.101 14.4392C71.9282 14.4392 72.6276 14.2361 73.1992 13.83C73.7858 13.4089 74.1768 12.8298 74.3723 12.0928H76.2449C76.0193 13.266 75.4553 14.2286 74.5528 14.9806C73.6504 15.7176 72.5449 16.0861 71.2363 16.0861C70.0932 16.0861 69.093 15.8455 68.2357 15.3642C67.3784 14.8678 66.7091 14.176 66.2278 13.2885C65.7615 12.3861 65.5284 11.3558 65.5284 10.1977ZM74.5303 9.02448C74.44 8.04683 74.0715 7.28727 73.4248 6.7458C72.7931 6.20433 71.9884 5.9336 71.0107 5.9336C70.1384 5.9336 69.3713 6.21937 68.7095 6.79092C68.0477 7.36247 67.6717 8.10699 67.5814 9.02448H74.5303Z" fill="currentColor"></path>
+  <path d="M84.0314 4.44456V6.24945H83.1064C82.0084 6.24945 81.2188 6.61795 80.7374 7.35495C80.2712 8.09195 80.038 9.00192 80.038 10.0849V15.9508H78.188V4.44456H79.7899L80.038 6.18177C80.3689 5.65534 80.7976 5.2342 81.324 4.91834C81.8505 4.60249 82.5799 4.44456 83.5125 4.44456H84.0314Z" fill="currentColor"></path>
+  <path d="M102.875 14.2587V15.9508H101.859C101.122 15.9508 100.596 15.8004 100.28 15.4995C99.9643 15.1987 99.7988 14.755 99.7838 14.1684C98.8964 15.4469 97.6104 16.0861 95.9258 16.0861C94.6473 16.0861 93.6171 15.7853 92.8349 15.1837C92.0678 14.5821 91.6843 13.7623 91.6843 12.7245C91.6843 11.5664 92.0754 10.679 92.8575 10.0623C93.6546 9.44562 94.8053 9.13729 96.3093 9.13729H99.6935V8.34764C99.6935 7.5956 99.4378 7.00901 98.9265 6.58787C98.4301 6.16673 97.7307 5.95616 96.8283 5.95616C96.0311 5.95616 95.3693 6.13665 94.8429 6.49763C94.3315 6.84356 94.0156 7.30983 93.8953 7.89642H92.0453C92.1806 6.76836 92.677 5.88847 93.5343 5.25676C94.4067 4.62505 95.5348 4.30919 96.9185 4.30919C98.3925 4.30919 99.5281 4.67017 100.325 5.39213C101.137 6.09905 101.544 7.12182 101.544 8.46045V13.4239C101.544 13.9804 101.799 14.2587 102.311 14.2587H102.875ZM99.6935 10.694H96.1289C94.4142 10.694 93.5569 11.3332 93.5569 12.6117C93.5569 13.1833 93.7825 13.642 94.2337 13.9879C94.685 14.3339 95.2941 14.5069 96.0612 14.5069C97.1892 14.5069 98.0766 14.2136 98.7234 13.627C99.3702 13.0253 99.6935 12.2357 99.6935 11.258V10.694Z" fill="currentColor"></path>
+  <path d="M104.851 0C105.197 0 105.483 0.112806 105.708 0.338418C105.934 0.56403 106.047 0.849805 106.047 1.19574C106.047 1.54168 105.934 1.82746 105.708 2.05307C105.483 2.27868 105.197 2.39149 104.851 2.39149C104.505 2.39149 104.219 2.27868 103.994 2.05307C103.768 1.82746 103.655 1.54168 103.655 1.19574C103.655 0.849805 103.768 0.56403 103.994 0.338418C104.219 0.112806 104.505 0 104.851 0ZM103.926 4.44456H105.776V15.9508H103.926V4.44456Z" fill="currentColor"></path>
+  <path d="M88.067 14.1988C88.067 15.1664 87.2731 15.9508 86.2937 15.9508C85.3144 15.9508 84.5205 15.1664 84.5205 14.1988C84.5205 13.2312 85.3144 12.4468 86.2937 12.4468C87.2731 12.4468 88.067 13.2312 88.067 14.1988Z" fill="#0F6FFF"></path>
+</svg>
\ No newline at end of file

From 378ccb6e70b48695e05490ec7d29fc0c8603253a Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 18:52:33 +0100
Subject: [PATCH 22/69] switch name and description

---
 .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index d02c2c62c3..1b2b4b99a7 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -25,7 +25,7 @@
     {/if}
   </div>
   <div class="header">
-    <Body>{config.name}</Body>
+    <Body>{config.provider}</Body>
     <Label>{config.name}</Label>
   </div>
   <div class="controls">

From 0d5e776424b5c12a9588ffbc2f921b33d39e4e8b Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 19:12:13 +0100
Subject: [PATCH 23/69] preserve apikeys on save

---
 .../portal/settings/ai/ConfigModal.svelte     |  3 ++-
 .../builder/portal/settings/ai/index.svelte   | 10 ++++-----
 packages/types/src/documents/global/config.ts | 22 ++++++++++---------
 .../src/api/controllers/global/configs.ts     | 11 ++++++----
 4 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 703f1e8129..747a070c6b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -22,6 +22,7 @@
     const {provider, defaultModel, name, apiKey} = config
     validation = provider && defaultModel && name && apiKey
   }
+  $: canEditBaseUrl = config.provider && ConfigMap[config.provider].baseUrl === ""
 
   function prefillConfig(evt) {
     const provider = evt.detail
@@ -70,7 +71,7 @@
   </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
-    <Input placeholder={"www.google.com"} bind:value={config.baseUrl}/>
+    <Input disabled={!canEditBaseUrl} placeholder={"https://budibase.ai"} bind:value={config.baseUrl}/>
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index d0ced7041a..57435168ac 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -60,10 +60,9 @@
     }
 
     try {
-      const savedConfig = await API.saveConfig(fullAIConfig)
-      fullAIConfig._rev = savedConfig._rev
-      fullAIConfig._id = savedConfig._id
+      await API.saveConfig(fullAIConfig)
       notifications.success(`Successfully saved and activated AI Configuration`)
+      await fetchAIConfig()
     } catch (error) {
       notifications.error(
         `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
@@ -76,10 +75,9 @@
     delete fullAIConfig.config[key]
 
     try {
-      const savedConfig = await API.saveConfig(fullAIConfig)
-      fullAIConfig._rev = savedConfig._rev
-      fullAIConfig._id = savedConfig._id
+      await API.saveConfig(fullAIConfig)
       notifications.success(`Deleted config`)
+      await fetchAIConfig()
     } catch (error) {
       notifications.error(
         `Failed to delete config, reason: ${error?.message || "Unknown"}`
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 249893520c..35caedef63 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -112,18 +112,20 @@ export interface SCIMInnerConfig {
 export interface SCIMConfig extends Config<SCIMInnerConfig> {}
 
 export interface AIInnerConfig {
-  // TODO: should be enum
-  provider: string
-  isDefault: boolean
-  name: string
-  active: boolean
-  baseUrl: string
-  apiKey: string
-  // TODO: should be enum
-  defaultModel: string
+  [key: string]: {
+    // TODO: should be enum
+    provider: string
+    isDefault: boolean
+    name: string
+    active: boolean
+    baseUrl: string
+    apiKey: string
+    // TODO: should be enum
+    defaultModel: string
+  }
 }
 
-export interface AIConfig extends Config<AIInnerConfig[]> {}
+export interface AIConfig extends Config<AIInnerConfig> {}
 
 export const isSettingsConfig = (config: Config): config is SettingsConfig =>
   config.type === ConfigType.SETTINGS
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 0b8e525eb8..1b66289f47 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -200,11 +200,14 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
 }
 
 async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) {
+  if (!existingConfig) return
+
   // ensure that the redacted API keys are not overwritten in the DB
-  // for (let aiConfig of existingConfig) {
-  //
-  // }
-  return true
+  for (let uuid in existingConfig.config) {
+    if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
+      config[uuid].apiKey = existingConfig.config[uuid].apiKey
+    }
+  }
 }
 
 export async function save(ctx: UserCtx<Config>) {

From be4b6aaceb47a1b6048e9c6b4cd9edf007b090d6 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 19:19:33 +0100
Subject: [PATCH 24/69] adding config for azure openai and more customisation
 for custom ai models

---
 .../builder/portal/settings/ai/ConfigModal.svelte  | 14 +++++++++-----
 .../pages/builder/portal/settings/ai/constants.ts  | 11 +++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 747a070c6b..cac8f74786 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -63,11 +63,15 @@
   </div>
   <div class="form-row">
     <Label size="M">Default Model</Label>
-    <Select
-      placeholder={config.provider ? "Choose an option" : "Select a provider first"}
-      bind:value={config.defaultModel}
-      options={config.provider ? Providers[config.provider].models : []}
-    />
+    {#if config.provider !== Providers.Custom.name}
+      <Select
+        placeholder={config.provider ? "Choose an option" : "Select a provider first"}
+        bind:value={config.defaultModel}
+        options={config.provider ? Providers[config.provider].models : []}
+      />
+    {:else}
+      <Input bind:value={config.defaultModel} />
+    {/if}
   </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index 9c82f714a4..d0db0d0d9b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -22,6 +22,17 @@ export const Providers = {
   TogetherAI: {
     name: "Together AI",
     // TODO: too many - probably need to use an autocomplete for this
+    models: [
+      "gpt-4o-mini",
+      "gpt-4o",
+      "gpt-3.5-turbo",
+      "chatgpt-4o-latest",
+      "gpt-4-turbo",
+      "gpt-4",
+    ]
+  },
+  AzureOpenAI: {
+    name: "Azure Open AI",
     models: ["whatever"]
   },
   Custom: {

From d3b90a02539c50a1dfb7a97ca6b52d103407bce3 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 19:47:36 +0100
Subject: [PATCH 25/69] initial impl of default Budibase AI config through env
 vars

---
 .../portal/settings/ai/AIConfigTile.svelte    |  6 +++++-
 .../portal/settings/ai/ConfigModal.svelte     | 10 +++++-----
 .../builder/portal/settings/ai/constants.ts   | 19 +++++++++++++------
 packages/types/src/documents/global/config.ts | 10 +++++-----
 .../src/api/controllers/global/configs.ts     | 19 ++++++++++++++++---
 packages/worker/src/environment.ts            |  3 +++
 6 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 1b2b4b99a7..39c0972027 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -1,5 +1,6 @@
 <script>
   import { Body, Label, Icon } from "@budibase/bbui"
+  import BudibaseLogo from "./logos/Budibase.svelte"
   import OpenAILogo from "./logos/OpenAI.svelte"
   import AnthropicLogo from "./logos/Anthropic.svelte"
   import TogetherAILogo from "./logos/TogetherAI.svelte"
@@ -7,6 +8,7 @@
 
   export let config
   export let disabled
+  export let budibaseAIDefault
 
   export let editHandler
   export let deleteHandler
@@ -16,7 +18,9 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div on:click class:disabled class="option">
   <div class="icon">
-    {#if config.provider === Providers.OpenAI.name}
+    {#if config.name === "Budibase AI"}
+      <BudibaseLogo height="30" width="30"/>
+    {:else if config.provider === Providers.OpenAI.name}
       <OpenAILogo height="30" width="30"/>
     {:else if config.provider === Providers.Anthropic.name}
       <AnthropicLogo height="30" width="30"/>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index cac8f74786..73db61161b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -19,7 +19,7 @@
   let validation
 
   $: {
-    const {provider, defaultModel, name, apiKey} = config
+    const { provider, defaultModel, name, apiKey } = config
     validation = provider && defaultModel && name && apiKey
   }
   $: canEditBaseUrl = config.provider && ConfigMap[config.provider].baseUrl === ""
@@ -49,10 +49,6 @@
   title="Custom AI Configuration"
 >
   <div class="form-row">
-    <div class="form-row">
-      <Label size="M">Name</Label>
-      <Input placeholder={"Test 1"} bind:value={config.name}/>
-    </div>
     <Label size="M">Provider</Label>
     <Select
       placeholder={null}
@@ -61,6 +57,10 @@
       on:change={prefillConfig}
     />
   </div>
+  <div class="form-row">
+    <Label size="M">Name</Label>
+    <Input placeholder={"Enter a name"} bind:value={config.name}/>
+  </div>
   <div class="form-row">
     <Label size="M">Default Model</Label>
     {#if config.provider !== Providers.Custom.name}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index d0db0d0d9b..f4f93f3114 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -33,24 +33,31 @@ export const Providers = {
   },
   AzureOpenAI: {
     name: "Azure Open AI",
-    models: ["whatever"]
+    models: [
+      "gpt-4o-mini",
+    ]
   },
   Custom: {
     name: "Custom",
-    // TODO: too many - probably need to use an autocomplete for this
-    models: [""]
   },
 }
 
 export const ConfigMap = {
   OpenAI: {
-    baseUrl: "https://api.openai.com"
+    name: "OpenAI",
+    baseUrl: "https://api.openai.com",
   },
   Anthropic: {
-    baseUrl: ""
+    name: "Anthropic",
+    baseUrl: "",
   },
   TogetherAI: {
-    baseUrl: "https://api.together.xyz/v1"
+    name: "TogetherAI",
+    baseUrl: "https://api.together.xyz/v1",
+  },
+  AzureOpenAI: {
+    name: "Azure OpenAI",
+    baseUrl: "",
   },
   Custom: {
     baseUrl: ""
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 35caedef63..2b0b97ccc0 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -111,16 +111,16 @@ export interface SCIMInnerConfig {
 
 export interface SCIMConfig extends Config<SCIMInnerConfig> {}
 
+type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom"
+
 export interface AIInnerConfig {
   [key: string]: {
-    // TODO: should be enum
-    provider: string
+    provider: AIProvider
     isDefault: boolean
     name: string
     active: boolean
-    baseUrl: string
-    apiKey: string
-    // TODO: should be enum
+    baseUrl?: string
+    apiKey?: string
     defaultModel: string
   }
 }
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 1b66289f47..087feede3c 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -320,12 +320,26 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
   )
 }
 
-function sanitizeAIConfig(aiConfig: AIConfig) {
+async function enrichAIConfig(aiConfig: AIConfig) {
+  // Strip out the API Keys from the response so they don't show in the UI
   for (let key in aiConfig.config) {
     if (aiConfig.config[key].apiKey) {
       aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
     }
   }
+
+  // Return the Budibase AI data source as part of the response if licensing allows
+  const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
+  if (budibaseAIEnabled) {
+    aiConfig.config["budibase_ai"] = {
+      provider: "OpenAI",
+      active: true,
+      isDefault: true,
+      defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL,
+      name: "Budibase AI",
+    }
+  }
+
   return aiConfig
 }
 
@@ -341,8 +355,7 @@ export async function find(ctx: UserCtx) {
       }
 
       if (type === ConfigType.AI) {
-        // TODO: do the licensing checks here and return the right things based on the license
-        sanitizeAIConfig(scopedConfig)
+        await enrichAIConfig(scopedConfig)
       }
       ctx.body = scopedConfig
     } else {
diff --git a/packages/worker/src/environment.ts b/packages/worker/src/environment.ts
index ed618f7c71..3a9efd70ce 100644
--- a/packages/worker/src/environment.ts
+++ b/packages/worker/src/environment.ts
@@ -70,6 +70,9 @@ const environment = {
   PASSPORT_OIDCAUTH_FAILURE_REDIRECT:
     process.env.PASSPORT_OIDCAUTH_FAILURE_REDIRECT || "/error",
 
+  // Budibase AI
+  BUDIBASE_AI_API_KEY: process.env.BUDIBASE_AI_API_KEY,
+  BUDIBASE_AI_DEFAULT_MODEL: process.env.BUDIBASE_AI_DEFAULT_MODEL,
   _set(key: any, value: any) {
     process.env[key] = value
     // @ts-ignore

From 4d873ae75ee3f62e216948270a81dde1e7f90a92 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 3 Sep 2024 19:51:53 +0100
Subject: [PATCH 26/69] cleaner select options for managing models

---
 .../builder/portal/settings/ai/constants.ts   | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index f4f93f3114..abefafd8f5 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -2,39 +2,41 @@ export const Providers = {
   OpenAI: {
     name: "OpenAI",
     models: [
-      "gpt-4o-mini",
-      "gpt-4o",
-      "gpt-3.5-turbo",
-      "chatgpt-4o-latest",
-      "gpt-4-turbo",
-      "gpt-4",
+      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
+      {label: "GPT 4o", value: "gpt-4o" },
+      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
+      {label: "GPT 4", value: "gpt-4" },
+      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
     ]
   },
   Anthropic: {
     name: "Anthropic",
     models: [
-      "claude-3-5-sonnet-20240620",
-      "claude-3-sonnet-20240229",
-      "claude-3-opus-20240229",
-      "claude-3-haiku-20240307"
+      {label: "Claude 3.5 Sonnet", value: "claude-3-5-sonnet-20240620" },
+      {label: "Claude 3 Sonnet", value: "claude-3-sonnet-20240229" },
+      {label: "Claude 3 Opus", value: "claude-3-opus-20240229" },
+      {label: "Claude 3 Haiku", value: "claude-3-haiku-20240307" },
     ]
   },
   TogetherAI: {
     name: "Together AI",
     // TODO: too many - probably need to use an autocomplete for this
     models: [
-      "gpt-4o-mini",
-      "gpt-4o",
-      "gpt-3.5-turbo",
-      "chatgpt-4o-latest",
-      "gpt-4-turbo",
-      "gpt-4",
+      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
+      {label: "GPT 4o", value: "gpt-4o" },
+      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
+      {label: "GPT 4", value: "gpt-4" },
+      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
     ]
   },
   AzureOpenAI: {
     name: "Azure Open AI",
     models: [
-      "gpt-4o-mini",
+      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
+      {label: "GPT 4o", value: "gpt-4o" },
+      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
+      {label: "GPT 4", value: "gpt-4" },
+      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
     ]
   },
   Custom: {

From 9f75f2a9fccae5686437b9a2a12e622e9c45e140 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 4 Sep 2024 09:22:00 +0100
Subject: [PATCH 27/69] account for setting a default config and unset the rest

---
 .../portal/settings/ai/AIConfigTile.svelte    | 43 +++++++++++--------
 .../portal/settings/ai/ConfigModal.svelte     |  2 +-
 .../builder/portal/settings/ai/index.svelte   | 10 ++++-
 .../portal/settings/ai/logos/OpenAI.svelte    | 10 +++--
 packages/types/src/sdk/licensing/quota.ts     |  4 ++
 5 files changed, 45 insertions(+), 24 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 39c0972027..0b7a006312 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -8,7 +8,6 @@
 
   export let config
   export let disabled
-  export let budibaseAIDefault
 
   export let editHandler
   export let deleteHandler
@@ -33,25 +32,30 @@
     <Label>{config.name}</Label>
   </div>
   <div class="controls">
-    <Icon
-      on:click={editHandler}
-      color="var(--grey-6)"
-      size="S"
-      hoverable
-      name="Edit"
-    />
-    <Icon
-      on:click={deleteHandler}
-      color="var(--grey-6)"
-      size="S"
-      hoverable
-      name="Delete"
-    />
+    {#if config.name !== "Budibase AI"}
+      <Icon
+        on:click={editHandler}
+        color="var(--grey-6)"
+        size="S"
+        hoverable
+        name="Edit"
+      />
+      <Icon
+        on:click={deleteHandler}
+        color="var(--grey-6)"
+        size="S"
+        hoverable
+        name="Delete"
+      />
+    {/if}
     {#if config.active}
       <div class="tag active">Activated</div>
     {:else if !config.active}
       <div class="tag disabled">Disabled</div>
     {/if}
+    {#if config.isDefault}
+      <div class="tag default">Default</div>
+    {/if}
   </div>
 </div>
 
@@ -63,7 +67,7 @@
     border-radius: 4px;
     cursor: pointer;
     display: grid;
-    grid-template-columns: 6% 1fr 20%;
+    grid-template-columns: 6% 1fr auto;
     grid-gap: 20px;
     align-items: center;
   }
@@ -109,15 +113,18 @@
     align-items: center;
     gap: 8px;
     font-size: 12px;
+    border-radius: 5px;
+  }
+
+  .default {
+    background: var(--grey-6);
   }
 
   .active {
-    border-radius: 5px;
     background: var(--spectrum-global-color-green-600);
   }
 
   .disabled {
-    border-radius: 5px;
     background: var(--spectrum-global-color-red-600);
   }
 </style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 73db61161b..99f3aa8e7b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -65,7 +65,7 @@
     <Label size="M">Default Model</Label>
     {#if config.provider !== Providers.Custom.name}
       <Select
-        placeholder={config.provider ? "Choose an option" : "Select a provider first"}
+        placeholder={config.provider ? "Choose an option" : "Select a provider"}
         bind:value={config.defaultModel}
         options={config.provider ? Providers[config.provider].models : []}
       />
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 57435168ac..83b24a5b97 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -55,6 +55,14 @@
         }
       }
     } else {
+      // We don't store the default BB AI config in the DB
+      delete fullAIConfig.config.budibase_ai
+      // unset the default value from other configs if default is set
+      if (editingAIConfig.isDefault) {
+        for (let key in fullAIConfig.config) {
+          fullAIConfig.config[key].isDefault = false
+        }
+      }
       // Add new or update existing custom AI Config
       fullAIConfig.config[id] = editingAIConfig
     }
@@ -102,8 +110,6 @@
   })
 </script>
 
-<!-- svelte-ignore a11y-no-static-element-interactions -->
-<!-- svelte-ignore a11y-click-events-have-key-events -->
 <Modal bind:this={modal}>
   <AIConfigModal
     saveHandler={saveConfig}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
index a7a2927dbc..fa60b33cfd 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
@@ -1,5 +1,9 @@
 <script>
- export let height
- export let width
+  export let height
+  export let width
 </script>
-<svg fill="#000000" {width} {height} viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>OpenAI icon</title><path d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"/></svg>
\ No newline at end of file
+<svg fill="#000000" {width} {height} viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg">
+  <title>OpenAI icon</title>
+  <path
+    d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"/>
+</svg>
\ No newline at end of file
diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 85700f167b..748d90fda7 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -17,12 +17,14 @@ export enum StaticQuotaName {
   CREATORS = "creators",
   USER_GROUPS = "userGroups",
   PLUGINS = "plugins",
+  AI_CUSTOM_CONFIGS = "aiCustomConfigs",
 }
 
 export enum MonthlyQuotaName {
   QUERIES = "queries",
   AUTOMATIONS = "automations",
   DAY_PASSES = "dayPasses",
+  BUDIBASE_AI_CREDITS = "budibaseAICredits"
 }
 
 export enum ConstantQuotaName {
@@ -62,6 +64,7 @@ export type MonthlyQuotas = {
   [MonthlyQuotaName.QUERIES]: Quota
   [MonthlyQuotaName.AUTOMATIONS]: Quota
   [MonthlyQuotaName.DAY_PASSES]: Quota
+  [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: Quota
 }
 
 export type StaticQuotas = {
@@ -71,6 +74,7 @@ export type StaticQuotas = {
   [StaticQuotaName.CREATORS]: Quota
   [StaticQuotaName.USER_GROUPS]: Quota
   [StaticQuotaName.PLUGINS]: Quota
+  [StaticQuotaName.AI_CUSTOM_CONFIGS]: Quota
 }
 
 export type ConstantQuotas = {

From fed82dffaf0e62997d6d030e0df134b4e6071248 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 4 Sep 2024 13:11:03 +0100
Subject: [PATCH 28/69] Linting.

---
 packages/backend-core/src/sql/sql.ts             |  2 +-
 .../src/api/controllers/row/utils/sqlUtils.ts    | 16 ----------------
 2 files changed, 1 insertion(+), 17 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 6181d18046..0bc9126924 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -128,7 +128,7 @@ class InternalBuilder {
   }
 
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { endpoint, resource, meta, tableAliases } = this.query
+    const { endpoint, resource, tableAliases } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
index d6e5c3e8f1..249bb43bbc 100644
--- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts
+++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts
@@ -7,11 +7,9 @@ import {
   ManyToManyRelationshipFieldMetadata,
   RelationshipFieldMetadata,
   RelationshipsJson,
-  Row,
   Table,
 } from "@budibase/types"
 import { breakExternalTableId } from "../../../../integrations/utils"
-import { basicProcessing } from "./basic"
 import { generateJunctionTableID } from "../../../../db/utils"
 
 type TableMap = Record<string, Table>
@@ -22,20 +20,6 @@ export function isManyToMany(
   return !!(field as ManyToManyRelationshipFieldMetadata).through
 }
 
-function isCorrectRelationship(
-  relationship: RelationshipsJson,
-  table1: Table,
-  table2: Table,
-  row: Row
-): boolean {
-  const junctionTableId = generateJunctionTableID(table1._id!, table2._id!)
-  const possibleColumns = [
-    `${junctionTableId}.doc1.fieldName`,
-    `${junctionTableId}.doc2.fieldName`,
-  ]
-  return !!possibleColumns.find(col => row[col] === relationship.column)
-}
-
 /**
  * Gets the list of relationship JSON structures based on the columns in the table,
  * this will be used by the underlying library to build whatever relationship mechanism

From eefb1f01a3521a052f8b0c16ef6366053cbb6319 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 4 Sep 2024 13:18:54 +0100
Subject: [PATCH 29/69] Fix for generic sql test.

---
 .../src/api/routes/tests/queries/generic-sql.spec.ts   | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
index a84b243e2d..4bd1951d67 100644
--- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
+++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
@@ -832,10 +832,12 @@ describe.each(
         },
       })
       expect(res).toHaveLength(1)
-      expect(res[0]).toEqual({
-        id: 2,
-        name: "two",
-      })
+      expect(res[0]).toEqual(
+        expect.objectContaining({
+          id: 2,
+          name: "two",
+        })
+      )
     })
 
     // this parameter really only impacts SQL queries

From 79de7b2c4578d82eba71e3643c48422e849548b0 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 4 Sep 2024 16:17:25 +0100
Subject: [PATCH 30/69] Updating to use a sub-query with a wrapper to get the
 JSON aggregations.

---
 packages/backend-core/src/sql/sql.ts | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 0bc9126924..7b11f4ce37 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -913,30 +913,26 @@ class InternalBuilder {
       const fieldList: string = relationshipFields
         .map(field => jsonField(field))
         .join(",")
-      let rawJsonArray: Knex.Raw, limit: number
+      let rawJsonArray: Knex.Raw
       switch (sqlClient) {
         case SqlClient.SQL_LITE:
           rawJsonArray = this.knex.raw(
             `json_group_array(json_object(${fieldList}))`
           )
-          limit = getBaseLimit()
           break
         case SqlClient.POSTGRES:
           rawJsonArray = this.knex.raw(
             `json_agg(json_build_object(${fieldList}))`
           )
-          limit = 1
           break
         case SqlClient.MY_SQL:
         case SqlClient.ORACLE:
           rawJsonArray = this.knex.raw(
             `json_arrayagg(json_object(${fieldList}))`
           )
-          limit = getBaseLimit()
           break
         case SqlClient.MS_SQL:
           rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`)
-          limit = 1
           break
         default:
           throw new Error(`JSON relationships not implement for ${this.client}`)
@@ -945,16 +941,12 @@ class InternalBuilder {
       // it reduces the result set rather than limiting how much data it filters over
       const primaryKey = `${toAlias}.${toPrimary || toKey}`
       let subQuery = this.knex
-        .select(rawJsonArray)
+        .select(`${toAlias}.*`)
         .from(toTableWithSchema)
-        .limit(limit)
+        .limit(getBaseLimit())
         // add sorting to get consistent order
         .orderBy(primaryKey)
 
-      if (sqlClient === SqlClient.POSTGRES) {
-        subQuery = subQuery.groupBy(primaryKey)
-      }
-
       // many-to-many relationship with junction table
       if (throughTable && toPrimary && fromPrimary) {
         const throughAlias = aliases?.[throughTable] || throughTable
@@ -985,7 +977,11 @@ class InternalBuilder {
       if (this.client === SqlClient.SQL_LITE) {
         subQuery = this.addJoinFieldCheck(subQuery, relationship)
       }
-      query = query.select({ [relationship.column]: subQuery })
+      // @ts-ignore - the from alias syntax isn't in Knex typing
+      const wrapperQuery = this.knex.select(rawJsonArray).from({
+        [toAlias]: subQuery,
+      })
+      query = query.select({ [relationship.column]: wrapperQuery })
     }
     return query
   }

From 12db64513ba2597397daf2db9fedcab71d875c71 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 4 Sep 2024 16:21:32 +0100
Subject: [PATCH 31/69] Revert to testing against mssql 2017, attempt to get
 relationship aggreggation working.

---
 packages/backend-core/src/sql/sql.ts          | 30 ++++++++++---------
 packages/server/datasource-sha.env            |  2 +-
 .../src/integrations/microsoftSqlServer.ts    |  4 +--
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 0bc9126924..b51079ac2d 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -913,30 +913,25 @@ class InternalBuilder {
       const fieldList: string = relationshipFields
         .map(field => jsonField(field))
         .join(",")
-      let rawJsonArray: Knex.Raw, limit: number
+      let select: Knex.Raw, limit: number
       switch (sqlClient) {
         case SqlClient.SQL_LITE:
-          rawJsonArray = this.knex.raw(
-            `json_group_array(json_object(${fieldList}))`
-          )
+          select = this.knex.raw(`json_group_array(json_object(${fieldList}))`)
           limit = getBaseLimit()
           break
         case SqlClient.POSTGRES:
-          rawJsonArray = this.knex.raw(
-            `json_agg(json_build_object(${fieldList}))`
-          )
+          select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`)
           limit = 1
           break
         case SqlClient.MY_SQL:
         case SqlClient.ORACLE:
-          rawJsonArray = this.knex.raw(
-            `json_arrayagg(json_object(${fieldList}))`
-          )
+          select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`)
           limit = getBaseLimit()
           break
         case SqlClient.MS_SQL:
-          rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`)
-          limit = 1
+          // Cursed, needs some code later instead
+          select = this.knex.raw(`*`)
+          limit = getBaseLimit()
           break
         default:
           throw new Error(`JSON relationships not implement for ${this.client}`)
@@ -944,8 +939,8 @@ class InternalBuilder {
       // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
       // it reduces the result set rather than limiting how much data it filters over
       const primaryKey = `${toAlias}.${toPrimary || toKey}`
-      let subQuery = this.knex
-        .select(rawJsonArray)
+      let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex
+        .select(select)
         .from(toTableWithSchema)
         .limit(limit)
         // add sorting to get consistent order
@@ -985,6 +980,13 @@ class InternalBuilder {
       if (this.client === SqlClient.SQL_LITE) {
         subQuery = this.addJoinFieldCheck(subQuery, relationship)
       }
+
+      if (this.client === SqlClient.MS_SQL) {
+        subQuery = this.knex.raw(
+          `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)`
+        )
+      }
+
       query = query.select({ [relationship.column]: subQuery })
     }
     return query
diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env
index 9b935ed8eb..61249d530c 100644
--- a/packages/server/datasource-sha.env
+++ b/packages/server/datasource-sha.env
@@ -1,4 +1,4 @@
-MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8
+MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545
 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts
index 88c75891e6..0a07371cd3 100644
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         err.number
       )
       if (readableMessage) {
-        throw new Error(readableMessage)
+        throw new Error(readableMessage, { cause: err })
       } else {
-        throw new Error(err.message as string)
+        throw new Error(err.message as string, { cause: err })
       }
     }
   }

From cda778598d8e4e9cb77f7a34bfc932b0ee4939ca Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 4 Sep 2024 16:41:36 +0100
Subject: [PATCH 32/69] Fix some MSSQL test cases.

---
 packages/backend-core/src/sql/sql.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 60cfdea9d9..b283ed2106 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -973,16 +973,18 @@ class InternalBuilder {
         subQuery = this.addJoinFieldCheck(subQuery, relationship)
       }
 
+      let wrapperQuery: Knex.QueryBuilder | Knex.Raw
       if (this.client === SqlClient.MS_SQL) {
-        subQuery = this.knex.raw(
-          `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)`
+        wrapperQuery = this.knex.raw(
+          `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))`
         )
+      } else {
+        // @ts-ignore - the from alias syntax isn't in Knex typing
+        wrapperQuery = this.knex.select(select).from({
+          [toAlias]: subQuery,
+        })
       }
 
-      // @ts-ignore - the from alias syntax isn't in Knex typing
-      const wrapperQuery = this.knex.select(select).from({
-        [toAlias]: subQuery,
-      })
       query = query.select({ [relationship.column]: wrapperQuery })
     }
     return query

From 637ac55a9f87f10bf2f9e954667888cd68f81383 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 4 Sep 2024 17:42:30 +0100
Subject: [PATCH 33/69] Slight refactor.

---
 packages/backend-core/src/sql/sql.ts | 73 +++++++++++++++-------------
 1 file changed, 39 insertions(+), 34 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index b283ed2106..5dd94456f4 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -859,6 +859,7 @@ class InternalBuilder {
     relationships: RelationshipsJson[]
   ): Knex.QueryBuilder {
     const sqlClient = this.client
+    const knex = this.knex
     const { resource, tableAliases: aliases, endpoint } = this.query
     const fields = resource?.fields || []
     const jsonField = (field: string) => {
@@ -913,29 +914,10 @@ class InternalBuilder {
       const fieldList: string = relationshipFields
         .map(field => jsonField(field))
         .join(",")
-      let select: Knex.Raw
-      switch (sqlClient) {
-        case SqlClient.SQL_LITE:
-          select = this.knex.raw(`json_group_array(json_object(${fieldList}))`)
-          break
-        case SqlClient.POSTGRES:
-          select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`)
-          break
-        case SqlClient.MY_SQL:
-        case SqlClient.ORACLE:
-          select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`)
-          break
-        case SqlClient.MS_SQL:
-          // Cursed, needs some code later instead
-          select = this.knex.raw(`*`)
-          break
-        default:
-          throw new Error(`JSON relationships not implement for ${this.client}`)
-      }
       // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
       // it reduces the result set rather than limiting how much data it filters over
       const primaryKey = `${toAlias}.${toPrimary || toKey}`
-      let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex
+      let subQuery: Knex.QueryBuilder | Knex.Raw = knex
         .select(`${toAlias}.*`)
         .from(toTableWithSchema)
         .limit(getBaseLimit())
@@ -956,7 +938,7 @@ class InternalBuilder {
           .where(
             `${throughAlias}.${fromKey}`,
             "=",
-            this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
+            knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`))
           )
       }
       // one-to-many relationship with foreign key
@@ -964,26 +946,49 @@ class InternalBuilder {
         subQuery = subQuery.where(
           `${toAlias}.${toKey}`,
           "=",
-          this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
+          knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`))
         )
       }
 
-      // need to check the junction table document is to the right column, this is just for SQS
-      if (this.client === SqlClient.SQL_LITE) {
-        subQuery = this.addJoinFieldCheck(subQuery, relationship)
-      }
-
-      let wrapperQuery: Knex.QueryBuilder | Knex.Raw
-      if (this.client === SqlClient.MS_SQL) {
-        wrapperQuery = this.knex.raw(
-          `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))`
-        )
-      } else {
+      const standardWrap = (select: string): Knex.QueryBuilder => {
         // @ts-ignore - the from alias syntax isn't in Knex typing
-        wrapperQuery = this.knex.select(select).from({
+        return knex.select(knex.raw(select)).from({
           [toAlias]: subQuery,
         })
       }
+      let wrapperQuery: Knex.QueryBuilder | Knex.Raw
+      switch (sqlClient) {
+        case SqlClient.SQL_LITE:
+          // need to check the junction table document is to the right column, this is just for SQS
+          subQuery = this.addJoinFieldCheck(subQuery, relationship)
+          wrapperQuery = standardWrap(
+            `json_group_array(json_object(${fieldList}))`
+          )
+          break
+        case SqlClient.POSTGRES:
+          wrapperQuery = standardWrap(
+            `json_agg(json_build_object(${fieldList}))`
+          )
+          break
+        case SqlClient.MY_SQL:
+        case SqlClient.ORACLE:
+          wrapperQuery = standardWrap(
+            `json_arrayagg(json_object(${fieldList}))`
+          )
+          break
+        case SqlClient.MS_SQL:
+          wrapperQuery = knex.raw(
+            `(SELECT ${this.quote(toAlias)} = (${knex
+              .select(`${fromAlias}.*`)
+              // @ts-ignore - from alias syntax not TS supported
+              .from({
+                [fromAlias]: subQuery,
+              })} FOR JSON PATH))`
+          )
+          break
+        default:
+          throw new Error(`JSON relationships not implement for ${sqlClient}`)
+      }
 
       query = query.select({ [relationship.column]: wrapperQuery })
     }

From e30469ce16fec403b4ba1b2881842b5e913de890 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 4 Sep 2024 18:14:24 +0100
Subject: [PATCH 34/69] Getting MariaDB to work again.

---
 packages/backend-core/src/sql/sql.ts | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index 5dd94456f4..ec8ebe0106 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -917,8 +917,7 @@ class InternalBuilder {
       // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax
       // it reduces the result set rather than limiting how much data it filters over
       const primaryKey = `${toAlias}.${toPrimary || toKey}`
-      let subQuery: Knex.QueryBuilder | Knex.Raw = knex
-        .select(`${toAlias}.*`)
+      let subQuery: Knex.QueryBuilder = knex
         .from(toTableWithSchema)
         .limit(getBaseLimit())
         // add sorting to get consistent order
@@ -951,6 +950,7 @@ class InternalBuilder {
       }
 
       const standardWrap = (select: string): Knex.QueryBuilder => {
+        subQuery = subQuery.select(`${toAlias}.*`)
         // @ts-ignore - the from alias syntax isn't in Knex typing
         return knex.select(knex.raw(select)).from({
           [toAlias]: subQuery,
@@ -971,6 +971,10 @@ class InternalBuilder {
           )
           break
         case SqlClient.MY_SQL:
+          wrapperQuery = subQuery.select(
+            knex.raw(`json_arrayagg(json_object(${fieldList}))`)
+          )
+          break
         case SqlClient.ORACLE:
           wrapperQuery = standardWrap(
             `json_arrayagg(json_object(${fieldList}))`
@@ -982,7 +986,7 @@ class InternalBuilder {
               .select(`${fromAlias}.*`)
               // @ts-ignore - from alias syntax not TS supported
               .from({
-                [fromAlias]: subQuery,
+                [fromAlias]: subQuery.select(`${toAlias}.*`),
               })} FOR JSON PATH))`
           )
           break

From 7cdf8137c52d42419af2db13a344e6c256c7d9d5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 5 Sep 2024 17:57:15 +0100
Subject: [PATCH 35/69] Fixing aliasing test cases.

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 91 ++++++++++---------
 1 file changed, 48 insertions(+), 43 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 6f34f4eb89..1ba37018dc 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -32,8 +32,8 @@ function multiline(sql: string) {
 }
 
 describe("Captures of real examples", () => {
-  const limit = 5000
-  const relationshipLimit = 100
+  const baseLimit = 5000
+  const primaryLimit = 100
 
   function getJson(name: string): QueryJson {
     return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
@@ -42,7 +42,7 @@ describe("Captures of real examples", () => {
   describe("create", () => {
     it("should create a row with relationships", () => {
       const queryJson = getJson("createWithRelationships.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
         sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") 
@@ -54,40 +54,42 @@ describe("Captures of real examples", () => {
   describe("read", () => {
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [relationshipLimit, limit],
+        bindings: [baseLimit, baseLimit, primaryLimit],
         sql: expect.stringContaining(
-          multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", 
-              "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", 
-              "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", 
-              "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", 
-              "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
-              "b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`)
+          multiline(
+            `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
+          )
         ),
       })
     })
 
     it("should handle filtering by relationship", () => {
       const queryJson = getJson("filterByRelationship.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [relationshipLimit, "assembling", limit],
+        bindings: [baseLimit, "assembling", primaryLimit],
         sql: expect.stringContaining(
-          multiline(`where COALESCE("b"."taskname" = $2, FALSE) 
-            order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`)
+          multiline(
+            `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" 
+                 where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)`
+          )
         ),
       })
     })
 
     it("should handle fetching many to many relationships", () => {
       const queryJson = getJson("fetchManyToMany.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [relationshipLimit, limit],
+        bindings: [baseLimit, primaryLimit],
         sql: expect.stringContaining(
-          multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
-              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `)
+          multiline(
+            `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) 
+                 from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" 
+                 where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1`
+          )
         ),
       })
     })
@@ -95,22 +97,21 @@ describe("Captures of real examples", () => {
     it("should handle enrichment of rows", () => {
       const queryJson = getJson("enrichRelationship.json")
       const filters = queryJson.filters?.oneOf?.taskid as number[]
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [...filters, limit, ...filters, limit],
+        bindings: [baseLimit, ...filters, baseLimit],
         sql: multiline(
-          `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid",
-          "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid"
-          from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a"
-          left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid"
-          where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6`
+          `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) 
+               from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
+               where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" 
+               from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4`
         ),
       })
     })
 
     it("should manage query with many relationship filters", () => {
       const queryJson = getJson("manyRelationshipFilters.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       const filters = queryJson.filters
       const notEqualsValue = Object.values(filters?.notEqual!)[0]
       const rangeValue: { high?: string | number; low?: string | number } =
@@ -119,17 +120,18 @@ describe("Captures of real examples", () => {
 
       expect(query).toEqual({
         bindings: [
-          notEqualsValue,
-          relationshipLimit,
+          baseLimit,
+          baseLimit,
+          baseLimit,
           rangeValue.low,
           rangeValue.high,
           equalValue,
-          true,
-          limit,
+          notEqualsValue,
+          primaryLimit,
         ],
         sql: expect.stringContaining(
           multiline(
-            `where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)`
+            `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)`
           )
         ),
       })
@@ -139,17 +141,19 @@ describe("Captures of real examples", () => {
   describe("update", () => {
     it("should handle performing a simple update", () => {
       const queryJson = getJson("updateSimple.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
-        sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
-             "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`),
+        sql: multiline(
+          `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
+               "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`
+        ),
       })
     })
 
     it("should handle performing an update of relationships", () => {
       const queryJson = getJson("updateRelationship.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
@@ -161,12 +165,12 @@ describe("Captures of real examples", () => {
   describe("delete", () => {
     it("should handle deleting with relationships", () => {
       const queryJson = getJson("deleteSimple.json")
-      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["ddd", ""],
         sql: multiline(`delete from "compositetable" as "a" 
           where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE)
-          returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
+          returning "a".*`),
       })
     })
   })
@@ -174,7 +178,7 @@ describe("Captures of real examples", () => {
   describe("returning (everything bar Postgres)", () => {
     it("should be able to handle row returning", () => {
       const queryJson = getJson("createSimple.json")
-      const SQL = new Sql(SqlClient.MS_SQL, limit)
+      const SQL = new Sql(SqlClient.MS_SQL, baseLimit)
       let query = SQL._query(queryJson, { disableReturning: true })
       expect(query).toEqual({
         sql: "insert into [people] ([age], [name]) values (@p0, @p1)",
@@ -187,10 +191,11 @@ describe("Captures of real examples", () => {
         returningQuery = input
       }, queryJson)
       expect(returningQuery).toEqual({
-        sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 
-             THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people] 
-             where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`),
-        bindings: [5000, 1, "Test", 22, "Test", 22],
+        sql: multiline(
+          `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 
+               and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc`
+        ),
+        bindings: [1, "Test", 22],
       })
     })
   })

From 888c4214bdbef1c5202144ff1c9ca0449e07072e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 5 Sep 2024 18:12:53 +0100
Subject: [PATCH 36/69] Fixing SQL unit tests.

---
 .../server/src/integrations/tests/sql.spec.ts | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index a6e63c434d..c434ec42cb 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -160,16 +160,16 @@ describe("SQL query builder", () => {
   it("should add the schema to the LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
-      bindings: [500, 5000],
-      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
+      bindings: [5000, limit],
+      sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`,
     })
   })
 
   it("should handle if the schema is not present when doing a LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson())
     expect(query).toEqual({
-      bindings: [500, 5000],
-      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`,
+      bindings: [5000, limit],
+      sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`,
     })
   })
 
@@ -178,8 +178,8 @@ describe("SQL query builder", () => {
       generateManyRelationshipJson({ schema: "production" })
     )
     expect(query).toEqual({
-      bindings: [500, 5000],
-      sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`,
+      bindings: [5000, limit],
+      sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`,
     })
   })
 
@@ -194,8 +194,8 @@ describe("SQL query builder", () => {
       })
     )
     expect(query).toEqual({
-      bindings: ["john%", limit, "john%", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
+      bindings: ["john%", limit],
+      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(
@@ -210,8 +210,8 @@ describe("SQL query builder", () => {
     )
     const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`]
     expect(query).toEqual({
-      bindings: [...filterSet, limit, ...filterSet, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`,
+      bindings: [...filterSet, limit],
+      sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`,
     })
 
     query = new Sql(SqlClient.ORACLE, limit)._query(
@@ -224,8 +224,8 @@ describe("SQL query builder", () => {
       })
     )
     expect(query).toEqual({
-      bindings: [`%jo%`, limit, `%jo%`, 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`,
+      bindings: [`%jo%`, limit],
+      sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`,
     })
   })
 
@@ -242,8 +242,8 @@ describe("SQL query builder", () => {
     )
 
     expect(query).toEqual({
-      bindings: ["John", limit, "John", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`,
+      bindings: ["John", limit],
+      sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`,
     })
   })
 
@@ -260,8 +260,8 @@ describe("SQL query builder", () => {
     )
 
     expect(query).toEqual({
-      bindings: ["John", limit, "John", 5000],
-      sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`,
+      bindings: ["John", limit],
+      sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`,
     })
   })
 })

From f7d9b8a9b31da5ee8357e42398c5bf733d738e9c Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 5 Sep 2024 19:04:45 +0100
Subject: [PATCH 37/69] Updating select statement generation.

---
 packages/backend-core/src/sql/sql.ts | 142 ++++++++++++++-------------
 1 file changed, 72 insertions(+), 70 deletions(-)

diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index ec8ebe0106..b1a2bc060a 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -94,6 +94,23 @@ class InternalBuilder {
     })
   }
 
+  // states the various situations in which we need a full mapped select statement
+  private readonly SPECIAL_SELECT_CASES = {
+    POSTGRES_MONEY: (field: FieldSchema | undefined) => {
+      return (
+        this.client === SqlClient.POSTGRES &&
+        field?.externalType?.includes("money")
+      )
+    },
+    MSSQL_DATES: (field: FieldSchema | undefined) => {
+      return (
+        this.client === SqlClient.MS_SQL &&
+        field?.type === FieldType.DATETIME &&
+        field.timeOnly
+      )
+    },
+  }
+
   get table(): Table {
     return this.query.meta.table
   }
@@ -127,8 +144,20 @@ class InternalBuilder {
       .join(".")
   }
 
+  private isFullSelectStatementRequired(): boolean {
+    const { meta } = this.query
+    for (let column of Object.values(meta.table.schema)) {
+      if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) {
+        return true
+      } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) {
+        return true
+      }
+    }
+    return false
+  }
+
   private generateSelectStatement(): (string | Knex.Raw)[] | "*" {
-    const { endpoint, resource, tableAliases } = this.query
+    const { meta, endpoint, resource, tableAliases } = this.query
 
     if (!resource || !resource.fields || resource.fields.length === 0) {
       return "*"
@@ -137,75 +166,48 @@ class InternalBuilder {
     const alias = tableAliases?.[endpoint.entityId]
       ? tableAliases?.[endpoint.entityId]
       : endpoint.entityId
-    return [this.knex.raw(`${this.quote(alias)}.*`)]
-    //
-    //
-    // const schema = meta.table.schema
-    // return resource.fields.map(field => {
-    //   const parts = field.split(/\./g)
-    //   let table: string | undefined = undefined
-    //   let column: string | undefined = undefined
-    //
-    //   // Just a column name, e.g.: "column"
-    //   if (parts.length === 1) {
-    //     column = parts[0]
-    //   }
-    //
-    //   // A table name and a column name, e.g.: "table.column"
-    //   if (parts.length === 2) {
-    //     table = parts[0]
-    //     column = parts[1]
-    //   }
-    //
-    //   // A link doc, e.g.: "table.doc1.fieldName"
-    //   if (parts.length > 2) {
-    //     table = parts[0]
-    //     column = parts.slice(1).join(".")
-    //   }
-    //
-    //   if (!column) {
-    //     throw new Error(`Invalid field name: ${field}`)
-    //   }
-    //
-    //   const columnSchema = schema[column]
-    //
-    //   if (
-    //     this.client === SqlClient.POSTGRES &&
-    //     columnSchema?.externalType?.includes("money")
-    //   ) {
-    //     return this.knex.raw(
-    //       `${this.quotedIdentifier(
-    //         [table, column].join(".")
-    //       )}::money::numeric as ${this.quote(field)}`
-    //     )
-    //   }
-    //
-    //   if (
-    //     this.client === SqlClient.MS_SQL &&
-    //     columnSchema?.type === FieldType.DATETIME &&
-    //     columnSchema.timeOnly
-    //   ) {
-    //     // Time gets returned as timestamp from mssql, not matching the expected
-    //     // HH:mm format
-    //     return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
-    //   }
-    //
-    //   // There's at least two edge cases being handled in the expression below.
-    //   //  1. The column name could start/end with a space, and in that case we
-    //   //     want to preseve that space.
-    //   //  2. Almost all column names are specified in the form table.column, except
-    //   //     in the case of relationships, where it's table.doc1.column. In that
-    //   //     case, we want to split it into `table`.`doc1.column` for reasons that
-    //   //     aren't actually clear to me, but `table`.`doc1` breaks things with the
-    //   //     sample data tests.
-    //   if (table) {
-    //     return this.knex.raw(
-    //       `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}`
-    //     )
-    //   } else {
-    //     return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`)
-    //   }
-    // })
+    const schema = meta.table.schema
+    if (!this.isFullSelectStatementRequired()) {
+      return [this.knex.raw(`${this.quote(alias)}.*`)]
+    }
+    // get just the fields for this table
+    return resource.fields
+      .map(field => {
+        const parts = field.split(/\./g)
+        let table: string | undefined = undefined
+        let column = parts[0]
+
+        // Just a column name, e.g.: "column"
+        if (parts.length > 1) {
+          table = parts[0]
+          column = parts.slice(1).join(".")
+        }
+
+        return { table, column, field }
+      })
+      .filter(({ table }) => !table || table === alias)
+      .map(({ table, column, field }) => {
+        const columnSchema = schema[column]
+
+        if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) {
+          return this.knex.raw(
+            `${this.quotedIdentifier(
+              [table, column].join(".")
+            )}::money::numeric as ${this.quote(field)}`
+          )
+        }
+
+        if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) {
+          // Time gets returned as timestamp from mssql, not matching the expected
+          // HH:mm format
+          return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`)
+        }
+
+        const quoted = table
+          ? `${this.quote(table)}.${this.quote(column)}`
+          : this.quote(field)
+        return this.knex.raw(quoted)
+      })
   }
 
   // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses,

From cdb0e2509c2067db0dc8a9f84e3e775c2b76a2ae Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 6 Sep 2024 14:53:26 +0100
Subject: [PATCH 38/69] testing configuration, unit tests for AI config

---
 .../core/utilities/structures/licenses.ts     |  10 +
 packages/builder/package.json                 |   1 +
 .../portal/settings/ai/AISettings.spec.js     |  96 ++++
 .../builder/portal/settings/ai/ai.spec.ts     |   0
 .../builder/portal/settings/ai/index.svelte   |   5 +-
 .../{vite.config.js => vite.config.mjs}       |   4 +-
 packages/builder/vitest.setup.js              |   5 +-
 yarn.lock                                     | 476 +++++++++---------
 8 files changed, 357 insertions(+), 240 deletions(-)
 create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
 delete mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts
 rename packages/builder/{vite.config.js => vite.config.mjs} (98%)

diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts
index bb452f9ad5..cf422637d6 100644
--- a/packages/backend-core/tests/core/utilities/structures/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts
@@ -55,6 +55,11 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
+        budibaseAICredits: {
+          name: "Budibase AI Credits",
+          value: 1,
+          triggers: [],
+        }
       },
       static: {
         rows: {
@@ -87,6 +92,11 @@ export function quotas(): Quotas {
           value: 1,
           triggers: [],
         },
+        aiCustomConfigs: {
+          name: "Plugins",
+          value: 1,
+          triggers: [],
+        },
       },
     },
     constant: {
diff --git a/packages/builder/package.json b/packages/builder/package.json
index f44c2ea549..f9e6becbab 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -93,6 +93,7 @@
     "identity-obj-proxy": "^3.0.0",
     "jest": "29.7.0",
     "jsdom": "^21.1.1",
+    "resize-observer-polyfill": "^1.5.1",
     "svelte-jester": "^1.3.2",
     "vite": "^4.5.0",
     "vite-plugin-static-copy": "^0.17.0",
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
new file mode 100644
index 0000000000..5ca14c7898
--- /dev/null
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
@@ -0,0 +1,96 @@
+import { it, expect, describe, vi } from "vitest"
+import AISettings from "./index.svelte"
+import { render } from "@testing-library/svelte"
+import { admin, licensing } from "stores/portal"
+
+vi.spyOn(notifications, "error").mockImplementation(vi.fn)
+vi.spyOn(notifications, "success").mockImplementation(vi.fn)
+
+const Hosting = {
+  Cloud: "cloud",
+  Self: "self",
+}
+
+function setupEnv(
+  hosting,
+  features = {}) {
+  const defaultFeatures = {
+    budibaseAIEnabled: false,
+    customAIConfigsEnabled: false,
+    ...features
+  }
+  admin.subscribe = vi.fn().mockImplementation(callback => {
+    callback({cloud: hosting === Hosting.Cloud})
+    return () => {
+    }
+  })
+  licensing.subscribe = vi.fn().mockImplementation(callback => {
+    callback(defaultFeatures)
+    return () => {
+    }
+  })
+}
+
+describe("AISettings", () => {
+  let instance = null
+
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it("that the AISettings is rendered", () => {
+    instance = render(AISettings, {})
+    expect(instance).toBeDefined()
+  })
+
+  describe("Licensing", () => {
+    it("should show the premium label on self host for custom configs", async () => {
+      setupEnv(Hosting.Self)
+      instance = render(AISettings, {})
+      const premiumTag = instance.queryByText("Premium")
+      expect(premiumTag).toBeInTheDocument()
+    })
+
+    it("should show the enterprise label on cloud for custom configs", async () => {
+      setupEnv(Hosting.Cloud)
+      instance = render(AISettings, {})
+      const enterpriseTag = instance.queryByText("Enterprise")
+      expect(enterpriseTag).toBeInTheDocument()
+    })
+
+    it("should show the premium label on cloud when Budibase AI isn't enabled", async () => {
+      setupEnv(Hosting.Cloud)
+      instance = render(AISettings, {})
+      const premiumTag = instance.queryByText("Premium")
+      expect(premiumTag).toBeInTheDocument()
+    })
+
+    it("should not show the add configuration button if the user doesn't have the correct license on cloud", async () => {
+      let addConfigurationButton
+
+      setupEnv(Hosting.Cloud)
+      instance = render(AISettings)
+      addConfigurationButton = instance.queryByText("Add configuration")
+      expect(addConfigurationButton).not.toBeInTheDocument()
+
+      setupEnv(Hosting.Cloud, {customAIConfigsEnabled: true})
+      instance = render(AISettings)
+      addConfigurationButton = instance.queryByText("Add configuration")
+      expect(addConfigurationButton).toBeInTheDocument()
+    })
+
+    it("should not show the add configuration button if the user doesn't have the correct license on self host", async () => {
+      let addConfigurationButton
+
+      setupEnv(Hosting.Self)
+      instance = render(AISettings)
+      addConfigurationButton = instance.queryByText("Add configuration")
+      expect(addConfigurationButton).not.toBeInTheDocument()
+
+      setupEnv(Hosting.Self, {customAIConfigsEnabled: true})
+      instance = render(AISettings, {})
+      addConfigurationButton = instance.queryByText("Add configuration")
+      expect(addConfigurationButton).toBeInTheDocument()
+    })
+  })
+})
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts b/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 83b24a5b97..dc7853b398 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -32,7 +32,6 @@
 
   async function fetchAIConfig() {
     try {
-      // Fetch the AI configs
       const aiDoc = await API.getConfig(ConfigTypes.AI)
       if (aiDoc._id) {
         fullAIConfig = aiDoc
@@ -105,8 +104,8 @@
     modal.show()
   }
 
-  onMount(async () => {
-    await fetchAIConfig()
+  onMount(() => {
+    fetchAIConfig()
   })
 </script>
 
diff --git a/packages/builder/vite.config.js b/packages/builder/vite.config.mjs
similarity index 98%
rename from packages/builder/vite.config.js
rename to packages/builder/vite.config.mjs
index 995cd36adb..a84fba2dae 100644
--- a/packages/builder/vite.config.js
+++ b/packages/builder/vite.config.mjs
@@ -35,7 +35,7 @@ export default defineConfig(({ mode }) => {
     // Copy fonts to an additional path so that svelte's automatic
     // prefixing of the base URL path can still resolve assets
     copyFonts("builder/fonts"),
-  ]
+]
 
   return {
     test: {
@@ -79,7 +79,7 @@ export default defineConfig(({ mode }) => {
         "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN),
       }),
       copyFonts("fonts"),
-      ...(isProduction ? [] : devOnlyPlugins),
+    ...(isProduction ? [] : devOnlyPlugins),
     ],
     optimizeDeps: {
       exclude: ["@roxi/routify", "fsevents"],
diff --git a/packages/builder/vitest.setup.js b/packages/builder/vitest.setup.js
index 6581df83e1..92df5e9600 100644
--- a/packages/builder/vitest.setup.js
+++ b/packages/builder/vitest.setup.js
@@ -1,4 +1,7 @@
-import { expect } from "vitest"
+import { expect, vi } from "vitest"
+import "@testing-library/jest-dom/vitest"
+
+global.ResizeObserver = require("resize-observer-polyfill")
 
 expect.extend({
   toBeFunc: received => {
diff --git a/yarn.lock b/yarn.lock
index 655d7d78fc..4e4e528334 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -8,9 +8,9 @@
   integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
 
 "@adobe/css-tools@^4.3.2":
-  version "4.3.3"
-  resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.3.tgz#90749bde8b89cd41764224f5aac29cd4138f75ff"
-  integrity sha512-rE0Pygv0sEZ4vBWHlAgJLGDU7Pm8xoO6p3wsEceb7GYAjScrOHpEo8KK/eVkAcnSM+slAEtXjA2JpdjLp4fJQQ==
+  version "4.4.0"
+  resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.4.0.tgz#728c484f4e10df03d5a3acd0d8adcbbebff8ad63"
+  integrity sha512-Ff9+ksdQQB3rMncgqDK78uLznstjyfIf2Arnh22pW8kBpLs6rpKDwgnZT46hin5Hl1WzazzK64DOrhSwYpS7bQ==
 
 "@adobe/spectrum-css-workflow-icons@1.2.1":
   version "1.2.1"
@@ -1995,14 +1995,14 @@
   resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310"
   integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
-"@babel/runtime@^7.10.5", "@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
+"@babel/runtime@^7.10.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4":
   version "7.24.7"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.7.tgz#f4f0d5530e8dbdf59b3451b9b3e594b6ba082e12"
   integrity sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw==
   dependencies:
     regenerator-runtime "^0.14.0"
 
-"@babel/runtime@^7.13.10":
+"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.9.2":
   version "7.25.6"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2"
   integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ==
@@ -2053,44 +2053,6 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.29.24":
-  version "0.0.0"
-  dependencies:
-    "@budibase/nano" "10.1.5"
-    "@budibase/pouchdb-replication-stream" "1.2.11"
-    "@budibase/shared-core" "0.0.0"
-    "@budibase/types" "0.0.0"
-    aws-cloudfront-sign "3.0.2"
-    aws-sdk "2.1030.0"
-    bcrypt "5.1.0"
-    bcryptjs "2.4.3"
-    bull "4.10.1"
-    correlation-id "4.0.0"
-    dd-trace "5.2.0"
-    dotenv "16.0.1"
-    ioredis "5.3.2"
-    joi "17.6.0"
-    jsonwebtoken "9.0.2"
-    knex "2.4.2"
-    koa-passport "^6.0.0"
-    koa-pino-logger "4.0.0"
-    lodash "4.17.21"
-    node-fetch "2.6.7"
-    passport-google-oauth "2.0.0"
-    passport-local "1.0.0"
-    passport-oauth2-refresh "^2.1.0"
-    pino "8.11.0"
-    pino-http "8.3.3"
-    posthog-node "4.0.1"
-    pouchdb "7.3.0"
-    pouchdb-find "7.2.2"
-    redlock "4.2.0"
-    rotating-file-stream "3.1.0"
-    sanitize-s3-objectkey "0.0.1"
-    semver "^7.5.4"
-    tar-fs "2.1.1"
-    uuid "^8.3.2"
-
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2133,44 +2095,6 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
-"@budibase/pro@npm:@budibase/pro@latest":
-  version "2.29.24"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.24.tgz#2dbd4c6c0f757aab7e17c413c6d6e4520086f9ac"
-  integrity sha512-m1v24UD6O21Vbrfsuo5kC5oeg7FzjWO2w8TQMw1VvPKmdIqqclaKDPTPytxwllTMkapMDRNzM5cQzqnQ3yHf6A==
-  dependencies:
-    "@budibase/backend-core" "2.29.24"
-    "@budibase/shared-core" "2.29.24"
-    "@budibase/string-templates" "2.29.24"
-    "@budibase/types" "2.29.24"
-    "@koa/router" "8.0.8"
-    bull "4.10.1"
-    joi "17.6.0"
-    jsonwebtoken "9.0.2"
-    lru-cache "^7.14.1"
-    memorystream "^0.3.1"
-    node-fetch "2.6.7"
-    scim-patch "^0.8.1"
-    scim2-parse-filter "^0.2.8"
-
-"@budibase/shared-core@2.29.24":
-  version "0.0.0"
-  dependencies:
-    "@budibase/types" "0.0.0"
-    cron-validate "1.4.5"
-
-"@budibase/string-templates@2.29.24":
-  version "0.0.0"
-  dependencies:
-    "@budibase/handlebars-helpers" "^0.13.2"
-    dayjs "^1.10.8"
-    handlebars "^4.7.8"
-    lodash.clonedeep "^4.5.0"
-
-"@budibase/types@2.29.24":
-  version "0.0.0"
-  dependencies:
-    scim-patch "^0.8.1"
-
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -5338,9 +5262,9 @@
     redent "^3.0.0"
 
 "@testing-library/svelte@^4.1.0":
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/@testing-library/svelte/-/svelte-4.1.0.tgz#de6fa34d13d99505e68134ef2acfbafdc03ed39a"
-  integrity sha512-MJqe7x9WowkiAVdk9mvazEC2ktFZdmK2OqFVoO557PC37aBemQ4ozqdK3yrG34Zg9kuln3qgTVeLSh08e69AMw==
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/@testing-library/svelte/-/svelte-4.2.3.tgz#bd0fd3dde5c63012da95d58ee640c0b44fd27a04"
+  integrity sha512-8vM2+JSPc6wZWkO9ICPmHvzacjy8jBw+iVjmNs+0VsPV3AO3v4P8qCLWTaQ9nYW/e+IR1BCy3MM3Uqg21dlBkw==
   dependencies:
     "@testing-library/dom" "^9.3.1"
 
@@ -5412,9 +5336,9 @@
     "@types/readdir-glob" "*"
 
 "@types/aria-query@^5.0.1":
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.1.tgz#3286741fb8f1e1580ac28784add4c7a1d49bdfbc"
-  integrity sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==
+  version "5.0.4"
+  resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708"
+  integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==
 
 "@types/babel__core@^7.1.14":
   version "7.20.0"
@@ -5485,9 +5409,9 @@
     "@types/chai" "*"
 
 "@types/chai@*", "@types/chai@^4.3.4":
-  version "4.3.16"
-  resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.16.tgz#b1572967f0b8b60bf3f87fe1d854a5604ea70c82"
-  integrity sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ==
+  version "4.3.19"
+  resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.19.tgz#14519f437361d41e84102ed3fbc922ddace3e228"
+  integrity sha512-2hHHvQBVE2FiSK4eN0Br6snX9MtolHaTo/batnLjlGRhoQzlCL61iVpxoqO7SfFyOw+P/pwv+0zNHzKoGWz9Cw==
 
 "@types/chance@1.1.3":
   version "1.1.3"
@@ -6922,16 +6846,16 @@ acorn@^7.1.1:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
   integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==
 
-acorn@^8.1.0, acorn@^8.10.0, acorn@^8.11.0, acorn@^8.11.3, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0:
-  version "8.12.0"
-  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c"
-  integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==
-
-acorn@^8.12.0, acorn@^8.12.1:
+acorn@^8.1.0, acorn@^8.11.3, acorn@^8.12.0, acorn@^8.12.1, acorn@^8.8.1:
   version "8.12.1"
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248"
   integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==
 
+acorn@^8.10.0, acorn@^8.11.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.2, acorn@^8.9.0:
+  version "8.12.0"
+  resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c"
+  integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==
+
 add-stream@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa"
@@ -7259,7 +7183,7 @@ aria-query@^5.0.0, aria-query@^5.3.0:
   dependencies:
     dequal "^2.0.3"
 
-array-buffer-byte-length@^1.0.1:
+array-buffer-byte-length@^1.0.0, array-buffer-byte-length@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f"
   integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg==
@@ -7524,7 +7448,30 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0:
+  version "0.24.0"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
+  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
+  dependencies:
+    follow-redirects "^1.14.4"
+
+axios@1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
+  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
+  dependencies:
+    follow-redirects "^1.15.0"
+    form-data "^4.0.0"
+    proxy-from-env "^1.1.0"
+
+axios@^0.21.1:
+  version "0.21.4"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
+  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
+  dependencies:
+    follow-redirects "^1.14.0"
+
+axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -8306,9 +8253,9 @@ caseless@~0.12.0:
   integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==
 
 chai@^4.3.7:
-  version "4.4.1"
-  resolved "https://registry.yarnpkg.com/chai/-/chai-4.4.1.tgz#3603fa6eba35425b0f2ac91a009fe924106e50d1"
-  integrity sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g==
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/chai/-/chai-4.5.0.tgz#707e49923afdd9b13a8b0b47d33d732d13812fd8"
+  integrity sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==
   dependencies:
     assertion-error "^1.1.0"
     check-error "^1.0.3"
@@ -8316,7 +8263,7 @@ chai@^4.3.7:
     get-func-name "^2.0.2"
     loupe "^2.3.6"
     pathval "^1.1.1"
-    type-detect "^4.0.8"
+    type-detect "^4.1.0"
 
 chai@^5.1.1:
   version "5.1.1"
@@ -9625,9 +9572,9 @@ dedent@^1.0.0:
   integrity sha512-7glNLfvdsMzZm3FpRY1CHuI2lbYDR+71YmrhmTZjYFD5pfT0ACgnGRdrrC9Mk2uICnzkcdelCx5at787UDGOvg==
 
 deep-eql@^4.1.3:
-  version "4.1.3"
-  resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.1.3.tgz#7c7775513092f7df98d8df9996dd085eb668cc6d"
-  integrity sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw==
+  version "4.1.4"
+  resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.1.4.tgz#d0d3912865911bb8fac5afb4e3acfa6a28dc72b7"
+  integrity sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==
   dependencies:
     type-detect "^4.0.0"
 
@@ -9637,15 +9584,16 @@ deep-eql@^5.0.1:
   integrity sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==
 
 deep-equal@^2.0.5:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.0.tgz#5caeace9c781028b9ff459f33b779346637c43e6"
-  integrity sha512-RdpzE0Hv4lhowpIUKKMJfeH6C1pXdtT1/it80ubgWqwI3qpuxUBpC1S4hnHg+zjnuOoDkzUtUCEEkG+XG5l3Mw==
+  version "2.2.3"
+  resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.3.tgz#af89dafb23a396c7da3e862abc0be27cf51d56e1"
+  integrity sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA==
   dependencies:
-    call-bind "^1.0.2"
-    es-get-iterator "^1.1.2"
-    get-intrinsic "^1.1.3"
+    array-buffer-byte-length "^1.0.0"
+    call-bind "^1.0.5"
+    es-get-iterator "^1.1.3"
+    get-intrinsic "^1.2.2"
     is-arguments "^1.1.1"
-    is-array-buffer "^3.0.1"
+    is-array-buffer "^3.0.2"
     is-date-object "^1.0.5"
     is-regex "^1.1.4"
     is-shared-array-buffer "^1.0.2"
@@ -9653,11 +9601,11 @@ deep-equal@^2.0.5:
     object-is "^1.1.5"
     object-keys "^1.1.1"
     object.assign "^4.1.4"
-    regexp.prototype.flags "^1.4.3"
+    regexp.prototype.flags "^1.5.1"
     side-channel "^1.0.4"
     which-boxed-primitive "^1.0.2"
     which-collection "^1.0.1"
-    which-typed-array "^1.1.9"
+    which-typed-array "^1.1.13"
 
 deep-equal@~1.0.1:
   version "1.0.1"
@@ -10619,7 +10567,7 @@ es-errors@^1.2.1, es-errors@^1.3.0:
   resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f"
   integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==
 
-es-get-iterator@^1.1.2:
+es-get-iterator@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.3.tgz#3ef87523c5d464d41084b2c3c9c214f1199763d6"
   integrity sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==
@@ -11633,6 +11581,11 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
+follow-redirects@^1.14.0, follow-redirects@^1.14.4:
+  version "1.15.9"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
+  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
+
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -11936,7 +11889,7 @@ get-func-name@^2.0.1, get-func-name@^2.0.2:
   resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41"
   integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==
 
-get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4:
+get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd"
   integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==
@@ -12726,7 +12679,12 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1:
+  version "3.8.1"
+  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
+  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
+
+http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13211,7 +13169,7 @@ is-arguments@^1.1.1:
     call-bind "^1.0.2"
     has-tostringtag "^1.0.0"
 
-is-array-buffer@^3.0.1, is-array-buffer@^3.0.4:
+is-array-buffer@^3.0.2, is-array-buffer@^3.0.4:
   version "3.0.4"
   resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98"
   integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw==
@@ -13402,10 +13360,10 @@ is-lambda@^1.0.1:
   resolved "https://registry.yarnpkg.com/is-lambda/-/is-lambda-1.0.1.tgz#3d9877899e6a53efc0160504cde15f82e6f061d5"
   integrity sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==
 
-is-map@^2.0.1, is-map@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.2.tgz#00922db8c9bf73e81b7a335827bc2a43f2b91127"
-  integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==
+is-map@^2.0.2, is-map@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e"
+  integrity sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==
 
 is-module@^1.0.0:
   version "1.0.0"
@@ -13552,10 +13510,10 @@ is-self-closing@^1.0.1:
   dependencies:
     self-closing-tags "^1.0.1"
 
-is-set@^2.0.1, is-set@^2.0.2:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.2.tgz#90755fa4c2562dc1c5d4024760d6119b94ca18ec"
-  integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==
+is-set@^2.0.2, is-set@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.3.tgz#8ab209ea424608141372ded6e0cb200ef1d9d01d"
+  integrity sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg==
 
 is-shared-array-buffer@^1.0.2, is-shared-array-buffer@^1.0.3:
   version "1.0.3"
@@ -13653,10 +13611,10 @@ is-utf8@^0.2.0:
   resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72"
   integrity sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==
 
-is-weakmap@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2"
-  integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==
+is-weakmap@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.2.tgz#bf72615d649dfe5f699079c54b83e47d1ae19cfd"
+  integrity sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w==
 
 is-weakref@^1.0.2:
   version "1.0.2"
@@ -13665,13 +13623,13 @@ is-weakref@^1.0.2:
   dependencies:
     call-bind "^1.0.2"
 
-is-weakset@^2.0.1:
-  version "2.0.2"
-  resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.2.tgz#4569d67a747a1ce5a994dfd4ef6dcea76e7c0a1d"
-  integrity sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==
+is-weakset@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.3.tgz#e801519df8c0c43e12ff2834eead84ec9e624007"
+  integrity sha512-LvIm3/KWzS9oRFHugab7d+M/GcBXuXX5xZkzPmN+NxihdQlZUQ4dWuSV1xR/sq6upL1TJEDrfBgRepHFdBtSNQ==
   dependencies:
-    call-bind "^1.0.2"
-    get-intrinsic "^1.1.1"
+    call-bind "^1.0.7"
+    get-intrinsic "^1.2.4"
 
 is-whitespace@^0.3.0:
   version "0.3.0"
@@ -13737,11 +13695,6 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
-isobject@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
-  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
-
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -14395,9 +14348,9 @@ jsdom@^16.0.1:
     xml-name-validator "^3.0.0"
 
 jsdom@^21.1.1:
-  version "21.1.1"
-  resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.1.tgz#ab796361e3f6c01bcfaeda1fea3c06197ac9d8ae"
-  integrity sha512-Jjgdmw48RKcdAIQyUD1UdBh2ecH7VqwaXPN3ehoZN6MqgVbMn+lRm1aAT1AsdJRAJpwfa4IpwgzySn61h2qu3w==
+  version "21.1.2"
+  resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.2.tgz#6433f751b8718248d646af1cdf6662dc8a1ca7f9"
+  integrity sha512-sCpFmK2jv+1sjff4u7fzft+pUh2KSUbUrEHYHyfSIbGTIcmnjyp83qg6qLwdJ/I3LpTXx33ACxeRL7Lsyc6lGQ==
   dependencies:
     abab "^2.0.6"
     acorn "^8.8.2"
@@ -14412,7 +14365,7 @@ jsdom@^21.1.1:
     http-proxy-agent "^5.0.0"
     https-proxy-agent "^5.0.1"
     is-potential-custom-element-name "^1.0.1"
-    nwsapi "^2.2.2"
+    nwsapi "^2.2.4"
     parse5 "^7.1.2"
     rrweb-cssom "^0.6.0"
     saxes "^6.0.0"
@@ -16191,7 +16144,7 @@ mkdirp@^1.0.3, mkdirp@^1.0.4:
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
   integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==
 
-mlly@^1.1.0, mlly@^1.7.0:
+mlly@^1.1.0, mlly@^1.7.1:
   version "1.7.1"
   resolved "https://registry.yarnpkg.com/mlly/-/mlly-1.7.1.tgz#e0336429bb0731b6a8e887b438cbdae522c8f32f"
   integrity sha512-rrVRZRELyQzrIUAVMHxP97kv+G786pHmOKzuFII8zDYahFBS7qnHh2AlYSl1GAHhaMPCz6/oHjVMcfFYgFYHgA==
@@ -16301,7 +16254,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@1.10.1, msgpackr@^1.5.2:
+msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16495,13 +16448,25 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
+node-fetch@2.6.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
+  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
+
+node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.9, node-fetch@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
+  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -16849,11 +16814,16 @@ nunjucks@^3.2.3:
     asap "^2.0.3"
     commander "^5.1.0"
 
-nwsapi@^2.2.0, nwsapi@^2.2.2:
+nwsapi@^2.2.0:
   version "2.2.4"
   resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.4.tgz#fd59d5e904e8e1f03c25a7d5a15cfa16c714a1e5"
   integrity sha512-NHj4rzRo0tQdijE9ZqAx6kYDcoRwYwSYzCA8MY3JzfxlrvEU0jhnhJT9BhqhJs7I/dKcrDm6TyulaRqZPIhN5g==
 
+nwsapi@^2.2.4:
+  version "2.2.12"
+  resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.12.tgz#fb6af5c0ec35b27b4581eb3bbad34ec9e5c696f8"
+  integrity sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w==
+
 nx-cloud@16.0.5:
   version "16.0.5"
   resolved "https://registry.yarnpkg.com/nx-cloud/-/nx-cloud-16.0.5.tgz#fa0b0185d254405ec47fcbcdbbd8b12ff1add096"
@@ -16954,12 +16924,12 @@ object-inspect@^1.13.1:
   integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==
 
 object-is@^1.1.5:
-  version "1.1.5"
-  resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac"
-  integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.6.tgz#1a6a53aed2dd8f7e6775ff870bea58545956ab07"
+  integrity sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q==
   dependencies:
-    call-bind "^1.0.2"
-    define-properties "^1.1.3"
+    call-bind "^1.0.7"
+    define-properties "^1.2.1"
 
 object-keys@^1.1.1:
   version "1.1.1"
@@ -17647,7 +17617,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
+passport@^0.4.0:
+  version "0.4.1"
+  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
+  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
+  dependencies:
+    passport-strategy "1.x.x"
+    pause "0.0.1"
+
+passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18024,12 +18002,12 @@ pkg-dir@^4.2.0:
     find-up "^4.0.0"
 
 pkg-types@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-1.1.1.tgz#07b626880749beb607b0c817af63aac1845a73f2"
-  integrity sha512-ko14TjmDuQJ14zsotODv7dBlwxKhUKQEhuhmbqo1uCi9BB0Z2alo/wAXg6q1dTR5TyuqYyWhjtfe/Tsh+X28jQ==
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-1.2.0.tgz#d0268e894e93acff11a6279de147e83354ebd42d"
+  integrity sha512-+ifYuSSqOQ8CqP4MbZA5hDpb97n3E8SVWdJe+Wms9kj745lmd3b7EZJiqvmLwAlmRfjrI7Hi5z3kdBJ93lFNPA==
   dependencies:
     confbox "^0.1.7"
-    mlly "^1.7.0"
+    mlly "^1.7.1"
     pathe "^1.1.2"
 
 pkginfo@0.4.x:
@@ -18920,7 +18898,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.33:
+psl@^1.1.28, psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -19396,7 +19374,7 @@ regexp-tree@~0.1.1:
   resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd"
   integrity sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==
 
-regexp.prototype.flags@^1.4.3, regexp.prototype.flags@^1.5.2:
+regexp.prototype.flags@^1.5.1, regexp.prototype.flags@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334"
   integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw==
@@ -19520,6 +19498,11 @@ requires-port@^1.0.0:
   resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
   integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==
 
+resize-observer-polyfill@^1.5.1:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464"
+  integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg==
+
 resolve-alpn@^1.0.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9"
@@ -19987,6 +19970,11 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
+sax@>=0.1.1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
+  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
+
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20059,13 +20047,33 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
+"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
+  version "5.7.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
+  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
+
+semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
+semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
+  version "6.3.1"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
+  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
+
+semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
+  version "7.6.3"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
+  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
+
+semver@~2.3.1:
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
+  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
+
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -20743,16 +20751,7 @@ string-similarity@^4.0.4:
   resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0":
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -20843,7 +20842,7 @@ stringify-object@^3.2.1:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -20857,13 +20856,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
-strip-ansi@^6.0.0, strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -21523,12 +21515,7 @@ tiny-queue@^0.2.0:
   resolved "https://registry.yarnpkg.com/tiny-queue/-/tiny-queue-0.2.1.tgz#25a67f2c6e253b2ca941977b5ef7442ef97a6046"
   integrity sha512-EijGsv7kzd9I9g0ByCl6h42BWNGUZrlCSejfrb3AKeHC33SGbASu1VDf5O3rRiiUOhAC9CHdZxFPbZu0HmR70A==
 
-tinybench@^2.3.1:
-  version "2.8.0"
-  resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.8.0.tgz#30e19ae3a27508ee18273ffed9ac7018949acd7b"
-  integrity sha512-1/eK7zUnIklz4JUUlL+658n58XO2hHLQfSk1Zf2LKieUjxidN16eKFEoDEfjHc3ohofSSqK3X5yO6VGb6iW8Lw==
-
-tinybench@^2.8.0:
+tinybench@^2.3.1, tinybench@^2.8.0:
   version "2.9.0"
   resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.9.0.tgz#103c9f8ba6d7237a47ab6dd1dcff77251863426b"
   integrity sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==
@@ -21649,7 +21636,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
+"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21659,6 +21646,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
+tough-cookie@~2.5.0:
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
+  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
+  dependencies:
+    psl "^1.1.28"
+    punycode "^2.1.1"
+
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -21873,11 +21868,16 @@ type-check@~0.3.2:
   dependencies:
     prelude-ls "~1.1.2"
 
-type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.8:
+type-detect@4.0.8:
   version "4.0.8"
   resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c"
   integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==
 
+type-detect@^4.0.0, type-detect@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c"
+  integrity sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==
+
 type-fest@^0.13.1:
   version "0.13.1"
   resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
@@ -22042,9 +22042,9 @@ typo-js@*:
   integrity sha512-C7pYBQK17EjSg8tVNY91KHdUt5Nf6FMJ+c3js076quPmBML57PmNMzAcIq/2kf/hSYtFABNDIYNYlJRl5BJhGw==
 
 ufo@^1.5.3:
-  version "1.5.3"
-  resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.3.tgz#3325bd3c977b6c6cd3160bf4ff52989adc9d3344"
-  integrity sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw==
+  version "1.5.4"
+  resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.4.tgz#16d6949674ca0c9e0fbbae1fa20a71d7b1ded754"
+  integrity sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ==
 
 uglify-js@^3.1.4:
   version "3.17.4"
@@ -22182,14 +22182,6 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
-unset-value@2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
-  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
-  dependencies:
-    has-value "^2.0.2"
-    isobject "^4.0.0"
-
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -22703,21 +22695,21 @@ which-boxed-primitive@^1.0.2:
     is-symbol "^1.0.3"
 
 which-collection@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906"
-  integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.2.tgz#627ef76243920a107e7ce8e96191debe4b16c2a0"
+  integrity sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw==
   dependencies:
-    is-map "^2.0.1"
-    is-set "^2.0.1"
-    is-weakmap "^2.0.1"
-    is-weakset "^2.0.1"
+    is-map "^2.0.3"
+    is-set "^2.0.3"
+    is-weakmap "^2.0.2"
+    is-weakset "^2.0.3"
 
 which-module@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
   integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==
 
-which-typed-array@^1.1.14, which-typed-array@^1.1.15, which-typed-array@^1.1.9:
+which-typed-array@^1.1.13, which-typed-array@^1.1.14, which-typed-array@^1.1.15:
   version "1.1.15"
   resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.15.tgz#264859e9b11a649b388bfaaf4f767df1f779b38d"
   integrity sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==
@@ -22749,15 +22741,7 @@ which@^3.0.0:
   dependencies:
     isexe "^2.0.0"
 
-why-is-node-running@^2.2.2:
-  version "2.2.2"
-  resolved "https://registry.yarnpkg.com/why-is-node-running/-/why-is-node-running-2.2.2.tgz#4185b2b4699117819e7154594271e7e344c9973e"
-  integrity sha512-6tSwToZxTOcotxHeA+qGCq1mVzKR3CwcJGmVcY+QE8SHy6TnpFnh8PAvPNHYr7EcuVeG0QSMxtYCuO1ta/G/oA==
-  dependencies:
-    siginfo "^2.0.0"
-    stackback "0.0.2"
-
-why-is-node-running@^2.3.0:
+why-is-node-running@^2.2.2, why-is-node-running@^2.3.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/why-is-node-running/-/why-is-node-running-2.3.0.tgz#a3f69a97107f494b3cdc3bdddd883a7d65cebf04"
   integrity sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==
@@ -22832,7 +22816,7 @@ worker-farm@1.7.0:
   dependencies:
     errno "~0.1.7"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -22850,15 +22834,6 @@ wrap-ansi@^5.1.0:
     string-width "^3.0.0"
     strip-ansi "^5.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
@@ -22941,7 +22916,12 @@ ws@^7.4.6:
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9"
   integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ==
 
-ws@^8.13.0, ws@~8.17.1:
+ws@^8.13.0:
+  version "8.18.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
+  integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
+
+ws@~8.17.1:
   version "8.17.1"
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b"
   integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==
@@ -22981,10 +22961,33 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.6.2"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
-  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
+xml2js@0.1.x:
+  version "0.1.14"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
+  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
+  dependencies:
+    sax ">=0.1.1"
+
+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
+xml2js@0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
+  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~11.0.0"
+
+xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.4.23"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
+  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -22994,6 +22997,11 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
+
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"
@@ -23172,9 +23180,9 @@ yocto-queue@^0.1.0:
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
 yocto-queue@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
-  integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.1.1.tgz#fef65ce3ac9f8a32ceac5a634f74e17e5b232110"
+  integrity sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==
 
 yup@0.32.9:
   version "0.32.9"

From 891f73681539239f6930374cfb4fba885683e7ab Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 6 Sep 2024 15:37:09 +0100
Subject: [PATCH 39/69] unit tests for secrets mapping

---
 packages/builder/vite.config.mjs              |  2 +-
 .../functions/backfill/global/configs.ts      |  7 +-
 .../src/api/controllers/global/configs.ts     |  2 +-
 .../controllers/global/tests/configs.spec.ts  | 76 +++++++++++++++++++
 4 files changed, 81 insertions(+), 6 deletions(-)
 create mode 100644 packages/worker/src/api/controllers/global/tests/configs.spec.ts

diff --git a/packages/builder/vite.config.mjs b/packages/builder/vite.config.mjs
index a84fba2dae..f5ff388952 100644
--- a/packages/builder/vite.config.mjs
+++ b/packages/builder/vite.config.mjs
@@ -79,7 +79,7 @@ export default defineConfig(({ mode }) => {
         "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN),
       }),
       copyFonts("fonts"),
-    ...(isProduction ? [] : devOnlyPlugins),
+      ...(isProduction ? [] : devOnlyPlugins),
     ],
     optimizeDeps: {
       exclude: ["@roxi/routify", "fsevents"],
diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts
index bb0781eba3..04eb9caff2 100644
--- a/packages/server/src/migrations/functions/backfill/global/configs.ts
+++ b/packages/server/src/migrations/functions/backfill/global/configs.ts
@@ -15,12 +15,11 @@ import {
 } from "@budibase/types"
 import env from "./../../../../environment"
 
-export function getConfigParams(type?: ConfigType): DatabaseQueryOpts {
-  const configType = type || ""
+export function getConfigParams(): DatabaseQueryOpts {
   return {
     include_docs: true,
-    startkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}`,
-    endkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}${UNICODE_MAX}`,
+    startkey: `${DocumentType.CONFIG}${SEPARATOR}`,
+    endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`,
   }
 }
 
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 087feede3c..4a19168c4f 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -199,7 +199,7 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
 }
 
-async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) {
+export async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) {
   if (!existingConfig) return
 
   // ensure that the redacted API keys are not overwritten in the DB
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
new file mode 100644
index 0000000000..1a1d78757d
--- /dev/null
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -0,0 +1,76 @@
+import { TestConfiguration, structures } from "../../../../tests"
+import { constants, configs } from "@budibase/backend-core"
+import { AIConfig, UserCtx } from "@budibase/types"
+import { find, verifyAIConfig } from "../configs"
+
+jest.mock("@budibase/backend-core", () => ({
+  ...jest.requireActual("@budibase/backend-core"),
+  configs: {
+    getConfig: jest.fn(),
+    save: jest.fn()
+  },
+}))
+
+describe("Global configs controller", () => {
+  it("Should strip secrets when pulling AI config", async () => {
+    configs.getConfig.mockResolvedValue({
+      config: {
+        ai: {
+          apiKey: "abc123APIKey",
+          baseUrl: "https://api.example.com",
+        }
+      }
+    })
+    const ctx = {
+      params: {
+        type: "ai"
+      },
+      throw: jest.fn()
+    } as UserCtx
+
+    await find(ctx)
+
+    expect(ctx.body).toEqual({
+      config: {
+        ai: {
+          apiKey: "--secret-value--",
+          "baseUrl": "https://api.example.com"
+        }
+      }
+    })
+  })
+
+  it("Should not update existing secrets when updating an existing AI Config", async () => {
+    const newConfig = {
+      type: "ai",
+      config: {
+        aiconfig: {
+          provider: "OpenAI",
+          isDefault: true,
+          name: "MyConfig",
+          active: true,
+          defaultModel: "gpt4",
+          apiKey: "--secret-value--"
+        }
+      }
+    }
+
+    const existingConfig = {
+      type: "ai",
+      config: {
+        aiconfig: {
+          provider: "OpenAI",
+          isDefault: true,
+          name: "MyConfig",
+          active: true,
+          defaultModel: "gpt4",
+          apiKey: "myapikey"
+        }
+      }
+    }
+
+    await verifyAIConfig(newConfig, existingConfig)
+    // should be unchanged
+    expect(newConfig.config.aiconfig.apiKey === "myapikey")
+  })
+})

From 6bf4f47110fae08a78b1d2413aa11f13a0936d52 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 6 Sep 2024 15:42:37 +0100
Subject: [PATCH 40/69] further tests around default BB AI config

---
 .../controllers/global/tests/configs.spec.ts  | 47 +++++++++++++++++--
 1 file changed, 44 insertions(+), 3 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 1a1d78757d..393a11a043 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,6 +1,6 @@
-import { TestConfiguration, structures } from "../../../../tests"
-import { constants, configs } from "@budibase/backend-core"
-import { AIConfig, UserCtx } from "@budibase/types"
+import { configs } from "@budibase/backend-core"
+import { UserCtx } from "@budibase/types"
+import * as pro from "@budibase/pro"
 import { find, verifyAIConfig } from "../configs"
 
 jest.mock("@budibase/backend-core", () => ({
@@ -11,7 +11,12 @@ jest.mock("@budibase/backend-core", () => ({
   },
 }))
 
+
 describe("Global configs controller", () => {
+  afterEach(() => {
+    jest.resetAllMocks()
+  })
+
   it("Should strip secrets when pulling AI config", async () => {
     configs.getConfig.mockResolvedValue({
       config: {
@@ -40,6 +45,42 @@ describe("Global configs controller", () => {
     })
   })
 
+  it("Should return the default BB AI config when the feature is turned on", async () => {
+    pro.features.isBudibaseAIEnabled = jest.fn(() => true)
+    configs.getConfig.mockResolvedValue({
+      config: {
+        ai: {
+          apiKey: "abc123APIKey",
+          baseUrl: "https://api.example.com",
+        }
+      }
+    })
+    const ctx = {
+      params: {
+        type: "ai"
+      },
+      throw: jest.fn()
+    } as UserCtx
+
+    await find(ctx)
+
+    expect(ctx.body).toEqual({
+      config: {
+        budibase_ai: {
+          provider: "OpenAI",
+          active: true,
+          isDefault: true,
+          defaultModel: undefined,
+          name: "Budibase AI",
+        },
+        ai: {
+          apiKey: "--secret-value--",
+          "baseUrl": "https://api.example.com"
+        }
+      }
+    })
+  })
+
   it("Should not update existing secrets when updating an existing AI Config", async () => {
     const newConfig = {
       type: "ai",

From 073017a0d6bdbb9b84ae762c23905b2ff6bb55c1 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 6 Sep 2024 15:43:34 +0100
Subject: [PATCH 41/69] remove todo comment

---
 packages/worker/src/api/routes/global/configs.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts
index 409e0a7797..471005c2aa 100644
--- a/packages/worker/src/api/routes/global/configs.ts
+++ b/packages/worker/src/api/routes/global/configs.ts
@@ -76,7 +76,6 @@ function aiValidation() {
        active: Joi.boolean().required(),
        baseUrl: Joi.string().optional().allow("", null),
        apiKey: Joi.string().required(),
-       // TODO: should be enum
        defaultModel: Joi.string().optional(),
      }).required()
   )

From b9499406534216549593aa555d6b7bddd1b68689 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Fri, 6 Sep 2024 15:44:23 +0100
Subject: [PATCH 42/69] remove comments

---
 packages/backend-core/src/configs/configs.ts | 39 --------------------
 1 file changed, 39 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 17f4abfe24..1eafd9596e 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -256,42 +256,3 @@ export async function getSCIMConfig(): Promise<SCIMInnerConfig | undefined> {
   const config = await getConfig<SCIMConfig>(ConfigType.SCIM)
   return config?.config
 }
-
-
-// AI
-
-// TODO: Can we assume that you are licensed when you hit this endpoint?
-// export async function getAIConfig(): Promise<AIConfig | undefined> {
-  // if (!env.SELF_HOSTED) {
-  //   // always use the env vars in cloud
-  //   // TODO: Licensing stuff - make this right
-  //   if (env.OPENAI_API_KEY) {
-  //     return getDefaultBudibaseAIConfig()
-  //   }
-  // }
-
-  // prefer the config in self-host
-  // let config = await getConfig<AIConfig>(ConfigType.AI)
-
-  // fallback to env vars
-  // if (!config || !config.activated) {
-  //   config = getDefaultBudibaseAIConfig()
-  // }
-
-  // return config
-// }
-
-// export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined {
-//   if (env.OPENAI_API_KEY) {
-//     return {
-//       provider: "",
-//       isDefault: true,
-//       name: "Budibase AI",
-//       active: true,
-//       baseUrl: "",
-//       apiKey: env.OPENAI_API_KEY,
-//       // TODO: should be enum
-//       defaultModel: ""
-//     }
-//   }
-// }

From 57698170a8d2983b3396919aa6c3266f8835a0e5 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 9 Sep 2024 13:55:26 +0100
Subject: [PATCH 43/69] lint

---
 packages/backend-core/src/configs/configs.ts  |   2 -
 .../core/utilities/structures/licenses.ts     |   2 +-
 .../portal/settings/ai/AIConfigTile.svelte    |   8 +-
 .../portal/settings/ai/AISettings.spec.js     |  18 +-
 .../portal/settings/ai/ConfigModal.svelte     |  27 +--
 .../builder/portal/settings/ai/constants.ts   |  49 ++--
 .../builder/portal/settings/ai/index.svelte   |  14 +-
 .../portal/settings/ai/logos/Anthropic.svelte |  18 +-
 .../portal/settings/ai/logos/Budibase.svelte  |  31 ++-
 .../portal/settings/ai/logos/OpenAI.svelte    |  15 +-
 .../settings/ai/logos/TogetherAI.svelte       |  66 ++++-
 .../builder/portal/users/users/index.svelte   |   2 +-
 packages/builder/vitest.setup.js              |   2 +-
 packages/types/src/documents/global/config.ts |   2 +-
 packages/types/src/sdk/licensing/feature.ts   |   2 +-
 packages/types/src/sdk/licensing/quota.ts     |   2 +-
 .../src/api/controllers/global/configs.ts     |   5 +-
 .../controllers/global/tests/configs.spec.ts  |  44 ++--
 yarn.lock                                     | 228 ++++++++----------
 19 files changed, 293 insertions(+), 244 deletions(-)

diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts
index 1eafd9596e..0d189e3f7d 100644
--- a/packages/backend-core/src/configs/configs.ts
+++ b/packages/backend-core/src/configs/configs.ts
@@ -1,6 +1,4 @@
 import {
-  AIConfig,
-  AIInnerConfig,
   Config,
   ConfigType,
   GoogleConfig,
diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts
index cf422637d6..b4d209a7ee 100644
--- a/packages/backend-core/tests/core/utilities/structures/licenses.ts
+++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts
@@ -59,7 +59,7 @@ export function quotas(): Quotas {
           name: "Budibase AI Credits",
           value: 1,
           triggers: [],
-        }
+        },
       },
       static: {
         rows: {
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 0b7a006312..d4d5279d3e 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -18,13 +18,13 @@
 <div on:click class:disabled class="option">
   <div class="icon">
     {#if config.name === "Budibase AI"}
-      <BudibaseLogo height="30" width="30"/>
+      <BudibaseLogo height="30" width="30" />
     {:else if config.provider === Providers.OpenAI.name}
-      <OpenAILogo height="30" width="30"/>
+      <OpenAILogo height="30" width="30" />
     {:else if config.provider === Providers.Anthropic.name}
-      <AnthropicLogo height="30" width="30"/>
+      <AnthropicLogo height="30" width="30" />
     {:else if config.provider === Providers.TogetherAI.name}
-      <TogetherAILogo height="30" width="30"/>
+      <TogetherAILogo height="30" width="30" />
     {/if}
   </div>
   <div class="header">
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
index 5ca14c7898..805739957b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js
@@ -11,23 +11,19 @@ const Hosting = {
   Self: "self",
 }
 
-function setupEnv(
-  hosting,
-  features = {}) {
+function setupEnv(hosting, features = {}) {
   const defaultFeatures = {
     budibaseAIEnabled: false,
     customAIConfigsEnabled: false,
-    ...features
+    ...features,
   }
   admin.subscribe = vi.fn().mockImplementation(callback => {
-    callback({cloud: hosting === Hosting.Cloud})
-    return () => {
-    }
+    callback({ cloud: hosting === Hosting.Cloud })
+    return () => {}
   })
   licensing.subscribe = vi.fn().mockImplementation(callback => {
     callback(defaultFeatures)
-    return () => {
-    }
+    return () => {}
   })
 }
 
@@ -73,7 +69,7 @@ describe("AISettings", () => {
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).not.toBeInTheDocument()
 
-      setupEnv(Hosting.Cloud, {customAIConfigsEnabled: true})
+      setupEnv(Hosting.Cloud, { customAIConfigsEnabled: true })
       instance = render(AISettings)
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).toBeInTheDocument()
@@ -87,7 +83,7 @@ describe("AISettings", () => {
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).not.toBeInTheDocument()
 
-      setupEnv(Hosting.Self, {customAIConfigsEnabled: true})
+      setupEnv(Hosting.Self, { customAIConfigsEnabled: true })
       instance = render(AISettings, {})
       addConfigurationButton = instance.queryByText("Add configuration")
       expect(addConfigurationButton).toBeInTheDocument()
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 99f3aa8e7b..2f7486d087 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -1,12 +1,6 @@
 <script>
-  import {
-    ModalContent,
-    Label,
-    Input,
-    Select,
-    Toggle,
-  } from "@budibase/bbui"
-  import {ConfigMap, Providers} from "./constants"
+  import { ModalContent, Label, Input, Select, Toggle } from "@budibase/bbui"
+  import { ConfigMap, Providers } from "./constants"
 
   export let config = {
     active: false,
@@ -22,7 +16,8 @@
     const { provider, defaultModel, name, apiKey } = config
     validation = provider && defaultModel && name && apiKey
   }
-  $: canEditBaseUrl = config.provider && ConfigMap[config.provider].baseUrl === ""
+  $: canEditBaseUrl =
+    config.provider && ConfigMap[config.provider].baseUrl === ""
 
   function prefillConfig(evt) {
     const provider = evt.detail
@@ -59,7 +54,7 @@
   </div>
   <div class="form-row">
     <Label size="M">Name</Label>
-    <Input placeholder={"Enter a name"} bind:value={config.name}/>
+    <Input placeholder={"Enter a name"} bind:value={config.name} />
   </div>
   <div class="form-row">
     <Label size="M">Default Model</Label>
@@ -75,14 +70,18 @@
   </div>
   <div class="form-row">
     <Label size="M">Base URL</Label>
-    <Input disabled={!canEditBaseUrl} placeholder={"https://budibase.ai"} bind:value={config.baseUrl}/>
+    <Input
+      disabled={!canEditBaseUrl}
+      placeholder={"https://budibase.ai"}
+      bind:value={config.baseUrl}
+    />
   </div>
   <div class="form-row">
     <Label size="M">API Key</Label>
-    <Input type="password" bind:value={config.apiKey}/>
+    <Input type="password" bind:value={config.apiKey} />
   </div>
-  <Toggle text="Active" bind:value={config.active}/>
-  <Toggle text="Set as default" bind:value={config.isDefault}/>
+  <Toggle text="Active" bind:value={config.active} />
+  <Toggle text="Set as default" bind:value={config.isDefault} />
 </ModalContent>
 
 <style>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
index abefafd8f5..7ff43582e0 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
@@ -2,42 +2,35 @@ export const Providers = {
   OpenAI: {
     name: "OpenAI",
     models: [
-      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
-      {label: "GPT 4o", value: "gpt-4o" },
-      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
-      {label: "GPT 4", value: "gpt-4" },
-      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
-    ]
+      { label: "GPT 4o Mini", value: "gpt-4o-mini" },
+      { label: "GPT 4o", value: "gpt-4o" },
+      { label: "GPT 4 Turbo", value: "gpt-4-turbo" },
+      { label: "GPT 4", value: "gpt-4" },
+      { label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
+    ],
   },
   Anthropic: {
     name: "Anthropic",
     models: [
-      {label: "Claude 3.5 Sonnet", value: "claude-3-5-sonnet-20240620" },
-      {label: "Claude 3 Sonnet", value: "claude-3-sonnet-20240229" },
-      {label: "Claude 3 Opus", value: "claude-3-opus-20240229" },
-      {label: "Claude 3 Haiku", value: "claude-3-haiku-20240307" },
-    ]
+      { label: "Claude 3.5 Sonnet", value: "claude-3-5-sonnet-20240620" },
+      { label: "Claude 3 Sonnet", value: "claude-3-sonnet-20240229" },
+      { label: "Claude 3 Opus", value: "claude-3-opus-20240229" },
+      { label: "Claude 3 Haiku", value: "claude-3-haiku-20240307" },
+    ],
   },
   TogetherAI: {
     name: "Together AI",
-    // TODO: too many - probably need to use an autocomplete for this
-    models: [
-      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
-      {label: "GPT 4o", value: "gpt-4o" },
-      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
-      {label: "GPT 4", value: "gpt-4" },
-      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
-    ]
+    models: [{ label: "Llama 3 8B", value: "meta-llama/Meta-Llama-3-8B" }],
   },
   AzureOpenAI: {
     name: "Azure Open AI",
     models: [
-      {label: "GPT 4o Mini", value: "gpt-4o-mini" },
-      {label: "GPT 4o", value: "gpt-4o" },
-      {label: "GPT 4 Turbo", value: "gpt-4-turbo" },
-      {label: "GPT 4", value: "gpt-4" },
-      {label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
-    ]
+      { label: "GPT 4o Mini", value: "gpt-4o-mini" },
+      { label: "GPT 4o", value: "gpt-4o" },
+      { label: "GPT 4 Turbo", value: "gpt-4-turbo" },
+      { label: "GPT 4", value: "gpt-4" },
+      { label: "GPT 3.5 Turbo", value: "gpt-3.5-turbo" },
+    ],
   },
   Custom: {
     name: "Custom",
@@ -62,6 +55,6 @@ export const ConfigMap = {
     baseUrl: "",
   },
   Custom: {
-    baseUrl: ""
-  }
-}
\ No newline at end of file
+    baseUrl: "",
+  },
+}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index dc7853b398..77d6aa952a 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -10,7 +10,7 @@
     notifications,
     Modal,
     Tags,
-    Tag
+    Tag,
   } from "@budibase/bbui"
   import { admin, licensing } from "stores/portal"
   import { API } from "api"
@@ -50,8 +50,8 @@
       fullAIConfig = {
         type: ConfigTypes.AI,
         config: {
-          [id]: editingAIConfig
-        }
+          [id]: editingAIConfig,
+        },
       }
     } else {
       // We don't store the default BB AI config in the DB
@@ -72,7 +72,9 @@
       await fetchAIConfig()
     } catch (error) {
       notifications.error(
-        `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}`
+        `Failed to save AI Configuration, reason: ${
+          error?.message || "Unknown"
+        }`
       )
     }
   }
@@ -142,7 +144,9 @@
         <Button size="S" cta on:click={newConfig}>Add configuration</Button>
       {/if}
     </div>
-    <Body size="S">Use the following interface to select your preferred AI configuration.</Body>
+    <Body size="S"
+      >Use the following interface to select your preferred AI configuration.</Body
+    >
     <Body size="S">Select your AI Model:</Body>
     {#if fullAIConfig?.config}
       {#each Object.keys(fullAIConfig.config) as key}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
index 4c598872a2..d16a37ba2b 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Anthropic.svelte
@@ -2,7 +2,16 @@
   export let height
   export let width
 </script>
-<svg id="katman_1" {height} {width} data-name="katman 1" xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="0 0 841.89 595.28">
+
+<svg
+  id="katman_1"
+  {height}
+  {width}
+  data-name="katman 1"
+  xmlns="http://www.w3.org/2000/svg"
+  version="1.1"
+  viewBox="0 0 841.89 595.28"
+>
   <defs>
     <style>
       .cls-1 {
@@ -11,5 +20,8 @@
       }
     </style>
   </defs>
-  <path class="cls-1" d="M552.13,91.34h-90.99l162.93,412.61h88.87l-160.81-412.61ZM289.76,91.34l-160.81,412.61h90.99l35.97-86.75h169.28l33.86,84.64h90.99L384.97,91.34h-95.22ZM281.29,341.02l55.01-146,57.13,146h-112.14Z"/>
-</svg>
\ No newline at end of file
+  <path
+    class="cls-1"
+    d="M552.13,91.34h-90.99l162.93,412.61h88.87l-160.81-412.61ZM289.76,91.34l-160.81,412.61h90.99l35.97-86.75h169.28l33.86,84.64h90.99L384.97,91.34h-95.22ZM281.29,341.02l55.01-146,57.13,146h-112.14Z"
+  />
+</svg>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
index f69c7d8281..dc19facb1a 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte
@@ -2,16 +2,35 @@
   export let width
   export let height
 </script>
-<svg {width} {height} viewBox="0 0 265 265" fill="none" xmlns="http://www.w3.org/2000/svg">
+
+<svg
+  {width}
+  {height}
+  viewBox="0 0 265 265"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
   <g clip-path="url(#clip0_1_1799)">
-    <path d="M158.2 8.6V116.6C158.2 121.3 162 125.2 166.8 125.2H213.8C218 125.2 222 123.2 224.6 119.8L262.9 68.9C265.7 65.2 265.7 60.1 262.9 56.4L224.6 5.4C222 2 218 0 213.8 0H166.8C162 0 158.2 3.8 158.2 8.6Z" fill="#FF4E4E"/>
-    <path d="M158.2 148.4V256.4C158.2 261.1 162 265 166.8 265H213.8C218 265 222 263 224.6 259.6L262.9 208.7C265.7 205 265.7 199.9 262.9 196.2L224.6 145.3C222.1 141.9 218.1 139.9 213.8 139.9H166.8C162 139.8 158.2 143.7 158.2 148.4Z" fill="#6E56FF"/>
-    <path d="M0 8.6V116.6C0 121.3 3.8 125.2 8.6 125.2H109.6C113.8 125.2 117.8 123.2 120.4 119.8L155.9 72.5C160.3 66.6 160.3 58.5 155.9 52.6L120.3 5.4C117.8 2 113.8 0 109.5 0H8.6C3.8 0 0 3.8 0 8.6Z" fill="#F97777"/>
-    <path d="M0 148.4V256.4C0 261.1 3.8 265 8.6 265H109.6C113.8 265 117.8 263 120.4 259.6L155.9 212.3C160.3 206.4 160.3 198.3 155.9 192.4L120.4 145.1C117.9 141.7 113.9 139.7 109.6 139.7H8.6C3.8 139.8 0 143.7 0 148.4Z" fill="#9F8FFF"/>
+    <path
+      d="M158.2 8.6V116.6C158.2 121.3 162 125.2 166.8 125.2H213.8C218 125.2 222 123.2 224.6 119.8L262.9 68.9C265.7 65.2 265.7 60.1 262.9 56.4L224.6 5.4C222 2 218 0 213.8 0H166.8C162 0 158.2 3.8 158.2 8.6Z"
+      fill="#FF4E4E"
+    />
+    <path
+      d="M158.2 148.4V256.4C158.2 261.1 162 265 166.8 265H213.8C218 265 222 263 224.6 259.6L262.9 208.7C265.7 205 265.7 199.9 262.9 196.2L224.6 145.3C222.1 141.9 218.1 139.9 213.8 139.9H166.8C162 139.8 158.2 143.7 158.2 148.4Z"
+      fill="#6E56FF"
+    />
+    <path
+      d="M0 8.6V116.6C0 121.3 3.8 125.2 8.6 125.2H109.6C113.8 125.2 117.8 123.2 120.4 119.8L155.9 72.5C160.3 66.6 160.3 58.5 155.9 52.6L120.3 5.4C117.8 2 113.8 0 109.5 0H8.6C3.8 0 0 3.8 0 8.6Z"
+      fill="#F97777"
+    />
+    <path
+      d="M0 148.4V256.4C0 261.1 3.8 265 8.6 265H109.6C113.8 265 117.8 263 120.4 259.6L155.9 212.3C160.3 206.4 160.3 198.3 155.9 192.4L120.4 145.1C117.9 141.7 113.9 139.7 109.6 139.7H8.6C3.8 139.8 0 143.7 0 148.4Z"
+      fill="#9F8FFF"
+    />
   </g>
   <defs>
     <clipPath id="clip0_1_1799">
-      <rect width="265" height="265" fill="white"/>
+      <rect width="265" height="265" fill="white" />
     </clipPath>
   </defs>
 </svg>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
index fa60b33cfd..a049cd537f 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte
@@ -2,8 +2,17 @@
   export let height
   export let width
 </script>
-<svg fill="#000000" {width} {height} viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg">
+
+<svg
+  fill="#000000"
+  {width}
+  {height}
+  viewBox="0 0 24 24"
+  role="img"
+  xmlns="http://www.w3.org/2000/svg"
+>
   <title>OpenAI icon</title>
   <path
-    d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"/>
-</svg>
\ No newline at end of file
+    d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z"
+  />
+</svg>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte
index 1afd5626b1..b81450f768 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte
@@ -2,16 +2,56 @@
   export let height
   export let width
 </script>
-<svg {width} {height} viewBox="0 0 107 21" fill="none" xmlns="http://www.w3.org/2000/svg">
-  <path d="M3.0727 6.13665H0.99707V4.44456H3.0727V1.2183H4.92272V4.44456H7.83312V6.13665H4.92272V12.9276C4.92272 13.4089 5.01297 13.7548 5.19346 13.9654C5.38899 14.1609 5.71988 14.2587 6.18615 14.2587H8.1941V15.9508H6.07334C4.9904 15.9508 4.2158 15.7101 3.74954 15.2288C3.29831 14.7475 3.0727 13.9879 3.0727 12.9501V6.13665Z" fill="currentColor"></path>
-  <path d="M14.1547 16.0861C13.0417 16.0861 12.049 15.838 11.1766 15.3416C10.3193 14.8453 9.64995 14.1534 9.16864 13.266C8.68734 12.3786 8.44669 11.3558 8.44669 10.1977C8.44669 9.03952 8.68734 8.01675 9.16864 7.12934C9.64995 6.24193 10.3193 5.55006 11.1766 5.05371C12.049 4.55736 13.0417 4.30919 14.1547 4.30919C15.2677 4.30919 16.2529 4.55736 17.1102 5.05371C17.9826 5.55006 18.6594 6.24193 19.1407 7.12934C19.622 8.01675 19.8627 9.03952 19.8627 10.1977C19.8627 11.3558 19.622 12.3786 19.1407 13.266C18.6594 14.1534 17.9826 14.8453 17.1102 15.3416C16.2529 15.838 15.2677 16.0861 14.1547 16.0861ZM14.1547 14.4392C14.9067 14.4392 15.5685 14.2587 16.1401 13.8977C16.7266 13.5367 17.1779 13.0329 17.4937 12.3861C17.8096 11.7393 17.9675 11.0099 17.9675 10.1977C17.9675 9.38546 17.8096 8.65598 17.4937 8.00923C17.1779 7.36247 16.7266 6.85861 16.1401 6.49763C15.5685 6.13665 14.9067 5.95616 14.1547 5.95616C13.4026 5.95616 12.7333 6.13665 12.1467 6.49763C11.5752 6.85861 11.1315 7.36247 10.8156 8.00923C10.4998 8.65598 10.3418 9.38546 10.3418 10.1977C10.3418 11.0099 10.4998 11.7393 10.8156 12.3861C11.1315 13.0329 11.5752 13.5367 12.1467 13.8977C12.7333 14.2587 13.4026 14.4392 14.1547 14.4392Z" fill="currentColor"></path>
-  <path d="M32.5327 4.44456V15.3867C32.5327 18.8612 30.6301 20.5984 26.8247 20.5984C25.3357 20.5984 24.1174 20.26 23.1698 19.5831C22.2373 18.9063 21.6958 17.9437 21.5454 16.6953H23.4406C23.591 17.4172 23.9745 17.9738 24.5912 18.3648C25.2079 18.7559 25.9975 18.9514 26.9601 18.9514C29.4418 18.9514 30.6827 17.7406 30.6827 15.3191V13.9654C29.8555 15.3792 28.5394 16.0861 26.7345 16.0861C25.6666 16.0861 24.7115 15.853 23.8692 15.3867C23.042 14.9205 22.3877 14.2436 21.9064 13.3562C21.4401 12.4688 21.207 11.416 21.207 10.1977C21.207 9.03952 21.4401 8.01675 21.9064 7.12934C22.3877 6.24193 23.0495 5.55006 23.8918 5.05371C24.7341 4.55736 25.6816 4.30919 26.7345 4.30919C27.667 4.30919 28.4642 4.4972 29.126 4.87322C29.7878 5.2342 30.3067 5.75311 30.6827 6.42994L30.9309 4.44456H32.5327ZM26.915 14.4392C27.652 14.4392 28.3063 14.2662 28.8778 13.9203C29.4644 13.5593 29.9156 13.0629 30.2315 12.4312C30.5473 11.7845 30.7053 11.055 30.7053 10.2428C30.7053 9.41554 30.5473 8.67854 30.2315 8.03179C29.9156 7.36999 29.4644 6.85861 28.8778 6.49763C28.3063 6.13665 27.652 5.95616 26.915 5.95616C25.7719 5.95616 24.8469 6.35474 24.14 7.1519C23.4481 7.93402 23.1021 8.94928 23.1021 10.1977C23.1021 11.4461 23.4481 12.4688 24.14 13.266C24.8469 14.0481 25.7719 14.4392 26.915 14.4392Z" fill="currentColor"></path>
-  <path d="M34.2758 10.1977C34.2758 9.02448 34.5014 7.99419 34.9526 7.10678C35.4189 6.21937 36.0656 5.53502 36.8929 5.05371C37.7201 4.55736 38.6677 4.30919 39.7356 4.30919C40.7884 4.30919 41.721 4.52728 42.5332 4.96346C43.3454 5.39965 43.9846 6.03136 44.4509 6.85861C44.9171 7.68585 45.1653 8.65598 45.1954 9.769C45.1954 9.93445 45.1803 10.1977 45.1503 10.5586H36.216V10.7166C36.2461 11.8446 36.592 12.7471 37.2538 13.4239C37.9156 14.1008 38.7805 14.4392 39.8484 14.4392C40.6756 14.4392 41.375 14.2361 41.9466 13.83C42.5332 13.4089 42.9242 12.8298 43.1198 12.0928H44.9923C44.7667 13.266 44.2027 14.2286 43.3002 14.9806C42.3978 15.7176 41.2923 16.0861 39.9837 16.0861C38.8406 16.0861 37.8404 15.8455 36.9831 15.3642C36.1258 14.8678 35.4565 14.176 34.9752 13.2885C34.5089 12.3861 34.2758 11.3558 34.2758 10.1977ZM43.2777 9.02448C43.1874 8.04683 42.8189 7.28727 42.1722 6.7458C41.5405 6.20433 40.7358 5.9336 39.7581 5.9336C38.8858 5.9336 38.1187 6.21937 37.4569 6.79092C36.7951 7.36247 36.4191 8.10699 36.3288 9.02448H43.2777Z" fill="currentColor"></path>
-  <path d="M47.5703 6.13665H45.4947V4.44456H47.5703V1.2183H49.4203V4.44456H52.3307V6.13665H49.4203V12.9276C49.4203 13.4089 49.5106 13.7548 49.6911 13.9654C49.8866 14.1609 50.2175 14.2587 50.6838 14.2587H52.6917V15.9508H50.571C49.488 15.9508 48.7134 15.7101 48.2472 15.2288C47.7959 14.7475 47.5703 13.9879 47.5703 12.9501V6.13665Z" fill="currentColor"></path>
-  <path d="M59.2771 4.30919C60.6759 4.30919 61.804 4.73033 62.6613 5.57262C63.5186 6.4149 63.9473 7.73849 63.9473 9.54339V15.9508H62.0973V9.6562C62.0973 8.46797 61.8265 7.56552 61.2851 6.94885C60.7436 6.31714 59.969 6.00128 58.9613 6.00128C57.8934 6.00128 57.0436 6.3773 56.4119 7.12934C55.7952 7.88138 55.4868 8.90415 55.4868 10.1977V15.9508H53.6368V0.157928H55.4868V6.3397C55.8629 5.73807 56.3592 5.24924 56.9759 4.87322C57.6076 4.4972 58.3747 4.30919 59.2771 4.30919Z" fill="currentColor"></path>
-  <path d="M65.5284 10.1977C65.5284 9.02448 65.754 7.99419 66.2052 7.10678C66.6715 6.21937 67.3182 5.53502 68.1455 5.05371C68.9727 4.55736 69.9203 4.30919 70.9882 4.30919C72.041 4.30919 72.9736 4.52728 73.7858 4.96346C74.598 5.39965 75.2372 6.03136 75.7035 6.85861C76.1697 7.68585 76.4179 8.65598 76.448 9.769C76.448 9.93445 76.4329 10.1977 76.4029 10.5586H67.4686V10.7166C67.4987 11.8446 67.8446 12.7471 68.5064 13.4239C69.1682 14.1008 70.0331 14.4392 71.101 14.4392C71.9282 14.4392 72.6276 14.2361 73.1992 13.83C73.7858 13.4089 74.1768 12.8298 74.3723 12.0928H76.2449C76.0193 13.266 75.4553 14.2286 74.5528 14.9806C73.6504 15.7176 72.5449 16.0861 71.2363 16.0861C70.0932 16.0861 69.093 15.8455 68.2357 15.3642C67.3784 14.8678 66.7091 14.176 66.2278 13.2885C65.7615 12.3861 65.5284 11.3558 65.5284 10.1977ZM74.5303 9.02448C74.44 8.04683 74.0715 7.28727 73.4248 6.7458C72.7931 6.20433 71.9884 5.9336 71.0107 5.9336C70.1384 5.9336 69.3713 6.21937 68.7095 6.79092C68.0477 7.36247 67.6717 8.10699 67.5814 9.02448H74.5303Z" fill="currentColor"></path>
-  <path d="M84.0314 4.44456V6.24945H83.1064C82.0084 6.24945 81.2188 6.61795 80.7374 7.35495C80.2712 8.09195 80.038 9.00192 80.038 10.0849V15.9508H78.188V4.44456H79.7899L80.038 6.18177C80.3689 5.65534 80.7976 5.2342 81.324 4.91834C81.8505 4.60249 82.5799 4.44456 83.5125 4.44456H84.0314Z" fill="currentColor"></path>
-  <path d="M102.875 14.2587V15.9508H101.859C101.122 15.9508 100.596 15.8004 100.28 15.4995C99.9643 15.1987 99.7988 14.755 99.7838 14.1684C98.8964 15.4469 97.6104 16.0861 95.9258 16.0861C94.6473 16.0861 93.6171 15.7853 92.8349 15.1837C92.0678 14.5821 91.6843 13.7623 91.6843 12.7245C91.6843 11.5664 92.0754 10.679 92.8575 10.0623C93.6546 9.44562 94.8053 9.13729 96.3093 9.13729H99.6935V8.34764C99.6935 7.5956 99.4378 7.00901 98.9265 6.58787C98.4301 6.16673 97.7307 5.95616 96.8283 5.95616C96.0311 5.95616 95.3693 6.13665 94.8429 6.49763C94.3315 6.84356 94.0156 7.30983 93.8953 7.89642H92.0453C92.1806 6.76836 92.677 5.88847 93.5343 5.25676C94.4067 4.62505 95.5348 4.30919 96.9185 4.30919C98.3925 4.30919 99.5281 4.67017 100.325 5.39213C101.137 6.09905 101.544 7.12182 101.544 8.46045V13.4239C101.544 13.9804 101.799 14.2587 102.311 14.2587H102.875ZM99.6935 10.694H96.1289C94.4142 10.694 93.5569 11.3332 93.5569 12.6117C93.5569 13.1833 93.7825 13.642 94.2337 13.9879C94.685 14.3339 95.2941 14.5069 96.0612 14.5069C97.1892 14.5069 98.0766 14.2136 98.7234 13.627C99.3702 13.0253 99.6935 12.2357 99.6935 11.258V10.694Z" fill="currentColor"></path>
-  <path d="M104.851 0C105.197 0 105.483 0.112806 105.708 0.338418C105.934 0.56403 106.047 0.849805 106.047 1.19574C106.047 1.54168 105.934 1.82746 105.708 2.05307C105.483 2.27868 105.197 2.39149 104.851 2.39149C104.505 2.39149 104.219 2.27868 103.994 2.05307C103.768 1.82746 103.655 1.54168 103.655 1.19574C103.655 0.849805 103.768 0.56403 103.994 0.338418C104.219 0.112806 104.505 0 104.851 0ZM103.926 4.44456H105.776V15.9508H103.926V4.44456Z" fill="currentColor"></path>
-  <path d="M88.067 14.1988C88.067 15.1664 87.2731 15.9508 86.2937 15.9508C85.3144 15.9508 84.5205 15.1664 84.5205 14.1988C84.5205 13.2312 85.3144 12.4468 86.2937 12.4468C87.2731 12.4468 88.067 13.2312 88.067 14.1988Z" fill="#0F6FFF"></path>
-</svg>
\ No newline at end of file
+
+<svg
+  {width}
+  {height}
+  viewBox="0 0 107 21"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="M3.0727 6.13665H0.99707V4.44456H3.0727V1.2183H4.92272V4.44456H7.83312V6.13665H4.92272V12.9276C4.92272 13.4089 5.01297 13.7548 5.19346 13.9654C5.38899 14.1609 5.71988 14.2587 6.18615 14.2587H8.1941V15.9508H6.07334C4.9904 15.9508 4.2158 15.7101 3.74954 15.2288C3.29831 14.7475 3.0727 13.9879 3.0727 12.9501V6.13665Z"
+    fill="currentColor"
+  />
+  <path
+    d="M14.1547 16.0861C13.0417 16.0861 12.049 15.838 11.1766 15.3416C10.3193 14.8453 9.64995 14.1534 9.16864 13.266C8.68734 12.3786 8.44669 11.3558 8.44669 10.1977C8.44669 9.03952 8.68734 8.01675 9.16864 7.12934C9.64995 6.24193 10.3193 5.55006 11.1766 5.05371C12.049 4.55736 13.0417 4.30919 14.1547 4.30919C15.2677 4.30919 16.2529 4.55736 17.1102 5.05371C17.9826 5.55006 18.6594 6.24193 19.1407 7.12934C19.622 8.01675 19.8627 9.03952 19.8627 10.1977C19.8627 11.3558 19.622 12.3786 19.1407 13.266C18.6594 14.1534 17.9826 14.8453 17.1102 15.3416C16.2529 15.838 15.2677 16.0861 14.1547 16.0861ZM14.1547 14.4392C14.9067 14.4392 15.5685 14.2587 16.1401 13.8977C16.7266 13.5367 17.1779 13.0329 17.4937 12.3861C17.8096 11.7393 17.9675 11.0099 17.9675 10.1977C17.9675 9.38546 17.8096 8.65598 17.4937 8.00923C17.1779 7.36247 16.7266 6.85861 16.1401 6.49763C15.5685 6.13665 14.9067 5.95616 14.1547 5.95616C13.4026 5.95616 12.7333 6.13665 12.1467 6.49763C11.5752 6.85861 11.1315 7.36247 10.8156 8.00923C10.4998 8.65598 10.3418 9.38546 10.3418 10.1977C10.3418 11.0099 10.4998 11.7393 10.8156 12.3861C11.1315 13.0329 11.5752 13.5367 12.1467 13.8977C12.7333 14.2587 13.4026 14.4392 14.1547 14.4392Z"
+    fill="currentColor"
+  />
+  <path
+    d="M32.5327 4.44456V15.3867C32.5327 18.8612 30.6301 20.5984 26.8247 20.5984C25.3357 20.5984 24.1174 20.26 23.1698 19.5831C22.2373 18.9063 21.6958 17.9437 21.5454 16.6953H23.4406C23.591 17.4172 23.9745 17.9738 24.5912 18.3648C25.2079 18.7559 25.9975 18.9514 26.9601 18.9514C29.4418 18.9514 30.6827 17.7406 30.6827 15.3191V13.9654C29.8555 15.3792 28.5394 16.0861 26.7345 16.0861C25.6666 16.0861 24.7115 15.853 23.8692 15.3867C23.042 14.9205 22.3877 14.2436 21.9064 13.3562C21.4401 12.4688 21.207 11.416 21.207 10.1977C21.207 9.03952 21.4401 8.01675 21.9064 7.12934C22.3877 6.24193 23.0495 5.55006 23.8918 5.05371C24.7341 4.55736 25.6816 4.30919 26.7345 4.30919C27.667 4.30919 28.4642 4.4972 29.126 4.87322C29.7878 5.2342 30.3067 5.75311 30.6827 6.42994L30.9309 4.44456H32.5327ZM26.915 14.4392C27.652 14.4392 28.3063 14.2662 28.8778 13.9203C29.4644 13.5593 29.9156 13.0629 30.2315 12.4312C30.5473 11.7845 30.7053 11.055 30.7053 10.2428C30.7053 9.41554 30.5473 8.67854 30.2315 8.03179C29.9156 7.36999 29.4644 6.85861 28.8778 6.49763C28.3063 6.13665 27.652 5.95616 26.915 5.95616C25.7719 5.95616 24.8469 6.35474 24.14 7.1519C23.4481 7.93402 23.1021 8.94928 23.1021 10.1977C23.1021 11.4461 23.4481 12.4688 24.14 13.266C24.8469 14.0481 25.7719 14.4392 26.915 14.4392Z"
+    fill="currentColor"
+  />
+  <path
+    d="M34.2758 10.1977C34.2758 9.02448 34.5014 7.99419 34.9526 7.10678C35.4189 6.21937 36.0656 5.53502 36.8929 5.05371C37.7201 4.55736 38.6677 4.30919 39.7356 4.30919C40.7884 4.30919 41.721 4.52728 42.5332 4.96346C43.3454 5.39965 43.9846 6.03136 44.4509 6.85861C44.9171 7.68585 45.1653 8.65598 45.1954 9.769C45.1954 9.93445 45.1803 10.1977 45.1503 10.5586H36.216V10.7166C36.2461 11.8446 36.592 12.7471 37.2538 13.4239C37.9156 14.1008 38.7805 14.4392 39.8484 14.4392C40.6756 14.4392 41.375 14.2361 41.9466 13.83C42.5332 13.4089 42.9242 12.8298 43.1198 12.0928H44.9923C44.7667 13.266 44.2027 14.2286 43.3002 14.9806C42.3978 15.7176 41.2923 16.0861 39.9837 16.0861C38.8406 16.0861 37.8404 15.8455 36.9831 15.3642C36.1258 14.8678 35.4565 14.176 34.9752 13.2885C34.5089 12.3861 34.2758 11.3558 34.2758 10.1977ZM43.2777 9.02448C43.1874 8.04683 42.8189 7.28727 42.1722 6.7458C41.5405 6.20433 40.7358 5.9336 39.7581 5.9336C38.8858 5.9336 38.1187 6.21937 37.4569 6.79092C36.7951 7.36247 36.4191 8.10699 36.3288 9.02448H43.2777Z"
+    fill="currentColor"
+  />
+  <path
+    d="M47.5703 6.13665H45.4947V4.44456H47.5703V1.2183H49.4203V4.44456H52.3307V6.13665H49.4203V12.9276C49.4203 13.4089 49.5106 13.7548 49.6911 13.9654C49.8866 14.1609 50.2175 14.2587 50.6838 14.2587H52.6917V15.9508H50.571C49.488 15.9508 48.7134 15.7101 48.2472 15.2288C47.7959 14.7475 47.5703 13.9879 47.5703 12.9501V6.13665Z"
+    fill="currentColor"
+  />
+  <path
+    d="M59.2771 4.30919C60.6759 4.30919 61.804 4.73033 62.6613 5.57262C63.5186 6.4149 63.9473 7.73849 63.9473 9.54339V15.9508H62.0973V9.6562C62.0973 8.46797 61.8265 7.56552 61.2851 6.94885C60.7436 6.31714 59.969 6.00128 58.9613 6.00128C57.8934 6.00128 57.0436 6.3773 56.4119 7.12934C55.7952 7.88138 55.4868 8.90415 55.4868 10.1977V15.9508H53.6368V0.157928H55.4868V6.3397C55.8629 5.73807 56.3592 5.24924 56.9759 4.87322C57.6076 4.4972 58.3747 4.30919 59.2771 4.30919Z"
+    fill="currentColor"
+  />
+  <path
+    d="M65.5284 10.1977C65.5284 9.02448 65.754 7.99419 66.2052 7.10678C66.6715 6.21937 67.3182 5.53502 68.1455 5.05371C68.9727 4.55736 69.9203 4.30919 70.9882 4.30919C72.041 4.30919 72.9736 4.52728 73.7858 4.96346C74.598 5.39965 75.2372 6.03136 75.7035 6.85861C76.1697 7.68585 76.4179 8.65598 76.448 9.769C76.448 9.93445 76.4329 10.1977 76.4029 10.5586H67.4686V10.7166C67.4987 11.8446 67.8446 12.7471 68.5064 13.4239C69.1682 14.1008 70.0331 14.4392 71.101 14.4392C71.9282 14.4392 72.6276 14.2361 73.1992 13.83C73.7858 13.4089 74.1768 12.8298 74.3723 12.0928H76.2449C76.0193 13.266 75.4553 14.2286 74.5528 14.9806C73.6504 15.7176 72.5449 16.0861 71.2363 16.0861C70.0932 16.0861 69.093 15.8455 68.2357 15.3642C67.3784 14.8678 66.7091 14.176 66.2278 13.2885C65.7615 12.3861 65.5284 11.3558 65.5284 10.1977ZM74.5303 9.02448C74.44 8.04683 74.0715 7.28727 73.4248 6.7458C72.7931 6.20433 71.9884 5.9336 71.0107 5.9336C70.1384 5.9336 69.3713 6.21937 68.7095 6.79092C68.0477 7.36247 67.6717 8.10699 67.5814 9.02448H74.5303Z"
+    fill="currentColor"
+  />
+  <path
+    d="M84.0314 4.44456V6.24945H83.1064C82.0084 6.24945 81.2188 6.61795 80.7374 7.35495C80.2712 8.09195 80.038 9.00192 80.038 10.0849V15.9508H78.188V4.44456H79.7899L80.038 6.18177C80.3689 5.65534 80.7976 5.2342 81.324 4.91834C81.8505 4.60249 82.5799 4.44456 83.5125 4.44456H84.0314Z"
+    fill="currentColor"
+  />
+  <path
+    d="M102.875 14.2587V15.9508H101.859C101.122 15.9508 100.596 15.8004 100.28 15.4995C99.9643 15.1987 99.7988 14.755 99.7838 14.1684C98.8964 15.4469 97.6104 16.0861 95.9258 16.0861C94.6473 16.0861 93.6171 15.7853 92.8349 15.1837C92.0678 14.5821 91.6843 13.7623 91.6843 12.7245C91.6843 11.5664 92.0754 10.679 92.8575 10.0623C93.6546 9.44562 94.8053 9.13729 96.3093 9.13729H99.6935V8.34764C99.6935 7.5956 99.4378 7.00901 98.9265 6.58787C98.4301 6.16673 97.7307 5.95616 96.8283 5.95616C96.0311 5.95616 95.3693 6.13665 94.8429 6.49763C94.3315 6.84356 94.0156 7.30983 93.8953 7.89642H92.0453C92.1806 6.76836 92.677 5.88847 93.5343 5.25676C94.4067 4.62505 95.5348 4.30919 96.9185 4.30919C98.3925 4.30919 99.5281 4.67017 100.325 5.39213C101.137 6.09905 101.544 7.12182 101.544 8.46045V13.4239C101.544 13.9804 101.799 14.2587 102.311 14.2587H102.875ZM99.6935 10.694H96.1289C94.4142 10.694 93.5569 11.3332 93.5569 12.6117C93.5569 13.1833 93.7825 13.642 94.2337 13.9879C94.685 14.3339 95.2941 14.5069 96.0612 14.5069C97.1892 14.5069 98.0766 14.2136 98.7234 13.627C99.3702 13.0253 99.6935 12.2357 99.6935 11.258V10.694Z"
+    fill="currentColor"
+  />
+  <path
+    d="M104.851 0C105.197 0 105.483 0.112806 105.708 0.338418C105.934 0.56403 106.047 0.849805 106.047 1.19574C106.047 1.54168 105.934 1.82746 105.708 2.05307C105.483 2.27868 105.197 2.39149 104.851 2.39149C104.505 2.39149 104.219 2.27868 103.994 2.05307C103.768 1.82746 103.655 1.54168 103.655 1.19574C103.655 0.849805 103.768 0.56403 103.994 0.338418C104.219 0.112806 104.505 0 104.851 0ZM103.926 4.44456H105.776V15.9508H103.926V4.44456Z"
+    fill="currentColor"
+  />
+  <path
+    d="M88.067 14.1988C88.067 15.1664 87.2731 15.9508 86.2937 15.9508C85.3144 15.9508 84.5205 15.1664 84.5205 14.1988C84.5205 13.2312 85.3144 12.4468 86.2937 12.4468C87.2731 12.4468 88.067 13.2312 88.067 14.1988Z"
+    fill="#0F6FFF"
+  />
+</svg>
diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index b91b2129e6..91d9c99af6 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -127,7 +127,7 @@
         name: user.firstName ? user.firstName + " " + user.lastName : "",
         userGroups,
         __selectable:
-          role.value === Constants.BudibaseRoles.Owner ? false : undefined,
+          role?.value === Constants.BudibaseRoles.Owner ? false : undefined,
         apps: [...new Set(Object.keys(user.roles))],
         access: role.sortOrder,
       }
diff --git a/packages/builder/vitest.setup.js b/packages/builder/vitest.setup.js
index 92df5e9600..0204a1a772 100644
--- a/packages/builder/vitest.setup.js
+++ b/packages/builder/vitest.setup.js
@@ -1,4 +1,4 @@
-import { expect, vi } from "vitest"
+import { expect } from "vitest"
 import "@testing-library/jest-dom/vitest"
 
 global.ResizeObserver = require("resize-observer-polyfill")
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 2b0b97ccc0..6e9c29af97 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -153,5 +153,5 @@ export enum ConfigType {
   OIDC = "oidc",
   OIDC_LOGOS = "logos_oidc",
   SCIM = "scim",
-  AI = "ai"
+  AI = "ai",
 }
diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts
index bf2bcb72b6..cbcb16f7b6 100644
--- a/packages/types/src/sdk/licensing/feature.ts
+++ b/packages/types/src/sdk/licensing/feature.ts
@@ -16,7 +16,7 @@ export enum Feature {
   VIEW_PERMISSIONS = "viewPermissions",
   VIEW_READONLY_COLUMNS = "viewReadonlyColumns",
   BUDIBASE_AI = "budibaseAI",
-  AI_CUSTOM_CONFIGS = "customAIConfigs"
+  AI_CUSTOM_CONFIGS = "customAIConfigs",
 }
 
 export type PlanFeatures = { [key in PlanType]: Feature[] | undefined }
diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts
index 748d90fda7..75fc18e8e6 100644
--- a/packages/types/src/sdk/licensing/quota.ts
+++ b/packages/types/src/sdk/licensing/quota.ts
@@ -24,7 +24,7 @@ export enum MonthlyQuotaName {
   QUERIES = "queries",
   AUTOMATIONS = "automations",
   DAY_PASSES = "dayPasses",
-  BUDIBASE_AI_CREDITS = "budibaseAICredits"
+  BUDIBASE_AI_CREDITS = "budibaseAICredits",
 }
 
 export enum ConstantQuotaName {
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 4a19168c4f..d3eef71240 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -199,7 +199,10 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
   await verifySSOConfig(ConfigType.OIDC, config.configs[0])
 }
 
-export async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) {
+export async function verifyAIConfig(
+  config: AIConfig,
+  existingConfig?: AIConfig
+) {
   if (!existingConfig) return
 
   // ensure that the redacted API keys are not overwritten in the DB
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 393a11a043..2ef00252ff 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,3 +1,4 @@
+import { expect } from "vitest"
 import { configs } from "@budibase/backend-core"
 import { UserCtx } from "@budibase/types"
 import * as pro from "@budibase/pro"
@@ -7,11 +8,10 @@ jest.mock("@budibase/backend-core", () => ({
   ...jest.requireActual("@budibase/backend-core"),
   configs: {
     getConfig: jest.fn(),
-    save: jest.fn()
+    save: jest.fn(),
   },
 }))
 
-
 describe("Global configs controller", () => {
   afterEach(() => {
     jest.resetAllMocks()
@@ -23,14 +23,14 @@ describe("Global configs controller", () => {
         ai: {
           apiKey: "abc123APIKey",
           baseUrl: "https://api.example.com",
-        }
-      }
+        },
+      },
     })
     const ctx = {
       params: {
-        type: "ai"
+        type: "ai",
       },
-      throw: jest.fn()
+      throw: jest.fn(),
     } as UserCtx
 
     await find(ctx)
@@ -39,9 +39,9 @@ describe("Global configs controller", () => {
       config: {
         ai: {
           apiKey: "--secret-value--",
-          "baseUrl": "https://api.example.com"
-        }
-      }
+          baseUrl: "https://api.example.com",
+        },
+      },
     })
   })
 
@@ -52,14 +52,14 @@ describe("Global configs controller", () => {
         ai: {
           apiKey: "abc123APIKey",
           baseUrl: "https://api.example.com",
-        }
-      }
+        },
+      },
     })
     const ctx = {
       params: {
-        type: "ai"
+        type: "ai",
       },
-      throw: jest.fn()
+      throw: jest.fn(),
     } as UserCtx
 
     await find(ctx)
@@ -75,9 +75,9 @@ describe("Global configs controller", () => {
         },
         ai: {
           apiKey: "--secret-value--",
-          "baseUrl": "https://api.example.com"
-        }
-      }
+          baseUrl: "https://api.example.com",
+        },
+      },
     })
   })
 
@@ -91,9 +91,9 @@ describe("Global configs controller", () => {
           name: "MyConfig",
           active: true,
           defaultModel: "gpt4",
-          apiKey: "--secret-value--"
-        }
-      }
+          apiKey: "--secret-value--",
+        },
+      },
     }
 
     const existingConfig = {
@@ -105,9 +105,9 @@ describe("Global configs controller", () => {
           name: "MyConfig",
           active: true,
           defaultModel: "gpt4",
-          apiKey: "myapikey"
-        }
-      }
+          apiKey: "myapikey",
+        },
+      },
     }
 
     await verifyAIConfig(newConfig, existingConfig)
diff --git a/yarn.lock b/yarn.lock
index 4e4e528334..146fa74d5b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2053,6 +2053,44 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
+"@budibase/backend-core@2.31.8":
+  version "0.0.0"
+  dependencies:
+    "@budibase/nano" "10.1.5"
+    "@budibase/pouchdb-replication-stream" "1.2.11"
+    "@budibase/shared-core" "0.0.0"
+    "@budibase/types" "0.0.0"
+    aws-cloudfront-sign "3.0.2"
+    aws-sdk "2.1030.0"
+    bcrypt "5.1.0"
+    bcryptjs "2.4.3"
+    bull "4.10.1"
+    correlation-id "4.0.0"
+    dd-trace "5.2.0"
+    dotenv "16.0.1"
+    ioredis "5.3.2"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    knex "2.4.2"
+    koa-passport "^6.0.0"
+    koa-pino-logger "4.0.0"
+    lodash "4.17.21"
+    node-fetch "2.6.7"
+    passport-google-oauth "2.0.0"
+    passport-local "1.0.0"
+    passport-oauth2-refresh "^2.1.0"
+    pino "8.11.0"
+    pino-http "8.3.3"
+    posthog-node "4.0.1"
+    pouchdb "7.3.0"
+    pouchdb-find "7.2.2"
+    redlock "4.2.0"
+    rotating-file-stream "3.1.0"
+    sanitize-s3-objectkey "0.0.1"
+    semver "^7.5.4"
+    tar-fs "2.1.1"
+    uuid "^8.3.2"
+
 "@budibase/handlebars-helpers@^0.13.2":
   version "0.13.2"
   resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77"
@@ -2095,6 +2133,45 @@
     pouchdb-promise "^6.0.4"
     through2 "^2.0.0"
 
+"@budibase/pro@npm:@budibase/pro@latest":
+  version "2.31.8"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.31.8.tgz#92b27f99f815f5d20bf58bfae916760b14a036da"
+  integrity sha512-nmNKVoMdUVqEIq6xqoBq0gVBCLkoPMszmn0Zu0SJ/Dc2SpsXhPz9S3n9xXfAA+FHUg9LgUAS+eKPCKPWZXtDHQ==
+  dependencies:
+    "@budibase/backend-core" "2.31.8"
+    "@budibase/shared-core" "2.31.8"
+    "@budibase/string-templates" "2.31.8"
+    "@budibase/types" "2.31.8"
+    "@koa/router" "8.0.8"
+    bull "4.10.1"
+    dd-trace "5.2.0"
+    joi "17.6.0"
+    jsonwebtoken "9.0.2"
+    lru-cache "^7.14.1"
+    memorystream "^0.3.1"
+    node-fetch "2.6.7"
+    scim-patch "^0.8.1"
+    scim2-parse-filter "^0.2.8"
+
+"@budibase/shared-core@2.31.8":
+  version "0.0.0"
+  dependencies:
+    "@budibase/types" "0.0.0"
+    cron-validate "1.4.5"
+
+"@budibase/string-templates@2.31.8":
+  version "0.0.0"
+  dependencies:
+    "@budibase/handlebars-helpers" "^0.13.2"
+    dayjs "^1.10.8"
+    handlebars "^4.7.8"
+    lodash.clonedeep "^4.5.0"
+
+"@budibase/types@2.31.8":
+  version "0.0.0"
+  dependencies:
+    scim-patch "^0.8.1"
+
 "@bull-board/api@5.10.2":
   version "5.10.2"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3"
@@ -7448,30 +7525,7 @@ axios-retry@^3.1.9:
     "@babel/runtime" "^7.15.4"
     is-retry-allowed "^2.2.0"
 
-axios@0.24.0:
-  version "0.24.0"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6"
-  integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA==
-  dependencies:
-    follow-redirects "^1.14.4"
-
-axios@1.1.3:
-  version "1.1.3"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35"
-  integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==
-  dependencies:
-    follow-redirects "^1.15.0"
-    form-data "^4.0.0"
-    proxy-from-env "^1.1.0"
-
-axios@^0.21.1:
-  version "0.21.4"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"
-  integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==
-  dependencies:
-    follow-redirects "^1.14.0"
-
-axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
+axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2:
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4"
   integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww==
@@ -11581,11 +11635,6 @@ fn.name@1.x.x:
   resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
   integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
 
-follow-redirects@^1.14.0, follow-redirects@^1.14.4:
-  version "1.15.9"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1"
-  integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==
-
 follow-redirects@^1.15.0:
   version "1.15.6"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b"
@@ -12679,12 +12728,7 @@ http-assert@^1.3.0:
     deep-equal "~1.0.1"
     http-errors "~1.8.0"
 
-http-cache-semantics@3.8.1:
-  version "3.8.1"
-  resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2"
-  integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==
-
-http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
+http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a"
   integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==
@@ -13695,6 +13739,11 @@ isobject@^3.0.1:
   resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df"
   integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==
 
+isobject@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0"
+  integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA==
+
 isolated-vm@^4.7.2:
   version "4.7.2"
   resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65"
@@ -16254,7 +16303,7 @@ msgpackr-extract@^3.0.2:
     "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2"
     "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2"
 
-msgpackr@^1.5.2:
+msgpackr@1.10.1, msgpackr@^1.5.2:
   version "1.10.1"
   resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555"
   integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==
@@ -16448,25 +16497,13 @@ node-domexception@1.0.0:
   resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
   integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
 
-node-fetch@2.6.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd"
-  integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==
-
-node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7:
+node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@^2.6.9, node-fetch@^2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
-  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^1.2.1, node-forge@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -17617,15 +17654,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0:
   resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4"
   integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA==
 
-passport@^0.4.0:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270"
-  integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg==
-  dependencies:
-    passport-strategy "1.x.x"
-    pause "0.0.1"
-
-passport@^0.6.0:
+passport@0.6.0, passport@^0.4.0, passport@^0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d"
   integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==
@@ -18898,7 +18927,7 @@ pseudomap@^1.0.2:
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
   integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==
 
-psl@^1.1.28, psl@^1.1.33:
+psl@^1.1.33:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7"
   integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==
@@ -19970,11 +19999,6 @@ sax@1.2.1:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
   integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==
 
-sax@>=0.1.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f"
-  integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==
-
 sax@>=0.6.0:
   version "1.2.4"
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
@@ -20047,33 +20071,13 @@ semver-diff@^3.1.1:
   dependencies:
     semver "^6.3.0"
 
-"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1:
-  version "5.7.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8"
-  integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==
-
-semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3:
+"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1:
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e"
   integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==
   dependencies:
     lru-cache "^6.0.0"
 
-semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1:
-  version "6.3.1"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
-  integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
-
-semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
-  version "7.6.3"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
-  integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
-
-semver@~2.3.1:
-  version "2.3.2"
-  resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52"
-  integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA==
-
 seq-queue@^0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e"
@@ -21636,7 +21640,7 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
-"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2:
+tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0:
   version "4.1.3"
   resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf"
   integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==
@@ -21646,14 +21650,6 @@ touch@^3.1.0:
     universalify "^0.2.0"
     url-parse "^1.5.3"
 
-tough-cookie@~2.5.0:
-  version "2.5.0"
-  resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2"
-  integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==
-  dependencies:
-    psl "^1.1.28"
-    punycode "^2.1.1"
-
 tr46@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240"
@@ -22182,6 +22178,14 @@ unpipe@1.0.0:
   resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
   integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
 
+unset-value@2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3"
+  integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg==
+  dependencies:
+    has-value "^2.0.2"
+    isobject "^4.0.0"
+
 untildify@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b"
@@ -22961,33 +22965,10 @@ xml-parse-from-string@^1.0.0:
   resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28"
   integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g==
 
-xml2js@0.1.x:
-  version "0.1.14"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c"
-  integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA==
-  dependencies:
-    sax ">=0.1.1"
-
-xml2js@0.4.19:
-  version "0.4.19"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
-  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~9.0.1"
-
-xml2js@0.5.0:
-  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7"
-  integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==
-  dependencies:
-    sax ">=0.6.0"
-    xmlbuilder "~11.0.0"
-
-xml2js@^0.4.19, xml2js@^0.4.5:
-  version "0.4.23"
-  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66"
-  integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==
+xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499"
+  integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==
   dependencies:
     sax ">=0.6.0"
     xmlbuilder "~11.0.0"
@@ -22997,11 +22978,6 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
-xmlbuilder@~9.0.1:
-  version "9.0.7"
-  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
-  integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ==
-
 xmlchars@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb"

From b4a4f81308fd6c6cccb91abaaee45a58828a5d48 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 9 Sep 2024 16:52:56 +0100
Subject: [PATCH 44/69] linting

---
 packages/account-portal                       |  2 +-
 .../backend-core/src/events/publishers/ai.ts  | 21 +++++++++++++++++++
 .../src/events/publishers/index.ts            |  1 +
 .../portal/settings/ai/ConfigModal.svelte     |  6 +++++-
 packages/pro                                  |  2 +-
 packages/types/src/documents/global/quotas.ts |  2 ++
 packages/types/src/sdk/events/ai.ts           |  5 +++++
 packages/types/src/sdk/events/event.ts        |  8 +++++++
 packages/types/src/sdk/events/index.ts        |  1 +
 .../src/api/controllers/global/configs.ts     | 11 ++++++++++
 10 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 packages/backend-core/src/events/publishers/ai.ts
 create mode 100644 packages/types/src/sdk/events/ai.ts

diff --git a/packages/account-portal b/packages/account-portal
index c24374879d..ce60bf9231 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit c24374879d2b61516fabc24d7404e7da235be05e
+Subproject commit ce60bf923115855a9cd0bb991297cccd308eae13
diff --git a/packages/backend-core/src/events/publishers/ai.ts b/packages/backend-core/src/events/publishers/ai.ts
new file mode 100644
index 0000000000..168ac9f796
--- /dev/null
+++ b/packages/backend-core/src/events/publishers/ai.ts
@@ -0,0 +1,21 @@
+import { publishEvent } from "../events"
+import {
+  Event,
+  AIConfigCreatedEvent,
+  AIConfigUpdatedEvent,
+} from "@budibase/types"
+
+async function AIConfigCreated(timestamp?: string | number) {
+  const properties: AIConfigCreatedEvent = {}
+  await publishEvent(Event.AI_CONFIG_CREATED, properties, timestamp)
+}
+
+async function AIConfigUpdated() {
+  const properties: AIConfigUpdatedEvent = {}
+  await publishEvent(Event.AI_CONFIG_UPDATED, properties)
+}
+
+export default {
+  AIConfigCreated,
+  AIConfigUpdated,
+}
diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts
index 87a34bf3f1..9c92b80499 100644
--- a/packages/backend-core/src/events/publishers/index.ts
+++ b/packages/backend-core/src/events/publishers/index.ts
@@ -4,6 +4,7 @@ export { default as auth } from "./auth"
 export { default as automation } from "./automation"
 export { default as datasource } from "./datasource"
 export { default as email } from "./email"
+export { default as ai } from "./ai"
 export { default as license } from "./license"
 export { default as layout } from "./layout"
 export { default as org } from "./org"
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
index 2f7486d087..d480689028 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte
@@ -54,7 +54,11 @@
   </div>
   <div class="form-row">
     <Label size="M">Name</Label>
-    <Input placeholder={"Enter a name"} bind:value={config.name} />
+    <Input
+      error={config.name === "Budibase AI" ? "Cannot use this name" : null}
+      placeholder={"Enter a name"}
+      bind:value={config.name}
+    />
   </div>
   <div class="form-row">
     <Label size="M">Default Model</Label>
diff --git a/packages/pro b/packages/pro
index a4d1d15d9c..62586bc47b 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758
+Subproject commit 62586bc47bf2af507356b0e1e64df8b0ae9fad43
diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts
index 4eb1168f7d..4726dabf0d 100644
--- a/packages/types/src/documents/global/quotas.ts
+++ b/packages/types/src/documents/global/quotas.ts
@@ -35,6 +35,7 @@ export interface StaticUsage {
   [StaticQuotaName.CREATORS]: number
   [StaticQuotaName.USER_GROUPS]: number
   [StaticQuotaName.ROWS]: number
+  [StaticQuotaName.AI_CUSTOM_CONFIGS]: number
   triggers: {
     [key in StaticQuotaName]?: QuotaTriggers
   }
@@ -44,6 +45,7 @@ export interface MonthlyUsage {
   [MonthlyQuotaName.QUERIES]: number
   [MonthlyQuotaName.AUTOMATIONS]: number
   [MonthlyQuotaName.DAY_PASSES]: number
+  [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: number
   triggers: {
     [key in MonthlyQuotaName]?: QuotaTriggers
   }
diff --git a/packages/types/src/sdk/events/ai.ts b/packages/types/src/sdk/events/ai.ts
new file mode 100644
index 0000000000..740c59d5f9
--- /dev/null
+++ b/packages/types/src/sdk/events/ai.ts
@@ -0,0 +1,5 @@
+import { BaseEvent } from "./event"
+
+export interface AIConfigCreatedEvent extends BaseEvent {}
+
+export interface AIConfigUpdatedEvent extends BaseEvent {}
diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts
index 05f4fe4be9..242b182dec 100644
--- a/packages/types/src/sdk/events/event.ts
+++ b/packages/types/src/sdk/events/event.ts
@@ -33,6 +33,10 @@ export enum Event {
   EMAIL_SMTP_CREATED = "email:smtp:created",
   EMAIL_SMTP_UPDATED = "email:smtp:updated",
 
+  // AI
+  AI_CONFIG_CREATED = "ai:config:created",
+  AI_CONFIG_UPDATED = "ai:config:updated",
+
   // AUTH
   AUTH_SSO_CREATED = "auth:sso:created",
   AUTH_SSO_UPDATED = "auth:sso:updated",
@@ -243,6 +247,10 @@ export const AuditedEventFriendlyName: Record<Event, string | undefined> = {
   [Event.EMAIL_SMTP_CREATED]: `Email configuration created`,
   [Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`,
 
+  // AI
+  [Event.AI_CONFIG_CREATED]: `AI configuration created`,
+  [Event.AI_CONFIG_UPDATED]: `AI configuration updated`,
+
   // AUTH
   [Event.AUTH_SSO_CREATED]: `SSO configuration created`,
   [Event.AUTH_SSO_UPDATED]: `SSO configuration updated`,
diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts
index 745f84d2a3..043e62faa4 100644
--- a/packages/types/src/sdk/events/index.ts
+++ b/packages/types/src/sdk/events/index.ts
@@ -2,6 +2,7 @@ export * from "./app"
 export * from "./auth"
 export * from "./automation"
 export * from "./email"
+export * from "./ai"
 export * from "./datasource"
 export * from "./event"
 export * from "./layout"
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index d3eef71240..b8feb6158f 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -31,6 +31,7 @@ import {
   OIDCLogosConfig,
   AIConfig,
   PASSWORD_REPLACEMENT,
+  isAIConfig,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
 
@@ -40,6 +41,11 @@ const getEventFns = async (config: Config, existing?: Config) => {
   if (!existing) {
     if (isSMTPConfig(config)) {
       fns.push(events.email.SMTPCreated)
+    } else if (isAIConfig(config)) {
+      fns.push(() => events.ai.AIConfigCreated)
+      fns.push(() =>
+        pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length)
+      )
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE))
       if (config.config.activated) {
@@ -76,6 +82,11 @@ const getEventFns = async (config: Config, existing?: Config) => {
   } else {
     if (isSMTPConfig(config)) {
       fns.push(events.email.SMTPUpdated)
+    } else if (isAIConfig(config)) {
+      fns.push(() => events.ai.AIConfigUpdated)
+      fns.push(() =>
+        pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length)
+      )
     } else if (isGoogleConfig(config)) {
       fns.push(() => events.auth.SSOUpdated(ConfigType.GOOGLE))
       if (!existing.config.activated && config.config.activated) {

From 96fbc8fff093fb5d49e6fa5a99441246c45b389e Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 9 Sep 2024 18:07:47 +0100
Subject: [PATCH 45/69] feature flag support

---
 packages/backend-core/src/features/index.ts   |  1 +
 .../tests/core/utilities/structures/quotas.ts |  4 +++
 .../builder/portal/settings/ai/index.svelte   |  4 ++-
 packages/builder/src/stores/portal/menu.js    | 15 ++++++---
 packages/types/src/sdk/featureFlag.ts         |  1 +
 .../src/api/controllers/global/configs.ts     |  4 +--
 .../controllers/global/tests/configs.spec.ts  | 31 ++++++++++++++++++-
 7 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts
index b84237b1f0..f855daf8a5 100644
--- a/packages/backend-core/src/features/index.ts
+++ b/packages/backend-core/src/features/index.ts
@@ -268,4 +268,5 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
 export const flags = new FlagSet({
   DEFAULT_VALUES: Flag.boolean(env.isDev()),
   SQS: Flag.boolean(env.isDev()),
+  AI_CUSTOM_CONFIGS: Flag.boolean(env.isDev()),
 })
diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts
index 8d0b05fe1e..58817e4831 100644
--- a/packages/backend-core/tests/core/utilities/structures/quotas.ts
+++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts
@@ -17,6 +17,7 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
         automations: 0,
         dayPasses: 0,
         queries: 0,
+        budibaseAICredits: 0,
         triggers: {},
         breakdown: {
           rowQueries: {
@@ -46,12 +47,14 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
         automations: 0,
         dayPasses: 0,
         queries: 0,
+        budibaseAICredits: 0,
         triggers: {},
       },
       current: {
         automations: 0,
         dayPasses: 0,
         queries: 0,
+        budibaseAICredits: 0,
         triggers: {},
       },
     },
@@ -62,6 +65,7 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => {
       creators,
       userGroups: 0,
       rows: 0,
+      aiCustomConfigs: 0,
       triggers: {},
     },
   }
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index 77d6aa952a..c4403e5045 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -80,7 +80,9 @@
   }
 
   async function deleteConfig(key) {
-    // Delete a configuration
+    // We don't store the default BB AI config in the DB
+    delete fullAIConfig.config.budibase_ai
+    // Delete the configuration
     delete fullAIConfig.config[key]
 
     try {
diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index ec40f6d4ee..1e6282894e 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -1,7 +1,9 @@
 import { derived } from "svelte/store"
 import { admin } from "./admin"
 import { auth } from "./auth"
+import { isEnabled } from "helpers/featureFlags"
 import { sdk } from "@budibase/shared-core"
+import { FeatureFlag } from "@budibase/types";
 
 export const menu = derived([admin, auth], ([$admin, $auth]) => {
   const user = $auth?.user
@@ -45,10 +47,6 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         title: "Auth",
         href: "/builder/portal/settings/auth",
       },
-      {
-        title: "AI",
-        href: "/builder/portal/settings/ai",
-      },
       {
         title: "Email",
         href: "/builder/portal/settings/email",
@@ -66,6 +64,15 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
         href: "/builder/portal/settings/environment",
       },
     ]
+    if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) {
+      settingsSubPages.push(
+        {
+          title: "AI",
+          href: "/builder/portal/settings/ai",
+        }
+      )
+    }
+
     if (!cloud) {
       settingsSubPages.push({
         title: "Version",
diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts
index 257b4ee576..69b2a3b1b2 100644
--- a/packages/types/src/sdk/featureFlag.ts
+++ b/packages/types/src/sdk/featureFlag.ts
@@ -1,6 +1,7 @@
 export enum FeatureFlag {
   PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE",
   PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT",
+  AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS",
 }
 
 export interface TenantFeatureFlags {
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index b8feb6158f..ffe7a1c717 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -135,7 +135,6 @@ const getEventFns = async (config: Config, existing?: Config) => {
       }
     }
   }
-
   return fns
 }
 
@@ -344,11 +343,12 @@ async function enrichAIConfig(aiConfig: AIConfig) {
 
   // Return the Budibase AI data source as part of the response if licensing allows
   const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
+  const defaultConfigExists = Object.keys(aiConfig.config).some(key => aiConfig.config[key].isDefault)
   if (budibaseAIEnabled) {
     aiConfig.config["budibase_ai"] = {
       provider: "OpenAI",
       active: true,
-      isDefault: true,
+      isDefault: !defaultConfigExists,
       defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL,
       name: "Budibase AI",
     }
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 2ef00252ff..4bb3d598a6 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,4 +1,3 @@
-import { expect } from "vitest"
 import { configs } from "@budibase/backend-core"
 import { UserCtx } from "@budibase/types"
 import * as pro from "@budibase/pro"
@@ -81,6 +80,35 @@ describe("Global configs controller", () => {
     })
   })
 
+  it("Should not not return the default Budibase AI config when on self host", async () => {
+    pro.features.isBudibaseAIEnabled = jest.fn(() => false)
+    configs.getConfig.mockResolvedValue({
+      config: {
+        ai: {
+          apiKey: "abc123APIKey",
+          baseUrl: "https://api.example.com",
+        },
+      },
+    })
+    const ctx = {
+      params: {
+        type: "ai",
+      },
+      throw: jest.fn(),
+    } as UserCtx
+
+    await find(ctx)
+
+    expect(ctx.body).toEqual({
+      config: {
+        ai: {
+          apiKey: "--secret-value--",
+          baseUrl: "https://api.example.com",
+        },
+      },
+    })
+  })
+
   it("Should not update existing secrets when updating an existing AI Config", async () => {
     const newConfig = {
       type: "ai",
@@ -114,4 +142,5 @@ describe("Global configs controller", () => {
     // should be unchanged
     expect(newConfig.config.aiconfig.apiKey === "myapikey")
   })
+
 })

From 43ae48cc822cd2a5164ea20921d6208d81e8772b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 9 Sep 2024 18:14:02 +0100
Subject: [PATCH 46/69] Test cases for return none/all with the logical
 operators, as well as updates to the hasFilter to support this correctly.

---
 .../src/api/routes/tests/search.spec.ts       | 46 ++++++++++++++++-
 packages/shared-core/src/filters.ts           | 49 +++++++++++++++----
 2 files changed, 84 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index 4b8c3eb5a1..08e22e7409 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -9,10 +9,10 @@ import {
   db as dbCore,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
+  setEnv as setCoreEnv,
   SQLITE_DESIGN_DOC_ID,
   utils,
   withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
@@ -2921,6 +2921,28 @@ describe.each([
             'Invalid body - "query.$and.conditions[1].$and.conditions" is required'
           )
         })
+
+      it("returns no rows when onEmptyFilter set to none", async () => {
+        await expectSearch({
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+            $and: {
+              conditions: [{ equal: { name: "" } }],
+            },
+          },
+        }).toFindNothing()
+      })
+
+      it("returns all rows when onEmptyFilter set to all", async () => {
+        await expectSearch({
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            $and: {
+              conditions: [{ equal: { name: "" } }],
+            },
+          },
+        }).toHaveLength(4)
+      })
     })
 
   !isLucene &&
@@ -3049,5 +3071,27 @@ describe.each([
           },
         }).toContainExactly([{ age: 1, name: "Jane" }])
       })
+
+      it("returns no rows when onEmptyFilter set to none", async () => {
+        await expectSearch({
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+            $or: {
+              conditions: [{ equal: { name: "" } }],
+            },
+          },
+        }).toFindNothing()
+      })
+
+      it("returns all rows when onEmptyFilter set to all", async () => {
+        await expectSearch({
+          query: {
+            onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+            $or: {
+              conditions: [{ equal: { name: "" } }],
+            },
+          },
+        }).toHaveLength(4)
+      })
     })
 })
diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts
index 31934371f6..a6520d24a9 100644
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@@ -27,6 +27,12 @@ import { isPlainObject, isEmpty } from "lodash"
 import { decodeNonAscii } from "./helpers/schema"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
+const LOGICAL_OPERATORS = Object.values(LogicalOperator)
+const SEARCH_OPERATORS = [
+  ...Object.values(BasicOperator),
+  ...Object.values(ArrayOperator),
+  ...Object.values(RangeOperator),
+]
 
 /**
  * Returns the valid operator options for a certain data type
@@ -117,7 +123,7 @@ export function recurseLogicalOperators(
   filters: SearchFilters,
   fn: (f: SearchFilters) => SearchFilters
 ) {
-  for (const logical of Object.values(LogicalOperator)) {
+  for (const logical of LOGICAL_OPERATORS) {
     if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => fn(condition)
@@ -135,7 +141,7 @@ export function recurseSearchFilters(
   filters = processFn(filters)
 
   // Recurse through logical operators
-  for (const logical of Object.values(LogicalOperator)) {
+  for (const logical of LOGICAL_OPERATORS) {
     if (filters[logical]) {
       filters[logical]!.conditions = filters[logical]!.conditions.map(
         condition => recurseSearchFilters(condition, processFn)
@@ -773,12 +779,16 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
         return filterFunctions[key as SearchFilterOperator]?.(doc) ?? false
       })
 
-    if (query.allOr) {
+    // there are no filters - logical operators can cover this up
+    if (!hasFilters(query)) {
+      return true
+    } else if (query.allOr) {
       return results.some(result => result === true)
     } else {
       return results.every(result => result === true)
     }
   }
+
   return docs.filter(docMatch)
 }
 
@@ -843,14 +853,33 @@ export const hasFilters = (query?: SearchFilters) => {
   if (!query) {
     return false
   }
-  const skipped = ["allOr", "onEmptyFilter"]
-  for (let [key, value] of Object.entries(query)) {
-    if (skipped.includes(key) || typeof value !== "object") {
-      continue
+  const check = (filters: SearchFilters): boolean => {
+    for (const logical of LOGICAL_OPERATORS) {
+      if (filters[logical]) {
+        for (const condition of filters[logical]?.conditions || []) {
+          const result = check(condition)
+          if (result) {
+            return result
+          }
+        }
+      }
     }
-    if (Object.keys(value || {}).length !== 0) {
-      return true
+    for (const search of SEARCH_OPERATORS) {
+      const searchValue = filters[search]
+      if (!searchValue || typeof searchValue !== "object") {
+        continue
+      }
+      const filtered = Object.entries(searchValue).filter(entry => {
+        const valueDefined =
+          entry[1] !== undefined || entry[1] !== null || entry[1] !== ""
+        // not empty is an edge case, null is allowed for it - this is covered by test cases
+        return search === BasicOperator.NOT_EMPTY || valueDefined
+      })
+      if (filtered.length !== 0) {
+        return true
+      }
     }
+    return false
   }
-  return false
+  return check(query)
 }

From 8b26ab422f91de19bdc57f5ff5976b9db02d0d38 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Mon, 9 Sep 2024 18:19:10 +0100
Subject: [PATCH 47/69] remove conditional chain

---
 .../builder/src/pages/builder/portal/users/users/index.svelte   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte
index 91d9c99af6..b91b2129e6 100644
--- a/packages/builder/src/pages/builder/portal/users/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte
@@ -127,7 +127,7 @@
         name: user.firstName ? user.firstName + " " + user.lastName : "",
         userGroups,
         __selectable:
-          role?.value === Constants.BudibaseRoles.Owner ? false : undefined,
+          role.value === Constants.BudibaseRoles.Owner ? false : undefined,
         apps: [...new Set(Object.keys(user.roles))],
         access: role.sortOrder,
       }

From 7a376d96295b1fd3d8d0367e2faf7d898c0dce53 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Tue, 10 Sep 2024 10:26:18 +0100
Subject: [PATCH 48/69] Update
 packages/worker/src/api/controllers/global/configs.ts

Co-authored-by: Sam Rose <hello@samwho.dev>
---
 packages/worker/src/api/controllers/global/configs.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index ffe7a1c717..5345c25bc2 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -335,7 +335,7 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) {
 
 async function enrichAIConfig(aiConfig: AIConfig) {
   // Strip out the API Keys from the response so they don't show in the UI
-  for (let key in aiConfig.config) {
+  for (const key in aiConfig.config) {
     if (aiConfig.config[key].apiKey) {
       aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT
     }

From 4fed5787ac1a11d47ebd82fa3c69c7accc3cec47 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 10:54:50 +0100
Subject: [PATCH 49/69] review comments

---
 .../portal/settings/ai/AIConfigTile.svelte      | 17 ++++++++---------
 .../builder/portal/settings/ai/index.svelte     |  6 ++++--
 .../src/api/controllers/global/configs.ts       |  8 ++++----
 .../controllers/global/tests/configs.spec.ts    |  4 ++--
 4 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index d4d5279d3e..693fbe7431 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -6,6 +6,13 @@
   import TogetherAILogo from "./logos/TogetherAI.svelte"
   import { Providers } from "./constants"
 
+  const logos = {
+    ["Budibase AI"]: BudibaseLogo,
+    [Providers.OpenAI.name]: OpenAILogo,
+    [Providers.Anthropic.name]: AnthropicLogo,
+    [Providers.TogetherAI.name]: TogetherAILogo,
+  }
+
   export let config
   export let disabled
 
@@ -17,15 +24,7 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div on:click class:disabled class="option">
   <div class="icon">
-    {#if config.name === "Budibase AI"}
-      <BudibaseLogo height="30" width="30" />
-    {:else if config.provider === Providers.OpenAI.name}
-      <OpenAILogo height="30" width="30" />
-    {:else if config.provider === Providers.Anthropic.name}
-      <AnthropicLogo height="30" width="30" />
-    {:else if config.provider === Providers.TogetherAI.name}
-      <TogetherAILogo height="30" width="30" />
-    {/if}
+    <svelte:component this={logos[config.name || config.provider]} height="30" width="30" />
   </div>
   <div class="header">
     <Body>{config.provider}</Body>
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
index c4403e5045..2ac1609e7c 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte
@@ -69,13 +69,14 @@
     try {
       await API.saveConfig(fullAIConfig)
       notifications.success(`Successfully saved and activated AI Configuration`)
-      await fetchAIConfig()
     } catch (error) {
       notifications.error(
         `Failed to save AI Configuration, reason: ${
           error?.message || "Unknown"
         }`
       )
+    } finally {
+      await fetchAIConfig()
     }
   }
 
@@ -88,11 +89,12 @@
     try {
       await API.saveConfig(fullAIConfig)
       notifications.success(`Deleted config`)
-      await fetchAIConfig()
     } catch (error) {
       notifications.error(
         `Failed to delete config, reason: ${error?.message || "Unknown"}`
       )
+    } finally {
+      await fetchAIConfig()
     }
   }
 
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index ffe7a1c717..95e0ba471f 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -211,10 +211,8 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
 
 export async function verifyAIConfig(
   config: AIConfig,
-  existingConfig?: AIConfig
+  existingConfig: AIConfig
 ) {
-  if (!existingConfig) return
-
   // ensure that the redacted API keys are not overwritten in the DB
   for (let uuid in existingConfig.config) {
     if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
@@ -251,7 +249,9 @@ export async function save(ctx: UserCtx<Config>) {
         await verifyOIDCConfig(config)
         break
       case ConfigType.AI:
-        await verifyAIConfig(config, existingConfig)
+        if (existingConfig) {
+          await verifyAIConfig(config, existingConfig)
+        }
         break
     }
   } catch (err: any) {
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 4bb3d598a6..8b859dcb89 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -45,7 +45,7 @@ describe("Global configs controller", () => {
   })
 
   it("Should return the default BB AI config when the feature is turned on", async () => {
-    pro.features.isBudibaseAIEnabled = jest.fn(() => true)
+    jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => true)
     configs.getConfig.mockResolvedValue({
       config: {
         ai: {
@@ -81,7 +81,7 @@ describe("Global configs controller", () => {
   })
 
   it("Should not not return the default Budibase AI config when on self host", async () => {
-    pro.features.isBudibaseAIEnabled = jest.fn(() => false)
+    jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => false)
     configs.getConfig.mockResolvedValue({
       config: {
         ai: {

From f63c95e44c282743b31b80a0306feeeb90e30452 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 13:59:21 +0100
Subject: [PATCH 50/69] Adding SQL_MAX_RELATED_ROWS environment variable,
 defaults to 500, allows for 500 rows per relationship.

---
 packages/backend-core/src/environment.ts | 1 +
 packages/backend-core/src/sql/sql.ts     | 9 ++++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts
index 3d931a8c67..6bef6efeb3 100644
--- a/packages/backend-core/src/environment.ts
+++ b/packages/backend-core/src/environment.ts
@@ -171,6 +171,7 @@ const environment = {
   // Couch/search
   SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
+  SQL_MAX_RELATED_ROWS: process.env.MAX_RELATED_ROWS,
   // smtp
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   SMTP_USER: process.env.SMTP_USER,
diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts
index b1a2bc060a..d8ad791829 100644
--- a/packages/backend-core/src/sql/sql.ts
+++ b/packages/backend-core/src/sql/sql.ts
@@ -49,6 +49,13 @@ function getBaseLimit() {
   return envLimit || 5000
 }
 
+function getRelationshipLimit() {
+  const envLimit = environment.SQL_MAX_RELATED_ROWS
+    ? parseInt(environment.SQL_MAX_RELATED_ROWS)
+    : null
+  return envLimit || 500
+}
+
 function getTableName(table?: Table): string | undefined {
   // SQS uses the table ID rather than the table name
   if (
@@ -921,7 +928,7 @@ class InternalBuilder {
       const primaryKey = `${toAlias}.${toPrimary || toKey}`
       let subQuery: Knex.QueryBuilder = knex
         .from(toTableWithSchema)
-        .limit(getBaseLimit())
+        .limit(getRelationshipLimit())
         // add sorting to get consistent order
         .orderBy(primaryKey)
 

From 933ca952d2e26ade754c9973099b676265774ec1 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 14:50:17 +0100
Subject: [PATCH 51/69] lint

---
 .../builder/portal/settings/ai/AIConfigTile.svelte   |  6 +++++-
 packages/builder/src/stores/portal/menu.js           | 12 +++++-------
 .../worker/src/api/controllers/global/configs.ts     |  4 +++-
 .../src/api/controllers/global/tests/configs.spec.ts | 11 +++++++----
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
index 693fbe7431..2907919ce7 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte
@@ -24,7 +24,11 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div on:click class:disabled class="option">
   <div class="icon">
-    <svelte:component this={logos[config.name || config.provider]} height="30" width="30" />
+    <svelte:component
+      this={logos[config.name || config.provider]}
+      height="30"
+      width="30"
+    />
   </div>
   <div class="header">
     <Body>{config.provider}</Body>
diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index 1e6282894e..a572539863 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -3,7 +3,7 @@ import { admin } from "./admin"
 import { auth } from "./auth"
 import { isEnabled } from "helpers/featureFlags"
 import { sdk } from "@budibase/shared-core"
-import { FeatureFlag } from "@budibase/types";
+import { FeatureFlag } from "@budibase/types"
 
 export const menu = derived([admin, auth], ([$admin, $auth]) => {
   const user = $auth?.user
@@ -65,12 +65,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
       },
     ]
     if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) {
-      settingsSubPages.push(
-        {
-          title: "AI",
-          href: "/builder/portal/settings/ai",
-        }
-      )
+      settingsSubPages.push({
+        title: "AI",
+        href: "/builder/portal/settings/ai",
+      })
     }
 
     if (!cloud) {
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index a8dfbe6360..d51ae65c51 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -343,7 +343,9 @@ async function enrichAIConfig(aiConfig: AIConfig) {
 
   // Return the Budibase AI data source as part of the response if licensing allows
   const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled()
-  const defaultConfigExists = Object.keys(aiConfig.config).some(key => aiConfig.config[key].isDefault)
+  const defaultConfigExists = Object.keys(aiConfig.config).some(
+    key => aiConfig.config[key].isDefault
+  )
   if (budibaseAIEnabled) {
     aiConfig.config["budibase_ai"] = {
       provider: "OpenAI",
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 8b859dcb89..419c12ccb0 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -45,7 +45,9 @@ describe("Global configs controller", () => {
   })
 
   it("Should return the default BB AI config when the feature is turned on", async () => {
-    jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => true)
+    jest
+      .spyOn(pro.features, "isBudibaseAIEnabled")
+      .mockImplementation(() => true)
     configs.getConfig.mockResolvedValue({
       config: {
         ai: {
@@ -81,7 +83,9 @@ describe("Global configs controller", () => {
   })
 
   it("Should not not return the default Budibase AI config when on self host", async () => {
-    jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => false)
+    jest
+      .spyOn(pro.features, "isBudibaseAIEnabled")
+      .mockImplementation(() => false)
     configs.getConfig.mockResolvedValue({
       config: {
         ai: {
@@ -140,7 +144,6 @@ describe("Global configs controller", () => {
 
     await verifyAIConfig(newConfig, existingConfig)
     // should be unchanged
-    expect(newConfig.config.aiconfig.apiKey === "myapikey")
+    expect(newConfig.config.aiconfig.apiKey).toEqual("myapikey")
   })
-
 })

From 44a76d67309a64dd88d657731915e44dc133e7fd Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 15:12:15 +0100
Subject: [PATCH 52/69] add anthropic baseURL

---
 .../builder/portal/settings/ai/{constants.ts => constants.js}   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename packages/builder/src/pages/builder/portal/settings/ai/{constants.ts => constants.js} (97%)

diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.js
similarity index 97%
rename from packages/builder/src/pages/builder/portal/settings/ai/constants.ts
rename to packages/builder/src/pages/builder/portal/settings/ai/constants.js
index 7ff43582e0..deb647f90a 100644
--- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts
+++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.js
@@ -44,7 +44,7 @@ export const ConfigMap = {
   },
   Anthropic: {
     name: "Anthropic",
-    baseUrl: "",
+    baseUrl: "https://api.anthropic.com/v1",
   },
   TogetherAI: {
     name: "TogetherAI",

From 2eb53c018ae1bd19c397a52a758e4713ab160f24 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 15:19:41 +0100
Subject: [PATCH 53/69] sort settings pages

---
 packages/builder/src/stores/portal/menu.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index a572539863..19b190fd24 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -84,7 +84,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
     menu.push({
       title: "Settings",
       href: "/builder/portal/settings",
-      subPages: settingsSubPages,
+      subPages: [...settingsSubPages].sort((a, b) => a.title.localeCompare(b.title)),
     })
   }
 

From 2fd5c1a99f9b976793bb1830f1fa7ee031db1bd5 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 15:45:32 +0100
Subject: [PATCH 54/69] Handling JSON types within relationships, they need to
 be parsed as well.

---
 .../src/api/controllers/row/utils/basic.ts    | 54 +++++++++----------
 .../src/api/controllers/row/utils/utils.ts    | 24 +++------
 .../src/sdk/app/rows/search/internal/sqs.ts   | 25 +++++----
 3 files changed, 47 insertions(+), 56 deletions(-)

diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts
index b754e288ed..596cc90b20 100644
--- a/packages/server/src/api/controllers/row/utils/basic.ts
+++ b/packages/server/src/api/controllers/row/utils/basic.ts
@@ -1,5 +1,5 @@
 // need to handle table name + field or just field, depending on if relationships used
-import { FieldSchema, FieldType, Row, Table } from "@budibase/types"
+import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types"
 import {
   helpers,
   PROTECTED_EXTERNAL_COLUMNS,
@@ -62,6 +62,22 @@ export function generateIdForRow(
   return generateRowIdField(idParts)
 }
 
+function fixJsonTypes(row: Row, table: Table) {
+  for (let [fieldName, schema] of Object.entries(table.schema)) {
+    if (JsonTypes.includes(schema.type) && typeof row[fieldName] === "string") {
+      try {
+        row[fieldName] = JSON.parse(row[fieldName])
+      } catch (err) {
+        if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) {
+          // couldn't convert back to array, ignore
+          delete row[fieldName]
+        }
+      }
+    }
+  }
+  return row
+}
+
 export function basicProcessing({
   row,
   table,
@@ -136,12 +152,15 @@ export function basicProcessing({
         const sortField =
           relatedTable.primaryDisplay || relatedTable.primary![0]!
         thisRow[col] = (thisRow[col] as Row[])
-          .map(relatedRow => {
-            relatedRow._id = relatedRow._id
-              ? relatedRow._id
-              : generateIdForRow(relatedRow, relatedTable)
-            return relatedRow
-          })
+          .map(relatedRow =>
+            basicProcessing({
+              row: relatedRow,
+              table: relatedTable,
+              tables,
+              isLinked: false,
+              sqs,
+            })
+          )
           .sort((a, b) => {
             const aField = a?.[sortField],
               bField = b?.[sortField]
@@ -157,24 +176,5 @@ export function basicProcessing({
       }
     }
   }
-  return thisRow
-}
-
-export function fixArrayTypes(row: Row, table: Table) {
-  for (let [fieldName, schema] of Object.entries(table.schema)) {
-    if (
-      [FieldType.ARRAY, FieldType.BB_REFERENCE].includes(schema.type) &&
-      typeof row[fieldName] === "string"
-    ) {
-      try {
-        row[fieldName] = JSON.parse(row[fieldName])
-      } catch (err) {
-        if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) {
-          // couldn't convert back to array, ignore
-          delete row[fieldName]
-        }
-      }
-    }
-  }
-  return row
+  return fixJsonTypes(thisRow, table)
 }
diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts
index 45f0cef085..ac305e70b6 100644
--- a/packages/server/src/api/controllers/row/utils/utils.ts
+++ b/packages/server/src/api/controllers/row/utils/utils.ts
@@ -14,12 +14,7 @@ import {
   processFormulas,
 } from "../../../../utilities/rowProcessor"
 import { isKnexEmptyReadResponse } from "./sqlUtils"
-import {
-  basicProcessing,
-  generateIdForRow,
-  fixArrayTypes,
-  getInternalRowId,
-} from "./basic"
+import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic"
 import sdk from "../../../../sdk"
 import { processStringSync } from "@budibase/string-templates"
 import validateJs from "validate.js"
@@ -149,16 +144,13 @@ export async function sqlOutputProcessing(
       rowId = generateIdForRow(row, table)
       row._id = rowId
     }
-    const thisRow = fixArrayTypes(
-      basicProcessing({
-        row,
-        table,
-        tables: Object.values(tables),
-        isLinked: false,
-        sqs: opts?.sqs,
-      }),
-      table
-    )
+    const thisRow = basicProcessing({
+      row,
+      table,
+      tables: Object.values(tables),
+      isLinked: false,
+      sqs: opts?.sqs,
+    })
     if (thisRow._id == null) {
       throw new Error("Unable to generate row ID for SQL rows")
     }
diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index b5e25e02ea..cf540a244b 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -18,6 +18,7 @@ import {
 } from "@budibase/types"
 import {
   buildInternalRelationships,
+  fixJsonTypes,
   sqlOutputProcessing,
 } from "../../../../../api/controllers/row/utils"
 import sdk from "../../../../index"
@@ -182,11 +183,20 @@ function buildTableMap(tables: Table[]) {
   return tableMap
 }
 
-function reverseUserColumnMapping(rows: Row[]) {
+// table is only needed to handle relationships
+function reverseUserColumnMapping(rows: Row[], table?: Table) {
   const prefixLength = USER_COLUMN_PREFIX.length
   return rows.map(row => {
     const finalRow: Row = {}
     for (let key of Object.keys(row)) {
+      // handle relationships
+      if (
+        table?.schema[key]?.type === FieldType.LINK &&
+        typeof row[key] === "string"
+      ) {
+        // no table required, relationship rows don't contain relationships
+        row[key] = reverseUserColumnMapping(JSON.parse(row[key]))
+      }
       // it should be the first prefix
       const index = key.indexOf(USER_COLUMN_PREFIX)
       if (index !== -1) {
@@ -261,7 +271,7 @@ async function runSqlQuery(
   if (opts?.countTotalRows) {
     return processRowCountResponse(response)
   } else if (Array.isArray(response)) {
-    return reverseUserColumnMapping(response)
+    return reverseUserColumnMapping(response, json.meta.table)
   }
   return response
 }
@@ -368,17 +378,6 @@ export async function search(
       })
     )
 
-    // make sure relationships have columns reversed correctly
-    for (let columnName of Object.keys(table.schema)) {
-      if (table.schema[columnName].type !== FieldType.LINK) {
-        continue
-      }
-      // process the relationships (JSON generated by SQS)
-      for (let row of processed) {
-        row[columnName] = reverseUserColumnMapping(row[columnName])
-      }
-    }
-
     // check for pagination final row
     let nextRow: boolean = false
     if (paginate && params.limit && rows.length > params.limit) {

From d1b12b8d4a5c7a0c74c589597bb9aa73e0f7d163 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 15:52:41 +0100
Subject: [PATCH 55/69] Linting.

---
 packages/server/src/sdk/app/rows/search/internal/sqs.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
index cf540a244b..560a39e70a 100644
--- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts
+++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts
@@ -18,7 +18,6 @@ import {
 } from "@budibase/types"
 import {
   buildInternalRelationships,
-  fixJsonTypes,
   sqlOutputProcessing,
 } from "../../../../../api/controllers/row/utils"
 import sdk from "../../../../index"

From 1ee8a12e290505ad5664823d388b6783005b04a8 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 16:01:00 +0100
Subject: [PATCH 56/69] use real data instead of mocks

---
 .../controllers/global/tests/configs.spec.ts  | 159 +++++++-----------
 packages/worker/src/tests/api/configs.ts      |   9 +
 .../worker/src/tests/structures/configs.ts    |  19 ++-
 3 files changed, 86 insertions(+), 101 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 419c12ccb0..1a6e6b7075 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,46 +1,36 @@
-import { configs } from "@budibase/backend-core"
-import { UserCtx } from "@budibase/types"
 import * as pro from "@budibase/pro"
-import { find, verifyAIConfig } from "../configs"
-
-jest.mock("@budibase/backend-core", () => ({
-  ...jest.requireActual("@budibase/backend-core"),
-  configs: {
-    getConfig: jest.fn(),
-    save: jest.fn(),
-  },
-}))
+import { verifyAIConfig } from "../configs"
+import { TestConfiguration, structures } from "../../../../tests"
 
 describe("Global configs controller", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.beforeAll()
+  })
+
+  afterAll(async () => {
+    await config.afterAll()
+  })
+
   afterEach(() => {
     jest.resetAllMocks()
   })
 
   it("Should strip secrets when pulling AI config", async () => {
-    configs.getConfig.mockResolvedValue({
-      config: {
-        ai: {
-          apiKey: "abc123APIKey",
-          baseUrl: "https://api.example.com",
-        },
-      },
-    })
-    const ctx = {
-      params: {
-        type: "ai",
-      },
-      throw: jest.fn(),
-    } as UserCtx
-
-    await find(ctx)
-
-    expect(ctx.body).toEqual({
-      config: {
-        ai: {
-          apiKey: "--secret-value--",
-          baseUrl: "https://api.example.com",
-        },
-      },
+    const data = structures.configs.ai()
+    await config.api.configs.saveConfig(data)
+    const response = await config.api.configs.getAIConfig()
+    expect(response.body.config).toEqual({
+      ai: {
+        active: true,
+        apiKey: "--secret-value--",
+        baseUrl: "https://api.example.com",
+        defaultModel: "gpt4",
+        isDefault: false,
+        name: "Test",
+        provider: "OpenAI"
+      }
     })
   })
 
@@ -48,36 +38,25 @@ describe("Global configs controller", () => {
     jest
       .spyOn(pro.features, "isBudibaseAIEnabled")
       .mockImplementation(() => true)
-    configs.getConfig.mockResolvedValue({
-      config: {
-        ai: {
-          apiKey: "abc123APIKey",
-          baseUrl: "https://api.example.com",
-        },
-      },
-    })
-    const ctx = {
-      params: {
-        type: "ai",
-      },
-      throw: jest.fn(),
-    } as UserCtx
+    const data = structures.configs.ai()
+    await config.api.configs.saveConfig(data)
+    const response = await config.api.configs.getAIConfig()
 
-    await find(ctx)
-
-    expect(ctx.body).toEqual({
-      config: {
-        budibase_ai: {
-          provider: "OpenAI",
-          active: true,
-          isDefault: true,
-          defaultModel: undefined,
-          name: "Budibase AI",
-        },
-        ai: {
-          apiKey: "--secret-value--",
-          baseUrl: "https://api.example.com",
-        },
+    expect(response.body.config).toEqual({
+      budibase_ai: {
+        provider: "OpenAI",
+        active: true,
+        isDefault: true,
+        name: "Budibase AI",
+      },
+      ai: {
+        active: true,
+        apiKey: "--secret-value--",
+        baseUrl: "https://api.example.com",
+        defaultModel: "gpt4",
+        isDefault: false,
+        name: "Test",
+        provider: "OpenAI"
       },
     })
   })
@@ -86,49 +65,29 @@ describe("Global configs controller", () => {
     jest
       .spyOn(pro.features, "isBudibaseAIEnabled")
       .mockImplementation(() => false)
-    configs.getConfig.mockResolvedValue({
-      config: {
-        ai: {
-          apiKey: "abc123APIKey",
-          baseUrl: "https://api.example.com",
-        },
-      },
-    })
-    const ctx = {
-      params: {
-        type: "ai",
-      },
-      throw: jest.fn(),
-    } as UserCtx
+    const data = structures.configs.ai()
+    await config.api.configs.saveConfig(data)
+    const response = await config.api.configs.getAIConfig()
 
-    await find(ctx)
-
-    expect(ctx.body).toEqual({
-      config: {
-        ai: {
-          apiKey: "--secret-value--",
-          baseUrl: "https://api.example.com",
-        },
+    expect(response.body.config).toEqual({
+      ai: {
+        active: true,
+        apiKey: "--secret-value--",
+        baseUrl: "https://api.example.com",
+        defaultModel: "gpt4",
+        isDefault: false,
+        name: "Test",
+        provider: "OpenAI"
       },
     })
   })
 
   it("Should not update existing secrets when updating an existing AI Config", async () => {
-    const newConfig = {
-      type: "ai",
-      config: {
-        aiconfig: {
-          provider: "OpenAI",
-          isDefault: true,
-          name: "MyConfig",
-          active: true,
-          defaultModel: "gpt4",
-          apiKey: "--secret-value--",
-        },
-      },
-    }
+    const data = structures.configs.ai()
+    await config.api.configs.saveConfig(data)
+    const existingConfig = await config.api.configs.getAIConfig()
 
-    const existingConfig = {
+    const newConfig = {
       type: "ai",
       config: {
         aiconfig: {
diff --git a/packages/worker/src/tests/api/configs.ts b/packages/worker/src/tests/api/configs.ts
index 86fe0830ca..c64419b490 100644
--- a/packages/worker/src/tests/api/configs.ts
+++ b/packages/worker/src/tests/api/configs.ts
@@ -22,6 +22,15 @@ export class ConfigAPI extends TestAPI {
       .expect("Content-Type", /json/)
   }
 
+  getAIConfig = () => {
+    return this.request
+      .get(`/api/global/configs/ai`)
+      .set(this.config.defaultHeaders())
+      .expect(200)
+      .expect("Content-Type", /json/)
+  }
+
+
   saveConfig = (data: any) => {
     return this.request
       .post(`/api/global/configs`)
diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts
index bac94a4154..e1fd835d41 100644
--- a/packages/worker/src/tests/structures/configs.ts
+++ b/packages/worker/src/tests/structures/configs.ts
@@ -4,7 +4,7 @@ import {
   ConfigType,
   SMTPConfig,
   GoogleConfig,
-  OIDCConfig,
+  OIDCConfig, AIConfig,
 } from "@budibase/types"
 
 export function oidc(conf?: any): OIDCConfig {
@@ -81,3 +81,20 @@ export function settings(conf?: any): SettingsConfig {
     },
   }
 }
+
+export function ai(): AIConfig {
+  return {
+    type: ConfigType.AI,
+    config: {
+      ai: {
+        provider: "OpenAI",
+        isDefault: false,
+        name: "Test",
+        active: true,
+        defaultModel: "gpt4",
+        apiKey: "abc123APIKey",
+        baseUrl: "https://api.example.com",
+      },
+    }
+  }
+}

From 86a6664c84445861b76477b90871cfaf3e199843 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 16:06:53 +0100
Subject: [PATCH 57/69] Updating test case.

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 54 ++++++++++++-------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 1ba37018dc..528e782543 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -32,7 +32,7 @@ function multiline(sql: string) {
 }
 
 describe("Captures of real examples", () => {
-  const baseLimit = 5000
+  const relationshipLimit = 500
   const primaryLimit = 100
 
   function getJson(name: string): QueryJson {
@@ -42,7 +42,9 @@ describe("Captures of real examples", () => {
   describe("create", () => {
     it("should create a row with relationships", () => {
       const queryJson = getJson("createWithRelationships.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
         bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
         sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") 
@@ -54,9 +56,11 @@ describe("Captures of real examples", () => {
   describe("read", () => {
     it("should handle basic retrieval with relationships", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
-        bindings: [baseLimit, baseLimit, primaryLimit],
+        bindings: [relationshipLimit, relationshipLimit, primaryLimit],
         sql: expect.stringContaining(
           multiline(
             `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
@@ -67,9 +71,11 @@ describe("Captures of real examples", () => {
 
     it("should handle filtering by relationship", () => {
       const queryJson = getJson("filterByRelationship.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
-        bindings: [baseLimit, "assembling", primaryLimit],
+        bindings: [relationshipLimit, "assembling", primaryLimit],
         sql: expect.stringContaining(
           multiline(
             `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" 
@@ -81,9 +87,11 @@ describe("Captures of real examples", () => {
 
     it("should handle fetching many to many relationships", () => {
       const queryJson = getJson("fetchManyToMany.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
-        bindings: [baseLimit, primaryLimit],
+        bindings: [relationshipLimit, primaryLimit],
         sql: expect.stringContaining(
           multiline(
             `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) 
@@ -97,9 +105,11 @@ describe("Captures of real examples", () => {
     it("should handle enrichment of rows", () => {
       const queryJson = getJson("enrichRelationship.json")
       const filters = queryJson.filters?.oneOf?.taskid as number[]
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
-        bindings: [baseLimit, ...filters, baseLimit],
+        bindings: [relationshipLimit, ...filters, relationshipLimit],
         sql: multiline(
           `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) 
                from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" 
@@ -111,7 +121,9 @@ describe("Captures of real examples", () => {
 
     it("should manage query with many relationship filters", () => {
       const queryJson = getJson("manyRelationshipFilters.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       const filters = queryJson.filters
       const notEqualsValue = Object.values(filters?.notEqual!)[0]
       const rangeValue: { high?: string | number; low?: string | number } =
@@ -120,9 +132,9 @@ describe("Captures of real examples", () => {
 
       expect(query).toEqual({
         bindings: [
-          baseLimit,
-          baseLimit,
-          baseLimit,
+          relationshipLimit,
+          relationshipLimit,
+          relationshipLimit,
           rangeValue.low,
           rangeValue.high,
           equalValue,
@@ -141,7 +153,9 @@ describe("Captures of real examples", () => {
   describe("update", () => {
     it("should handle performing a simple update", () => {
       const queryJson = getJson("updateSimple.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(
@@ -153,7 +167,9 @@ describe("Captures of real examples", () => {
 
     it("should handle performing an update of relationships", () => {
       const queryJson = getJson("updateRelationship.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
         sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
@@ -165,7 +181,9 @@ describe("Captures of real examples", () => {
   describe("delete", () => {
     it("should handle deleting with relationships", () => {
       const queryJson = getJson("deleteSimple.json")
-      let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson)
+      let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query(
+        queryJson
+      )
       expect(query).toEqual({
         bindings: ["ddd", ""],
         sql: multiline(`delete from "compositetable" as "a" 
@@ -178,7 +196,7 @@ describe("Captures of real examples", () => {
   describe("returning (everything bar Postgres)", () => {
     it("should be able to handle row returning", () => {
       const queryJson = getJson("createSimple.json")
-      const SQL = new Sql(SqlClient.MS_SQL, baseLimit)
+      const SQL = new Sql(SqlClient.MS_SQL, relationshipLimit)
       let query = SQL._query(queryJson, { disableReturning: true })
       expect(query).toEqual({
         sql: "insert into [people] ([age], [name]) values (@p0, @p1)",

From c6907eb09e325b167dd83db93cf7fdb12ea1602d Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 16:41:44 +0100
Subject: [PATCH 58/69] fix types

---
 packages/builder/src/stores/portal/menu.js    |  4 ++-
 packages/types/src/documents/global/config.ts |  2 +-
 .../src/api/controllers/global/configs.ts     |  9 +++---
 .../controllers/global/tests/configs.spec.ts  | 31 +++++++++----------
 packages/worker/src/tests/api/configs.ts      |  1 -
 .../worker/src/tests/structures/configs.ts    |  5 +--
 6 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js
index 19b190fd24..a1b5450ea4 100644
--- a/packages/builder/src/stores/portal/menu.js
+++ b/packages/builder/src/stores/portal/menu.js
@@ -84,7 +84,9 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => {
     menu.push({
       title: "Settings",
       href: "/builder/portal/settings",
-      subPages: [...settingsSubPages].sort((a, b) => a.title.localeCompare(b.title)),
+      subPages: [...settingsSubPages].sort((a, b) =>
+        a.title.localeCompare(b.title)
+      ),
     })
   }
 
diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts
index 6e9c29af97..8d64b49ee9 100644
--- a/packages/types/src/documents/global/config.ts
+++ b/packages/types/src/documents/global/config.ts
@@ -121,7 +121,7 @@ export interface AIInnerConfig {
     active: boolean
     baseUrl?: string
     apiKey?: string
-    defaultModel: string
+    defaultModel?: string
   }
 }
 
diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index d51ae65c51..0760a6c692 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -32,6 +32,7 @@ import {
   AIConfig,
   PASSWORD_REPLACEMENT,
   isAIConfig,
+  AIInnerConfig,
 } from "@budibase/types"
 import * as pro from "@budibase/pro"
 
@@ -210,13 +211,13 @@ async function verifyOIDCConfig(config: OIDCConfigs) {
 }
 
 export async function verifyAIConfig(
-  config: AIConfig,
+  configToSave: AIInnerConfig,
   existingConfig: AIConfig
 ) {
   // ensure that the redacted API keys are not overwritten in the DB
   for (let uuid in existingConfig.config) {
-    if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
-      config[uuid].apiKey = existingConfig.config[uuid].apiKey
+    if (configToSave[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
+      configToSave[uuid].apiKey = existingConfig.config[uuid].apiKey
     }
   }
 }
@@ -351,7 +352,7 @@ async function enrichAIConfig(aiConfig: AIConfig) {
       provider: "OpenAI",
       active: true,
       isDefault: !defaultConfigExists,
-      defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL,
+      defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "",
       name: "Budibase AI",
     }
   }
diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 1a6e6b7075..06d1ad0b2f 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -1,6 +1,7 @@
 import * as pro from "@budibase/pro"
 import { verifyAIConfig } from "../configs"
 import { TestConfiguration, structures } from "../../../../tests"
+import { AIInnerConfig } from "@budibase/types"
 
 describe("Global configs controller", () => {
   const config = new TestConfiguration()
@@ -29,8 +30,8 @@ describe("Global configs controller", () => {
         defaultModel: "gpt4",
         isDefault: false,
         name: "Test",
-        provider: "OpenAI"
-      }
+        provider: "OpenAI",
+      },
     })
   })
 
@@ -56,7 +57,7 @@ describe("Global configs controller", () => {
         defaultModel: "gpt4",
         isDefault: false,
         name: "Test",
-        provider: "OpenAI"
+        provider: "OpenAI",
       },
     })
   })
@@ -77,7 +78,7 @@ describe("Global configs controller", () => {
         defaultModel: "gpt4",
         isDefault: false,
         name: "Test",
-        provider: "OpenAI"
+        provider: "OpenAI",
       },
     })
   })
@@ -87,22 +88,18 @@ describe("Global configs controller", () => {
     await config.api.configs.saveConfig(data)
     const existingConfig = await config.api.configs.getAIConfig()
 
-    const newConfig = {
-      type: "ai",
-      config: {
-        aiconfig: {
-          provider: "OpenAI",
-          isDefault: true,
-          name: "MyConfig",
-          active: true,
-          defaultModel: "gpt4",
-          apiKey: "myapikey",
-        },
+    const newConfig: AIInnerConfig = {
+      aiconfig: {
+        provider: "OpenAI",
+        isDefault: true,
+        name: "MyConfig",
+        active: true,
+        defaultModel: "gpt4",
       },
     }
 
-    await verifyAIConfig(newConfig, existingConfig)
+    await verifyAIConfig(newConfig, existingConfig.body)
     // should be unchanged
-    expect(newConfig.config.aiconfig.apiKey).toEqual("myapikey")
+    expect(newConfig.aiconfig.apiKey).toEqual("myapikey")
   })
 })
diff --git a/packages/worker/src/tests/api/configs.ts b/packages/worker/src/tests/api/configs.ts
index c64419b490..e0c1c806af 100644
--- a/packages/worker/src/tests/api/configs.ts
+++ b/packages/worker/src/tests/api/configs.ts
@@ -30,7 +30,6 @@ export class ConfigAPI extends TestAPI {
       .expect("Content-Type", /json/)
   }
 
-
   saveConfig = (data: any) => {
     return this.request
       .post(`/api/global/configs`)
diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts
index e1fd835d41..9257e813cb 100644
--- a/packages/worker/src/tests/structures/configs.ts
+++ b/packages/worker/src/tests/structures/configs.ts
@@ -4,7 +4,8 @@ import {
   ConfigType,
   SMTPConfig,
   GoogleConfig,
-  OIDCConfig, AIConfig,
+  OIDCConfig,
+  AIConfig,
 } from "@budibase/types"
 
 export function oidc(conf?: any): OIDCConfig {
@@ -95,6 +96,6 @@ export function ai(): AIConfig {
         apiKey: "abc123APIKey",
         baseUrl: "https://api.example.com",
       },
-    }
+    },
   }
 }

From 87adb99073e31cc4fdca7149cd07d9b7c74285e2 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 16:42:56 +0100
Subject: [PATCH 59/69] update pro ref

---
 packages/pro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/pro b/packages/pro
index 62586bc47b..ec1d2bda75 160000
--- a/packages/pro
+++ b/packages/pro
@@ -1 +1 @@
-Subproject commit 62586bc47bf2af507356b0e1e64df8b0ae9fad43
+Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c

From 304b2a5c61853e4e71f0216a086d1344688408bb Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Tue, 10 Sep 2024 16:45:32 +0100
Subject: [PATCH 60/69] Update
 packages/worker/src/api/controllers/global/configs.ts

Co-authored-by: Sam Rose <hello@samwho.dev>
---
 packages/worker/src/api/controllers/global/configs.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts
index 0760a6c692..70b2279f6c 100644
--- a/packages/worker/src/api/controllers/global/configs.ts
+++ b/packages/worker/src/api/controllers/global/configs.ts
@@ -215,7 +215,7 @@ export async function verifyAIConfig(
   existingConfig: AIConfig
 ) {
   // ensure that the redacted API keys are not overwritten in the DB
-  for (let uuid in existingConfig.config) {
+  for (const uuid in existingConfig.config) {
     if (configToSave[uuid]?.apiKey === PASSWORD_REPLACEMENT) {
       configToSave[uuid].apiKey = existingConfig.config[uuid].apiKey
     }

From 5c9f88de59c8d84aaf90be316d7a0446eaecd698 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 16:51:22 +0100
Subject: [PATCH 61/69] account portal ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index ce60bf9231..4bef13d035 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit ce60bf923115855a9cd0bb991297cccd308eae13
+Subproject commit 4bef13d035217b65c7f2107e1f8dfc04e7a56787

From 1582e3221faa8f7eac1d87e9eb9eb3e5a11622fa Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 17:04:59 +0100
Subject: [PATCH 62/69] Adding test case for getting related array column in a
 JS formula.

---
 .../src/api/routes/tests/search.spec.ts       | 63 ++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index f5c5ade2f8..edac4c6662 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -9,10 +9,10 @@ import {
   db as dbCore,
   MAX_VALID_DATE,
   MIN_VALID_DATE,
+  setEnv as setCoreEnv,
   SQLITE_DESIGN_DOC_ID,
   utils,
   withEnv as withCoreEnv,
-  setEnv as setCoreEnv,
 } from "@budibase/backend-core"
 
 import * as setup from "./utilities"
@@ -1937,6 +1937,67 @@ describe.each([
     })
   })
 
+  isSql &&
+    describe("related formulas", () => {
+      beforeAll(async () => {
+        const arrayTable = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+            array: {
+              name: "array",
+              type: FieldType.ARRAY,
+              constraints: {
+                type: JsonFieldSubType.ARRAY,
+                inclusion: ["option 1", "option 2"],
+              },
+            },
+          },
+          "array"
+        )
+        table = await createTable(
+          {
+            relationship: {
+              type: FieldType.LINK,
+              relationshipType: RelationshipType.ONE_TO_MANY,
+              name: "relationship",
+              fieldName: "relate",
+              tableId: arrayTable._id!,
+              constraints: {
+                type: "array",
+              },
+            },
+            formula: {
+              type: FieldType.FORMULA,
+              name: "formula",
+              formula: encodeJSBinding(
+                `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")`
+              ),
+            },
+          },
+          "main"
+        )
+        const arrayRows = await Promise.all([
+          config.api.row.save(arrayTable._id!, {
+            name: "foo",
+            array: ["option 1"],
+          }),
+          config.api.row.save(arrayTable._id!, {
+            name: "bar",
+            array: ["option 2"],
+          }),
+        ])
+        await Promise.all([
+          config.api.row.save(table._id!, {
+            relationship: [arrayRows[0]._id, arrayRows[1]._id],
+          }),
+        ])
+      })
+
+      it("formula is correct with relationship arrays", async () => {
+        await expectQuery({}).toContain([{ formula: "option 1,option 2" }])
+      })
+    })
+
   describe("user", () => {
     let user1: User
     let user2: User

From 9a61ec5950ecb63e89f586d59c73be14408a1d6b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 17:07:31 +0100
Subject: [PATCH 63/69] More incorrect limits.

---
 packages/server/src/integrations/tests/sql.spec.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index c434ec42cb..f2bcc0bde0 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -149,6 +149,7 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
 }
 
 describe("SQL query builder", () => {
+  const relationshipLimit = 500
   const limit = 500
   const client = SqlClient.POSTGRES
   let sql: any
@@ -160,7 +161,7 @@ describe("SQL query builder", () => {
   it("should add the schema to the LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
-      bindings: [5000, limit],
+      bindings: [relationshipLimit, limit],
       sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`,
     })
   })
@@ -168,7 +169,7 @@ describe("SQL query builder", () => {
   it("should handle if the schema is not present when doing a LEFT JOIN", () => {
     const query = sql._query(generateRelationshipJson())
     expect(query).toEqual({
-      bindings: [5000, limit],
+      bindings: [relationshipLimit, limit],
       sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`,
     })
   })
@@ -178,7 +179,7 @@ describe("SQL query builder", () => {
       generateManyRelationshipJson({ schema: "production" })
     )
     expect(query).toEqual({
-      bindings: [5000, limit],
+      bindings: [relationshipLimit, limit],
       sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`,
     })
   })

From 595dd7ea6d31cd092f2adb7ccb0f54e3ffd86d3b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 10 Sep 2024 17:21:36 +0100
Subject: [PATCH 64/69] Fix for test case.

---
 packages/server/src/api/routes/tests/search.spec.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts
index edac4c6662..26bb1582dd 100644
--- a/packages/server/src/api/routes/tests/search.spec.ts
+++ b/packages/server/src/api/routes/tests/search.spec.ts
@@ -1958,7 +1958,7 @@ describe.each([
           {
             relationship: {
               type: FieldType.LINK,
-              relationshipType: RelationshipType.ONE_TO_MANY,
+              relationshipType: RelationshipType.MANY_TO_ONE,
               name: "relationship",
               fieldName: "relate",
               tableId: arrayTable._id!,
@@ -1970,7 +1970,7 @@ describe.each([
               type: FieldType.FORMULA,
               name: "formula",
               formula: encodeJSBinding(
-                `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")`
+                `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")`
               ),
             },
           },

From 38fc65c5419082dd106c8fab764f9f6a5de61842 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 22:25:47 +0100
Subject: [PATCH 65/69] fix tests

---
 .../src/api/controllers/global/tests/configs.spec.ts     | 9 +++++----
 packages/worker/src/tests/structures/configs.ts          | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index 06d1ad0b2f..ad9efc4398 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -49,6 +49,7 @@ describe("Global configs controller", () => {
         active: true,
         isDefault: true,
         name: "Budibase AI",
+        defaultModel: "",
       },
       ai: {
         active: true,
@@ -86,20 +87,20 @@ describe("Global configs controller", () => {
   it("Should not update existing secrets when updating an existing AI Config", async () => {
     const data = structures.configs.ai()
     await config.api.configs.saveConfig(data)
-    const existingConfig = await config.api.configs.getAIConfig()
 
     const newConfig: AIInnerConfig = {
-      aiconfig: {
+      ai: {
         provider: "OpenAI",
         isDefault: true,
+        apiKey: "--secret-value--",
         name: "MyConfig",
         active: true,
         defaultModel: "gpt4",
       },
     }
 
-    await verifyAIConfig(newConfig, existingConfig.body)
+    await verifyAIConfig(newConfig, data)
     // should be unchanged
-    expect(newConfig.aiconfig.apiKey).toEqual("myapikey")
+    expect(newConfig.ai.apiKey).toEqual("myapikey")
   })
 })
diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts
index 9257e813cb..5aa073408c 100644
--- a/packages/worker/src/tests/structures/configs.ts
+++ b/packages/worker/src/tests/structures/configs.ts
@@ -93,7 +93,7 @@ export function ai(): AIConfig {
         name: "Test",
         active: true,
         defaultModel: "gpt4",
-        apiKey: "abc123APIKey",
+        apiKey: "myapikey",
         baseUrl: "https://api.example.com",
       },
     },

From a27be54b691336f7edcd128fc2b2e7466f2035ef Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 22:38:45 +0100
Subject: [PATCH 66/69] make mocks async

---
 .../worker/src/api/controllers/global/tests/configs.spec.ts   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
index ad9efc4398..3ff6a5298c 100644
--- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts
+++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts
@@ -38,7 +38,7 @@ describe("Global configs controller", () => {
   it("Should return the default BB AI config when the feature is turned on", async () => {
     jest
       .spyOn(pro.features, "isBudibaseAIEnabled")
-      .mockImplementation(() => true)
+      .mockImplementation(() => Promise.resolve(true))
     const data = structures.configs.ai()
     await config.api.configs.saveConfig(data)
     const response = await config.api.configs.getAIConfig()
@@ -66,7 +66,7 @@ describe("Global configs controller", () => {
   it("Should not not return the default Budibase AI config when on self host", async () => {
     jest
       .spyOn(pro.features, "isBudibaseAIEnabled")
-      .mockImplementation(() => false)
+      .mockImplementation(() => Promise.resolve(false))
     const data = structures.configs.ai()
     await config.api.configs.saveConfig(data)
     const response = await config.api.configs.getAIConfig()

From a5ff0d5ddc9c597bc8bb32fbe03c2c50cd799666 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Tue, 10 Sep 2024 22:55:14 +0100
Subject: [PATCH 67/69] update acct portal ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index 4bef13d035..c587ebc17a 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit 4bef13d035217b65c7f2107e1f8dfc04e7a56787
+Subproject commit c587ebc17adf778bb02a77732cc0b2cc63399a93

From fcc03814744fcb55ac261de41f29697309f40f38 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martinmckeaveney@gmail.com>
Date: Wed, 11 Sep 2024 09:42:29 +0100
Subject: [PATCH 68/69] update account portal ref

---
 packages/account-portal                     | 2 +-
 packages/types/src/sdk/licensing/feature.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index c587ebc17a..048c37ecd9 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit c587ebc17adf778bb02a77732cc0b2cc63399a93
+Subproject commit 048c37ecd921340614bf61a76a774aaa46176569
diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts
index cbcb16f7b6..9d09f1d14f 100644
--- a/packages/types/src/sdk/licensing/feature.ts
+++ b/packages/types/src/sdk/licensing/feature.ts
@@ -16,7 +16,7 @@ export enum Feature {
   VIEW_PERMISSIONS = "viewPermissions",
   VIEW_READONLY_COLUMNS = "viewReadonlyColumns",
   BUDIBASE_AI = "budibaseAI",
-  AI_CUSTOM_CONFIGS = "customAIConfigs",
+  AI_CUSTOM_CONFIGS = "aiCustomConfigs",
 }
 
 export type PlanFeatures = { [key in PlanType]: Feature[] | undefined }

From 4f740c001b95ec05e7c7a966e095d5ac7d71e1dc Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 11 Sep 2024 10:57:09 +0000
Subject: [PATCH 69/69] Bump version to 2.32.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 0292b863e8..4a7cfc95cc 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.31.8",
+  "version": "2.32.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*",