From 478e297e9e3c2944fb20b750463db89f2dea1b72 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 24 Nov 2023 18:11:53 +0000
Subject: [PATCH 01/47] Initial work towards aliasing queries for SQL.

---
 .../server/src/api/controllers/row/alias.ts   | 101 ++++++++++++++++++
 packages/server/src/integrations/base/sql.ts  |   1 +
 2 files changed, 102 insertions(+)
 create mode 100644 packages/server/src/api/controllers/row/alias.ts

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
new file mode 100644
index 0000000000..8111396ea9
--- /dev/null
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -0,0 +1,101 @@
+import { QueryJson, SearchFilters, Table, Row } from "@budibase/types"
+import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
+import { cloneDeep } from "lodash"
+
+class AliasTables {
+  character: string
+  aliases: Record<string, string>
+  tableAliases: Record<string, string>
+
+  constructor() {
+    this.character = "a"
+    this.aliases = {}
+    this.tableAliases = {}
+  }
+
+  getAlias(tableName: string) {
+    if (this.aliases[tableName]) {
+      return this.aliases[tableName]
+    }
+    this.character = String.fromCharCode(this.character.charCodeAt(0) + 1)
+    this.aliases[tableName] = this.character
+    this.tableAliases[this.character] = tableName
+    return this.character
+  }
+
+  aliasField(tableNames: string[], field: string) {
+    if (field.includes(".")) {
+      const [tableName, column] = field.split(".")
+      if (tableNames.includes(tableName)) {
+        return `${this.getAlias(tableName)}.${column}`
+      }
+    }
+    return field
+  }
+
+  reverse<T extends Row | Row[]>(tableNames: string[], rows: T): T {
+    const process = (row: Row) => {
+      const final: Row = {}
+      for (let [key, value] of Object.entries(row)) {
+        if (!key.includes(".")) {
+          final[key] = value
+        } else {
+          const [alias, column] = key.split(".")
+          const tableName = this.tableAliases[alias] || alias
+          final[`${tableName}.${column}`] = value
+        }
+      }
+      return final
+    }
+    if (Array.isArray(rows)) {
+      return rows.map(row => process(row)) as T
+    } else {
+      return process(rows) as T
+    }
+  }
+
+  async queryWithAliasing(tableNames: string[], json: QueryJson) {
+    json = cloneDeep(json)
+    const aliasField = (field: string) => this.aliasField(tableNames, field)
+    const aliasTable = (table: Table) => ({
+      ...table,
+      name: this.getAlias(table.name),
+    })
+    // run through the query json to update anywhere a table may be used
+    if (json.resource?.fields) {
+      json.resource.fields = json.resource.fields.map(field =>
+        aliasField(field)
+      )
+    }
+    if (json.filters) {
+      for (let [filterKey, filter] of Object.entries(json.filters)) {
+        if (typeof filter !== "object") {
+          continue
+        }
+        const aliasedFilters: typeof filter = {}
+        for (let key of Object.keys(filter)) {
+          aliasedFilters[aliasField(key)] = filter
+        }
+        json.filters[filterKey as keyof SearchFilters] = aliasedFilters
+      }
+    }
+    if (json.relationships) {
+      json.relationships = json.relationships.map(relationship => ({
+        ...relationship,
+        tableName: this.getAlias(relationship.tableName),
+      }))
+    }
+    if (json.meta?.table) {
+      json.meta.table = aliasTable(json.meta.table)
+    }
+    if (json.meta?.tables) {
+      const aliasedTables: Record<string, Table> = {}
+      for (let [tableName, table] of Object.entries(json.meta.tables)) {
+        aliasedTables[this.getAlias(tableName)] = aliasTable(table)
+      }
+      json.meta.tables = aliasedTables
+    }
+    const response = await getDatasourceAndQuery(json)
+    return this.reverse(tableNames, response)
+  }
+}
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 29c8416b34..630c962a15 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -437,6 +437,7 @@ class InternalBuilder {
 
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     let { endpoint, resource, filters, paginate, relationships } = json
+
     const tableName = endpoint.entityId
     // select all if not specified
     if (!resource) {

From c16ad8614240cbb70f21179aeb2e6239916fcce2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 24 Nov 2023 18:12:35 +0000
Subject: [PATCH 02/47] Updating reverse function.

---
 packages/server/src/api/controllers/row/alias.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 8111396ea9..d4937186d9 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -33,7 +33,7 @@ class AliasTables {
     return field
   }
 
-  reverse<T extends Row | Row[]>(tableNames: string[], rows: T): T {
+  reverse<T extends Row | Row[]>(rows: T): T {
     const process = (row: Row) => {
       const final: Row = {}
       for (let [key, value] of Object.entries(row)) {
@@ -96,6 +96,6 @@ class AliasTables {
       json.meta.tables = aliasedTables
     }
     const response = await getDatasourceAndQuery(json)
-    return this.reverse(tableNames, response)
+    return this.reverse(response)
   }
 }

From cb7c1898f2d29ac52f6b943dc04dd716b9ba128e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 27 Nov 2023 19:02:06 +0000
Subject: [PATCH 03/47] Getting basic aliasing working after some testing.

---
 .../api/controllers/row/ExternalRequest.ts    | 19 +++++++-------
 .../server/src/api/controllers/row/alias.ts   | 25 +++++++++++--------
 packages/server/src/integrations/base/sql.ts  | 11 +++++---
 packages/server/src/sdk/app/rows/utils.ts     |  4 +--
 packages/types/src/sdk/search.ts              |  2 ++
 5 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 7c98fecb9b..29851e457f 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -32,6 +32,7 @@ import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
+import AliasTables from "./alias"
 import sdk from "../../../sdk"
 
 export interface ManyRelationship {
@@ -178,13 +179,13 @@ function generateIdForRow(
 
 function getEndpoint(tableId: string | undefined, operation: string) {
   if (!tableId) {
-    return {}
+    throw new Error("Cannot get endpoint information - no table ID specified")
   }
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   return {
-    datasourceId,
-    entityId: tableName,
-    operation,
+    datasourceId: datasourceId!,
+    entityId: tableName!,
+    operation: operation as Operation,
   }
 }
 
@@ -704,7 +705,7 @@ export class ExternalRequest<T extends Operation> {
         // safety check, if there are no filters on deletion bad things happen
         if (Object.keys(filters).length !== 0) {
           const op = isMany ? Operation.DELETE : Operation.UPDATE
-          const body = isMany ? null : { [colName]: null }
+          const body = isMany ? undefined : { [colName]: null }
           promises.push(
             getDatasourceAndQuery({
               endpoint: getEndpoint(tableId, op),
@@ -807,7 +808,7 @@ export class ExternalRequest<T extends Operation> {
     }
     let json = {
       endpoint: {
-        datasourceId,
+        datasourceId: datasourceId!,
         entityId: tableName,
         operation,
       },
@@ -829,9 +830,9 @@ export class ExternalRequest<T extends Operation> {
       },
     }
 
-    // can't really use response right now
-    const response = await getDatasourceAndQuery(json)
-    // handle many to many relationships now if we know the ID (could be auto increment)
+    const aliasing = new AliasTables(Object.keys(this.tables))
+    const response = await aliasing.queryWithAliasing(json)
+    // handle many-to-many relationships now if we know the ID (could be auto increment)
     if (operation !== Operation.READ) {
       await this.handleManyRelationships(
         table._id || "",
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index d4937186d9..19be8db654 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -2,12 +2,14 @@ import { QueryJson, SearchFilters, Table, Row } from "@budibase/types"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { cloneDeep } from "lodash"
 
-class AliasTables {
+export default class AliasTables {
   character: string
   aliases: Record<string, string>
   tableAliases: Record<string, string>
+  tableNames: string[]
 
-  constructor() {
+  constructor(tableNames: string[]) {
+    this.tableNames = tableNames
     this.character = "a"
     this.aliases = {}
     this.tableAliases = {}
@@ -17,13 +19,15 @@ class AliasTables {
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
     }
-    this.character = String.fromCharCode(this.character.charCodeAt(0) + 1)
-    this.aliases[tableName] = this.character
-    this.tableAliases[this.character] = tableName
-    return this.character
+    const char = this.character
+    this.aliases[tableName] = char
+    this.tableAliases[char] = tableName
+    this.character = String.fromCharCode(char.charCodeAt(0) + 1)
+    return char
   }
 
-  aliasField(tableNames: string[], field: string) {
+  aliasField(field: string) {
+    const tableNames = this.tableNames
     if (field.includes(".")) {
       const [tableName, column] = field.split(".")
       if (tableNames.includes(tableName)) {
@@ -54,9 +58,9 @@ class AliasTables {
     }
   }
 
-  async queryWithAliasing(tableNames: string[], json: QueryJson) {
+  async queryWithAliasing(json: QueryJson) {
     json = cloneDeep(json)
-    const aliasField = (field: string) => this.aliasField(tableNames, field)
+    const aliasField = (field: string) => this.aliasField(field)
     const aliasTable = (table: Table) => ({
       ...table,
       name: this.getAlias(table.name),
@@ -82,7 +86,7 @@ class AliasTables {
     if (json.relationships) {
       json.relationships = json.relationships.map(relationship => ({
         ...relationship,
-        tableName: this.getAlias(relationship.tableName),
+        alias: this.getAlias(relationship.tableName),
       }))
     }
     if (json.meta?.table) {
@@ -95,6 +99,7 @@ class AliasTables {
       }
       json.meta.tables = aliasedTables
     }
+    json.endpoint.alias = this.getAlias(json.endpoint.entityId)
     const response = await getDatasourceAndQuery(json)
     return this.reverse(response)
   }
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 630c962a15..3147e8c670 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -439,6 +439,9 @@ class InternalBuilder {
     let { endpoint, resource, filters, paginate, relationships } = json
 
     const tableName = endpoint.entityId
+    const alias = endpoint.alias
+    const aliased = alias ? alias : tableName
+    const tableAliased = alias ? `${tableName} as ${alias}` : tableName
     // select all if not specified
     if (!resource) {
       resource = { fields: [] }
@@ -463,20 +466,20 @@ class InternalBuilder {
       foundLimit = paginate.limit
     }
     // start building the query
-    let query: KnexQuery = knex(tableName).limit(foundLimit)
+    let query: KnexQuery = knex(tableAliased).limit(foundLimit)
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, { tableName })
+    query = this.addFilters(query, filters, { tableName: aliased })
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
     let preQuery: KnexQuery = knex({
       // @ts-ignore
-      [tableName]: query,
+      [aliased]: query,
     }).select(selectStatement)
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL) {
@@ -485,7 +488,7 @@ class InternalBuilder {
     // handle joins
     query = this.addRelationships(
       preQuery,
-      tableName,
+      aliased,
       relationships,
       endpoint.schema
     )
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index d0227c7c6b..c160aaba3f 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -1,13 +1,13 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
-import { Row, Table, TableSchema } from "@budibase/types"
+import { QueryJson, Row, Table, TableSchema } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
 
-export async function getDatasourceAndQuery(json: any) {
+export async function getDatasourceAndQuery(json: QueryJson) {
   const datasourceId = json.endpoint.datasourceId
   const datasource = await sdk.datasources.get(datasourceId)
   return makeExternalQuery(datasource, json)
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 35fd148c05..1f9aa6c375 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -67,6 +67,7 @@ export interface RelationshipsJson {
   fromPrimary?: string
   toPrimary?: string
   tableName: string
+  alias?: string
   column: string
 }
 
@@ -74,6 +75,7 @@ export interface QueryJson {
   endpoint: {
     datasourceId: string
     entityId: string
+    alias?: string
     operation: Operation
     schema?: string
   }

From 65cddae9dac4f511c70634e4885987b672989c13 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 28 Nov 2023 18:43:38 +0000
Subject: [PATCH 04/47] Getting relationship aliasing working.

---
 .../server/src/api/controllers/row/alias.ts   | 16 +++++++-
 packages/server/src/integrations/base/sql.ts  | 39 ++++++++++++++-----
 packages/types/src/sdk/search.ts              |  2 +-
 3 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 19be8db654..0c7a4bb8a0 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -58,6 +58,16 @@ export default class AliasTables {
     }
   }
 
+  aliasMap(tableNames: (string | undefined)[]) {
+    const map: Record<string, string> = {}
+    for (let tableName of tableNames) {
+      if (tableName) {
+        map[tableName] = this.getAlias(tableName)
+      }
+    }
+    return map
+  }
+
   async queryWithAliasing(json: QueryJson) {
     json = cloneDeep(json)
     const aliasField = (field: string) => this.aliasField(field)
@@ -86,7 +96,11 @@ export default class AliasTables {
     if (json.relationships) {
       json.relationships = json.relationships.map(relationship => ({
         ...relationship,
-        alias: this.getAlias(relationship.tableName),
+        aliases: this.aliasMap([
+          relationship.through,
+          relationship.tableName,
+          json.endpoint.entityId,
+        ]),
       }))
     }
     if (json.meta?.table) {
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 3147e8c670..f3f574b1af 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -330,6 +330,17 @@ class InternalBuilder {
     return query
   }
 
+  tableNameWithSchema(
+    tableName: string,
+    opts?: { alias?: string; schema?: string }
+  ) {
+    let withSchema = opts?.schema ? `${opts.schema}.${tableName}` : tableName
+    if (opts?.alias) {
+      withSchema += ` as ${opts.alias}`
+    }
+    return withSchema
+  }
+
   addRelationships(
     query: KnexQuery,
     fromTable: string,
@@ -339,9 +350,12 @@ class InternalBuilder {
     if (!relationships) {
       return query
     }
-    const tableSets: Record<string, [any]> = {}
+    const tableSets: Record<string, [RelationshipsJson]> = {}
+    // add up all aliases
+    let aliases: Record<string, string> = {}
     // aggregate into table sets (all the same to tables)
     for (let relationship of relationships) {
+      aliases = { ...aliases, ...relationship.aliases }
       const keyObj: { toTable: string; throughTable: string | undefined } = {
         toTable: relationship.tableName,
         throughTable: undefined,
@@ -358,10 +372,17 @@ class InternalBuilder {
     }
     for (let [key, relationships] of Object.entries(tableSets)) {
       const { toTable, throughTable } = JSON.parse(key)
-      const toTableWithSchema = schema ? `${schema}.${toTable}` : toTable
-      const throughTableWithSchema = schema
-        ? `${schema}.${throughTable}`
-        : throughTable
+      const toAlias = aliases[toTable],
+        throughAlias = aliases[throughTable],
+        fromAlias = aliases[fromTable]
+      let toTableWithSchema = this.tableNameWithSchema(toTable, {
+        alias: toAlias,
+        schema,
+      })
+      let throughTableWithSchema = this.tableNameWithSchema(throughTable, {
+        alias: throughAlias,
+        schema,
+      })
       if (!throughTable) {
         // @ts-ignore
         query = query.leftJoin(toTableWithSchema, function () {
@@ -369,7 +390,7 @@ class InternalBuilder {
             const from = relationship.from,
               to = relationship.to
             // @ts-ignore
-            this.orOn(`${fromTable}.${from}`, "=", `${toTable}.${to}`)
+            this.orOn(`${fromTable}.${from}`, "=", `${toAlias}.${to}`)
           }
         })
       } else {
@@ -381,9 +402,9 @@ class InternalBuilder {
               const from = relationship.from
               // @ts-ignore
               this.orOn(
-                `${fromTable}.${fromPrimary}`,
+                `${fromAlias}.${fromPrimary}`,
                 "=",
-                `${throughTable}.${from}`
+                `${throughAlias}.${from}`
               )
             }
           })
@@ -392,7 +413,7 @@ class InternalBuilder {
               const toPrimary = relationship.toPrimary
               const to = relationship.to
               // @ts-ignore
-              this.orOn(`${toTable}.${toPrimary}`, `${throughTable}.${to}`)
+              this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`)
             }
           })
       }
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index 1f9aa6c375..a4045c2558 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -67,7 +67,7 @@ export interface RelationshipsJson {
   fromPrimary?: string
   toPrimary?: string
   tableName: string
-  alias?: string
+  aliases?: Record<string, string>
   column: string
 }
 

From 649025ca124a4b9b7f0714d9621edcb3c4ae3424 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 28 Nov 2023 18:45:05 +0000
Subject: [PATCH 05/47] Fixing missed from.

---
 packages/server/src/integrations/base/sql.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index f3f574b1af..c419edc805 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -390,7 +390,7 @@ class InternalBuilder {
             const from = relationship.from,
               to = relationship.to
             // @ts-ignore
-            this.orOn(`${fromTable}.${from}`, "=", `${toAlias}.${to}`)
+            this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`)
           }
         })
       } else {

From 5c4dc0dc8351310f6ab9c022594386a4543c47f6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 1 Dec 2023 14:14:44 +0000
Subject: [PATCH 06/47] Fixing issue with aliasing.

---
 packages/server/src/integrations/base/sql.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index c419edc805..57af95eabb 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -372,9 +372,9 @@ class InternalBuilder {
     }
     for (let [key, relationships] of Object.entries(tableSets)) {
       const { toTable, throughTable } = JSON.parse(key)
-      const toAlias = aliases[toTable],
-        throughAlias = aliases[throughTable],
-        fromAlias = aliases[fromTable]
+      const toAlias = aliases[toTable] || toTable,
+        throughAlias = aliases[throughTable] || throughTable,
+        fromAlias = aliases[fromTable] || fromTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
         schema,

From 7eccbb851dac11b36e936ae52a67500f99cc7a52 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 1 Dec 2023 15:27:49 +0000
Subject: [PATCH 07/47] Fixing issues with other SQL functions than just
 reading.

---
 .../server/src/api/controllers/row/alias.ts   |  2 +-
 packages/server/src/integrations/base/sql.ts  | 46 +++++++++----------
 2 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 0c7a4bb8a0..fc00b505c4 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -88,7 +88,7 @@ export default class AliasTables {
         }
         const aliasedFilters: typeof filter = {}
         for (let key of Object.keys(filter)) {
-          aliasedFilters[aliasField(key)] = filter
+          aliasedFilters[aliasField(key)] = filter[key]
         }
         json.filters[filterKey as keyof SearchFilters] = aliasedFilters
       }
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 57af95eabb..14bcb532cc 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -421,12 +421,24 @@ class InternalBuilder {
     return query.limit(BASE_LIMIT)
   }
 
-  create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
-    const { endpoint, body } = json
-    let query: KnexQuery = knex(endpoint.entityId)
+  knexWithAlias(
+    knex: Knex,
+    endpoint: { entityId: string; alias?: string; schema?: string }
+  ): { query: KnexQuery; name: string } {
+    const tableName = endpoint.entityId
+    const alias = endpoint.alias
+    const aliased = alias ? alias : tableName
+    const tableAliased = alias ? `${tableName} as ${alias}` : tableName
+    let query = knex(tableAliased)
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }
+    return { query, name: aliased }
+  }
+
+  create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+    const { endpoint, body } = json
+    let { query } = this.knexWithAlias(knex, endpoint)
     const parsedBody = parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
@@ -445,10 +457,7 @@ class InternalBuilder {
 
   bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
     const { endpoint, body } = json
-    let query: KnexQuery = knex(endpoint.entityId)
-    if (endpoint.schema) {
-      query = query.withSchema(endpoint.schema)
-    }
+    let { query } = this.knexWithAlias(knex, endpoint)
     if (!Array.isArray(body)) {
       return query
     }
@@ -459,10 +468,6 @@ class InternalBuilder {
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
     let { endpoint, resource, filters, paginate, relationships } = json
 
-    const tableName = endpoint.entityId
-    const alias = endpoint.alias
-    const aliased = alias ? alias : tableName
-    const tableAliased = alias ? `${tableName} as ${alias}` : tableName
     // select all if not specified
     if (!resource) {
       resource = { fields: [] }
@@ -487,10 +492,9 @@ class InternalBuilder {
       foundLimit = paginate.limit
     }
     // start building the query
-    let query: KnexQuery = knex(tableAliased).limit(foundLimit)
-    if (endpoint.schema) {
-      query = query.withSchema(endpoint.schema)
-    }
+
+    let { query, name: aliased } = this.knexWithAlias(knex, endpoint)
+    query = query.limit(foundLimit)
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
@@ -518,10 +522,7 @@ class InternalBuilder {
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
-    let query: KnexQuery = knex(endpoint.entityId)
-    if (endpoint.schema) {
-      query = query.withSchema(endpoint.schema)
-    }
+    let { query } = this.knexWithAlias(knex, endpoint)
     const parsedBody = parseBody(body)
     query = this.addFilters(query, filters, { tableName: endpoint.entityId })
     // mysql can't use returning
@@ -534,11 +535,8 @@ class InternalBuilder {
 
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
-    let query: KnexQuery = knex(endpoint.entityId)
-    if (endpoint.schema) {
-      query = query.withSchema(endpoint.schema)
-    }
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    let { query, name: aliased } = this.knexWithAlias(knex, endpoint)
+    query = this.addFilters(query, filters, { tableName: aliased })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

From 3ce00c42a2e9751bcfb17d906b8b8a8c85f04752 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 18 Jan 2024 18:13:11 +0000
Subject: [PATCH 08/47] Adding SQL logging capabilities.

---
 packages/server/src/environment.ts            |   1 +
 packages/server/src/integrations/base/sql.ts  |  12 +
 .../src/integrations/microsoftSqlServer.ts    |   1 +
 packages/server/src/integrations/mysql.ts     |   1 +
 packages/server/src/integrations/oracle.ts    |   1 +
 packages/server/src/integrations/postgres.ts  |   4 +-
 yarn.lock                                     | 660 +-----------------
 7 files changed, 55 insertions(+), 625 deletions(-)

diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index f692a8b6cf..f46abe5b16 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -67,6 +67,7 @@ const environment = {
   DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
+  ENABLE_SQL_LOGGING: process.env.ENABLE_SQL_LOGGING,
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
   FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 14bcb532cc..3375e175e6 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -671,6 +671,18 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
     }
     return results.length ? results : [{ [operation.toLowerCase()]: true }]
   }
+
+  log(query: string, values?: any[]) {
+    if (!environment.ENABLE_SQL_LOGGING) {
+      return
+    }
+    const sqlClient = this.getSqlClient()
+    let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"`
+    if (values) {
+      string += ` values="${values.join(", ")}"`
+    }
+    console.log(string)
+  }
 }
 
 export default SqlQueryBuilder
diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts
index d0a06d4476..e063933503 100644
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@@ -329,6 +329,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
         operation === Operation.CREATE
           ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
           : query.sql
+      this.log(sql, query.bindings)
       return await request.query(sql)
     } catch (err: any) {
       let readableMessage = getReadableErrorMessage(
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index 8ec73307f4..6eebda8df5 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -261,6 +261,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
       const bindings = opts?.disableCoercion
         ? baseBindings
         : bindingTypeCoerce(baseBindings)
+      this.log(query.sql, bindings)
       // Node MySQL is callback based, so we must wrap our call in a promise
       const response = await this.client!.query(query.sql, bindings)
       return response[0]
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index e9a2dc7998..1a1e440410 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -368,6 +368,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       const options: ExecuteOptions = { autoCommit: true }
       const bindings: BindParameters = query.bindings || []
 
+      this.log(query.sql, bindings)
       return await connection.execute<T>(query.sql, bindings, options)
     } finally {
       if (connection) {
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 78955c06dc..f8cd2b62fc 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -262,7 +262,9 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       }
     }
     try {
-      return await client.query(query.sql, query.bindings || [])
+      const bindings = query.bindings || []
+      this.log(query.sql, bindings)
+      return await client.query(query.sql, bindings)
     } catch (err: any) {
       await this.closeConnection()
       let readableMessage = getReadableErrorMessage(
diff --git a/yarn.lock b/yarn.lock
index 91697cd151..fa746b9d72 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -625,13 +625,6 @@
   dependencies:
     tslib "^2.5.0"
 
-"@aws/dynamodb-auto-marshaller@^0.7.1":
-  version "0.7.1"
-  resolved "https://registry.yarnpkg.com/@aws/dynamodb-auto-marshaller/-/dynamodb-auto-marshaller-0.7.1.tgz#70676c056e4ecb798c08ec2e398a3d93e703858d"
-  integrity sha512-LeURlf6/avrfFo9+4Yht9J3CUTJ72yoBpm1FOUmlexuHNW4Ka61tG30w3ZDCXXXmCO2rG0k3ywAgNJEo3WPbyw==
-  dependencies:
-    tslib "^1.8.1"
-
 "@azure/abort-controller@^1.0.0", "@azure/abort-controller@^1.0.4":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.1.0.tgz#788ee78457a55af8a1ad342acb182383d2119249"
@@ -1980,7 +1973,7 @@
   resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310"
   integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==
 
-"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
+"@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.7.2", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2":
   version "7.23.8"
   resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.8.tgz#8ee6fe1ac47add7122902f257b8ddf55c898f650"
   integrity sha512-Y7KbAP984rn1VGMbGqKmBLio9V7y5Je9GvU4rQPCPinCyNfUcToxIXl06d59URp/F3LwinvODxab5N/G6qggkw==
@@ -2638,14 +2631,6 @@
     teeny-request "^8.0.0"
     uuid "^8.0.0"
 
-"@grpc/grpc-js@1.9.7":
-  version "1.9.7"
-  resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.9.7.tgz#7d0e29bc162287bee2523901c9bc9320d8402397"
-  integrity sha512-yMaA/cIsRhGzW3ymCNpdlPcInXcovztlgu/rirThj2b87u3RzWUszliOqZ/pldy7yhmJPS8uwog+kZSTa4A0PQ==
-  dependencies:
-    "@grpc/proto-loader" "^0.7.8"
-    "@types/node" ">=12.12.47"
-
 "@grpc/grpc-js@~1.8.0":
   version "1.8.21"
   resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.21.tgz#d282b122c71227859bf6c5866f4c40f4a2696513"
@@ -2654,7 +2639,7 @@
     "@grpc/proto-loader" "^0.7.0"
     "@types/node" ">=12.12.47"
 
-"@grpc/proto-loader@0.7.10", "@grpc/proto-loader@^0.7.0", "@grpc/proto-loader@^0.7.8":
+"@grpc/proto-loader@^0.7.0":
   version "0.7.10"
   resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.10.tgz#6bf26742b1b54d0a473067743da5d3189d06d720"
   integrity sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==
@@ -2676,20 +2661,6 @@
   dependencies:
     "@hapi/hoek" "^9.0.0"
 
-"@hubspot/api-client@7.1.2":
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/@hubspot/api-client/-/api-client-7.1.2.tgz#a405b0a18b8caa27f129fd510b2555e5a5cc2708"
-  integrity sha512-JVQqh0fdHf97ePk0Hg/7BJsiXNlS9HQRPiM/CLgvVWt5CIviSLQ/kHLZXREmZqTWu7BisjCgHxnSx/d7gRdr2g==
-  dependencies:
-    bluebird "^3.7.2"
-    bottleneck "^2.19.5"
-    btoa "^1.2.1"
-    es6-promise "^4.2.4"
-    form-data "^2.5.0"
-    lodash "^4.17.21"
-    node-fetch "^2.6.0"
-    url-parse "^1.4.3"
-
 "@humanwhocodes/config-array@^0.11.13":
   version "0.11.13"
   resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297"
@@ -3341,13 +3312,6 @@
   dependencies:
     lodash "^4.17.21"
 
-"@koa/cors@^3.1.0":
-  version "3.4.3"
-  resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.4.3.tgz#d669ee6e8d6e4f0ec4a7a7b0a17e7a3ed3752ebb"
-  integrity sha512-WPXQUaAeAMVaLTEFpoq3T2O1C+FstkjJnDQqy95Ck1UdILajsRhu6mhJ8H2f4NFPRBoCNN+qywTJfq/gGki5mw==
-  dependencies:
-    vary "^1.1.2"
-
 "@koa/router@8.0.8":
   version "8.0.8"
   resolved "https://registry.yarnpkg.com/@koa/router/-/router-8.0.8.tgz#95f32d11373d03d89dcb63fabe9ac6f471095236"
@@ -3956,14 +3920,6 @@
     is-module "^1.0.0"
     resolve "^1.19.0"
 
-"@rollup/plugin-replace@^2.4.2":
-  version "2.4.2"
-  resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-2.4.2.tgz#a2d539314fbc77c244858faa523012825068510a"
-  integrity sha512-IGcu+cydlUMZ5En85jxHH4qj2hta/11BHq95iHEyb2sbgiN0eCdzvUcHw5gt9pBL5lTi4JDYJ1acCoMGpTvEZg==
-  dependencies:
-    "@rollup/pluginutils" "^3.1.0"
-    magic-string "^0.25.7"
-
 "@rollup/plugin-replace@^5.0.2", "@rollup/plugin-replace@^5.0.3":
   version "5.0.5"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-replace/-/plugin-replace-5.0.5.tgz#33d5653dce6d03cb24ef98bef7f6d25b57faefdf"
@@ -4006,23 +3962,6 @@
     estree-walker "^2.0.2"
     picomatch "^2.3.1"
 
-"@roxi/routify@2.18.0":
-  version "2.18.0"
-  resolved "https://registry.yarnpkg.com/@roxi/routify/-/routify-2.18.0.tgz#8f88bedd936312d0dbe44cbc11ab179b1f938ec2"
-  integrity sha512-MVB50HN+VQWLzfjLplcBjsSBvwOiExKOmht2DuWR3WQ60JxQi9pSejkB06tFVkFKNXz2X5iYtKDqKBTdae/gRg==
-  dependencies:
-    "@roxi/ssr" "^0.2.1"
-    "@types/node" ">=4.2.0 < 13"
-    chalk "^4.0.0"
-    cheap-watch "^1.0.2"
-    commander "^7.1.0"
-    configent "^2.1.4"
-    esm "^3.2.25"
-    fs-extra "^9.0.1"
-    log-symbols "^3.0.0"
-    picomatch "^2.2.2"
-    rollup-pluginutils "^2.8.2"
-
 "@roxi/routify@2.18.12":
   version "2.18.12"
   resolved "https://registry.yarnpkg.com/@roxi/routify/-/routify-2.18.12.tgz#901ca95b96f274ddddaefbf18424557ee1ae3fae"
@@ -4104,11 +4043,6 @@
   resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd"
   integrity sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==
 
-"@sindresorhus/is@^4.0.0":
-  version "4.6.0"
-  resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f"
-  integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==
-
 "@sinonjs/commons@^2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-2.0.0.tgz#fd4ca5b063554307e8327b4564bd56d3b73924a3"
@@ -4932,13 +4866,6 @@
   dependencies:
     defer-to-connect "^1.0.1"
 
-"@szmarczak/http-timer@^4.0.5":
-  version "4.0.6"
-  resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807"
-  integrity sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==
-  dependencies:
-    defer-to-connect "^2.0.0"
-
 "@techpass/passport-openidconnect@0.3.2":
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.2.tgz#f8fd5d97256286665dbf26dac92431f977ab1e63"
@@ -4950,17 +4877,6 @@
     request "^2.88.0"
     webfinger "^0.4.2"
 
-"@techpass/passport-openidconnect@^0.3.0":
-  version "0.3.3"
-  resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.3.tgz#6c01c78bd8da0ca8917378dfbe18024702620352"
-  integrity sha512-i2X/CofjnGBqpTmw6b+Ex3Co/NrR2xjnIHvnOJk62XIlJJHNSTwmhJ1PkXoA5RGKlxZWchADFGjLTJnebvRj7A==
-  dependencies:
-    base64url "^3.0.1"
-    oauth "^0.9.15"
-    passport-strategy "^1.0.0"
-    request "^2.88.0"
-    webfinger "^0.4.2"
-
 "@techteamer/ocsp@1.0.0":
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/@techteamer/ocsp/-/ocsp-1.0.0.tgz#7b82b02093fbe351e915bb37685ac1ac5a1233d3"
@@ -5133,16 +5049,6 @@
     "@types/connect" "*"
     "@types/node" "*"
 
-"@types/cacheable-request@^6.0.1":
-  version "6.0.3"
-  resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183"
-  integrity sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==
-  dependencies:
-    "@types/http-cache-semantics" "*"
-    "@types/keyv" "^3.1.4"
-    "@types/node" "*"
-    "@types/responselike" "^1.0.0"
-
 "@types/caseless@*":
   version "0.12.2"
   resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8"
@@ -5307,11 +5213,6 @@
   resolved "https://registry.yarnpkg.com/@types/http-assert/-/http-assert-1.5.3.tgz#ef8e3d1a8d46c387f04ab0f2e8ab8cb0c5078661"
   integrity sha512-FyAOrDuQmBi8/or3ns4rwPno7/9tJTijVW6aQQjK02+kOQ8zmoNg2XJtAuQhvQcy1ASJq38wirX5//9J1EqoUA==
 
-"@types/http-cache-semantics@*":
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz#b979ebad3919799c979b17c72621c0bc0a31c6c4"
-  integrity sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==
-
 "@types/http-errors@*":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.1.tgz#20172f9578b225f6c7da63446f56d4ce108d5a65"
@@ -5373,13 +5274,6 @@
   resolved "https://registry.yarnpkg.com/@types/keygrip/-/keygrip-1.0.2.tgz#513abfd256d7ad0bf1ee1873606317b33b1b2a72"
   integrity sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==
 
-"@types/keyv@^3.1.4":
-  version "3.1.4"
-  resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6"
-  integrity sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==
-  dependencies:
-    "@types/node" "*"
-
 "@types/koa-compose@*":
   version "3.2.5"
   resolved "https://registry.yarnpkg.com/@types/koa-compose/-/koa-compose-3.2.5.tgz#85eb2e80ac50be95f37ccf8c407c09bbe3468e9d"
@@ -5387,29 +5281,13 @@
   dependencies:
     "@types/koa" "*"
 
-"@types/koa-passport@^4.0.3":
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/@types/koa-passport/-/koa-passport-4.0.3.tgz#063ec6310edee76cf854aadaa717b97f04b104fb"
-  integrity sha512-tNMYd/bcv0Zw7fc0CzEBYM9uUzVtn4XWzdUYfkTgSkEljP6nap7eI4E5x43ukrUQvztgXSYFkz3Uk+ujFeUzTg==
-  dependencies:
-    "@types/koa" "*"
-    "@types/passport" "*"
-
-"@types/koa-send@*", "@types/koa-send@^4.1.6":
+"@types/koa-send@^4.1.6":
   version "4.1.6"
   resolved "https://registry.yarnpkg.com/@types/koa-send/-/koa-send-4.1.6.tgz#15d90e95e3ccce669a15b6a3c56c3a650a167cea"
   integrity sha512-vgnNGoOJkx7FrF0Jl6rbK1f8bBecqAchKpXtKuXzqIEdXTDO6dsSTjr+eZ5m7ltSjH4K/E7auNJEQCAd0McUPA==
   dependencies:
     "@types/koa" "*"
 
-"@types/koa-static@^4.0.2":
-  version "4.0.4"
-  resolved "https://registry.yarnpkg.com/@types/koa-static/-/koa-static-4.0.4.tgz#ce6f2a5d14cc7ef19f9bf6ee8e4f3eadfcc77323"
-  integrity sha512-j1AUzzl7eJYEk9g01hNTlhmipFh8RFbOQmaMNLvLcNNAkPw0bdTs3XTa3V045XFlrWN0QYnblbDJv2RzawTn6A==
-  dependencies:
-    "@types/koa" "*"
-    "@types/koa-send" "*"
-
 "@types/koa@*":
   version "2.13.5"
   resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.5.tgz#64b3ca4d54e08c0062e89ec666c9f45443b21a61"
@@ -5438,13 +5316,6 @@
     "@types/koa-compose" "*"
     "@types/node" "*"
 
-"@types/koa__cors@^3.1.1":
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.1.tgz#0ec7543c4c620fd23451bfdd3e21b9a6aadedccd"
-  integrity sha512-aFGYhTFW7651KhmZZ05VG0QZJre7QxBxDj2LF1lf6GA/wSXEfKVAJxiQQWzRV4ZoMzQIO8vJBXKsUcRuvYK9qw==
-  dependencies:
-    "@types/koa" "*"
-
 "@types/koa__router@8.0.8":
   version "8.0.8"
   resolved "https://registry.yarnpkg.com/@types/koa__router/-/koa__router-8.0.8.tgz#b1e0e9a512498777d3366bbdf0e853df27ec831c"
@@ -5546,42 +5417,21 @@
   dependencies:
     undici-types "~5.26.4"
 
-"@types/node@>=4.2.0 < 13", "@types/node@^12.20.52":
-  version "12.20.55"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240"
-  integrity sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==
-
 "@types/node@>=8.0.0 <15":
   version "14.18.37"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.37.tgz#0bfcd173e8e1e328337473a8317e37b3b14fd30d"
   integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
 
-"@types/node@>=8.1.0":
-  version "20.11.2"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.2.tgz#39cea3fe02fbbc2f80ed283e94e1d24f2d3856fb"
-  integrity sha512-cZShBaVa+UO1LjWWBPmWRR4+/eY/JR/UIEcDlVsw3okjWEu+rB7/mH6X3B/L+qJVHDLjk9QW/y2upp9wp1yDXA==
-  dependencies:
-    undici-types "~5.26.4"
-
-"@types/nodemailer@^6.4.4":
-  version "6.4.14"
-  resolved "https://registry.yarnpkg.com/@types/nodemailer/-/nodemailer-6.4.14.tgz#5c81a5e856db7f8ede80013e6dbad7c5fb2283e2"
-  integrity sha512-fUWthHO9k9DSdPCSPRqcu6TWhYyxTBg382vlNIttSe9M7XfsT06y0f24KHXtbnijPGGRIcVvdKHTNikOI6qiHA==
-  dependencies:
-    "@types/node" "*"
+"@types/node@^12.20.52":
+  version "12.20.55"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240"
+  integrity sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==
 
 "@types/normalize-package-data@^2.4.0":
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301"
   integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==
 
-"@types/oauth@*":
-  version "0.9.4"
-  resolved "https://registry.yarnpkg.com/@types/oauth/-/oauth-0.9.4.tgz#dcbab5efa2f34f312b915f80685760ccc8111e0a"
-  integrity sha512-qk9orhti499fq5XxKCCEbd0OzdPZuancneyse3KtR+vgMiHRbh+mn8M4G6t64ob/Fg+GZGpa565MF/2dKWY32A==
-  dependencies:
-    "@types/node" "*"
-
 "@types/oracledb@5.2.2":
   version "5.2.2"
   resolved "https://registry.yarnpkg.com/@types/oracledb/-/oracledb-5.2.2.tgz#ae7ba795969e3bbd8d57ab141873a1aa012b86cd"
@@ -5590,37 +5440,6 @@
     "@types/node" "*"
     dotenv "^8.2.0"
 
-"@types/passport-google-oauth@^1.0.42":
-  version "1.0.45"
-  resolved "https://registry.yarnpkg.com/@types/passport-google-oauth/-/passport-google-oauth-1.0.45.tgz#c986c787ec9706b4a596d2bae43342b50b54973d"
-  integrity sha512-O3Y3DDKnf9lR8+DSaUOCEGF6aFjVYdI8TLhQYtySZ3Sq75c5tGYJ0KJRDZw0GsyLD/Que0nqFkP/GnDVwZZL9w==
-  dependencies:
-    "@types/express" "*"
-    "@types/passport" "*"
-
-"@types/passport-microsoft@1.0.0":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/@types/passport-microsoft/-/passport-microsoft-1.0.0.tgz#a2ddc2200843570d38c35c53f6388e33df915b58"
-  integrity sha512-vD9ajSUc9Sz/8gdCj0ODUbPYQDxcI/imIDdgMPh//c5yMK/PgV6SNUXFLBzJo89Y30LU6bYAfXKn40WJqtMBiA==
-  dependencies:
-    "@types/passport-oauth2" "*"
-
-"@types/passport-oauth2@*":
-  version "1.4.15"
-  resolved "https://registry.yarnpkg.com/@types/passport-oauth2/-/passport-oauth2-1.4.15.tgz#34f2684f53aad36e664cd01ca9879224229f47e7"
-  integrity sha512-9cUTP/HStNSZmhxXGuRrBJfEWzIEJRub2eyJu3CvkA+8HAMc9W3aKdFhVq+Qz1hi42qn+GvSAnz3zwacDSYWpw==
-  dependencies:
-    "@types/express" "*"
-    "@types/oauth" "*"
-    "@types/passport" "*"
-
-"@types/passport@*":
-  version "1.0.16"
-  resolved "https://registry.yarnpkg.com/@types/passport/-/passport-1.0.16.tgz#5a2918b180a16924c4d75c31254c31cdca5ce6cf"
-  integrity sha512-FD0qD5hbPWQzaM0wHUnJ/T0BBCJBxCeemtnCwc/ThhTg3x9jfrAcRUmj5Dopza+MfFS9acTe3wk7rcVnRIp/0A==
-  dependencies:
-    "@types/express" "*"
-
 "@types/pg@8.6.6":
   version "8.6.6"
   resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.6.6.tgz#21cdf873a3e345a6e78f394677e3b3b1b543cb80"
@@ -5825,13 +5644,6 @@
   dependencies:
     "@types/node" "*"
 
-"@types/responselike@^1.0.0":
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.3.tgz#cc29706f0a397cfe6df89debfe4bf5cea159db50"
-  integrity sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==
-  dependencies:
-    "@types/node" "*"
-
 "@types/rimraf@^3.0.2":
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/@types/rimraf/-/rimraf-3.0.2.tgz#a63d175b331748e5220ad48c901d7bbf1f44eef8"
@@ -5860,13 +5672,6 @@
   dependencies:
     "@types/node" "*"
 
-"@types/server-destroy@^1.0.1":
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/@types/server-destroy/-/server-destroy-1.0.3.tgz#2460932ea3a02a70ec99669c8f40ff089a5b8a2b"
-  integrity sha512-Qq0fn70C7TLDG1W9FCblKufNWW1OckQ41dVKV2Dku5KdZF7bexezG4e2WBaBKhdwL3HZ+cYCEIKwg2BRgzrWmA==
-  dependencies:
-    "@types/node" "*"
-
 "@types/stack-utils@^2.0.0":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
@@ -5948,7 +5753,7 @@
   dependencies:
     "@types/node" "*"
 
-"@types/uuid@8.3.4", "@types/uuid@^8.3.4":
+"@types/uuid@8.3.4":
   version "8.3.4"
   resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc"
   integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==
@@ -6290,6 +6095,11 @@
     js-yaml "^3.10.0"
     tslib "^2.4.0"
 
+"@zerodevx/svelte-json-view@^1.0.7":
+  version "1.0.7"
+  resolved "https://registry.yarnpkg.com/@zerodevx/svelte-json-view/-/svelte-json-view-1.0.7.tgz#abf3efa71dedcb3e9d16bc9cc61d5ea98c8d00b1"
+  integrity sha512-yW0MV+9BCKOwzt3h86y3xDqYdI5st+Rxk+L5pa0Utq7nlPD+VvxyhL7R1gJoLxQvWwjyAvY/fyUCFTdwDyI14w==
+
 "@zkochan/js-yaml@0.0.6":
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/@zkochan/js-yaml/-/js-yaml-0.0.6.tgz#975f0b306e705e28b8068a07737fa46d3fc04826"
@@ -6332,7 +6142,7 @@ abortcontroller-polyfill@^1.4.0:
   resolved "https://registry.yarnpkg.com/abortcontroller-polyfill/-/abortcontroller-polyfill-1.7.5.tgz#6738495f4e901fbb57b6c0611d0c75f76c485bed"
   integrity sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ==
 
-abstract-leveldown@^6.2.1, abstract-leveldown@^6.3.0:
+abstract-leveldown@^6.2.1:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz#d25221d1e6612f820c35963ba4bd739928f6026a"
   integrity sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ==
@@ -6854,13 +6664,6 @@ async-retry@^1.3.3:
   dependencies:
     retry "0.13.1"
 
-async@^2.6.3:
-  version "2.6.4"
-  resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"
-  integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==
-  dependencies:
-    lodash "^4.17.14"
-
 async@^3.2.1, async@^3.2.3:
   version "3.2.4"
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c"
@@ -7234,11 +7037,6 @@ bootstrap@3.4.1:
   resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.4.1.tgz#c3a347d419e289ad11f4033e3c4132b87c081d72"
   integrity sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==
 
-bottleneck@^2.19.5:
-  version "2.19.5"
-  resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.19.5.tgz#5df0b90f59fd47656ebe63c78a98419205cadd91"
-  integrity sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==
-
 bowser@^2.11.0:
   version "2.11.0"
   resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f"
@@ -7387,11 +7185,6 @@ bson@^5.4.0:
   resolved "https://registry.yarnpkg.com/bson/-/bson-5.4.0.tgz#0eea77276d490953ad8616b483298dbff07384c6"
   integrity sha512-WRZ5SQI5GfUuKnPTNmAYPiKIof3ORXAF4IRU5UcgmivNIon01rWQlw5RUH954dpu8yGL8T59YShVddIPaU/gFA==
 
-btoa@^1.2.1:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/btoa/-/btoa-1.2.1.tgz#01a9909f8b2c93f6bf680ba26131eb30f7fa3d73"
-  integrity sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g==
-
 buffer-alloc-unsafe@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0"
@@ -7405,7 +7198,7 @@ buffer-alloc@^1.2.0:
     buffer-alloc-unsafe "^1.1.0"
     buffer-fill "^1.0.0"
 
-buffer-crc32@^0.2.13, buffer-crc32@~0.2.3:
+buffer-crc32@~0.2.3:
   version "0.2.13"
   resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
   integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
@@ -7590,11 +7383,6 @@ cache-content-type@^1.0.0:
     mime-types "^2.1.18"
     ylru "^1.2.0"
 
-cacheable-lookup@^5.0.3:
-  version "5.0.4"
-  resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005"
-  integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==
-
 cacheable-request@^2.1.1:
   version "2.1.4"
   resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-2.1.4.tgz#0d808801b6342ad33c91df9d0b44dc09b91e5c3d"
@@ -7621,19 +7409,6 @@ cacheable-request@^6.0.0:
     normalize-url "^4.1.0"
     responselike "^1.0.2"
 
-cacheable-request@^7.0.2:
-  version "7.0.4"
-  resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.4.tgz#7a33ebf08613178b403635be7b899d3e69bbe817"
-  integrity sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==
-  dependencies:
-    clone-response "^1.0.2"
-    get-stream "^5.1.0"
-    http-cache-semantics "^4.0.0"
-    keyv "^4.0.0"
-    lowercase-keys "^2.0.0"
-    normalize-url "^6.0.1"
-    responselike "^2.0.0"
-
 call-bind@^1.0.0, call-bind@^1.0.2, call-bind@^1.0.4, call-bind@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.5.tgz#6fa2b7845ce0ea49bf4d8b9ef64727a2c2e2e513"
@@ -7672,52 +7447,6 @@ camelcase@^6.2.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
   integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
 
-camunda-8-credentials-from-env@^1.1.1, camunda-8-credentials-from-env@^1.2.2:
-  version "1.2.2"
-  resolved "https://registry.yarnpkg.com/camunda-8-credentials-from-env/-/camunda-8-credentials-from-env-1.2.2.tgz#abe5d216e7e4cfc970e0463e9aa5e802487b1062"
-  integrity sha512-uj2PY5/IoAgu0cHmeEUp+qmSXCtpQafStzGJ8ORYvyupBN/gVpdP9X+A+UlQRCGmApcaIuPUw8/9FsXig5NWXg==
-  dependencies:
-    neon-env "^0.1.1"
-
-camunda-8-sdk@^0.15.0:
-  version "0.15.0"
-  resolved "https://registry.yarnpkg.com/camunda-8-sdk/-/camunda-8-sdk-0.15.0.tgz#13754dca499d16802675b6f2790e2d06bd8034d6"
-  integrity sha512-felyQU+rD8uupPjBArmyy0E/k9mrmeZvfFliF3y/pxYkGBoaC5kjDHDsx+hNpbnIwShET0RLjklit7f+98yIBw==
-  dependencies:
-    camunda-console-client "^0.9.1"
-    camunda-tasklist-client "0.9.5"
-    operate-api-client "1.2.3"
-    optimize-api-client "^1.0.3"
-    zeebe-node "^8.2.5"
-
-camunda-console-client@^0.9.1:
-  version "0.9.2"
-  resolved "https://registry.yarnpkg.com/camunda-console-client/-/camunda-console-client-0.9.2.tgz#137dbd2e61bb5bbfff38aebe5d53e775653aabb8"
-  integrity sha512-ni+7lSc5oG0FevCagrBV6juZzwcQ4ciATBZxyOMFQK0yVTmZxOUz5efN9XWP4E36PGpuqALQXsViUDlGZcfZBA==
-  dependencies:
-    camunda-8-credentials-from-env "^1.2.2"
-    camunda-saas-oauth "^1.2.4"
-    debug "^4.3.4"
-    dotenv "^16.3.1"
-    got "^11.8.6"
-
-camunda-saas-oauth@^1.2.0, camunda-saas-oauth@^1.2.4:
-  version "1.2.4"
-  resolved "https://registry.yarnpkg.com/camunda-saas-oauth/-/camunda-saas-oauth-1.2.4.tgz#348a8422f266dafed98cf2a73046aa62c89d03f2"
-  integrity sha512-AO/kcnZXcsodwM3qgMZj/5wn8SBoKmSDpuFYUpPS+HqQhG9GvWY8noBx/4pvX3gYPKiPTYi9/e9ApAe02NARzA==
-  dependencies:
-    camunda-8-credentials-from-env "^1.2.2"
-    got "^11.8.5"
-
-camunda-tasklist-client@0.9.5:
-  version "0.9.5"
-  resolved "https://registry.yarnpkg.com/camunda-tasklist-client/-/camunda-tasklist-client-0.9.5.tgz#c0f2685ef7fb7fdb198a37e5b35a911e3b233b28"
-  integrity sha512-gipH8ON/ttTgLfleWecQith1g9SpC5Q8CoLXFq2yw3cVJ1JVrcn0ArtgCxA1QCgtZBlV7EuGt9QWGc9UCfbbGw==
-  dependencies:
-    camunda-8-credentials-from-env "^1.1.1"
-    camunda-saas-oauth "^1.2.0"
-    gotql "^2.1.0-alpha1"
-
 caniuse-api@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/caniuse-api/-/caniuse-api-3.0.0.tgz#5e4d90e2274961d46291997df599e3ed008ee4c0"
@@ -7811,7 +7540,7 @@ charenc@0.0.2:
   resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
   integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==
 
-cheap-watch@^1.0.2, cheap-watch@^1.0.4:
+cheap-watch@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/cheap-watch/-/cheap-watch-1.0.4.tgz#0bcb4a3a8fbd9d5327936493f6b56baa668d8fef"
   integrity sha512-QR/9FrtRL5fjfUJBhAKCdi0lSRQ3rVRRum3GF9wDKp2TJbEIMGhUEr2yU8lORzm9Isdjx7/k9S0DFDx+z5VGtw==
@@ -8138,7 +7867,7 @@ commander@^5.1.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae"
   integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
 
-commander@^7.0.0, commander@^7.1.0, commander@^7.2.0:
+commander@^7.0.0, commander@^7.2.0:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
   integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
@@ -8247,7 +7976,7 @@ config-chain@^1.1.13:
     ini "^1.3.4"
     proto-list "~1.2.1"
 
-configent@^2.1.4, configent@^2.2.0:
+configent@^2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/configent/-/configent-2.2.0.tgz#2de230fc43f22c47cfd99016aa6962d6f9546994"
   integrity sha512-yIN6zfOWk2nycNJ2JFNiWEai0oiqAhISIht8+pbEBP8bdcpwoQ74AhCZPbUv9aRVJwo7wh1MbCBDUV44UJa7Kw==
@@ -8271,14 +8000,6 @@ console-control-strings@^1.0.0, console-control-strings@^1.1.0:
   resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
   integrity sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==
 
-console-stamp@^3.0.2:
-  version "3.1.2"
-  resolved "https://registry.yarnpkg.com/console-stamp/-/console-stamp-3.1.2.tgz#35dac393e16069a4d9d37b71ca6d5d13d7f3f8fd"
-  integrity sha512-ab66x3NxOTxPuq71dI6gXEiw2X6ql4Le5gZz0bm7FW3FSCB00eztra/oQUuCoCGlsyKOxtULnHwphzMrRtzMBg==
-  dependencies:
-    chalk "^4.1.2"
-    dateformat "^4.6.3"
-
 consolidate@^0.16.0:
   version "0.16.0"
   resolved "https://registry.yarnpkg.com/consolidate/-/consolidate-0.16.0.tgz#a11864768930f2f19431660a65906668f5fbdc16"
@@ -8771,7 +8492,7 @@ dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-dayjs@^1.10.8, dayjs@^1.8.15:
+dayjs@^1.10.8:
   version "1.11.10"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0"
   integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==
@@ -8821,7 +8542,7 @@ dd-trace@5.0.0:
     semver "^7.5.4"
     tlhunter-sorted-set "^0.1.0"
 
-debug@4, debug@4.3.4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.2.0, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
+debug@4, debug@4.3.4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
   integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -8870,13 +8591,6 @@ decompress-response@^3.3.0:
   dependencies:
     mimic-response "^1.0.0"
 
-decompress-response@^6.0.0:
-  version "6.0.0"
-  resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc"
-  integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==
-  dependencies:
-    mimic-response "^3.1.0"
-
 decompress-tar@^4.0.0, decompress-tar@^4.1.0, decompress-tar@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/decompress-tar/-/decompress-tar-4.1.1.tgz#718cbd3fcb16209716e70a26b84e7ba4592e5af1"
@@ -9002,11 +8716,6 @@ defer-to-connect@^1.0.1:
   resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-1.1.3.tgz#331ae050c08dcf789f8c83a7b81f0ed94f4ac591"
   integrity sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==
 
-defer-to-connect@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587"
-  integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==
-
 deferred-leveldown@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/deferred-leveldown/-/deferred-leveldown-0.2.0.tgz#2cef1f111e1c57870d8bbb8af2650e587cd2f5b4"
@@ -9050,11 +8759,6 @@ defined@^1.0.0:
   resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.1.tgz#c0b9db27bfaffd95d6f61399419b893df0f91ebf"
   integrity sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==
 
-defined@~0.0.0:
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/defined/-/defined-0.0.0.tgz#f35eea7d705e933baf13b2f03b3f83d921403b3e"
-  integrity sha512-zpqiCT8bODLu3QSmLLic8xJnYWBFjOSu/fBCm189oAiTtPq/PSanNACKZDS7kgSyCJY7P+IcODzlIogBK/9RBg==
-
 delay@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d"
@@ -9350,7 +9054,7 @@ docker-compose@0.24.0:
   dependencies:
     yaml "^1.10.2"
 
-docker-compose@^0.23.5, docker-compose@^0.23.6:
+docker-compose@^0.23.5:
   version "0.23.19"
   resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.19.tgz#9947726e2fe67bdfa9e8efe1ff15aa0de2e10eb8"
   integrity sha512-v5vNLIdUqwj4my80wxFDkNH+4S85zsRuH29SO7dCWVWPCMt/ohZBsGN6g6KXWifT0pzQ7uOxqEKCYCDPJ8Vz4g==
@@ -9491,11 +9195,6 @@ dotenv@8.6.0, dotenv@^8.2.0:
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.6.0.tgz#061af664d19f7f4d8fc6e4ff9b584ce237adcb8b"
   integrity sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==
 
-dotenv@^16.3.1:
-  version "16.3.1"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"
-  integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==
-
 dotenv@~10.0.0:
   version "10.0.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81"
@@ -9548,24 +9247,6 @@ duplexify@^4.0.0, duplexify@^4.1.2:
     readable-stream "^3.1.1"
     stream-shift "^1.0.0"
 
-dynalite@^3.2.1:
-  version "3.2.2"
-  resolved "https://registry.yarnpkg.com/dynalite/-/dynalite-3.2.2.tgz#34b4f4dd69638f17c0f7551a867959972c892441"
-  integrity sha512-sx9ZjTgMs/D4gHnba4rnBkw29648dHwHmywJet132KAbiq1ZyWx9W1fMd/eP9cPwTKDXyCBuTYOChE0qMDjaXQ==
-  dependencies:
-    async "^2.6.3"
-    big.js "^5.2.2"
-    buffer-crc32 "^0.2.13"
-    lazy "^1.0.11"
-    levelup "^4.4.0"
-    lock "^1.1.0"
-    memdown "^5.1.0"
-    minimist "^1.2.5"
-    once "^1.4.0"
-    subleveldown "^5.0.1"
-  optionalDependencies:
-    leveldown "^5.6.0"
-
 eastasianwidth@^0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb"
@@ -9672,7 +9353,7 @@ encodeurl@^1.0.2:
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
   integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==
 
-encoding-down@^6.2.0, encoding-down@^6.3.0:
+encoding-down@^6.3.0:
   version "6.3.0"
   resolved "https://registry.yarnpkg.com/encoding-down/-/encoding-down-6.3.0.tgz#b1c4eb0e1728c146ecaef8e32963c549e76d082b"
   integrity sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==
@@ -9780,11 +9461,6 @@ envinfo@7.8.1, envinfo@^7.7.3:
   resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.8.1.tgz#06377e3e5f4d379fea7ac592d5ad8927e0c4d475"
   integrity sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw==
 
-err-code@^1.0.0:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/err-code/-/err-code-1.1.2.tgz#06e0116d3028f6aef4806849eb0ea6a748ae6960"
-  integrity sha512-CJAN+O0/yA1CKfRn9SXOGctSpEM7DCon/r/5r2eXFMY2zCCJBasFhcM5I+1kh3Ap11FsQCX+vGHceNPvpWKhoA==
-
 err-code@^2.0.2:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9"
@@ -9922,11 +9598,6 @@ es6-error@^4.0.1, es6-error@^4.1.1:
   resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d"
   integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==
 
-es6-promise@^4.2.4:
-  version "4.2.8"
-  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.8.tgz#4eb21594c972bc40553d276e510539143db53e0a"
-  integrity sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==
-
 esbuild-loader@^2.16.0:
   version "2.21.0"
   resolved "https://registry.yarnpkg.com/esbuild-loader/-/esbuild-loader-2.21.0.tgz#2698a3e565b0db2bb19a3dd91c2b6c9aad526c80"
@@ -10525,13 +10196,6 @@ fast-xml-parser@4.2.5:
   dependencies:
     strnum "^1.0.5"
 
-fast-xml-parser@^4.1.3:
-  version "4.3.3"
-  resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.3.3.tgz#aeaf5778392329f17168c40c51bcbfec8ff965be"
-  integrity sha512-coV/D1MhrShMvU6D0I+VAK3umz6hUaxxhL0yp/9RjfiYUfAv14rDhGQL+PLForhMdr0wq3PiV07WtkkNjJjNHg==
-  dependencies:
-    strnum "^1.0.5"
-
 fast-xml-parser@^4.2.2, fast-xml-parser@^4.2.5:
   version "4.3.2"
   resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.3.2.tgz#761e641260706d6e13251c4ef8e3f5694d4b0d79"
@@ -10872,11 +10536,6 @@ formidable@^2.1.2:
     once "^1.4.0"
     qs "^6.11.0"
 
-fp-ts@^2.5.1:
-  version "2.16.2"
-  resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.16.2.tgz#7faa90f6fc2e8cf84c711d2c4e606afe2be9e342"
-  integrity sha512-CkqAjnIKFqvo3sCyoBTqgJvF+bHrSik584S9nhTjtBESLx26cbtVMR/T9a6ApChOcSDAaM3JydDmWDUn4EEXng==
-
 fresh@^0.5.2, fresh@~0.5.2:
   version "0.5.2"
   resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7"
@@ -10918,7 +10577,7 @@ fs-extra@^11.1.0, fs-extra@^11.1.1:
     jsonfile "^6.0.1"
     universalify "^2.0.0"
 
-fs-extra@^9.0.0, fs-extra@^9.0.1, fs-extra@^9.1.0:
+fs-extra@^9.0.0, fs-extra@^9.1.0:
   version "9.1.0"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
   integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==
@@ -10967,7 +10626,7 @@ function.prototype.name@^1.1.6:
     es-abstract "^1.22.1"
     functions-have-names "^1.2.3"
 
-functional-red-black-tree@^1.0.1, functional-red-black-tree@~1.0.1:
+functional-red-black-tree@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
   integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
@@ -11561,23 +11220,6 @@ gopd@^1.0.1:
   dependencies:
     get-intrinsic "^1.1.3"
 
-got@^11.5.1, got@^11.8.5, got@^11.8.6:
-  version "11.8.6"
-  resolved "https://registry.yarnpkg.com/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a"
-  integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==
-  dependencies:
-    "@sindresorhus/is" "^4.0.0"
-    "@szmarczak/http-timer" "^4.0.5"
-    "@types/cacheable-request" "^6.0.1"
-    "@types/responselike" "^1.0.0"
-    cacheable-lookup "^5.0.3"
-    cacheable-request "^7.0.2"
-    decompress-response "^6.0.0"
-    http2-wrapper "^1.0.0-beta.5.2"
-    lowercase-keys "^2.0.0"
-    p-cancelable "^2.0.0"
-    responselike "^2.0.0"
-
 got@^8.3.1:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/got/-/got-8.3.2.tgz#1d23f64390e97f776cac52e5b936e5f514d2e937"
@@ -11618,15 +11260,6 @@ got@^9.6.0:
     to-readable-stream "^1.0.0"
     url-parse-lax "^3.0.0"
 
-gotql@^2.1.0-alpha1:
-  version "2.1.0-alpha1"
-  resolved "https://registry.yarnpkg.com/gotql/-/gotql-2.1.0-alpha1.tgz#b04e9adb0d1751a0c2de05bd4399f5c57aec79ba"
-  integrity sha512-4xG1AczSpK+tdKUDM4kB1ah/2LoNlmFU5IhGNktuYNBLgyWB5iDs4OE36NE7k59iTKYi2B7vudQz2Itw1ZXrRg==
-  dependencies:
-    debug "^4.1.1"
-    got "^11.5.1"
-    prepend-http "^3.0.1"
-
 graceful-fs@4.2.11, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9:
   version "4.2.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
@@ -11991,14 +11624,6 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-http2-wrapper@^1.0.0-beta.5.2:
-  version "1.0.3"
-  resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d"
-  integrity sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==
-  dependencies:
-    quick-lru "^5.1.1"
-    resolve-alpn "^1.0.0"
-
 https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6"
@@ -12118,11 +11743,6 @@ immediate@~3.0.5:
   resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
   integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==
 
-immediate@~3.2.3:
-  version "3.2.3"
-  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.2.3.tgz#d140fa8f614659bd6541233097ddaac25cdd991c"
-  integrity sha512-RrGCXRm/fRVqMIhqXrGEX9rRADavPiDFSoMb/k64i9XMk8uH4r/Omi5Ctierj6XzNecwDbO4WuFbDD1zmpl3Tg==
-
 import-cwd@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/import-cwd/-/import-cwd-3.0.0.tgz#20845547718015126ea9b3676b7592fb8bd4cf92"
@@ -13048,15 +12668,6 @@ jest-docblock@^29.7.0:
   dependencies:
     detect-newline "^3.0.0"
 
-jest-dynalite@^3.6.1:
-  version "3.6.1"
-  resolved "https://registry.yarnpkg.com/jest-dynalite/-/jest-dynalite-3.6.1.tgz#8bae305a3c33d9a8036f563827b173b54a323ca5"
-  integrity sha512-MERtTt8Pj39vFmbItMC3YuIaqLf1kh/pJIE0DRcjeP/2Fa8Nni9IxwN6XWIMgXNbFKtlOM6ppH+Bsy0rWIdPiw==
-  dependencies:
-    "@aws/dynamodb-auto-marshaller" "^0.7.1"
-    dynalite "^3.2.1"
-    setimmediate "^1.0.5"
-
 jest-each@^29.7.0:
   version "29.7.0"
   resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.7.0.tgz#162a9b3f2328bdd991beaabffbb74745e56577d1"
@@ -13576,11 +13187,6 @@ json-buffer@3.0.0:
   resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898"
   integrity sha512-CuUqjv0FUZIdXkHPI8MezCnFCdaTAacej1TZYulLoAg1h/PhwkdXFN4V/gzY4g+fMBCOV2xF+rp7t2XD2ns/NQ==
 
-json-buffer@3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13"
-  integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==
-
 json-parse-better-errors@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9"
@@ -13749,13 +13355,6 @@ keyv@^3.0.0:
   dependencies:
     json-buffer "3.0.0"
 
-keyv@^4.0.0:
-  version "4.5.4"
-  resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93"
-  integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==
-  dependencies:
-    json-buffer "3.0.1"
-
 kill-port@^1.6.1:
   version "1.6.1"
   resolved "https://registry.yarnpkg.com/kill-port/-/kill-port-1.6.1.tgz#560fe79484583bdf3a5e908557dae614447618aa"
@@ -13871,7 +13470,7 @@ koa-mount@^4.0.0:
     debug "^4.0.1"
     koa-compose "^4.1.0"
 
-koa-passport@4.1.4, koa-passport@^4.1.4:
+koa-passport@4.1.4:
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/koa-passport/-/koa-passport-4.1.4.tgz#5f1665c1c2a37ace79af9f970b770885ca30ccfa"
   integrity sha512-dJBCkl4X+zdYxbI2V2OtoGy0PUenpvp2ZLLWObc8UJhsId0iQpTFT8RVcuA0709AL2txGwRHnSPoT1bYNGa6Kg==
@@ -13905,7 +13504,7 @@ koa-send@5.0.1, koa-send@^5.0.0:
     http-errors "^1.7.3"
     resolve-path "^1.4.0"
 
-koa-session@5.13.1, koa-session@^5.12.0:
+koa-session@5.13.1:
   version "5.13.1"
   resolved "https://registry.yarnpkg.com/koa-session/-/koa-session-5.13.1.tgz#a47e39015a4b464e21e3e1e2deeca48eb83916ee"
   integrity sha512-TfYiun6xiFosyfIJKnEw0aoG5XmLIwM+K3OVWfkz84qY0NP2gbk0F/olRn0/Hrxq0f14s8amHVXeWyKYH3Cx3Q==
@@ -13923,7 +13522,7 @@ koa-static@5.0.0, koa-static@^5.0.0:
     debug "^3.1.0"
     koa-send "^5.0.0"
 
-koa-useragent@4.1.0, koa-useragent@^4.1.0:
+koa-useragent@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/koa-useragent/-/koa-useragent-4.1.0.tgz#d3f128b552c6da3e5e9e9e9c887b2922b16e4468"
   integrity sha512-x/HUDZ1zAmNNh5hA9hHbPm9p3UVg2prlpHzxCXQCzbibrNS0kmj7MkCResCbAbG7ZT6FVxNSMjR94ZGamdMwxA==
@@ -14023,11 +13622,6 @@ latest-version@^5.1.0:
   dependencies:
     package-json "^6.3.0"
 
-lazy@^1.0.11:
-  version "1.0.11"
-  resolved "https://registry.yarnpkg.com/lazy/-/lazy-1.0.11.tgz#daa068206282542c088288e975c297c1ae77b690"
-  integrity sha512-Y+CjUfLmIpoUCCRl0ub4smrYtGGr5AOa2AKOaWelGHOGz33X/Y/KizefGqbkwfz44+cnq/+9habclf8vOmu2LA==
-
 lcid@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/lcid/-/lcid-2.0.0.tgz#6ef5d2df60e52f82eb228a4c373e8d1f397253cf"
@@ -14213,13 +13807,6 @@ level-js@^5.0.0:
     inherits "^2.0.3"
     ltgt "^2.1.2"
 
-level-option-wrap@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/level-option-wrap/-/level-option-wrap-1.1.0.tgz#ad20e68d9f3c22c8897531cc6aa7af596b1ed129"
-  integrity sha512-gQouC22iCqHuBLNl4BHxEZUxLvUKALAtT/Q0c6ziOxZQ8c02G/gyxHWNbLbxUzRNfMrRnbt6TZT3gNe8VBqQeg==
-  dependencies:
-    defined "~0.0.0"
-
 level-packager@^5.1.0:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/level-packager/-/level-packager-5.1.1.tgz#323ec842d6babe7336f70299c14df2e329c18939"
@@ -14268,7 +13855,7 @@ level@6.0.1:
     level-packager "^5.1.0"
     leveldown "^5.4.0"
 
-leveldown@5.6.0, leveldown@^5.4.0, leveldown@^5.6.0:
+leveldown@5.6.0, leveldown@^5.4.0:
   version "5.6.0"
   resolved "https://registry.yarnpkg.com/leveldown/-/leveldown-5.6.0.tgz#16ba937bb2991c6094e13ac5a6898ee66d3eee98"
   integrity sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==
@@ -14277,7 +13864,7 @@ leveldown@5.6.0, leveldown@^5.4.0, leveldown@^5.6.0:
     napi-macros "~2.0.0"
     node-gyp-build "~4.1.0"
 
-levelup@4.4.0, levelup@^4.3.2, levelup@^4.4.0:
+levelup@4.4.0, levelup@^4.3.2:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/levelup/-/levelup-4.4.0.tgz#f89da3a228c38deb49c48f88a70fb71f01cafed6"
   integrity sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==
@@ -14466,11 +14053,6 @@ locate-path@^6.0.0:
   dependencies:
     p-locate "^5.0.0"
 
-lock@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/lock/-/lock-1.1.0.tgz#53157499d1653b136ca66451071fca615703fa55"
-  integrity sha512-NZQIJJL5Rb9lMJ0Yl1JoVr9GSdo4HTPsUEWsSFzB8dE8DSoiLCVavWZPi7Rnlv/o73u6I24S/XYc/NmG4l8EKA==
-
 lodash-es@^4.17.21:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
@@ -14611,7 +14193,7 @@ lodash.xor@^4.5.0:
   resolved "https://registry.yarnpkg.com/lodash.xor/-/lodash.xor-4.5.0.tgz#4d48ed7e98095b0632582ba714d3ff8ae8fb1db6"
   integrity sha512-sVN2zimthq7aZ5sPGXnSz32rZPuqcparVW50chJQe+mzTYV+IsxSsl/2gnkWWE2Of7K3myBQBqtLKOUEHJKRsQ==
 
-lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
+lodash@4.17.21, lodash@^4.17.11, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.17.3, lodash@^4.7.0:
   version "4.17.21"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -14965,18 +14547,6 @@ memdown@1.4.1:
     ltgt "~2.2.0"
     safe-buffer "~5.1.1"
 
-memdown@^5.1.0:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/memdown/-/memdown-5.1.0.tgz#608e91a9f10f37f5b5fe767667a8674129a833cb"
-  integrity sha512-B3J+UizMRAlEArDjWHTMmadet+UKwHd3UjMgGBkZcKAxAYVPS9o0Yeiha4qvz7iGiL2Sb3igUft6p7nbFWctpw==
-  dependencies:
-    abstract-leveldown "~6.2.1"
-    functional-red-black-tree "~1.0.1"
-    immediate "~3.2.3"
-    inherits "~2.0.1"
-    ltgt "~2.2.0"
-    safe-buffer "~5.2.0"
-
 memory-pager@^1.0.2:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5"
@@ -15085,11 +14655,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
   integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
 
-mimic-response@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
-  integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==
-
 min-document@^2.19.0:
   version "2.19.0"
   resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685"
@@ -15533,11 +15098,6 @@ neo-async@^2.6.0, neo-async@^2.6.2:
   resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f"
   integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==
 
-neon-env@^0.1.1:
-  version "0.1.3"
-  resolved "https://registry.yarnpkg.com/neon-env/-/neon-env-0.1.3.tgz#071e86fde3c698e9314f057d209e0b79ddab16e9"
-  integrity sha512-Zo+L6Nm19gJrjyfhxn/ZDm8eIIDzr75o64ZhijBau4LNuhLzjEAteRg3gchIvgaN8XTo5BxN6iTNP5clZQ0agA==
-
 nice-try@^1.0.4:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366"
@@ -16255,23 +15815,6 @@ opentracing@>=0.12.1:
   resolved "https://registry.yarnpkg.com/opentracing/-/opentracing-0.14.7.tgz#25d472bd0296dc0b64d7b94cbc995219031428f5"
   integrity sha512-vz9iS7MJ5+Bp1URw8Khvdyw1H/hGvzHWlKQ7eRrQojSCDL1/SrWfrY9QebLw97n2deyRtzHRC3MkQfVNUCo91Q==
 
-operate-api-client@1.2.3:
-  version "1.2.3"
-  resolved "https://registry.yarnpkg.com/operate-api-client/-/operate-api-client-1.2.3.tgz#c884ab09fe07360ac5ce5b58ae470ba1e91db879"
-  integrity sha512-8FWfDsHVxgYIBe4p4fB6e7SSiYdW/PPTCCLFcGnbqdUxlhcUq9ncYu6ZMMm6E3A3WKxagdInYQbxOhtTeVGhVQ==
-  dependencies:
-    camunda-saas-oauth "^1.2.0"
-    got "^11.8.5"
-
-optimize-api-client@^1.0.3:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/optimize-api-client/-/optimize-api-client-1.0.4.tgz#a2e653780fd1e9e54a38912418b0ea27bd0484ef"
-  integrity sha512-2XBW+sv6eENOCHMc5v0XmH2DaaSETAb/qH5BsfpTDD8Pmeu10ZR61W7Pc/rBqauy96vPP/MfgmMphx5CjHb2xg==
-  dependencies:
-    camunda-8-credentials-from-env "^1.1.1"
-    camunda-saas-oauth "^1.2.4"
-    got "^11.8.5"
-
 optionator@^0.8.1:
   version "0.8.3"
   resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495"
@@ -16345,11 +15888,6 @@ p-cancelable@^1.0.0:
   resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-1.1.0.tgz#d078d15a3af409220c886f1d9a0ca2e441ab26cc"
   integrity sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==
 
-p-cancelable@^2.0.0:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf"
-  integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==
-
 p-defer@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c"
@@ -16662,7 +16200,7 @@ passport-google-oauth20@2.x.x:
   dependencies:
     passport-oauth2 "1.x.x"
 
-passport-google-oauth@2.0.0, passport-google-oauth@^2.0.0:
+passport-google-oauth@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/passport-google-oauth/-/passport-google-oauth-2.0.0.tgz#f6eb4bc96dd6c16ec0ecfdf4e05ec48ca54d4dae"
   integrity sha512-JKxZpBx6wBQXX1/a1s7VmdBgwOugohH+IxCy84aPTZNq/iIPX6u7Mqov1zY7MKRz3niFPol0KJz8zPLBoHKtYA==
@@ -16677,14 +16215,6 @@ passport-local@1.0.0:
   dependencies:
     passport-strategy "1.x.x"
 
-passport-microsoft@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/passport-microsoft/-/passport-microsoft-1.0.0.tgz#78954cf3201fdce61beeb6587a3b158f8e9db86c"
-  integrity sha512-L1JHeCbSObSZZXiG7jU2KoKie6nzZLwGt38HXz1GasKrsCQdOnf5kH8ltV4BWNUfBL2Pt1csWn1iuBSerprrcg==
-  dependencies:
-    passport-oauth2 "1.6.1"
-    pkginfo "0.4.x"
-
 passport-oauth1@1.x.x:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/passport-oauth1/-/passport-oauth1-1.3.0.tgz#5d57f1415c8e28e46b461a12ec1b492934f7c354"
@@ -16699,17 +16229,6 @@ passport-oauth2-refresh@^2.1.0:
   resolved "https://registry.yarnpkg.com/passport-oauth2-refresh/-/passport-oauth2-refresh-2.1.0.tgz#c31cd133826383f5539d16ad8ab4f35ca73ce4a4"
   integrity sha512-4ML7ooCESCqiTgdDBzNUFTBcPR8zQq9iM6eppEUGMMvLdsjqRL93jKwWm4Az3OJcI+Q2eIVyI8sVRcPFvxcF/A==
 
-passport-oauth2@1.6.1:
-  version "1.6.1"
-  resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.6.1.tgz#c5aee8f849ce8bd436c7f81d904a3cd1666f181b"
-  integrity sha512-ZbV43Hq9d/SBSYQ22GOiglFsjsD1YY/qdiptA+8ej+9C1dL1TVB+mBE5kDH/D4AJo50+2i8f4bx0vg4/yDDZCQ==
-  dependencies:
-    base64url "3.x.x"
-    oauth "0.9.x"
-    passport-strategy "1.x.x"
-    uid2 "0.0.x"
-    utils-merge "1.x.x"
-
 passport-oauth2@1.x.x:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.7.0.tgz#5c4766c8531ac45ffe9ec2c09de9809e2c841fc4"
@@ -17086,11 +16605,6 @@ pkg-types@^1.0.3:
     mlly "^1.2.0"
     pathe "^1.1.0"
 
-pkginfo@0.4.x:
-  version "0.4.1"
-  resolved "https://registry.yarnpkg.com/pkginfo/-/pkginfo-0.4.1.tgz#b5418ef0439de5425fc4995042dced14fb2a84ff"
-  integrity sha512-8xCNE/aT/EXKenuMDZ+xTVwkT8gsoHN2z/Q29l80u0ppGEXVvsKRzNMbtKhg8LS8k1tJLAHHylf6p4VFmP6XUQ==
-
 pluralize@^8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1"
@@ -17420,13 +16934,6 @@ postgres-interval@^1.1.0:
   dependencies:
     xtend "^4.0.0"
 
-posthog-js@^1.13.4:
-  version "1.100.0"
-  resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.100.0.tgz#687b9a6e4ed226aa6572f4040b418ea0c8b3d353"
-  integrity sha512-r2XZEiHQ9mBK7D1G9k57I8uYZ2kZTAJ0OCX6K/OOdCWN8jKPhw3h5F9No5weilP6eVAn+hrsy7NvPV7SCX7gMg==
-  dependencies:
-    fflate "^0.4.1"
-
 posthog-js@^1.36.0:
   version "1.96.1"
   resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.96.1.tgz#4f9719a24e4e14037b0e72d430194d7cdb576447"
@@ -17723,11 +17230,6 @@ prepend-http@^2.0.0:
   resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897"
   integrity sha512-ravE6m9Atw9Z/jjttRUZ+clIXogdghyZAuWJ3qEzjT+jI/dL1ifAqhZeC5VHzQp1MSt1+jxKkFNemj/iO7tVUA==
 
-prepend-http@^3.0.1:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-3.0.1.tgz#3e724d58fd5867465b300bb9615009fa2f8ee3b6"
-  integrity sha512-BLxfZh+m6UiAiCPZFJ4+vYoL7NrRs5XgCTRrjseATAggXhdZKKxn+JUNmuVYWY23bDHgaEHodxw8mnmtVEDtHw==
-
 prettier-plugin-svelte@^2.3.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/prettier-plugin-svelte/-/prettier-plugin-svelte-2.6.0.tgz#0e845b560b55cd1d951d6c50431b4949f8591746"
@@ -17827,14 +17329,6 @@ promise-inflight@^1.0.1:
   resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"
   integrity sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==
 
-promise-retry@^1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/promise-retry/-/promise-retry-1.1.1.tgz#6739e968e3051da20ce6497fb2b50f6911df3d6d"
-  integrity sha512-StEy2osPr28o17bIW776GtwO6+Q+M9zPiZkYfosciUUMYqjhU/ffwRAH0zN2+uvGyUsn8/YICIHRzLbPacpZGw==
-  dependencies:
-    err-code "^1.0.0"
-    retry "^0.10.0"
-
 promise-retry@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/promise-retry/-/promise-retry-2.0.1.tgz#ff747a13620ab57ba688f5fc67855410c370da22"
@@ -18026,7 +17520,7 @@ q@^1.1.2:
   resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
   integrity sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==
 
-qs@^6.10.3, qs@^6.11.0, qs@^6.4.0:
+qs@^6.11.0, qs@^6.4.0:
   version "6.11.2"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.2.tgz#64bea51f12c1f5da1bc01496f48ffcff7c69d7d9"
   integrity sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==
@@ -18082,11 +17576,6 @@ quick-lru@^4.0.1:
   resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
   integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==
 
-quick-lru@^5.1.1:
-  version "5.1.1"
-  resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932"
-  integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==
-
 quote-unquote@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/quote-unquote/-/quote-unquote-1.0.0.tgz#67a9a77148effeaf81a4d428404a710baaac8a0b"
@@ -18139,11 +17628,6 @@ rc@1.2.8, rc@^1.2.7, rc@^1.2.8:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
-reachdown@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/reachdown/-/reachdown-1.1.0.tgz#c3b85b459dbd0fe2c79782233a0a38e66a9b5454"
-  integrity sha512-6LsdRe4cZyOjw4NnvbhUd/rGG7WQ9HMopPr+kyL018Uci4kijtxcGR5kVb5Ln13k4PEE+fEFQbjfOvNw7cnXmA==
-
 react-is@^17.0.1:
   version "17.0.2"
   resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0"
@@ -18541,11 +18025,6 @@ requizzle@^0.2.3:
   dependencies:
     lodash "^4.17.21"
 
-resolve-alpn@^1.0.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9"
-  integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==
-
 resolve-cwd@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d"
@@ -18605,13 +18084,6 @@ responselike@1.0.2, responselike@^1.0.2:
   dependencies:
     lowercase-keys "^1.0.0"
 
-responselike@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/responselike/-/responselike-2.0.1.tgz#9a0bc8fdc252f3fb1cca68b016591059ba1422bc"
-  integrity sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==
-  dependencies:
-    lowercase-keys "^2.0.0"
-
 restore-cursor@^3.1.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e"
@@ -18633,11 +18105,6 @@ retry@0.13.1, retry@^0.13.1:
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
   integrity "sha1-GFsVh6z2eRnWOzVzSeA1N7JIRlg= sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg=="
 
-retry@^0.10.0:
-  version "0.10.1"
-  resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4"
-  integrity sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==
-
 retry@^0.12.0:
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b"
@@ -19094,7 +18561,7 @@ serialize-javascript@^6.0.1:
   dependencies:
     randombytes "^2.1.0"
 
-server-destroy@1.0.1, server-destroy@^1.0.1:
+server-destroy@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/server-destroy/-/server-destroy-1.0.1.tgz#f13bf928e42b9c3e79383e61cc3998b5d14e6cdd"
   integrity sha512-rb+9B5YBIEzYcD6x2VKidaa+cqYBJQKnU4oe4E3ANwRRN56yk/ua1YCJT1n21NTS8w6CcOclAKNP3PhdCXKYtQ==
@@ -19123,11 +18590,6 @@ set-function-name@^2.0.0, set-function-name@^2.0.1:
     functions-have-names "^1.2.3"
     has-property-descriptors "^1.0.0"
 
-setimmediate@^1.0.5:
-  version "1.0.5"
-  resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285"
-  integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==
-
 setprototypeof@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656"
@@ -19603,7 +19065,7 @@ stable@^0.1.8:
   resolved "https://registry.yarnpkg.com/stable/-/stable-0.1.8.tgz#836eb3c8382fe2936feaf544631017ce7d47a3cf"
   integrity sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==
 
-stack-trace@0.0.10, stack-trace@0.0.x:
+stack-trace@0.0.x:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
   integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==
@@ -19885,14 +19347,6 @@ strip-outer@^1.0.0:
   dependencies:
     escape-string-regexp "^1.0.2"
 
-stripe@9.16.0:
-  version "9.16.0"
-  resolved "https://registry.yarnpkg.com/stripe/-/stripe-9.16.0.tgz#94c24549c91fced457b9e3259e8a1a1bdb6dbd0e"
-  integrity sha512-Dn8K+jSoQcXjxCobRI4HXUdHjOXsiF/KszK49fJnkbeCFjZ3EZxLG2JiM/CX+Hcq27NBDtv/Sxhvy+HhTmvyaQ==
-  dependencies:
-    "@types/node" ">=8.1.0"
-    qs "^6.10.3"
-
 striptags@^3.1.1:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/striptags/-/striptags-3.2.0.tgz#cc74a137db2de8b0b9a370006334161f7dd67052"
@@ -19966,18 +19420,6 @@ sublevel-pouchdb@7.2.2:
     ltgt "2.2.1"
     readable-stream "1.1.14"
 
-subleveldown@^5.0.1:
-  version "5.0.1"
-  resolved "https://registry.yarnpkg.com/subleveldown/-/subleveldown-5.0.1.tgz#aa2b4e4698a48d9a86856b2c4df1b6bce2d2ce53"
-  integrity sha512-cVqd/URpp7si1HWu5YqQ3vqQkjuolAwHypY1B4itPlS71/lsf6TQPZ2Y0ijT22EYVkvH5ove9JFJf4u7VGPuZw==
-  dependencies:
-    abstract-leveldown "^6.3.0"
-    encoding-down "^6.2.0"
-    inherits "^2.0.3"
-    level-option-wrap "^1.1.0"
-    levelup "^4.4.0"
-    reachdown "^1.1.0"
-
 superagent@^8.0.5:
   version "8.1.2"
   resolved "https://registry.yarnpkg.com/superagent/-/superagent-8.1.2.tgz#03cb7da3ec8b32472c9d20f6c2a57c7f3765f30b"
@@ -20844,11 +20286,6 @@ typed-array-length@^1.0.4:
     for-each "^0.3.3"
     is-typed-array "^1.1.9"
 
-typed-duration@^1.0.12:
-  version "1.0.13"
-  resolved "https://registry.yarnpkg.com/typed-duration/-/typed-duration-1.0.13.tgz#a40f9ba563b6e20674cac491e15ecbf6811d85a7"
-  integrity sha512-HLwA+hNq/2eXe03isJSfa7YJt6NikplBGdNKvlhyuR6WL5iZi2uXJIZv1SSOMEIukCZbeQ8QwIcQ801S0/Qulw==
-
 typedarray-to-buffer@^3.1.5:
   version "3.1.5"
   resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"
@@ -21062,7 +20499,7 @@ update-browserslist-db@^1.0.10:
     escalade "^3.1.1"
     picocolors "^1.0.0"
 
-update-dotenv@1.1.1, update-dotenv@^1.1.1:
+update-dotenv@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/update-dotenv/-/update-dotenv-1.1.1.tgz#17146f302f216c3c92419d5a327a45be910050ca"
   integrity sha512-3cIC18In/t0X/yH793c00qqxcKD8jVCgNOPif/fGQkFpYMGecM9YAc+kaAKXuZsM2dE9I9wFI7KvAuNX22SGMQ==
@@ -21106,7 +20543,7 @@ url-parse-lax@^3.0.0:
   dependencies:
     prepend-http "^2.0.0"
 
-url-parse@^1.4.3, url-parse@^1.5.3:
+url-parse@^1.5.3:
   version "1.5.10"
   resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1"
   integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==
@@ -21176,11 +20613,6 @@ uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-uuid@^7.0.3:
-  version "7.0.3"
-  resolved "https://registry.yarnpkg.com/uuid/-/uuid-7.0.3.tgz#c5c9f2c8cf25dc0a372c4df1441c41f5bd0c680b"
-  integrity sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==
-
 uuid@^9.0.0, uuid@^9.0.1:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
@@ -22079,23 +21511,3 @@ z-schema@^5.0.1:
     validator "^13.7.0"
   optionalDependencies:
     commander "^9.4.1"
-
-zeebe-node@^8.2.5:
-  version "8.3.1"
-  resolved "https://registry.yarnpkg.com/zeebe-node/-/zeebe-node-8.3.1.tgz#e100bf3708464e305305b4efa1ffde53f9786c45"
-  integrity sha512-68ascWO3g7g+9WwDzvfa3I9TkLKHku5auEgSINP+k5ktNfsfGW68ELDmEJA+XHZgzvGsdGILZqGRzVd5SC8aaQ==
-  dependencies:
-    "@grpc/grpc-js" "1.9.7"
-    "@grpc/proto-loader" "0.7.10"
-    chalk "^2.4.2"
-    console-stamp "^3.0.2"
-    dayjs "^1.8.15"
-    debug "^4.2.0"
-    fast-xml-parser "^4.1.3"
-    fp-ts "^2.5.1"
-    got "^11.8.5"
-    long "^4.0.0"
-    promise-retry "^1.1.1"
-    stack-trace "0.0.10"
-    typed-duration "^1.0.12"
-    uuid "^7.0.3"

From 6964e2d146de1a51eb55ffbf8a40052bce090575 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 29 Jan 2024 13:43:51 +0000
Subject: [PATCH 09/47] Fixing update aliasing.

---
 .../server/scripts/integrations/postgres/reset.sh    |  1 +
 packages/server/src/integrations/base/sql.ts         | 12 ++++++------
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/reset.sh b/packages/server/scripts/integrations/postgres/reset.sh
index 32778bd11f..29a5db0181 100755
--- a/packages/server/scripts/integrations/postgres/reset.sh
+++ b/packages/server/scripts/integrations/postgres/reset.sh
@@ -1,3 +1,4 @@
 #!/bin/bash
 docker-compose down
 docker volume prune -f
+docker volume rm postgres_pg_data
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 3375e175e6..d33c077ee5 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -424,7 +424,7 @@ class InternalBuilder {
   knexWithAlias(
     knex: Knex,
     endpoint: { entityId: string; alias?: string; schema?: string }
-  ): { query: KnexQuery; name: string } {
+  ): { query: KnexQuery; aliased: string } {
     const tableName = endpoint.entityId
     const alias = endpoint.alias
     const aliased = alias ? alias : tableName
@@ -433,7 +433,7 @@ class InternalBuilder {
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }
-    return { query, name: aliased }
+    return { query, aliased }
   }
 
   create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
@@ -493,7 +493,7 @@ class InternalBuilder {
     }
     // start building the query
 
-    let { query, name: aliased } = this.knexWithAlias(knex, endpoint)
+    let { query, aliased } = this.knexWithAlias(knex, endpoint)
     query = query.limit(foundLimit)
     if (foundOffset) {
       query = query.offset(foundOffset)
@@ -522,9 +522,9 @@ class InternalBuilder {
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
-    let { query } = this.knexWithAlias(knex, endpoint)
+    let { query, aliased } = this.knexWithAlias(knex, endpoint)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(query, filters, { tableName: aliased })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -535,7 +535,7 @@ class InternalBuilder {
 
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
-    let { query, name: aliased } = this.knexWithAlias(knex, endpoint)
+    let { query, aliased } = this.knexWithAlias(knex, endpoint)
     query = this.addFilters(query, filters, { tableName: aliased })
     // mysql can't use returning
     if (opts.disableReturning) {

From 5d2ba68fae4b39086686c41999b903c05c352ba4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jan 2024 13:35:45 +0000
Subject: [PATCH 10/47] Adding test case based on capture of real failing
 query.

---
 .../server/src/integrations/tests/sql.spec.ts | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 5cc4849d03..580a8117cb 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -683,3 +683,110 @@ describe("SQL query builder", () => {
     })
   })
 })
+
+describe("Captures of real examples", () => {
+  const limit = 5000
+
+  it("should handle filtering by relationship", () => {
+    const queryJson = {
+      endpoint: {
+        datasourceId: "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+        entityId: "products",
+        operation: "READ",
+        alias: "a",
+      },
+      resource: {
+        fields: [
+          "a.productname",
+          "a.productid",
+          "b.executorid",
+          "b.taskname",
+          "b.taskid",
+          "b.completed",
+          "b.qaid",
+        ],
+      },
+      filters: {
+        equal: {
+          "1:tasks.taskname": "assembling",
+        },
+        onEmptyFilter: "all",
+      },
+      sort: {
+        productname: {
+          direction: "ASCENDING",
+        },
+      },
+      paginate: {
+        limit: 100,
+        page: 1,
+      },
+      relationships: [
+        {
+          tableName: "tasks",
+          column: "tasks",
+          through: "products_tasks",
+          from: "productid",
+          to: "taskid",
+          fromPrimary: "productid",
+          toPrimary: "taskid",
+          aliases: {
+            products_tasks: "c",
+            tasks: "b",
+            products: "a",
+          },
+        },
+      ],
+      meta: {
+        table: {
+          type: "table",
+          _id: "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
+          primary: ["productid"],
+          name: "a",
+          schema: {
+            productname: {
+              type: "string",
+              externalType: "character varying",
+              autocolumn: false,
+              name: "productname",
+              constraints: {
+                presence: false,
+              },
+            },
+            productid: {
+              type: "number",
+              externalType: "integer",
+              autocolumn: true,
+              name: "productid",
+              constraints: {
+                presence: false,
+              },
+            },
+            tasks: {
+              tableId:
+                "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+              name: "tasks",
+              relationshipType: "many-to-many",
+              fieldName: "taskid",
+              through:
+                "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks",
+              throughFrom: "taskid",
+              throughTo: "productid",
+              type: "link",
+              main: true,
+              _id: "ca6862d9ba09146dd8a68e3b5b7055a09",
+            },
+          },
+          sourceId: "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+          sourceType: "external",
+          primaryDisplay: "productname",
+        },
+      },
+    }
+    let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+    expect(query).toEqual({
+      bindings: [100, "assembling", limit],
+      sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
+    })
+  })
+})

From 09a0d00aa7df535454cb3eafe49714dcd1adf3e9 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jan 2024 13:50:36 +0000
Subject: [PATCH 11/47] Fixing some test cases.

---
 packages/server/src/integrations/tests/sql.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 580a8117cb..bca0cf5422 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -502,7 +502,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson({ schema: "production" }))
     expect(query).toEqual({
       bindings: [500, 5000],
-      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" limit $1) as "brands" left join "production"."products" on "brands"."brand_id" = "products"."brand_id" limit $2`,
+      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" limit $2`,
     })
   })
 
@@ -510,7 +510,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateRelationshipJson())
     expect(query).toEqual({
       bindings: [500, 5000],
-      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" limit $1) as "brands" left join "products" on "brands"."brand_id" = "products"."brand_id" limit $2`,
+      sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" limit $2`,
     })
   })
 
@@ -520,7 +520,7 @@ describe("SQL query builder", () => {
     )
     expect(query).toEqual({
       bindings: [500, 5000],
-      sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" limit $1) as "stores" left join "production"."stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" on "products"."product_id" = "stocks"."product_id" limit $2`,
+      sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" limit $2`,
     })
   })
 

From bb0b776684e29a529ac5198451472e8981cadd1f Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 30 Jan 2024 17:57:10 +0000
Subject: [PATCH 12/47] Updating how aliasing is handled.

---
 .../server/src/api/controllers/row/alias.ts   |   2 +-
 packages/server/src/integrations/base/sql.ts  |  86 ++++++++------
 .../server/src/integrations/tests/sql.spec.ts | 110 ++----------------
 .../sqlQueryJson/filterByRelationship.json    |  94 +++++++++++++++
 packages/types/src/sdk/search.ts              |   3 +-
 5 files changed, 158 insertions(+), 137 deletions(-)
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index fc00b505c4..589431cc1a 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -113,7 +113,7 @@ export default class AliasTables {
       }
       json.meta.tables = aliasedTables
     }
-    json.endpoint.alias = this.getAlias(json.endpoint.entityId)
+    json.tableAliases = this.tableAliases
     const response = await getDatasourceAndQuery(json)
     return this.reverse(response)
   }
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index d33c077ee5..c9be2e1ae6 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -129,8 +129,13 @@ class InternalBuilder {
   addFilters(
     query: KnexQuery,
     filters: SearchFilters | undefined,
-    opts: { relationship?: boolean; tableName?: string }
+    tableName: string,
+    opts: { aliases?: Record<string, string>; relationship?: boolean }
   ): KnexQuery {
+    function getTableName(name: string) {
+      const alias = opts.aliases?.[name]
+      return alias || name
+    }
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
@@ -139,10 +144,11 @@ class InternalBuilder {
         const updatedKey = dbCore.removeKeyNumbering(key)
         const isRelationshipField = updatedKey.includes(".")
         if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${updatedKey}`, value)
+          fn(`${getTableName(tableName)}.${updatedKey}`, value)
         }
         if (opts.relationship && isRelationshipField) {
-          fn(updatedKey, value)
+          const [filterTableName, property] = updatedKey.split(".")
+          fn(`${getTableName(filterTableName)}.${property}`, value)
         }
       }
     }
@@ -345,17 +351,15 @@ class InternalBuilder {
     query: KnexQuery,
     fromTable: string,
     relationships: RelationshipsJson[] | undefined,
-    schema: string | undefined
+    schema: string | undefined,
+    aliases?: Record<string, string>
   ): KnexQuery {
     if (!relationships) {
       return query
     }
     const tableSets: Record<string, [RelationshipsJson]> = {}
-    // add up all aliases
-    let aliases: Record<string, string> = {}
     // aggregate into table sets (all the same to tables)
     for (let relationship of relationships) {
-      aliases = { ...aliases, ...relationship.aliases }
       const keyObj: { toTable: string; throughTable: string | undefined } = {
         toTable: relationship.tableName,
         throughTable: undefined,
@@ -372,9 +376,9 @@ class InternalBuilder {
     }
     for (let [key, relationships] of Object.entries(tableSets)) {
       const { toTable, throughTable } = JSON.parse(key)
-      const toAlias = aliases[toTable] || toTable,
-        throughAlias = aliases[throughTable] || throughTable,
-        fromAlias = aliases[fromTable] || fromTable
+      const toAlias = aliases?.[toTable] || toTable,
+        throughAlias = aliases?.[throughTable] || throughTable,
+        fromAlias = aliases?.[fromTable] || fromTable
       let toTableWithSchema = this.tableNameWithSchema(toTable, {
         alias: toAlias,
         schema,
@@ -423,22 +427,23 @@ class InternalBuilder {
 
   knexWithAlias(
     knex: Knex,
-    endpoint: { entityId: string; alias?: string; schema?: string }
-  ): { query: KnexQuery; aliased: string } {
+    endpoint: QueryJson["endpoint"],
+    aliases?: QueryJson["tableAliases"]
+  ): KnexQuery {
     const tableName = endpoint.entityId
-    const alias = endpoint.alias
-    const aliased = alias ? alias : tableName
-    const tableAliased = alias ? `${tableName} as ${alias}` : tableName
+    const tableAliased = aliases?.[tableName]
+      ? `${tableName} as ${aliases?.[tableName]}`
+      : tableName
     let query = knex(tableAliased)
     if (endpoint.schema) {
       query = query.withSchema(endpoint.schema)
     }
-    return { query, aliased }
+    return query
   }
 
   create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body } = json
-    let { query } = this.knexWithAlias(knex, endpoint)
+    let query = this.knexWithAlias(knex, endpoint)
     const parsedBody = parseBody(body)
     // make sure no null values in body for creation
     for (let [key, value] of Object.entries(parsedBody)) {
@@ -457,7 +462,7 @@ class InternalBuilder {
 
   bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
     const { endpoint, body } = json
-    let { query } = this.knexWithAlias(knex, endpoint)
+    let query = this.knexWithAlias(knex, endpoint)
     if (!Array.isArray(body)) {
       return query
     }
@@ -466,8 +471,10 @@ class InternalBuilder {
   }
 
   read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
-    let { endpoint, resource, filters, paginate, relationships } = json
+    let { endpoint, resource, filters, paginate, relationships, tableAliases } =
+      json
 
+    const tableName = endpoint.entityId
     // select all if not specified
     if (!resource) {
       resource = { fields: [] }
@@ -493,19 +500,20 @@ class InternalBuilder {
     }
     // start building the query
 
-    let { query, aliased } = this.knexWithAlias(knex, endpoint)
+    let query = this.knexWithAlias(knex, endpoint, tableAliases)
     query = query.limit(foundLimit)
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, { tableName: aliased })
+    query = this.addFilters(query, filters, tableName, {
+      aliases: tableAliases,
+    })
     // add sorting to pre-query
     query = this.addSorting(query, json)
-    // @ts-ignore
-    let preQuery: KnexQuery = knex({
-      // @ts-ignore
-      [aliased]: query,
-    }).select(selectStatement)
+    const alias = tableAliases?.[tableName] || tableName
+    let preQuery = knex({
+      [alias]: query,
+    } as any).select(selectStatement) as any
     // have to add after as well (this breaks MS-SQL)
     if (this.client !== SqlClient.MS_SQL) {
       preQuery = this.addSorting(preQuery, json)
@@ -513,18 +521,24 @@ class InternalBuilder {
     // handle joins
     query = this.addRelationships(
       preQuery,
-      aliased,
+      tableName,
       relationships,
-      endpoint.schema
+      endpoint.schema,
+      tableAliases
     )
-    return this.addFilters(query, filters, { relationship: true })
+    return this.addFilters(query, filters, tableName, {
+      relationship: true,
+      aliases: tableAliases,
+    })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
-    const { endpoint, body, filters } = json
-    let { query, aliased } = this.knexWithAlias(knex, endpoint)
+    const { endpoint, body, filters, tableAliases } = json
+    let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, { tableName: aliased })
+    query = this.addFilters(query, filters, endpoint.entityId, {
+      aliases: tableAliases,
+    })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -534,9 +548,11 @@ class InternalBuilder {
   }
 
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
-    const { endpoint, filters } = json
-    let { query, aliased } = this.knexWithAlias(knex, endpoint)
-    query = this.addFilters(query, filters, { tableName: aliased })
+    const { endpoint, filters, tableAliases } = json
+    let query = this.knexWithAlias(knex, endpoint, tableAliases)
+    query = this.addFilters(query, filters, endpoint.entityId, {
+      aliases: tableAliases,
+    })
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index bca0cf5422..0e7257242c 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -1,5 +1,7 @@
-const Sql = require("../base/sql").default
-const { SqlClient } = require("../utils")
+import { SqlClient } from "../utils"
+import Sql from "../base/sql"
+import { QueryJson } from "@budibase/types"
+import { join } from "path"
 
 const TABLE_NAME = "test"
 
@@ -17,7 +19,7 @@ function generateReadJson({
   filters,
   sort,
   paginate,
-}: any = {}) {
+}: any = {}): QueryJson {
   return {
     endpoint: endpoint(table || TABLE_NAME, "READ"),
     resource: {
@@ -30,7 +32,7 @@ function generateReadJson({
       table: {
         name: table || TABLE_NAME,
         primary: ["id"],
-      },
+      } as any,
     },
   }
 }
@@ -687,102 +689,12 @@ describe("SQL query builder", () => {
 describe("Captures of real examples", () => {
   const limit = 5000
 
+  function getJson(name: string): QueryJson {
+    return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
+  }
+
   it("should handle filtering by relationship", () => {
-    const queryJson = {
-      endpoint: {
-        datasourceId: "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-        entityId: "products",
-        operation: "READ",
-        alias: "a",
-      },
-      resource: {
-        fields: [
-          "a.productname",
-          "a.productid",
-          "b.executorid",
-          "b.taskname",
-          "b.taskid",
-          "b.completed",
-          "b.qaid",
-        ],
-      },
-      filters: {
-        equal: {
-          "1:tasks.taskname": "assembling",
-        },
-        onEmptyFilter: "all",
-      },
-      sort: {
-        productname: {
-          direction: "ASCENDING",
-        },
-      },
-      paginate: {
-        limit: 100,
-        page: 1,
-      },
-      relationships: [
-        {
-          tableName: "tasks",
-          column: "tasks",
-          through: "products_tasks",
-          from: "productid",
-          to: "taskid",
-          fromPrimary: "productid",
-          toPrimary: "taskid",
-          aliases: {
-            products_tasks: "c",
-            tasks: "b",
-            products: "a",
-          },
-        },
-      ],
-      meta: {
-        table: {
-          type: "table",
-          _id: "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
-          primary: ["productid"],
-          name: "a",
-          schema: {
-            productname: {
-              type: "string",
-              externalType: "character varying",
-              autocolumn: false,
-              name: "productname",
-              constraints: {
-                presence: false,
-              },
-            },
-            productid: {
-              type: "number",
-              externalType: "integer",
-              autocolumn: true,
-              name: "productid",
-              constraints: {
-                presence: false,
-              },
-            },
-            tasks: {
-              tableId:
-                "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
-              name: "tasks",
-              relationshipType: "many-to-many",
-              fieldName: "taskid",
-              through:
-                "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks",
-              throughFrom: "taskid",
-              throughTo: "productid",
-              type: "link",
-              main: true,
-              _id: "ca6862d9ba09146dd8a68e3b5b7055a09",
-            },
-          },
-          sourceId: "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
-          sourceType: "external",
-          primaryDisplay: "productname",
-        },
-      },
-    }
+    const queryJson = getJson(`filterByRelationship.json`)
     let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
     expect(query).toEqual({
       bindings: [100, "assembling", limit],
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json
new file mode 100644
index 0000000000..eb1025f382
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json
@@ -0,0 +1,94 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "products",
+    "operation": "READ"
+  },
+  "resource": {
+    "fields": [
+      "a.productname",
+      "a.productid",
+      "b.executorid",
+      "b.taskname",
+      "b.taskid",
+      "b.completed",
+      "b.qaid"
+    ]
+  },
+  "filters": {
+    "equal": {
+      "1:tasks.taskname": "assembling"
+    },
+    "onEmptyFilter": "all"
+  },
+  "sort": {
+    "productname": {
+      "direction": "ASCENDING"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "tasks",
+      "through": "products_tasks",
+      "from": "productid",
+      "to": "taskid",
+      "fromPrimary": "productid",
+      "toPrimary": "taskid"
+    }
+  ],
+  "tableAliases": {
+    "products_tasks": "c",
+    "tasks": "b",
+    "products": "a"
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products",
+      "primary": [
+        "productid"
+      ],
+      "name": "a",
+      "schema": {
+        "productname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "productname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "productid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "productid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "tasks": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "tasks",
+          "relationshipType": "many-to-many",
+          "fieldName": "taskid",
+          "through": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__products_tasks",
+          "throughFrom": "taskid",
+          "throughTo": "productid",
+          "type": "link",
+          "main": true,
+          "_id": "ca6862d9ba09146dd8a68e3b5b7055a09"
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "productname"
+    }
+  }
+}
\ No newline at end of file
diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts
index a4045c2558..67c344d845 100644
--- a/packages/types/src/sdk/search.ts
+++ b/packages/types/src/sdk/search.ts
@@ -67,7 +67,6 @@ export interface RelationshipsJson {
   fromPrimary?: string
   toPrimary?: string
   tableName: string
-  aliases?: Record<string, string>
   column: string
 }
 
@@ -75,7 +74,6 @@ export interface QueryJson {
   endpoint: {
     datasourceId: string
     entityId: string
-    alias?: string
     operation: Operation
     schema?: string
   }
@@ -96,6 +94,7 @@ export interface QueryJson {
     idFilter?: SearchFilters
   }
   relationships?: RelationshipsJson[]
+  tableAliases?: Record<string, string>
 }
 
 export interface SqlQuery {

From c4f4a46d7002e3b4321bb6cd7c22b0a608d481bf Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 5 Feb 2024 12:45:19 +0000
Subject: [PATCH 13/47] Quick fix based on testing.

---
 .../server/src/api/controllers/row/alias.ts   |  7 +++++-
 packages/server/src/integrations/base/sql.ts  |  1 -
 .../server/src/integrations/tests/sql.spec.ts | 17 -------------
 .../src/integrations/tests/sqlAlias.spec.ts   | 25 +++++++++++++++++++
 4 files changed, 31 insertions(+), 19 deletions(-)
 create mode 100644 packages/server/src/integrations/tests/sqlAlias.spec.ts

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 589431cc1a..6533e51728 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -113,7 +113,12 @@ export default class AliasTables {
       }
       json.meta.tables = aliasedTables
     }
-    json.tableAliases = this.tableAliases
+    // invert and return
+    const invertedTableAliases: Record<string, string> = {}
+    for (let [key, value] of Object.entries(this.tableAliases)) {
+      invertedTableAliases[value] = key
+    }
+    json.tableAliases = invertedTableAliases
     const response = await getDatasourceAndQuery(json)
     return this.reverse(response)
   }
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index c9be2e1ae6..cc2e1d94a8 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -499,7 +499,6 @@ class InternalBuilder {
       foundLimit = paginate.limit
     }
     // start building the query
-
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     query = query.limit(foundLimit)
     if (foundOffset) {
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 0e7257242c..cf22064cb7 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -685,20 +685,3 @@ describe("SQL query builder", () => {
     })
   })
 })
-
-describe("Captures of real examples", () => {
-  const limit = 5000
-
-  function getJson(name: string): QueryJson {
-    return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
-  }
-
-  it("should handle filtering by relationship", () => {
-    const queryJson = getJson(`filterByRelationship.json`)
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
-    expect(query).toEqual({
-      bindings: [100, "assembling", limit],
-      sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
-    })
-  })
-})
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
new file mode 100644
index 0000000000..cfd82cfd01
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -0,0 +1,25 @@
+import { QueryJson } from "@budibase/types"
+import { join } from "path"
+import Sql from "../base/sql"
+import { SqlClient } from "../utils"
+
+describe("Captures of real examples", () => {
+  const limit = 5000
+
+  function getJson(name: string): QueryJson {
+    return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
+  }
+
+  it("should handle basic retrieval", () => {
+    const queryJson = getJson("")
+  })
+
+  it("should handle filtering by relationship", () => {
+    const queryJson = getJson("filterByRelationship.json")
+    let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+    expect(query).toEqual({
+      bindings: [100, "assembling", limit],
+      sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
+    })
+  })
+})

From e8e7eea47a234241dd2c79196b7e101b183c56ca Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 5 Feb 2024 15:23:24 +0000
Subject: [PATCH 14/47] Adding some test cases for aliasing.

---
 .../src/integrations/tests/sqlAlias.spec.ts   |  65 ++++++-
 .../basicFetchWithRelationships.json          | 183 ++++++++++++++++++
 .../sqlQueryJson/createWithRelationships.json | 173 +++++++++++++++++
 .../tests/sqlQueryJson/deleteSimple.json      |  75 +++++++
 .../sqlQueryJson/updateRelationship.json      | 181 +++++++++++++++++
 .../tests/sqlQueryJson/updateSimple.json      | 181 +++++++++++++++++
 6 files changed, 850 insertions(+), 8 deletions(-)
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index cfd82cfd01..c91d988849 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -10,16 +10,65 @@ describe("Captures of real examples", () => {
     return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
   }
 
-  it("should handle basic retrieval", () => {
-    const queryJson = getJson("")
+  describe("create", () => {
+    it("should create a row with relationships", () => {
+      const queryJson = getJson("createWithRelationships.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
+        sql: `insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") values ($1, $2, $3, $4, $5, $6, $7) returning *`,
+      })
+    })
   })
 
-  it("should handle filtering by relationship", () => {
-    const queryJson = getJson("filterByRelationship.json")
-    let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
-    expect(query).toEqual({
-      bindings: [100, "assembling", limit],
-      sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
+  describe("read", () => {
+    it("should handle basic retrieval with relationships", () => {
+      const queryJson = getJson("basicFetchWithRelationships.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [100, limit],
+        sql: `select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" order by "a"."firstname" asc limit $2`,
+      })
+    })
+
+    it("should handle filtering by relationship", () => {
+      const queryJson = getJson("filterByRelationship.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [100, "assembling", limit],
+        sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
+      })
+    })
+  })
+
+  describe("update", () => {
+    it("should handle performing a simple update", () => {
+      const queryJson = getJson("updateSimple.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
+        sql: `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`,
+      })
+    })
+
+    it("should handle performing an update of relationships", () => {
+      const queryJson = getJson("updateRelationship.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
+        sql: `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`,
+      })
+    })
+  })
+
+  describe("delete", () => {
+    it("should handle deleting with relationships", () => {
+      const queryJson = getJson("deleteSimple.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: ["ddd", ""],
+        sql: `delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`,
+      })
     })
   })
 })
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json
new file mode 100644
index 0000000000..3445f5fe67
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json
@@ -0,0 +1,183 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "persons",
+    "operation": "READ"
+  },
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.address",
+      "a.age",
+      "a.type",
+      "a.city",
+      "a.lastname",
+      "b.executorid",
+      "b.taskname",
+      "b.taskid",
+      "b.completed",
+      "b.qaid",
+      "b.executorid",
+      "b.taskname",
+      "b.taskid",
+      "b.completed",
+      "b.qaid"
+    ]
+  },
+  "filters": {},
+  "sort": {
+    "firstname": {
+      "direction": "ASCENDING"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "QA",
+      "from": "personid",
+      "to": "qaid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    },
+    {
+      "tableName": "tasks",
+      "column": "executor",
+      "from": "personid",
+      "to": "executorid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    }
+  ],
+  "extra": {
+    "idFilter": {}
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+      "primary": [
+        "personid"
+      ],
+      "name": "a",
+      "schema": {
+        "year": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "year",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "firstname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "firstname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "personid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "personid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "address": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "address",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "age": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "age",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "type": {
+          "type": "options",
+          "externalType": "USER-DEFINED",
+          "autocolumn": false,
+          "name": "type",
+          "constraints": {
+            "presence": false,
+            "inclusion": [
+              "support",
+              "designer",
+              "programmer",
+              "qa"
+            ]
+          }
+        },
+        "city": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "city",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "lastname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "lastname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "QA": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "QA",
+          "relationshipType": "many-to-one",
+          "fieldName": "qaid",
+          "type": "link",
+          "main": true,
+          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+          "foreignKey": "personid"
+        },
+        "executor": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "executor",
+          "relationshipType": "many-to-one",
+          "fieldName": "executorid",
+          "type": "link",
+          "main": true,
+          "_id": "c89530b9770d94bec851e062b5cff3001",
+          "foreignKey": "personid",
+          "tableName": "persons"
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "firstname",
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json
new file mode 100644
index 0000000000..20331b949a
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json
@@ -0,0 +1,173 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "persons",
+    "operation": "CREATE"
+  },
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.address",
+      "a.age",
+      "a.type",
+      "a.city",
+      "a.lastname"
+    ]
+  },
+  "filters": {},
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "QA",
+      "from": "personid",
+      "to": "qaid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    },
+    {
+      "tableName": "tasks",
+      "column": "executor",
+      "from": "personid",
+      "to": "executorid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    }
+  ],
+  "body": {
+    "year": 1990,
+    "firstname": "A",
+    "address": "A Street",
+    "age": 34,
+    "type": "designer",
+    "city": "London",
+    "lastname": "B"
+  },
+  "extra": {
+    "idFilter": {}
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+      "primary": [
+        "personid"
+      ],
+      "name": "a",
+      "schema": {
+        "year": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "year",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "firstname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "firstname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "personid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "personid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "address": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "address",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "age": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "age",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "type": {
+          "type": "options",
+          "externalType": "USER-DEFINED",
+          "autocolumn": false,
+          "name": "type",
+          "constraints": {
+            "presence": false,
+            "inclusion": [
+              "support",
+              "designer",
+              "programmer",
+              "qa"
+            ]
+          }
+        },
+        "city": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "city",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "lastname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "lastname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "QA": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "QA",
+          "relationshipType": "many-to-one",
+          "fieldName": "qaid",
+          "type": "link",
+          "main": true,
+          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+          "foreignKey": "personid"
+        },
+        "executor": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "executor",
+          "relationshipType": "many-to-one",
+          "fieldName": "executorid",
+          "type": "link",
+          "main": true,
+          "_id": "c89530b9770d94bec851e062b5cff3001",
+          "foreignKey": "personid",
+          "tableName": "persons"
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "firstname",
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json
new file mode 100644
index 0000000000..2266b8c8be
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json
@@ -0,0 +1,75 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "compositetable",
+    "operation": "DELETE"
+  },
+  "resource": {
+    "fields": [
+      "a.keyparttwo",
+      "a.keypartone",
+      "a.name"
+    ]
+  },
+  "filters": {
+    "equal": {
+      "keypartone": "ddd",
+      "keyparttwo": ""
+    }
+  },
+  "relationships": [],
+  "extra": {
+    "idFilter": {
+      "equal": {
+        "keypartone": "ddd",
+        "keyparttwo": ""
+      }
+    }
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__compositetable",
+      "primary": [
+        "keypartone",
+        "keyparttwo"
+      ],
+      "name": "a",
+      "schema": {
+        "keyparttwo": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "keyparttwo",
+          "constraints": {
+            "presence": true
+          }
+        },
+        "keypartone": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "keypartone",
+          "constraints": {
+            "presence": true
+          }
+        },
+        "name": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "name",
+          "constraints": {
+            "presence": false
+          }
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "keypartone"
+    }
+  },
+  "tableAliases": {
+    "compositetable": "a"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json
new file mode 100644
index 0000000000..01e795bd6c
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json
@@ -0,0 +1,181 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "persons",
+    "operation": "UPDATE"
+  },
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.address",
+      "a.age",
+      "a.type",
+      "a.city",
+      "a.lastname"
+    ]
+  },
+  "filters": {
+    "equal": {
+      "personid": 5
+    }
+  },
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "QA",
+      "from": "personid",
+      "to": "qaid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    },
+    {
+      "tableName": "tasks",
+      "column": "executor",
+      "from": "personid",
+      "to": "executorid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    }
+  ],
+  "body": {
+    "year": 1990,
+    "firstname": "C",
+    "address": "A Street",
+    "age": 34,
+    "type": "designer",
+    "city": "London",
+    "lastname": "B"
+  },
+  "extra": {
+    "idFilter": {
+      "equal": {
+        "personid": 5
+      }
+    }
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+      "primary": [
+        "personid"
+      ],
+      "name": "a",
+      "schema": {
+        "year": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "year",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "firstname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "firstname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "personid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "personid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "address": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "address",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "age": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "age",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "type": {
+          "type": "options",
+          "externalType": "USER-DEFINED",
+          "autocolumn": false,
+          "name": "type",
+          "constraints": {
+            "presence": false,
+            "inclusion": [
+              "support",
+              "designer",
+              "programmer",
+              "qa"
+            ]
+          }
+        },
+        "city": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "city",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "lastname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "lastname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "QA": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "QA",
+          "relationshipType": "many-to-one",
+          "fieldName": "qaid",
+          "type": "link",
+          "main": true,
+          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+          "foreignKey": "personid"
+        },
+        "executor": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "executor",
+          "relationshipType": "many-to-one",
+          "fieldName": "executorid",
+          "type": "link",
+          "main": true,
+          "_id": "c89530b9770d94bec851e062b5cff3001",
+          "foreignKey": "personid",
+          "tableName": "persons"
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "firstname",
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json
new file mode 100644
index 0000000000..01e795bd6c
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json
@@ -0,0 +1,181 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+    "entityId": "persons",
+    "operation": "UPDATE"
+  },
+  "resource": {
+    "fields": [
+      "a.year",
+      "a.firstname",
+      "a.personid",
+      "a.address",
+      "a.age",
+      "a.type",
+      "a.city",
+      "a.lastname"
+    ]
+  },
+  "filters": {
+    "equal": {
+      "personid": 5
+    }
+  },
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "QA",
+      "from": "personid",
+      "to": "qaid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    },
+    {
+      "tableName": "tasks",
+      "column": "executor",
+      "from": "personid",
+      "to": "executorid",
+      "aliases": {
+        "tasks": "b",
+        "persons": "a"
+      }
+    }
+  ],
+  "body": {
+    "year": 1990,
+    "firstname": "C",
+    "address": "A Street",
+    "age": 34,
+    "type": "designer",
+    "city": "London",
+    "lastname": "B"
+  },
+  "extra": {
+    "idFilter": {
+      "equal": {
+        "personid": 5
+      }
+    }
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__persons",
+      "primary": [
+        "personid"
+      ],
+      "name": "a",
+      "schema": {
+        "year": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "year",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "firstname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "firstname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "personid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "personid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "address": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "address",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "age": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "age",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "type": {
+          "type": "options",
+          "externalType": "USER-DEFINED",
+          "autocolumn": false,
+          "name": "type",
+          "constraints": {
+            "presence": false,
+            "inclusion": [
+              "support",
+              "designer",
+              "programmer",
+              "qa"
+            ]
+          }
+        },
+        "city": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "city",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "lastname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "lastname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "QA": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "QA",
+          "relationshipType": "many-to-one",
+          "fieldName": "qaid",
+          "type": "link",
+          "main": true,
+          "_id": "ccb68481c80c34217a4540a2c6c27fe46",
+          "foreignKey": "personid"
+        },
+        "executor": {
+          "tableId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7__tasks",
+          "name": "executor",
+          "relationshipType": "many-to-one",
+          "fieldName": "executorid",
+          "type": "link",
+          "main": true,
+          "_id": "c89530b9770d94bec851e062b5cff3001",
+          "foreignKey": "personid",
+          "tableName": "persons"
+        }
+      },
+      "sourceId": "datasource_plus_8066e56456784eb2a00129d31be5c3e7",
+      "sourceType": "external",
+      "primaryDisplay": "firstname",
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "persons": "a",
+    "tasks": "b"
+  }
+}
\ No newline at end of file

From 9a8c31a2a42bc616d096b2d76e0e015c3ac18983 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 5 Feb 2024 18:57:16 +0000
Subject: [PATCH 15/47] Handling deletion of rows that violate constraints,
 this has been an issue in Budibase for some time and causes some confusion,
 attempting to resolve this when deleting rows.

---
 .../api/controllers/row/ExternalRequest.ts    | 91 ++++++++++++++++---
 1 file changed, 76 insertions(+), 15 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 2f3c1ad557..4f755845dc 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -7,6 +7,7 @@ import {
   FilterType,
   IncludeRelationship,
   ManyToManyRelationshipFieldMetadata,
+  ManyToOneRelationshipFieldMetadata,
   OneToManyRelationshipFieldMetadata,
   Operation,
   PaginationJson,
@@ -102,6 +103,26 @@ function buildFilters(
   }
 }
 
+function removeRelationships(
+  rowId: string,
+  table: Table,
+  isManyToMany: boolean,
+  colName?: string
+) {
+  const tableId = table._id!
+  const filters = buildFilters(rowId, {}, table)
+  // safety check, if there are no filters on deletion bad things happen
+  if (Object.keys(filters).length !== 0) {
+    const op = isManyToMany ? Operation.DELETE : Operation.UPDATE
+    const body = colName && !isManyToMany ? { [colName]: null } : undefined
+    return getDatasourceAndQuery({
+      endpoint: getEndpoint(tableId, op),
+      body,
+      filters,
+    })
+  }
+}
+
 /**
  * This function checks the incoming parameters to make sure all the inputs are
  * valid based on on the table schema. The main thing this is looking for is when a
@@ -305,6 +326,18 @@ export class ExternalRequest<T extends Operation> {
     }
   }
 
+  async getRow(table: Table, rowId: string): Promise<Row> {
+    const response = await getDatasourceAndQuery({
+      endpoint: getEndpoint(table._id!, Operation.READ),
+      filters: buildFilters(rowId, {}, table),
+    })
+    if (response.length > 0) {
+      return response[0]
+    } else {
+      throw new Error(`Cannot fetch row by ID "${rowId}"`)
+    }
+  }
+
   inputProcessing(row: Row | undefined, table: Table) {
     if (!row) {
       return { row, manyRelationships: [] }
@@ -572,7 +605,9 @@ export class ExternalRequest<T extends Operation> {
    * information.
    */
   async lookupRelations(tableId: string, row: Row) {
-    const related: { [key: string]: any } = {}
+    const related: {
+      [key: string]: { rows: Row[]; isMany: boolean; tableId: string }
+    } = {}
     const { tableName } = breakExternalTableId(tableId)
     if (!tableName) {
       return related
@@ -591,7 +626,7 @@ export class ExternalRequest<T extends Operation> {
         continue
       }
       const isMany = field.relationshipType === RelationshipType.MANY_TO_MANY
-      const tableId = isMany ? field.through : field.tableId
+      const tableId = isMany ? field.through! : field.tableId!
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
@@ -610,7 +645,7 @@ export class ExternalRequest<T extends Operation> {
         },
       })
       // this is the response from knex if no rows found
-      const rows = !response[0].read ? response : []
+      const rows: Row[] = !response[0].read ? response : []
       const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
       related[storeTo] = { rows, isMany, tableId }
     }
@@ -698,24 +733,46 @@ export class ExternalRequest<T extends Operation> {
         continue
       }
       for (let row of rows) {
-        const filters = buildFilters(generateIdForRow(row, table), {}, table)
-        // safety check, if there are no filters on deletion bad things happen
-        if (Object.keys(filters).length !== 0) {
-          const op = isMany ? Operation.DELETE : Operation.UPDATE
-          const body = isMany ? undefined : { [colName]: null }
-          promises.push(
-            getDatasourceAndQuery({
-              endpoint: getEndpoint(tableId, op),
-              body,
-              filters,
-            })
-          )
+        const promise = removeRelationships(
+          generateIdForRow(row, table),
+          table,
+          isMany,
+          colName
+        )
+        if (promise) {
+          promises.push(promise)
         }
       }
     }
     await Promise.all(promises)
   }
 
+  async removeRelationshipsToRow(table: Table, rowId: string) {
+    const row = await this.getRow(table, rowId)
+    const related = await this.lookupRelations(table._id!, row)
+    for (let column of Object.values(table.schema)) {
+      if (
+        column.type !== FieldType.LINK ||
+        column.relationshipType === RelationshipType.ONE_TO_MANY
+      ) {
+        continue
+      }
+      const relationshipColumn = column as ManyToOneRelationshipFieldMetadata
+      const { rows, isMany, tableId } = related[relationshipColumn.fieldName]
+      const table = this.getTable(tableId)!
+      await Promise.all(
+        rows.map(row =>
+          removeRelationships(
+            generateIdForRow(row, table),
+            table,
+            isMany,
+            relationshipColumn.fieldName
+          )
+        )
+      )
+    }
+  }
+
   /**
    * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
    * you have column overlap in relationships, e.g. we join a few different tables and they all have the
@@ -828,6 +885,10 @@ export class ExternalRequest<T extends Operation> {
     }
 
     const aliasing = new AliasTables(Object.keys(this.tables))
+    // remove any relationships that could block deletion
+    if (operation === Operation.DELETE && id) {
+      await this.removeRelationshipsToRow(table, generateRowIdField(id))
+    }
     const response = await aliasing.queryWithAliasing(json)
     // handle many-to-many relationships now if we know the ID (could be auto increment)
     if (operation !== Operation.READ) {

From b2c4f04aa60f3e19e70eb9eaf63b870f5843d961 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Wed, 21 Feb 2024 17:52:58 +0000
Subject: [PATCH 16/47] Typing and config.api'ing application.spec.ts, WIP

---
 .../server/src/api/controllers/application.ts | 25 +++---
 .../src/api/routes/tests/application.spec.ts  | 79 +++++++------------
 .../src/tests/utilities/api/application.ts    | 41 +++++++++-
 packages/types/src/documents/app/app.ts       | 12 +++
 packages/worker/scripts/test.sh               |  8 +-
 qa-core/src/internal-api/api/apis/AppAPI.ts   |  3 +-
 .../src/internal-api/fixtures/applications.ts |  2 +-
 .../internal-api/tests/tables/tables.spec.ts  | 11 ---
 .../src/shared/BudibaseTestConfiguration.ts   |  4 +-
 qa-core/src/types/app.ts                      | 10 ---
 qa-core/src/types/index.ts                    |  1 -
 11 files changed, 105 insertions(+), 91 deletions(-)
 delete mode 100644 qa-core/src/types/app.ts

diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 33582cf656..2d8b4b8686 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -47,6 +47,7 @@ import {
   PlanType,
   Screen,
   UserCtx,
+  CreateAppRequest,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -116,8 +117,8 @@ function checkAppName(
 }
 
 interface AppTemplate {
-  templateString: string
-  useTemplate: string
+  templateString?: string
+  useTemplate?: string
   file?: {
     type: string
     path: string
@@ -231,17 +232,21 @@ export async function fetchAppPackage(ctx: UserCtx) {
   }
 }
 
-async function performAppCreate(ctx: UserCtx) {
+async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const name = ctx.request.body.name,
-    possibleUrl = ctx.request.body.url,
-    encryptionPassword = ctx.request.body.encryptionPassword
+  const {
+    name,
+    url,
+    encryptionPassword,
+    useTemplate,
+    templateKey,
+    templateString,
+  } = ctx.request.body
 
   checkAppName(ctx, apps, name)
-  const url = sdk.applications.getAppUrl({ name, url: possibleUrl })
-  checkAppUrl(ctx, apps, url)
+  const appUrl = sdk.applications.getAppUrl({ name, url })
+  checkAppUrl(ctx, apps, appUrl)
 
-  const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig: AppTemplate = {
     useTemplate,
     key: templateKey,
@@ -268,7 +273,7 @@ async function performAppCreate(ctx: UserCtx) {
       version: envCore.VERSION,
       componentLibraries: ["@budibase/standard-components"],
       name: name,
-      url: url,
+      url: appUrl,
       template: templateKey,
       instance,
       tenantId: tenancy.getTenantId(),
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index fa5cb0a983..7340166e67 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -35,41 +35,30 @@ describe("/applications", () => {
 
   describe("create", () => {
     it("creates empty app", async () => {
-      const res = await request
-        .post("/api/applications")
-        .field("name", utils.newid())
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
+      const app = await config.api.application.create({ name: utils.newid() })
+      expect(app._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
     })
 
     it("creates app from template", async () => {
-      const res = await request
-        .post("/api/applications")
-        .field("name", utils.newid())
-        .field("useTemplate", "true")
-        .field("templateKey", "test")
-        .field("templateString", "{}") // override the file download
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
+      const app = await config.api.application.create({
+        name: utils.newid(),
+        useTemplate: "true",
+        templateKey: "test",
+        templateString: "{}",
+      })
+      expect(app._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
       expect(events.app.templateImported).toBeCalledTimes(1)
     })
 
     it("creates app from file", async () => {
-      const res = await request
-        .post("/api/applications")
-        .field("name", utils.newid())
-        .field("useTemplate", "true")
-        .set(config.defaultHeaders())
-        .attach("templateFile", "src/api/routes/tests/data/export.txt")
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
+      const app = await config.api.application.create({
+        name: utils.newid(),
+        useTemplate: "true",
+        templateFile: "src/api/routes/tests/data/export.txt",
+      })
+      expect(app._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
       expect(events.app.fileImported).toBeCalledTimes(1)
     })
@@ -84,24 +73,21 @@ describe("/applications", () => {
     })
 
     it("migrates navigation settings from old apps", async () => {
-      const res = await request
-        .post("/api/applications")
-        .field("name", "Old App")
-        .field("useTemplate", "true")
-        .set(config.defaultHeaders())
-        .attach("templateFile", "src/api/routes/tests/data/old-app.txt")
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
-      expect(res.body.navigation).toBeDefined()
-      expect(res.body.navigation.hideLogo).toBe(true)
-      expect(res.body.navigation.title).toBe("Custom Title")
-      expect(res.body.navigation.hideLogo).toBe(true)
-      expect(res.body.navigation.navigation).toBe("Left")
-      expect(res.body.navigation.navBackground).toBe(
+      const app = await config.api.application.create({
+        name: "Old App",
+        useTemplate: "true",
+        templateFile: "src/api/routes/tests/data/old-app.txt",
+      })
+      expect(app._id).toBeDefined()
+      expect(app.navigation).toBeDefined()
+      expect(app.navigation!.hideLogo).toBe(true)
+      expect(app.navigation!.title).toBe("Custom Title")
+      expect(app.navigation!.hideLogo).toBe(true)
+      expect(app.navigation!.navigation).toBe("Left")
+      expect(app.navigation!.navBackground).toBe(
         "var(--spectrum-global-color-blue-600)"
       )
-      expect(res.body.navigation.navTextColor).toBe(
+      expect(app.navigation!.navTextColor).toBe(
         "var(--spectrum-global-color-gray-50)"
       )
       expect(events.app.created).toBeCalledTimes(1)
@@ -118,15 +104,10 @@ describe("/applications", () => {
     it("lists all applications", async () => {
       await config.createApp("app1")
       await config.createApp("app2")
-
-      const res = await request
-        .get(`/api/applications?status=${AppStatus.DEV}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const apps = await config.api.application.fetch({ status: AppStatus.DEV })
 
       // two created apps + the inited app
-      expect(res.body.length).toBe(3)
+      expect(apps.length).toBe(3)
     })
   })
 
diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts
index 9c784bade1..7cc88d9eea 100644
--- a/packages/server/src/tests/utilities/api/application.ts
+++ b/packages/server/src/tests/utilities/api/application.ts
@@ -1,13 +1,38 @@
 import { Response } from "supertest"
-import { App } from "@budibase/types"
+import { App, type CreateAppRequest } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
+import { AppStatus } from "../../../db/utils"
+import { dbObjectAsPojo } from "oracledb"
 
 export class ApplicationAPI extends TestAPI {
   constructor(config: TestConfiguration) {
     super(config)
   }
 
+  create = async (app: CreateAppRequest): Promise<App> => {
+    const request = this.request
+      .post("/api/applications")
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    for (const key of Object.keys(app)) {
+      request.field(key, (app as any)[key])
+    }
+
+    if (app.templateFile) {
+      request.attach("templateFile", app.templateFile)
+    }
+
+    const result = await request
+
+    if (result.statusCode !== 200) {
+      fail(JSON.stringify(result.body))
+    }
+
+    return result.body as App
+  }
+
   getRaw = async (appId: string): Promise<Response> => {
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
@@ -21,4 +46,18 @@ export class ApplicationAPI extends TestAPI {
     const result = await this.getRaw(appId)
     return result.body.application as App
   }
+
+  fetch = async ({ status }: { status?: AppStatus } = {}): Promise<App[]> => {
+    let query = []
+    if (status) {
+      query.push(`status=${status}`)
+    }
+
+    const result = await this.request
+      .get(`/api/applications${query.length ? `?${query.join("&")}` : ""}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as App[]
+  }
 }
diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts
index 08aafc6527..8571895fcc 100644
--- a/packages/types/src/documents/app/app.ts
+++ b/packages/types/src/documents/app/app.ts
@@ -73,3 +73,15 @@ export interface AppFeatures {
 export interface AutomationSettings {
   chainAutomations?: boolean
 }
+
+export interface CreateAppRequest {
+  name: string
+  url?: string
+  useTemplate?: string
+  templateName?: string
+  templateKey?: string
+  templateFile?: string
+  includeSampleData?: boolean
+  encryptionPassword?: string
+  templateString?: string
+}
diff --git a/packages/worker/scripts/test.sh b/packages/worker/scripts/test.sh
index eba95c4916..17b3ee17f4 100644
--- a/packages/worker/scripts/test.sh
+++ b/packages/worker/scripts/test.sh
@@ -4,10 +4,10 @@ set -e
 if [[ -n $CI ]]
 then
   # Running in ci, where resources are limited
-  echo "jest --coverage --maxWorkers=2  --forceExit --bail"
-  jest --coverage --maxWorkers=2  --forceExit --bail
+  echo "jest --coverage --maxWorkers=2  --forceExit --bail $@"
+  jest --coverage --maxWorkers=2  --forceExit --bail $@
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --maxWorkers=2 --forceExit"
-  jest --coverage --maxWorkers=2 --forceExit
+  echo "jest --coverage --maxWorkers=2 --forceExit $@"
+  jest --coverage --maxWorkers=2 --forceExit $@
 fi
\ No newline at end of file
diff --git a/qa-core/src/internal-api/api/apis/AppAPI.ts b/qa-core/src/internal-api/api/apis/AppAPI.ts
index a9f9a6a841..8b291a628e 100644
--- a/qa-core/src/internal-api/api/apis/AppAPI.ts
+++ b/qa-core/src/internal-api/api/apis/AppAPI.ts
@@ -1,11 +1,10 @@
-import { App } from "@budibase/types"
+import { App, CreateAppRequest } from "@budibase/types"
 import { Response } from "node-fetch"
 import {
   RouteConfig,
   AppPackageResponse,
   DeployConfig,
   MessageResponse,
-  CreateAppRequest,
 } from "../../../types"
 import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
 import BaseAPI from "./BaseAPI"
diff --git a/qa-core/src/internal-api/fixtures/applications.ts b/qa-core/src/internal-api/fixtures/applications.ts
index 01dd18fc6a..59f73ba863 100644
--- a/qa-core/src/internal-api/fixtures/applications.ts
+++ b/qa-core/src/internal-api/fixtures/applications.ts
@@ -1,5 +1,5 @@
 import { generator } from "../../shared"
-import { CreateAppRequest } from "../../types"
+import { CreateAppRequest } from "@budibase/types"
 
 function uniqueWord() {
   return generator.word() + generator.hash()
diff --git a/qa-core/src/internal-api/tests/tables/tables.spec.ts b/qa-core/src/internal-api/tests/tables/tables.spec.ts
index 09d8f68e86..a38b8e6059 100644
--- a/qa-core/src/internal-api/tests/tables/tables.spec.ts
+++ b/qa-core/src/internal-api/tests/tables/tables.spec.ts
@@ -13,17 +13,6 @@ describe("Internal API - Table Operations", () => {
     await config.afterAll()
   })
 
-  async function createAppFromTemplate() {
-    return config.api.apps.create({
-      name: generator.word(),
-      url: `/${generator.word()}`,
-      useTemplate: "true",
-      templateName: "Near Miss Register",
-      templateKey: "app/near-miss-register",
-      templateFile: undefined,
-    })
-  }
-
   it("Create and delete table, columns and rows", async () => {
     // create the app
     await config.createApp(fixtures.apps.appFromTemplate())
diff --git a/qa-core/src/shared/BudibaseTestConfiguration.ts b/qa-core/src/shared/BudibaseTestConfiguration.ts
index 18b7c89ec8..9a12f3e65d 100644
--- a/qa-core/src/shared/BudibaseTestConfiguration.ts
+++ b/qa-core/src/shared/BudibaseTestConfiguration.ts
@@ -1,8 +1,8 @@
 import { BudibaseInternalAPI } from "../internal-api"
 import { AccountInternalAPI } from "../account-api"
-import { APIRequestOpts, CreateAppRequest, State } from "../types"
+import { APIRequestOpts, State } from "../types"
 import * as fixtures from "../internal-api/fixtures"
-import { CreateAccountRequest } from "@budibase/types"
+import { CreateAccountRequest, CreateAppRequest } from "@budibase/types"
 
 export default class BudibaseTestConfiguration {
   // apis
diff --git a/qa-core/src/types/app.ts b/qa-core/src/types/app.ts
deleted file mode 100644
index 7159112024..0000000000
--- a/qa-core/src/types/app.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-// TODO: Integrate with budibase
-export interface CreateAppRequest {
-  name: string
-  url: string
-  useTemplate?: string
-  templateName?: string
-  templateKey?: string
-  templateFile?: string
-  includeSampleData?: boolean
-}
diff --git a/qa-core/src/types/index.ts b/qa-core/src/types/index.ts
index 9bde46c66e..a44df4ef3c 100644
--- a/qa-core/src/types/index.ts
+++ b/qa-core/src/types/index.ts
@@ -1,6 +1,5 @@
 export * from "./api"
 export * from "./apiKeyResponse"
-export * from "./app"
 export * from "./appPackage"
 export * from "./deploy"
 export * from "./newAccount"

From 5f76f143bf1d8a561821317b3145256ac0a51fd3 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 23 Feb 2024 17:31:45 +0000
Subject: [PATCH 17/47] Adding the ability to disable SQL aliasing if
 necessary.

---
 .../src/api/controllers/row/ExternalRequest.ts      | 13 +++++++++++--
 packages/server/src/environment.ts                  |  6 ++++--
 packages/server/src/integrations/base/sql.ts        |  2 +-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 4f755845dc..0070e0bf24 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -35,6 +35,7 @@ import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import AliasTables from "./alias"
 import sdk from "../../../sdk"
+import env from "../../../environment"
 
 export interface ManyRelationship {
   tableId?: string
@@ -884,12 +885,20 @@ export class ExternalRequest<T extends Operation> {
       },
     }
 
-    const aliasing = new AliasTables(Object.keys(this.tables))
     // remove any relationships that could block deletion
     if (operation === Operation.DELETE && id) {
       await this.removeRelationshipsToRow(table, generateRowIdField(id))
     }
-    const response = await aliasing.queryWithAliasing(json)
+
+    // aliasing can be disabled fully if desired
+    let response
+    if (!env.SQL_ALIASING_DISABLE) {
+      const aliasing = new AliasTables(Object.keys(this.tables))
+      response = await aliasing.queryWithAliasing(json)
+    } else {
+      response = await getDatasourceAndQuery(json)
+    }
+
     // handle many-to-many relationships now if we know the ID (could be auto increment)
     if (operation !== Operation.READ) {
       await this.handleManyRelationships(
diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts
index 1d07eff1ce..a7c6df29ea 100644
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@@ -76,13 +76,16 @@ const environment = {
     DEFAULTS.AUTOMATION_THREAD_TIMEOUT > QUERY_THREAD_TIMEOUT
       ? DEFAULTS.AUTOMATION_THREAD_TIMEOUT
       : QUERY_THREAD_TIMEOUT,
-  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
   PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
   MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
   SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
+  // SQL
+  SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
+  SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
+  SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,
@@ -90,7 +93,6 @@ const environment = {
   DISABLE_RATE_LIMITING: process.env.DISABLE_RATE_LIMITING,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
-  ENABLE_SQL_LOGGING: process.env.ENABLE_SQL_LOGGING,
   SELF_HOSTED: process.env.SELF_HOSTED,
   HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
   FORKED_PROCESS_NAME:
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index cc2e1d94a8..172a10ea7f 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -688,7 +688,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   }
 
   log(query: string, values?: any[]) {
-    if (!environment.ENABLE_SQL_LOGGING) {
+    if (!environment.SQL_LOGGING_ENABLE) {
       return
     }
     const sqlClient = this.getSqlClient()

From b9600d83302b32c9d2cc9fad82c22c79995578c6 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 26 Feb 2024 11:57:56 +0000
Subject: [PATCH 18/47] More progress on modernising application tests.

---
 .../src/objectStore/buckets/plugins.ts        |   2 +-
 packages/server/scripts/test.sh               |   4 +-
 .../server/src/api/controllers/application.ts |  37 +++--
 .../src/api/routes/tests/application.spec.ts  | 153 +++++-------------
 .../src/tests/utilities/api/application.ts    |  96 ++++++++++-
 packages/types/src/documents/app/app.ts       |  21 ++-
 6 files changed, 179 insertions(+), 134 deletions(-)

diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index 6f1b7116ae..2d17a0562c 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins: Plugin[]) {
+export function enrichPluginURLs(plugins: Plugin[]): Plugin[] {
   if (!plugins || !plugins.length) {
     return []
   }
diff --git a/packages/server/scripts/test.sh b/packages/server/scripts/test.sh
index 9efef05526..3ecf8bb794 100644
--- a/packages/server/scripts/test.sh
+++ b/packages/server/scripts/test.sh
@@ -3,12 +3,12 @@ set -e
 
 if [[ -n $CI ]]
 then
-  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot"
+  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
   echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
   jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
-  export NODE_OPTIONS="--no-node-snapshot"
+  export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
   echo "jest --coverage --maxWorkers=2 --forceExit $@"
   jest --coverage --maxWorkers=2 --forceExit $@
 fi
\ No newline at end of file
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index 2d8b4b8686..f5a121fea2 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -48,6 +48,8 @@ import {
   Screen,
   UserCtx,
   CreateAppRequest,
+  FetchAppDefinitionResponse,
+  type FetchAppPackageResponse,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -59,23 +61,23 @@ import * as appMigrations from "../../appMigrations"
 async function getLayouts() {
   const db = context.getAppDB()
   return (
-    await db.allDocs(
+    await db.allDocs<Layout>(
       getLayoutParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map((row: any) => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
 async function getScreens() {
   const db = context.getAppDB()
   return (
-    await db.allDocs(
+    await db.allDocs<Screen>(
       getScreenParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map((row: any) => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
 function getUserRoleId(ctx: UserCtx) {
@@ -175,14 +177,16 @@ export const addSampleData = async (ctx: UserCtx) => {
   ctx.status = 200
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(ctx: UserCtx<null, App[]>) {
   ctx.body = await sdk.applications.fetch(
     ctx.query.status as AppStatus,
     ctx.user
   )
 }
 
-export async function fetchAppDefinition(ctx: UserCtx) {
+export async function fetchAppDefinition(
+  ctx: UserCtx<null, FetchAppDefinitionResponse>
+) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -197,17 +201,19 @@ export async function fetchAppDefinition(ctx: UserCtx) {
   }
 }
 
-export async function fetchAppPackage(ctx: UserCtx) {
+export async function fetchAppPackage(
+  ctx: UserCtx<null, FetchAppPackageResponse>
+) {
   const db = context.getAppDB()
   const appId = context.getAppId()
-  let application = await db.get<any>(DocumentType.APP_METADATA)
+  let application = await db.get<App>(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
   const license = await licensing.cache.getCachedLicense()
 
   // Enrich plugin URLs
   application.usedPlugins = objectStore.enrichPluginURLs(
-    application.usedPlugins
+    application.usedPlugins || []
   )
 
   // Only filter screens if the user is not a builder
@@ -425,7 +431,9 @@ export async function create(ctx: UserCtx) {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export async function update(ctx: UserCtx) {
+export async function update(
+  ctx: UserCtx<{ name?: string; url?: string }, App>
+) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
   const name = ctx.request.body.name,
@@ -498,7 +506,7 @@ export async function revertClient(ctx: UserCtx) {
   const revertedToVersion = application.revertableVersion
   const appPackageUpdates = {
     version: revertedToVersion,
-    revertableVersion: null,
+    revertableVersion: undefined,
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   await events.app.versionReverted(app, currentVersion, revertedToVersion)
@@ -618,12 +626,15 @@ export async function importToApp(ctx: UserCtx) {
   ctx.body = { message: "app updated" }
 }
 
-export async function updateAppPackage(appPackage: any, appId: any) {
+export async function updateAppPackage(
+  appPackage: Partial<App>,
+  appId: string
+) {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get<App>(DocumentType.APP_METADATA)
 
-    const newAppPackage = { ...application, ...appPackage }
+    const newAppPackage: App = { ...application, ...appPackage }
     if (appPackage._rev !== application._rev) {
       newAppPackage._rev = application._rev
     }
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index 7340166e67..5fcff9c770 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -11,25 +11,27 @@ jest.mock("../../../utilities/redis", () => ({
   checkDebounce: jest.fn(),
   shutdown: jest.fn(),
 }))
-import { clearAllApps, checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
+import type { App } from "@budibase/types"
 
-jest.setTimeout(15000)
+jest.setTimeout(150000000)
 
 describe("/applications", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
+  let app: App
 
   afterAll(setup.afterAll)
-
-  beforeAll(async () => {
-    await config.init()
-  })
+  beforeAll(async () => await config.init())
 
   beforeEach(async () => {
+    app = await config.api.application.create({ name: utils.newid() })
+    const deployment = await config.api.application.publish(app.appId)
+    expect(deployment.status).toBe("SUCCESS")
     jest.clearAllMocks()
   })
 
@@ -74,7 +76,7 @@ describe("/applications", () => {
 
     it("migrates navigation settings from old apps", async () => {
       const app = await config.api.application.create({
-        name: "Old App",
+        name: utils.newid(),
         useTemplate: "true",
         templateFile: "src/api/routes/tests/data/old-app.txt",
       })
@@ -96,77 +98,45 @@ describe("/applications", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(async () => {
-      // Clean all apps but the onde from config
-      await clearAllApps(config.getTenantId(), [config.getAppId()!])
-    })
-
     it("lists all applications", async () => {
-      await config.createApp("app1")
-      await config.createApp("app2")
       const apps = await config.api.application.fetch({ status: AppStatus.DEV })
-
-      // two created apps + the inited app
-      expect(apps.length).toBe(3)
+      expect(apps.length).toBeGreaterThan(0)
     })
   })
 
   describe("fetchAppDefinition", () => {
     it("should be able to get an apps definition", async () => {
-      const res = await request
-        .get(`/api/applications/${config.getAppId()}/definition`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.libraries.length).toEqual(1)
+      const res = await config.api.application.getDefinition(app.appId)
+      expect(res.libraries.length).toEqual(1)
     })
   })
 
   describe("fetchAppPackage", () => {
     it("should be able to fetch the app package", async () => {
-      const res = await request
-        .get(`/api/applications/${config.getAppId()}/appPackage`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.application).toBeDefined()
-      expect(res.body.application.appId).toEqual(config.getAppId())
+      const res = await config.api.application.getAppPackage(app.appId)
+      expect(res.application).toBeDefined()
+      expect(res.application.appId).toEqual(config.getAppId())
     })
   })
 
   describe("update", () => {
     it("should be able to update the app package", async () => {
-      const res = await request
-        .put(`/api/applications/${config.getAppId()}`)
-        .send({
-          name: "TEST_APP",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._rev).toBeDefined()
+      const updatedApp = await config.api.application.update(app.appId, {
+        name: "TEST_APP",
+      })
+      expect(updatedApp._rev).toBeDefined()
       expect(events.app.updated).toBeCalledTimes(1)
     })
   })
 
   describe("publish", () => {
     it("should publish app with dev app ID", async () => {
-      const appId = config.getAppId()
-      await request
-        .post(`/api/applications/${appId}/publish`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.publish(app.appId)
       expect(events.app.published).toBeCalledTimes(1)
     })
 
     it("should publish app with prod app ID", async () => {
-      const appId = config.getProdAppId()
-      await request
-        .post(`/api/applications/${appId}/publish`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.publish(app.appId.replace("_dev", ""))
       expect(events.app.published).toBeCalledTimes(1)
     })
   })
@@ -222,33 +192,25 @@ describe("/applications", () => {
 
   describe("sync", () => {
     it("app should sync correctly", async () => {
-      const res = await request
-        .post(`/api/applications/${config.getAppId()}/sync`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("App sync completed successfully.")
+      const { message } = await config.api.application.sync(app.appId)
+      expect(message).toEqual("App sync completed successfully.")
     })
 
     it("app should not sync if production", async () => {
-      const res = await request
-        .post(`/api/applications/app_123456/sync`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-      expect(res.body.message).toEqual(
+      const { message } = await config.api.application.sync(
+        app.appId.replace("_dev", ""),
+        { statusCode: 400 }
+      )
+
+      expect(message).toEqual(
         "This action cannot be performed for production apps"
       )
     })
 
     it("app should not sync if sync is disabled", async () => {
       env._set("DISABLE_AUTO_PROD_APP_SYNC", true)
-      const res = await request
-        .post(`/api/applications/${config.getAppId()}/sync`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual(
+      const { message } = await config.api.application.sync(app.appId)
+      expect(message).toEqual(
         "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable."
       )
       env._set("DISABLE_AUTO_PROD_APP_SYNC", false)
@@ -256,51 +218,26 @@ describe("/applications", () => {
   })
 
   describe("unpublish", () => {
-    beforeEach(async () => {
-      // We want to republish as the unpublish will delete the prod app
-      await config.publish()
-    })
-
     it("should unpublish app with dev app ID", async () => {
-      const appId = config.getAppId()
-      await request
-        .post(`/api/applications/${appId}/unpublish`)
-        .set(config.defaultHeaders())
-        .expect(204)
+      await config.api.application.unpublish(app.appId)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
 
     it("should unpublish app with prod app ID", async () => {
-      const appId = config.getProdAppId()
-      await request
-        .post(`/api/applications/${appId}/unpublish`)
-        .set(config.defaultHeaders())
-        .expect(204)
+      await config.api.application.unpublish(app.appId.replace("_dev", ""))
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
   })
 
   describe("delete", () => {
     it("should delete published app and dev apps with dev app ID", async () => {
-      await config.createApp("to-delete")
-      const appId = config.getAppId()
-      await request
-        .delete(`/api/applications/${appId}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.delete(app.appId)
       expect(events.app.deleted).toBeCalledTimes(1)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
 
     it("should delete published app and dev app with prod app ID", async () => {
-      await config.createApp("to-delete")
-      const appId = config.getProdAppId()
-      await request
-        .delete(`/api/applications/${appId}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.delete(app.appId.replace("_dev", ""))
       expect(events.app.deleted).toBeCalledTimes(1)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
@@ -308,28 +245,18 @@ describe("/applications", () => {
 
   describe("POST /api/applications/:appId/sync", () => {
     it("should not sync automation logs", async () => {
-      // setup the apps
-      await config.createApp("testing-auto-logs")
       const automation = await config.createAutomation()
-      await config.publish()
-      await context.doInAppContext(config.getProdAppId(), () => {
-        return config.createAutomationLog(automation)
-      })
+      await context.doInAppContext(app.appId, () =>
+        config.createAutomationLog(automation)
+      )
 
-      // do the sync
-      const appId = config.getAppId()
-      await request
-        .post(`/api/applications/${appId}/sync`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.sync(app.appId)
 
       // does exist in prod
       const prodLogs = await config.getAutomationLogs()
       expect(prodLogs.data.length).toBe(1)
 
-      // delete prod app so we revert to dev log search
-      await config.unpublish()
+      await config.api.application.unpublish(app.appId)
 
       // doesn't exist in dev
       const devLogs = await config.getAutomationLogs()
diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts
index 7cc88d9eea..83e42db1b2 100644
--- a/packages/server/src/tests/utilities/api/application.ts
+++ b/packages/server/src/tests/utilities/api/application.ts
@@ -1,9 +1,14 @@
 import { Response } from "supertest"
-import { App, type CreateAppRequest } from "@budibase/types"
+import {
+  App,
+  type CreateAppRequest,
+  type FetchAppDefinitionResponse,
+  type FetchAppPackageResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 import { AppStatus } from "../../../db/utils"
-import { dbObjectAsPojo } from "oracledb"
+import { constants } from "@budibase/backend-core"
 
 export class ApplicationAPI extends TestAPI {
   constructor(config: TestConfiguration) {
@@ -27,12 +32,55 @@ export class ApplicationAPI extends TestAPI {
     const result = await request
 
     if (result.statusCode !== 200) {
-      fail(JSON.stringify(result.body))
+      throw new Error(JSON.stringify(result.body))
     }
 
     return result.body as App
   }
 
+  delete = async (appId: string): Promise<void> => {
+    await this.request
+      .delete(`/api/applications/${appId}`)
+      .set(this.config.defaultHeaders())
+      .expect(200)
+  }
+
+  publish = async (
+    appId: string
+  ): Promise<{ _id: string; status: string; appUrl: string }> => {
+    // While the publsih endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const result = await this.request
+      .post(`/api/applications/${appId}/publish`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as { _id: string; status: string; appUrl: string }
+  }
+
+  unpublish = async (appId: string): Promise<void> => {
+    await this.request
+      .post(`/api/applications/${appId}/unpublish`)
+      .set(this.config.defaultHeaders())
+      .expect(204)
+  }
+
+  sync = async (
+    appId: string,
+    { statusCode }: { statusCode: number } = { statusCode: 200 }
+  ): Promise<{ message: string }> => {
+    const result = await this.request
+      .post(`/api/applications/${appId}/sync`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(statusCode)
+    return result.body
+  }
+
   getRaw = async (appId: string): Promise<Response> => {
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
@@ -47,6 +95,48 @@ export class ApplicationAPI extends TestAPI {
     return result.body.application as App
   }
 
+  getDefinition = async (
+    appId: string
+  ): Promise<FetchAppDefinitionResponse> => {
+    const result = await this.request
+      .get(`/api/applications/${appId}/definition`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as FetchAppDefinitionResponse
+  }
+
+  getAppPackage = async (appId: string): Promise<FetchAppPackageResponse> => {
+    const result = await this.request
+      .get(`/api/applications/${appId}/appPackage`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body
+  }
+
+  update = async (
+    appId: string,
+    app: { name?: string; url?: string }
+  ): Promise<App> => {
+    const request = this.request
+      .put(`/api/applications/${appId}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    for (const key of Object.keys(app)) {
+      request.field(key, (app as any)[key])
+    }
+
+    const result = await request
+
+    if (result.statusCode !== 200) {
+      throw new Error(JSON.stringify(result.body))
+    }
+
+    return result.body as App
+  }
+
   fetch = async ({ status }: { status?: AppStatus } = {}): Promise<App[]> => {
     let query = []
     if (status) {
diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts
index 8571895fcc..cdd825b777 100644
--- a/packages/types/src/documents/app/app.ts
+++ b/packages/types/src/documents/app/app.ts
@@ -1,5 +1,5 @@
-import { User, Document } from "../"
-import { SocketSession } from "../../sdk"
+import { User, Document, Layout, Screen, Plugin } from "../"
+import { SocketSession, PlanType } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
 
@@ -24,6 +24,8 @@ export interface App extends Document {
   icon?: AppIcon
   features?: AppFeatures
   automations?: AutomationSettings
+  usedPlugins?: Plugin[]
+  upgradableVersion?: string
 }
 
 export interface AppInstance {
@@ -85,3 +87,18 @@ export interface CreateAppRequest {
   encryptionPassword?: string
   templateString?: string
 }
+
+export interface FetchAppDefinitionResponse {
+  layouts: Layout[]
+  screens: Screen[]
+  libraries: string[]
+}
+
+export interface FetchAppPackageResponse {
+  application: App
+  licenseType: PlanType
+  screens: Screen[]
+  layouts: Layout[]
+  clientLibPath: string
+  hasLock: boolean
+}

From 45d2e6790539bd83bbb40da37e8709e2891ad76e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 13:50:15 +0000
Subject: [PATCH 19/47] Adding some new test cases based on finishing off
 testing.

---
 .../src/integrations/tests/sqlAlias.spec.ts   |  45 +++-
 .../sqlQueryJson/enrichRelationship.json      | 123 +++++++++++
 .../tests/sqlQueryJson/fetchManyToMany.json   | 109 ++++++++++
 .../sqlQueryJson/manyRelationshipFilters.json | 202 ++++++++++++++++++
 4 files changed, 477 insertions(+), 2 deletions(-)
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json
 create mode 100644 packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index c91d988849..c7c544be3c 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -5,6 +5,7 @@ import { SqlClient } from "../utils"
 
 describe("Captures of real examples", () => {
   const limit = 5000
+  const relationshipLimit = 100
 
   function getJson(name: string): QueryJson {
     return require(join(__dirname, "sqlQueryJson", name)) as QueryJson
@@ -26,7 +27,7 @@ describe("Captures of real examples", () => {
       const queryJson = getJson("basicFetchWithRelationships.json")
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [100, limit],
+        bindings: [relationshipLimit, limit],
         sql: `select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" order by "a"."firstname" asc limit $2`,
       })
     })
@@ -35,10 +36,50 @@ describe("Captures of real examples", () => {
       const queryJson = getJson("filterByRelationship.json")
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
-        bindings: [100, "assembling", limit],
+        bindings: [relationshipLimit, "assembling", limit],
         sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
       })
     })
+
+    it("should handle fetching many to many relationships", () => {
+      const queryJson = getJson("fetchManyToMany.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [relationshipLimit, limit],
+        sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" order by "a"."productname" asc limit $2`,
+      })
+    })
+
+    it("should handle enrichment of rows", () => {
+      const queryJson = getJson("enrichRelationship.json")
+      const filters = queryJson.filters?.oneOf?.taskid as number[]
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      expect(query).toEqual({
+        bindings: [...filters, limit, limit],
+        sql: `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid" from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) limit $3) as "a" left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid" limit $4`,
+      })
+    })
+
+    it("should manage query with many relationship filters", () => {
+      const queryJson = getJson("manyRelationshipFilters.json")
+      let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
+      const filters = queryJson.filters
+      const notEqualsValue = Object.values(filters?.notEqual!)[0]
+      const rangeValue = Object.values(filters?.range!)[0]
+      const equalValue = Object.values(filters?.equal!)[0]
+
+      expect(query).toEqual({
+        bindings: [
+          notEqualsValue,
+          relationshipLimit,
+          rangeValue.low,
+          rangeValue.high,
+          equalValue,
+          limit,
+        ],
+        sql: `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname" from (select * from "tasks" as "a" where not "a"."completed" = $1 order by "a"."taskname" asc limit $2) as "a" left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" left join "products" as "b" on "b"."productid" = "d"."productid" left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`,
+      })
+    })
   })
 
   describe("update", () => {
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json
new file mode 100644
index 0000000000..ee658aed18
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json
@@ -0,0 +1,123 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "entityId": "tasks",
+    "operation": "READ"
+  },
+  "resource": {
+    "fields": [
+      "a.executorid",
+      "a.taskname",
+      "a.taskid",
+      "a.completed",
+      "a.qaid",
+      "b.productname",
+      "b.productid"
+    ]
+  },
+  "filters": {
+    "oneOf": {
+      "taskid": [
+        1,
+        2
+      ]
+    }
+  },
+  "relationships": [
+    {
+      "tableName": "products",
+      "column": "products",
+      "through": "products_tasks",
+      "from": "taskid",
+      "to": "productid",
+      "fromPrimary": "taskid",
+      "toPrimary": "productid",
+      "aliases": {
+        "products_tasks": "c",
+        "products": "b",
+        "tasks": "a"
+      }
+    }
+  ],
+  "extra": {
+    "idFilter": {}
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+      "primary": [
+        "taskid"
+      ],
+      "name": "a",
+      "schema": {
+        "executorid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "executorid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "taskname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "taskname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "taskid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "taskid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "completed": {
+          "type": "boolean",
+          "externalType": "boolean",
+          "autocolumn": false,
+          "name": "completed",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "qaid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": false,
+          "name": "qaid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "products": {
+          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+          "name": "products",
+          "relationshipType": "many-to-many",
+          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+          "type": "link",
+          "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
+          "fieldName": "productid",
+          "throughFrom": "productid",
+          "throughTo": "taskid"
+        }
+      },
+      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "sourceType": "external",
+      "primaryDisplay": "taskname",
+      "sql": true,
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "tasks": "a",
+    "products": "b",
+    "products_tasks": "c"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json
new file mode 100644
index 0000000000..682ebaab2d
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json
@@ -0,0 +1,109 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "entityId": "products",
+    "operation": "READ"
+  },
+  "resource": {
+    "fields": [
+      "a.productname",
+      "a.productid",
+      "b.executorid",
+      "b.taskname",
+      "b.taskid",
+      "b.completed",
+      "b.qaid"
+    ]
+  },
+  "filters": {
+    "string": {},
+    "fuzzy": {},
+    "range": {},
+    "equal": {},
+    "notEqual": {},
+    "empty": {},
+    "notEmpty": {},
+    "contains": {},
+    "notContains": {},
+    "oneOf": {},
+    "containsAny": {}
+  },
+  "sort": {
+    "productname": {
+      "direction": "ASCENDING"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [
+    {
+      "tableName": "tasks",
+      "column": "tasks",
+      "through": "products_tasks",
+      "from": "productid",
+      "to": "taskid",
+      "fromPrimary": "productid",
+      "toPrimary": "taskid",
+      "aliases": {
+        "products_tasks": "c",
+        "tasks": "b",
+        "products": "a"
+      }
+    }
+  ],
+  "extra": {
+    "idFilter": {}
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+      "primary": [
+        "productid"
+      ],
+      "name": "a",
+      "schema": {
+        "productname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "productname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "productid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "productid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "tasks": {
+          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+          "name": "tasks",
+          "relationshipType": "many-to-many",
+          "fieldName": "taskid",
+          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+          "throughFrom": "taskid",
+          "throughTo": "productid",
+          "type": "link",
+          "main": true,
+          "_id": "c3b91d00cd36c4cc1a347794725b9adbd"
+        }
+      },
+      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "sourceType": "external",
+      "primaryDisplay": "productname"
+    }
+  },
+  "tableAliases": {
+    "products": "a",
+    "tasks": "b",
+    "products_tasks": "c"
+  }
+}
\ No newline at end of file
diff --git a/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json
new file mode 100644
index 0000000000..afa0889450
--- /dev/null
+++ b/packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json
@@ -0,0 +1,202 @@
+{
+  "endpoint": {
+    "datasourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+    "entityId": "tasks",
+    "operation": "READ"
+  },
+  "resource": {
+    "fields": [
+      "a.executorid",
+      "a.taskname",
+      "a.taskid",
+      "a.completed",
+      "a.qaid",
+      "b.productname",
+      "b.productid",
+      "c.year",
+      "c.firstname",
+      "c.personid",
+      "c.address",
+      "c.age",
+      "c.type",
+      "c.city",
+      "c.lastname",
+      "c.year",
+      "c.firstname",
+      "c.personid",
+      "c.address",
+      "c.age",
+      "c.type",
+      "c.city",
+      "c.lastname"
+    ]
+  },
+  "filters": {
+    "string": {},
+    "fuzzy": {},
+    "range": {
+      "1:persons.year": {
+        "low": 1990,
+        "high": 2147483647
+      }
+    },
+    "equal": {
+      "2:products.productname": "Computers"
+    },
+    "notEqual": {
+      "3:completed": true
+    },
+    "empty": {},
+    "notEmpty": {},
+    "contains": {},
+    "notContains": {},
+    "oneOf": {},
+    "containsAny": {},
+    "onEmptyFilter": "all"
+  },
+  "sort": {
+    "taskname": {
+      "direction": "ASCENDING"
+    }
+  },
+  "paginate": {
+    "limit": 100,
+    "page": 1
+  },
+  "relationships": [
+    {
+      "tableName": "products",
+      "column": "products",
+      "through": "products_tasks",
+      "from": "taskid",
+      "to": "productid",
+      "fromPrimary": "taskid",
+      "toPrimary": "productid",
+      "aliases": {
+        "products_tasks": "d",
+        "products": "b",
+        "tasks": "a"
+      }
+    },
+    {
+      "tableName": "persons",
+      "column": "tasksToExecute",
+      "from": "executorid",
+      "to": "personid",
+      "aliases": {
+        "persons": "c",
+        "tasks": "a"
+      }
+    },
+    {
+      "tableName": "persons",
+      "column": "tasksToQA",
+      "from": "qaid",
+      "to": "personid",
+      "aliases": {
+        "persons": "c",
+        "tasks": "a"
+      }
+    }
+  ],
+  "extra": {
+    "idFilter": {}
+  },
+  "meta": {
+    "table": {
+      "type": "table",
+      "_id": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__tasks",
+      "primary": [
+        "taskid"
+      ],
+      "name": "a",
+      "schema": {
+        "executorid": {
+          "type": "number",
+          "externalType": "integer",
+          "name": "executorid",
+          "constraints": {
+            "presence": false
+          },
+          "autocolumn": true,
+          "autoReason": "foreign_key"
+        },
+        "taskname": {
+          "type": "string",
+          "externalType": "character varying",
+          "autocolumn": false,
+          "name": "taskname",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "taskid": {
+          "type": "number",
+          "externalType": "integer",
+          "autocolumn": true,
+          "name": "taskid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "completed": {
+          "type": "boolean",
+          "externalType": "boolean",
+          "autocolumn": false,
+          "name": "completed",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "qaid": {
+          "type": "number",
+          "externalType": "integer",
+          "name": "qaid",
+          "constraints": {
+            "presence": false
+          }
+        },
+        "products": {
+          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products",
+          "name": "products",
+          "relationshipType": "many-to-many",
+          "through": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__products_tasks",
+          "type": "link",
+          "_id": "c3b91d00cd36c4cc1a347794725b9adbd",
+          "fieldName": "productid",
+          "throughFrom": "productid",
+          "throughTo": "taskid"
+        },
+        "tasksToExecute": {
+          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
+          "name": "tasksToExecute",
+          "relationshipType": "one-to-many",
+          "type": "link",
+          "_id": "c0f440590bda04f28846242156c1dd60b",
+          "foreignKey": "executorid",
+          "fieldName": "personid"
+        },
+        "tasksToQA": {
+          "tableId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81__persons",
+          "name": "tasksToQA",
+          "relationshipType": "one-to-many",
+          "type": "link",
+          "_id": "c5fdf453a0ba743d58e29491d174c974b",
+          "foreignKey": "qaid",
+          "fieldName": "personid"
+        }
+      },
+      "sourceId": "datasource_plus_44a967caf37a435f84fe01cd6dfe8f81",
+      "sourceType": "external",
+      "primaryDisplay": "taskname",
+      "sql": true,
+      "views": {}
+    }
+  },
+  "tableAliases": {
+    "tasks": "a",
+    "products": "b",
+    "persons": "c",
+    "products_tasks": "d"
+  }
+}
\ No newline at end of file

From 04e5699c9c222f17ee6e0a3aea095d020586ed08 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 26 Feb 2024 16:00:12 +0000
Subject: [PATCH 20/47] Finish modernising application.spec.ts

---
 packages/server/src/api/routes/application.ts |  1 -
 .../src/api/routes/tests/application.spec.ts  | 61 +++++++------------
 .../src/tests/utilities/api/application.ts    | 44 ++++++++++++-
 3 files changed, 65 insertions(+), 41 deletions(-)

diff --git a/packages/server/src/api/routes/application.ts b/packages/server/src/api/routes/application.ts
index babcb1b44b..7e01a3c2ef 100644
--- a/packages/server/src/api/routes/application.ts
+++ b/packages/server/src/api/routes/application.ts
@@ -4,7 +4,6 @@ import * as deploymentController from "../controllers/deploy"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import { applicationValidator } from "./utils/validators"
-import { importToApp } from "../controllers/application"
 
 const router: Router = new Router()
 
diff --git a/packages/server/src/api/routes/tests/application.spec.ts b/packages/server/src/api/routes/tests/application.spec.ts
index 5fcff9c770..dbe4eb51ae 100644
--- a/packages/server/src/api/routes/tests/application.spec.ts
+++ b/packages/server/src/api/routes/tests/application.spec.ts
@@ -17,11 +17,9 @@ import { AppStatus } from "../../../db/utils"
 import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
 import type { App } from "@budibase/types"
-
-jest.setTimeout(150000000)
+import tk from "timekeeper"
 
 describe("/applications", () => {
-  let request = setup.getRequest()
   let config = setup.getConfig()
   let app: App
 
@@ -143,50 +141,37 @@ describe("/applications", () => {
 
   describe("manage client library version", () => {
     it("should be able to update the app client library version", async () => {
-      await request
-        .post(`/api/applications/${config.getAppId()}/client/update`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.updateClient(app.appId)
       expect(events.app.versionUpdated).toBeCalledTimes(1)
     })
 
     it("should be able to revert the app client library version", async () => {
-      // We need to first update the version so that we can then revert
-      await request
-        .post(`/api/applications/${config.getAppId()}/client/update`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      await request
-        .post(`/api/applications/${config.getAppId()}/client/revert`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      await config.api.application.updateClient(app.appId)
+      await config.api.application.revertClient(app.appId)
       expect(events.app.versionReverted).toBeCalledTimes(1)
     })
   })
 
   describe("edited at", () => {
-    it("middleware should set edited at", async () => {
-      const headers = config.defaultHeaders()
-      headers["referer"] = `/${config.getAppId()}/test`
-      const res = await request
-        .put(`/api/applications/${config.getAppId()}`)
-        .send({
-          name: "UPDATED_NAME",
-        })
-        .set(headers)
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._rev).toBeDefined()
-      // retrieve the app to check it
-      const getRes = await request
-        .get(`/api/applications/${config.getAppId()}/appPackage`)
-        .set(headers)
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(getRes.body.application.updatedAt).toBeDefined()
+    it("middleware should set updatedAt", async () => {
+      const app = await tk.withFreeze(
+        "2021-01-01",
+        async () => await config.api.application.create({ name: utils.newid() })
+      )
+      expect(app.updatedAt).toEqual("2021-01-01T00:00:00.000Z")
+
+      const updatedApp = await tk.withFreeze(
+        "2021-02-01",
+        async () =>
+          await config.api.application.update(app.appId, {
+            name: "UPDATED_NAME",
+          })
+      )
+      expect(updatedApp._rev).toBeDefined()
+      expect(updatedApp.updatedAt).toEqual("2021-02-01T00:00:00.000Z")
+
+      const fetchedApp = await config.api.application.get(app.appId)
+      expect(fetchedApp.updatedAt).toEqual("2021-02-01T00:00:00.000Z")
     })
   })
 
diff --git a/packages/server/src/tests/utilities/api/application.ts b/packages/server/src/tests/utilities/api/application.ts
index 83e42db1b2..3951bba667 100644
--- a/packages/server/src/tests/utilities/api/application.ts
+++ b/packages/server/src/tests/utilities/api/application.ts
@@ -48,7 +48,7 @@ export class ApplicationAPI extends TestAPI {
   publish = async (
     appId: string
   ): Promise<{ _id: string; status: string; appUrl: string }> => {
-    // While the publsih endpoint does take an :appId parameter, it doesn't
+    // While the publish endpoint does take an :appId parameter, it doesn't
     // use it. It uses the appId from the context.
     let headers = {
       ...this.config.defaultHeaders(),
@@ -82,9 +82,15 @@ export class ApplicationAPI extends TestAPI {
   }
 
   getRaw = async (appId: string): Promise<Response> => {
+    // While the appPackage endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
-      .set(this.config.defaultHeaders())
+      .set(headers)
       .expect("Content-Type", /json/)
       .expect(200)
     return result
@@ -137,6 +143,40 @@ export class ApplicationAPI extends TestAPI {
     return result.body as App
   }
 
+  updateClient = async (appId: string): Promise<void> => {
+    // While the updateClient endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const response = await this.request
+      .post(`/api/applications/${appId}/client/update`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+
+    if (response.statusCode !== 200) {
+      throw new Error(JSON.stringify(response.body))
+    }
+  }
+
+  revertClient = async (appId: string): Promise<void> => {
+    // While the revertClient endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const response = await this.request
+      .post(`/api/applications/${appId}/client/revert`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+
+    if (response.statusCode !== 200) {
+      throw new Error(JSON.stringify(response.body))
+    }
+  }
+
   fetch = async ({ status }: { status?: AppStatus } = {}): Promise<App[]> => {
     let query = []
     if (status) {

From c15554547bd313b70b98811fbd102f7632da64b2 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 26 Feb 2024 17:28:37 +0000
Subject: [PATCH 21/47] Respond to PR feedback.

---
 .../src/objectStore/buckets/plugins.ts        |  2 +-
 .../server/src/api/controllers/application.ts | 10 +++---
 packages/types/src/api/web/application.ts     | 29 +++++++++++++++++
 packages/types/src/api/web/index.ts           |  1 +
 packages/types/src/documents/app/app.ts       | 31 ++-----------------
 5 files changed, 38 insertions(+), 35 deletions(-)
 create mode 100644 packages/types/src/api/web/application.ts

diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index 2d17a0562c..cade60aa09 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins: Plugin[]): Plugin[] {
+export function enrichPluginURLs(plugins: Plugin[] | undefined): Plugin[] {
   if (!plugins || !plugins.length) {
     return []
   }
diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts
index f5a121fea2..0bc93888ae 100644
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@@ -49,7 +49,7 @@ import {
   UserCtx,
   CreateAppRequest,
   FetchAppDefinitionResponse,
-  type FetchAppPackageResponse,
+  FetchAppPackageResponse,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -177,7 +177,7 @@ export const addSampleData = async (ctx: UserCtx) => {
   ctx.status = 200
 }
 
-export async function fetch(ctx: UserCtx<null, App[]>) {
+export async function fetch(ctx: UserCtx<void, App[]>) {
   ctx.body = await sdk.applications.fetch(
     ctx.query.status as AppStatus,
     ctx.user
@@ -185,7 +185,7 @@ export async function fetch(ctx: UserCtx<null, App[]>) {
 }
 
 export async function fetchAppDefinition(
-  ctx: UserCtx<null, FetchAppDefinitionResponse>
+  ctx: UserCtx<void, FetchAppDefinitionResponse>
 ) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
@@ -202,7 +202,7 @@ export async function fetchAppDefinition(
 }
 
 export async function fetchAppPackage(
-  ctx: UserCtx<null, FetchAppPackageResponse>
+  ctx: UserCtx<void, FetchAppPackageResponse>
 ) {
   const db = context.getAppDB()
   const appId = context.getAppId()
@@ -213,7 +213,7 @@ export async function fetchAppPackage(
 
   // Enrich plugin URLs
   application.usedPlugins = objectStore.enrichPluginURLs(
-    application.usedPlugins || []
+    application.usedPlugins
   )
 
   // Only filter screens if the user is not a builder
diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
new file mode 100644
index 0000000000..8b1db534bb
--- /dev/null
+++ b/packages/types/src/api/web/application.ts
@@ -0,0 +1,29 @@
+import type { PlanType } from "../../sdk"
+import type { Layout, App } from "../../documents"
+
+export interface CreateAppRequest {
+  name: string
+  url?: string
+  useTemplate?: string
+  templateName?: string
+  templateKey?: string
+  templateFile?: string
+  includeSampleData?: boolean
+  encryptionPassword?: string
+  templateString?: string
+}
+
+export interface FetchAppDefinitionResponse {
+  layouts: Layout[]
+  screens: Screen[]
+  libraries: string[]
+}
+
+export interface FetchAppPackageResponse {
+  application: App
+  licenseType: PlanType
+  screens: Screen[]
+  layouts: Layout[]
+  clientLibPath: string
+  hasLock: boolean
+}
diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts
index 75c246ab9b..ab18add208 100644
--- a/packages/types/src/api/web/index.ts
+++ b/packages/types/src/api/web/index.ts
@@ -1,3 +1,4 @@
+export * from "./application"
 export * from "./analytics"
 export * from "./auth"
 export * from "./user"
diff --git a/packages/types/src/documents/app/app.ts b/packages/types/src/documents/app/app.ts
index cdd825b777..ae4f3fa6da 100644
--- a/packages/types/src/documents/app/app.ts
+++ b/packages/types/src/documents/app/app.ts
@@ -1,5 +1,5 @@
-import { User, Document, Layout, Screen, Plugin } from "../"
-import { SocketSession, PlanType } from "../../sdk"
+import { User, Document, Plugin } from "../"
+import { SocketSession } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
 
@@ -75,30 +75,3 @@ export interface AppFeatures {
 export interface AutomationSettings {
   chainAutomations?: boolean
 }
-
-export interface CreateAppRequest {
-  name: string
-  url?: string
-  useTemplate?: string
-  templateName?: string
-  templateKey?: string
-  templateFile?: string
-  includeSampleData?: boolean
-  encryptionPassword?: string
-  templateString?: string
-}
-
-export interface FetchAppDefinitionResponse {
-  layouts: Layout[]
-  screens: Screen[]
-  libraries: string[]
-}
-
-export interface FetchAppPackageResponse {
-  application: App
-  licenseType: PlanType
-  screens: Screen[]
-  layouts: Layout[]
-  clientLibPath: string
-  hasLock: boolean
-}

From 2e8eda47f10fb47f59449df2a83d50354142264a Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 26 Feb 2024 17:38:33 +0000
Subject: [PATCH 22/47] Respond to PR feedback.

---
 packages/backend-core/src/objectStore/buckets/plugins.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts
index cade60aa09..02be9345ab 100644
--- a/packages/backend-core/src/objectStore/buckets/plugins.ts
+++ b/packages/backend-core/src/objectStore/buckets/plugins.ts
@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins: Plugin[] | undefined): Plugin[] {
+export function enrichPluginURLs(plugins?: Plugin[]): Plugin[] {
   if (!plugins || !plugins.length) {
     return []
   }

From c1a88f12795fff1ce4bcdc2c4922e80397d51b96 Mon Sep 17 00:00:00 2001
From: Sam Rose <hello@samwho.dev>
Date: Mon, 26 Feb 2024 17:55:32 +0000
Subject: [PATCH 23/47] Fix type checks.

---
 packages/types/src/api/web/application.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/types/src/api/web/application.ts b/packages/types/src/api/web/application.ts
index 8b1db534bb..87a0bd6ef9 100644
--- a/packages/types/src/api/web/application.ts
+++ b/packages/types/src/api/web/application.ts
@@ -1,5 +1,5 @@
 import type { PlanType } from "../../sdk"
-import type { Layout, App } from "../../documents"
+import type { Layout, App, Screen } from "../../documents"
 
 export interface CreateAppRequest {
   name: string

From cb19e1f24c0ab72989cf7058525492f1fe2a84c1 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 17:56:28 +0000
Subject: [PATCH 24/47] Fixing response types of DS+ query function.

---
 .../api/controllers/row/ExternalRequest.ts    |  9 ++++----
 .../server/src/api/controllers/row/alias.ts   | 21 +++++++++++++------
 .../server/src/integrations/base/query.ts     |  8 +++++--
 .../server/src/integrations/googlesheets.ts   |  7 ++++---
 .../src/integrations/microsoftSqlServer.ts    |  3 ++-
 packages/server/src/integrations/mysql.ts     |  4 ++--
 packages/server/src/integrations/oracle.ts    |  8 ++++---
 packages/server/src/integrations/postgres.ts  |  3 ++-
 packages/server/src/sdk/app/rows/utils.ts     | 13 ++++++++++--
 packages/types/src/sdk/datasources.ts         | 16 +++++++++++++-
 10 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 0070e0bf24..0dd17a86e6 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -332,7 +332,7 @@ export class ExternalRequest<T extends Operation> {
       endpoint: getEndpoint(table._id!, Operation.READ),
       filters: buildFilters(rowId, {}, table),
     })
-    if (response.length > 0) {
+    if (Array.isArray(response)) {
       return response[0]
     } else {
       throw new Error(`Cannot fetch row by ID "${rowId}"`)
@@ -646,7 +646,7 @@ export class ExternalRequest<T extends Operation> {
         },
       })
       // this is the response from knex if no rows found
-      const rows: Row[] = !response[0].read ? response : []
+      const rows: Row[] = response?.[0].read ? [] : (response as Row[])
       const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
       related[storeTo] = { rows, isMany, tableId }
     }
@@ -899,15 +899,16 @@ export class ExternalRequest<T extends Operation> {
       response = await getDatasourceAndQuery(json)
     }
 
+    const responseRows = Array.isArray(response) ? response : []
     // handle many-to-many relationships now if we know the ID (could be auto increment)
     if (operation !== Operation.READ) {
       await this.handleManyRelationships(
         table._id || "",
-        response[0],
+        responseRows[0],
         processed.manyRelationships
       )
     }
-    const output = this.outputProcessing(response, table, relationships)
+    const output = this.outputProcessing(responseRows, table, relationships)
     // if reading it'll just be an array of rows, return whole thing
     if (operation === Operation.READ) {
       return (
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 6533e51728..ca144ee518 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -1,4 +1,10 @@
-import { QueryJson, SearchFilters, Table, Row } from "@budibase/types"
+import {
+  QueryJson,
+  SearchFilters,
+  Table,
+  Row,
+  DatasourcePlusQueryResponse,
+} from "@budibase/types"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { cloneDeep } from "lodash"
 
@@ -68,9 +74,8 @@ export default class AliasTables {
     return map
   }
 
-  async queryWithAliasing(json: QueryJson) {
+  async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
     json = cloneDeep(json)
-    const aliasField = (field: string) => this.aliasField(field)
     const aliasTable = (table: Table) => ({
       ...table,
       name: this.getAlias(table.name),
@@ -78,7 +83,7 @@ export default class AliasTables {
     // run through the query json to update anywhere a table may be used
     if (json.resource?.fields) {
       json.resource.fields = json.resource.fields.map(field =>
-        aliasField(field)
+        this.aliasField(field)
       )
     }
     if (json.filters) {
@@ -88,7 +93,7 @@ export default class AliasTables {
         }
         const aliasedFilters: typeof filter = {}
         for (let key of Object.keys(filter)) {
-          aliasedFilters[aliasField(key)] = filter[key]
+          aliasedFilters[this.aliasField(key)] = filter[key]
         }
         json.filters[filterKey as keyof SearchFilters] = aliasedFilters
       }
@@ -120,6 +125,10 @@ export default class AliasTables {
     }
     json.tableAliases = invertedTableAliases
     const response = await getDatasourceAndQuery(json)
-    return this.reverse(response)
+    if (Array.isArray(response)) {
+      return this.reverse(response)
+    } else {
+      return response
+    }
   }
 }
diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts
index 4f31e37744..b906ecbb1b 100644
--- a/packages/server/src/integrations/base/query.ts
+++ b/packages/server/src/integrations/base/query.ts
@@ -1,11 +1,15 @@
-import { QueryJson, Datasource } from "@budibase/types"
+import {
+  QueryJson,
+  Datasource,
+  DatasourcePlusQueryResponse,
+} from "@budibase/types"
 import { getIntegration } from "../index"
 import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
-) {
+): DatasourcePlusQueryResponse {
   datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts
index 58c867ea0b..32398bde41 100644
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@@ -16,6 +16,7 @@ import {
   Table,
   TableRequest,
   TableSourceType,
+  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import {
@@ -334,7 +335,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     return { tables: externalTables, errors }
   }
 
-  async query(json: QueryJson) {
+  async query(json: QueryJson): DatasourcePlusQueryResponse {
     const sheet = json.endpoint.entityId
     switch (json.endpoint.operation) {
       case Operation.CREATE:
@@ -384,7 +385,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     }
     try {
       await this.connect()
-      return await this.client.addSheet({ title: name, headerValues: [name] })
+      await this.client.addSheet({ title: name, headerValues: [name] })
     } catch (err) {
       console.error("Error creating new table in google sheets", err)
       throw err
@@ -450,7 +451,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     try {
       await this.connect()
       const sheetToDelete = this.client.sheetsByTitle[sheet]
-      return await sheetToDelete.delete()
+      await sheetToDelete.delete()
     } catch (err) {
       console.error("Error deleting table in google sheets", err)
       throw err
diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts
index e063933503..f87e248ac0 100644
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@@ -13,6 +13,7 @@ import {
   SourceName,
   Schema,
   TableSourceType,
+  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -493,7 +494,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
     return response.recordset || [{ deleted: true }]
   }
 
-  async query(json: QueryJson) {
+  async query(json: QueryJson): DatasourcePlusQueryResponse {
     const schema = this.config.schema
     await this.connect()
     if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index 6eebda8df5..f629381807 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -12,7 +12,7 @@ import {
   SourceName,
   Schema,
   TableSourceType,
-  FieldType,
+  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -381,7 +381,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     return results.length ? results : [{ deleted: true }]
   }
 
-  async query(json: QueryJson) {
+  async query(json: QueryJson): DatasourcePlusQueryResponse {
     await this.connect()
     try {
       const queryFn = (query: any) =>
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index cdf37a9c83..86c4bf519d 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -12,6 +12,8 @@ import {
   ConnectionInfo,
   Schema,
   TableSourceType,
+  Row,
+  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
   buildExternalTableId,
@@ -420,7 +422,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       : [{ deleted: true }]
   }
 
-  async query(json: QueryJson) {
+  async query(json: QueryJson): DatasourcePlusQueryResponse {
     const operation = this._operation(json)
     const input = this._query(json, { disableReturning: true })
     if (Array.isArray(input)) {
@@ -444,7 +446,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
       if (deletedRows?.rows?.length) {
         return deletedRows.rows
       } else if (response.rows?.length) {
-        return response.rows
+        return response.rows as Row[]
       } else {
         // get the last row that was updated
         if (
@@ -455,7 +457,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           const lastRow = await this.internalQuery({
             sql: `SELECT * FROM \"${json.endpoint.entityId}\" WHERE ROWID = '${response.lastRowid}'`,
           })
-          return lastRow.rows
+          return lastRow.rows as Row[]
         } else {
           return [{ [operation.toLowerCase()]: true }]
         }
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 7fb75f5d9f..6e87f296e1 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -12,6 +12,7 @@ import {
   SourceName,
   Schema,
   TableSourceType,
+  DatasourcePlusQueryResponse,
 } from "@budibase/types"
 import {
   getSqlQuery,
@@ -419,7 +420,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     return response.rows.length ? response.rows : [{ deleted: true }]
   }
 
-  async query(json: QueryJson) {
+  async query(json: QueryJson): DatasourcePlusQueryResponse {
     const operation = this._operation(json).toLowerCase()
     const input = this._query(json)
     if (Array.isArray(input)) {
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 2320820b3e..75f980d9f9 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -1,12 +1,21 @@
 import cloneDeep from "lodash/cloneDeep"
 import validateJs from "validate.js"
-import { FieldType, QueryJson, Row, Table, TableSchema } from "@budibase/types"
+import {
+  FieldType,
+  QueryJson,
+  Row,
+  Table,
+  TableSchema,
+  DatasourcePlusQueryResponse,
+} from "@budibase/types"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
 import sdk from "../.."
 import { isRelationshipColumn } from "../../../db/utils"
 
-export async function getDatasourceAndQuery(json: QueryJson) {
+export async function getDatasourceAndQuery(
+  json: QueryJson
+): DatasourcePlusQueryResponse {
   const datasourceId = json.endpoint.datasourceId
   const datasource = await sdk.datasources.get(datasourceId)
   return makeExternalQuery(datasource, json)
diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts
index 7a335eb3b9..4cddb0c09e 100644
--- a/packages/types/src/sdk/datasources.ts
+++ b/packages/types/src/sdk/datasources.ts
@@ -1,4 +1,5 @@
-import { Table } from "../documents"
+import { Table, Row } from "../documents"
+import { QueryJson } from "./search"
 
 export const PASSWORD_REPLACEMENT = "--secret-value--"
 
@@ -180,11 +181,24 @@ export interface Schema {
   errors: Record<string, string>
 }
 
+// return these when an operation occurred but we got no response
+enum DSPlusOperation {
+  CREATE = "create",
+  READ = "read",
+  UPDATE = "update",
+  DELETE = "delete",
+}
+
+export type DatasourcePlusQueryResponse = Promise<
+  Row[] | Record<DSPlusOperation, boolean>[] | void
+>
+
 export interface DatasourcePlus extends IntegrationBase {
   // if the datasource supports the use of bindings directly (to protect against SQL injection)
   // this returns the format of the identifier
   getBindingIdentifier(): string
   getStringConcat(parts: string[]): string
+  query(json: QueryJson): DatasourcePlusQueryResponse
   buildSchema(
     datasourceId: string,
     entities: Record<string, Table>

From 1bb375a500557c73de17f8d457abf0b692dc2444 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 18:16:42 +0000
Subject: [PATCH 25/47] Further typing.

---
 .../scripts/integrations/postgres/reset.sh    |  3 +-
 .../api/controllers/row/ExternalRequest.ts    | 69 +++++++++++--------
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/reset.sh b/packages/server/scripts/integrations/postgres/reset.sh
index 29a5db0181..8deb01cdf8 100755
--- a/packages/server/scripts/integrations/postgres/reset.sh
+++ b/packages/server/scripts/integrations/postgres/reset.sh
@@ -1,4 +1,3 @@
 #!/bin/bash
-docker-compose down
+docker-compose down -v
 docker volume prune -f
-docker volume rm postgres_pg_data
diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 0dd17a86e6..dceadb3cd4 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -19,6 +19,7 @@ import {
   SortJson,
   SortType,
   Table,
+  isManyToOne,
 } from "@budibase/types"
 import {
   breakExternalTableId,
@@ -104,23 +105,36 @@ function buildFilters(
   }
 }
 
-function removeRelationships(
+async function removeManyToManyRelationships(
   rowId: string,
   table: Table,
-  isManyToMany: boolean,
-  colName?: string
+  colName: string
 ) {
   const tableId = table._id!
   const filters = buildFilters(rowId, {}, table)
   // safety check, if there are no filters on deletion bad things happen
   if (Object.keys(filters).length !== 0) {
-    const op = isManyToMany ? Operation.DELETE : Operation.UPDATE
-    const body = colName && !isManyToMany ? { [colName]: null } : undefined
     return getDatasourceAndQuery({
-      endpoint: getEndpoint(tableId, op),
-      body,
+      endpoint: getEndpoint(tableId, Operation.DELETE),
+      body: { [colName]: null },
       filters,
     })
+  } else {
+    return []
+  }
+}
+
+async function removeOneToManyRelationships(rowId: string, table: Table) {
+  const tableId = table._id!
+  const filters = buildFilters(rowId, {}, table)
+  // safety check, if there are no filters on deletion bad things happen
+  if (Object.keys(filters).length !== 0) {
+    return getDatasourceAndQuery({
+      endpoint: getEndpoint(tableId, Operation.UPDATE),
+      filters,
+    })
+  } else {
+    return []
   }
 }
 
@@ -734,12 +748,10 @@ export class ExternalRequest<T extends Operation> {
         continue
       }
       for (let row of rows) {
-        const promise = removeRelationships(
-          generateIdForRow(row, table),
-          table,
-          isMany,
-          colName
-        )
+        const rowId = generateIdForRow(row, table)
+        const promise: Promise<any> = isMany
+          ? removeManyToManyRelationships(rowId, table, colName)
+          : removeOneToManyRelationships(rowId, table)
         if (promise) {
           promises.push(promise)
         }
@@ -752,24 +764,23 @@ export class ExternalRequest<T extends Operation> {
     const row = await this.getRow(table, rowId)
     const related = await this.lookupRelations(table._id!, row)
     for (let column of Object.values(table.schema)) {
-      if (
-        column.type !== FieldType.LINK ||
-        column.relationshipType === RelationshipType.ONE_TO_MANY
-      ) {
+      const relationshipColumn = column as RelationshipFieldMetadata
+      if (!isManyToOne(relationshipColumn)) {
         continue
       }
-      const relationshipColumn = column as ManyToOneRelationshipFieldMetadata
       const { rows, isMany, tableId } = related[relationshipColumn.fieldName]
       const table = this.getTable(tableId)!
       await Promise.all(
-        rows.map(row =>
-          removeRelationships(
-            generateIdForRow(row, table),
-            table,
-            isMany,
-            relationshipColumn.fieldName
-          )
-        )
+        rows.map(row => {
+          const rowId = generateIdForRow(row, table)
+          return isMany
+            ? removeManyToManyRelationships(
+                rowId,
+                table,
+                relationshipColumn.fieldName
+              )
+            : removeOneToManyRelationships(rowId, table)
+        })
       )
     }
   }
@@ -892,11 +903,11 @@ export class ExternalRequest<T extends Operation> {
 
     // aliasing can be disabled fully if desired
     let response
-    if (!env.SQL_ALIASING_DISABLE) {
+    if (env.SQL_ALIASING_DISABLE) {
+      response = await getDatasourceAndQuery(json)
+    } else {
       const aliasing = new AliasTables(Object.keys(this.tables))
       response = await aliasing.queryWithAliasing(json)
-    } else {
-      response = await getDatasourceAndQuery(json)
     }
 
     const responseRows = Array.isArray(response) ? response : []

From 80dc0beeeda53446decf82528d2882e8b141bf4e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 18:22:19 +0000
Subject: [PATCH 26/47] Multiline SQL tests.

---
 .../src/integrations/tests/sqlAlias.spec.ts   | 63 ++++++++++++++++---
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index c7c544be3c..da88127b16 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -3,6 +3,10 @@ import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
 
+function multiline(sql: string) {
+  return sql.replace(/\n/g, "").replace(/ +/g, " ")
+}
+
 describe("Captures of real examples", () => {
   const limit = 5000
   const relationshipLimit = 100
@@ -17,7 +21,8 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["A Street", 34, "London", "A", "B", "designer", 1990],
-        sql: `insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") values ($1, $2, $3, $4, $5, $6, $7) returning *`,
+        sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") 
+              values ($1, $2, $3, $4, $5, $6, $7) returning *`),
       })
     })
   })
@@ -28,7 +33,15 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [relationshipLimit, limit],
-        sql: `select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" order by "a"."firstname" asc limit $2`,
+        sql: multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", 
+              "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", 
+              "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", 
+              "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", 
+              "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
+              "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
+              from (select * from "persons" as "a" order by "a"."firstname" asc limit $1) as "a" 
+              left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" 
+              order by "a"."firstname" asc limit $2`),
       })
     })
 
@@ -37,7 +50,13 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [relationshipLimit, "assembling", limit],
-        sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 order by "a"."productname" asc limit $3`,
+        sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
+              "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
+              "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
+              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
+              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where "b"."taskname" = $2 
+              order by "a"."productname" asc limit $3`),
       })
     })
 
@@ -46,7 +65,13 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [relationshipLimit, limit],
-        sql: `select "a"."productname" as "a.productname", "a"."productid" as "a.productid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" left join "products_tasks" as "c" on "a"."productid" = "c"."productid" left join "tasks" as "b" on "b"."taskid" = "c"."taskid" order by "a"."productname" asc limit $2`,
+        sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", 
+              "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", 
+              "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" 
+              from (select * from "products" as "a" order by "a"."productname" asc limit $1) as "a" 
+              left join "products_tasks" as "c" on "a"."productid" = "c"."productid" 
+              left join "tasks" as "b" on "b"."taskid" = "c"."taskid" 
+              order by "a"."productname" asc limit $2`),
       })
     })
 
@@ -56,7 +81,12 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [...filters, limit, limit],
-        sql: `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid" from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) limit $3) as "a" left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid" limit $4`,
+        sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", 
+             "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", 
+             "b"."productname" as "b.productname", "b"."productid" as "b.productid" 
+             from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) limit $3) as "a" 
+             left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" 
+             left join "products" as "b" on "b"."productid" = "c"."productid" limit $4`),
       })
     })
 
@@ -77,7 +107,19 @@ describe("Captures of real examples", () => {
           equalValue,
           limit,
         ],
-        sql: `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", "c"."city" as "c.city", "c"."lastname" as "c.lastname" from (select * from "tasks" as "a" where not "a"."completed" = $1 order by "a"."taskname" asc limit $2) as "a" left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" left join "products" as "b" on "b"."productid" = "d"."productid" left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`,
+        sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", 
+             "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", 
+             "b"."productid" as "b.productid", "c"."year" as "c.year", "c"."firstname" as "c.firstname", 
+             "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
+             "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", 
+             "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", 
+             "c"."city" as "c.city", "c"."lastname" as "c.lastname" 
+             from (select * from "tasks" as "a" where not "a"."completed" = $1 
+             order by "a"."taskname" asc limit $2) as "a" 
+             left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" 
+             left join "products" as "b" on "b"."productid" = "d"."productid" 
+             left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" 
+             where "c"."year" between $3 and $4 and "b"."productname" = $5 order by "a"."taskname" asc limit $6`),
       })
     })
   })
@@ -88,7 +130,8 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
-        sql: `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`,
+        sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
+             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
       })
     })
 
@@ -97,7 +140,8 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5],
-        sql: `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`,
+        sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, 
+             "type" = $5, "city" = $6, "lastname" = $7 where "a"."personid" = $8 returning *`),
       })
     })
   })
@@ -108,7 +152,8 @@ describe("Captures of real examples", () => {
       let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
       expect(query).toEqual({
         bindings: ["ddd", ""],
-        sql: `delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`,
+        sql: multiline(`delete from "compositetable" as "a" where "a"."keypartone" = $1 and "a"."keyparttwo" = $2 
+             returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`),
       })
     })
   })

From 59ab557a937a2d91ce3375a84aa23e76f7a1ff00 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 18:29:57 +0000
Subject: [PATCH 27/47] Looping characters used.

---
 packages/server/src/api/controllers/row/alias.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index ca144ee518..05a261af42 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -28,7 +28,13 @@ export default class AliasTables {
     const char = this.character
     this.aliases[tableName] = char
     this.tableAliases[char] = tableName
-    this.character = String.fromCharCode(char.charCodeAt(0) + 1)
+    this.character =
+      char.substring(0, char.length - 1) +
+      String.fromCharCode(char.charCodeAt(char.length - 1) + 1)
+    // reached end of characters, extend number of characters used
+    if (this.character === "z") {
+      this.character = new Array(this.character.length + 1).fill("a").join("")
+    }
     return char
   }
 

From d68fcbf8f7c7e6b3b1edc16497357b0a624dc2a4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Mon, 26 Feb 2024 18:36:34 +0000
Subject: [PATCH 28/47] Loop aliasing.

---
 .../server/src/api/controllers/row/alias.ts     |  2 +-
 .../src/integrations/tests/sqlAlias.spec.ts     | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 05a261af42..747166a7ba 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -32,7 +32,7 @@ export default class AliasTables {
       char.substring(0, char.length - 1) +
       String.fromCharCode(char.charCodeAt(char.length - 1) + 1)
     // reached end of characters, extend number of characters used
-    if (this.character === "z") {
+    if (this.character.charAt(this.character.length - 1) === "z") {
       this.character = new Array(this.character.length + 1).fill("a").join("")
     }
     return char
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index da88127b16..c35c2b45b6 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -2,6 +2,8 @@ import { QueryJson } from "@budibase/types"
 import { join } from "path"
 import Sql from "../base/sql"
 import { SqlClient } from "../utils"
+import AliasTables from "../../api/controllers/row/alias"
+import { generator } from "@budibase/backend-core/tests"
 
 function multiline(sql: string) {
   return sql.replace(/\n/g, "").replace(/ +/g, " ")
@@ -157,4 +159,19 @@ describe("Captures of real examples", () => {
       })
     })
   })
+
+  describe("check max character aliasing", () => {
+    it("should handle over 'z' max character alias", () => {
+      const tableNames = []
+      for (let i = 0; i < 100; i++) {
+        tableNames.push(generator.word())
+      }
+      const aliasing = new AliasTables(tableNames)
+      let alias: string = ""
+      for (let table of tableNames) {
+        alias = aliasing.getAlias(table)
+      }
+      expect(alias).toEqual("aaay")
+    })
+  })
 })

From 6e4c2b7242c7a67049a43eaf433543babf1a6a40 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Tue, 27 Feb 2024 09:23:49 +0000
Subject: [PATCH 29/47] Export data make CSV delimiter configurable (#13028)

* Add delimiter option

* Add custom delimiter

* external export delimiter

* Custom headers for row export

* External export rows custom headers

* Support custom JSON export labels

* Handle export table source switch

* update account portal

* Add space as delimiter

* Refactor

* update account portal
---
 packages/account-portal                       |   2 +-
 .../actions/ExportData.svelte                 | 102 +++++++++++++-----
 .../controls/ColumnEditor/ColumnEditor.svelte |   6 ++
 packages/client/src/utils/buttonActions.js    |   6 +-
 packages/frontend-core/src/api/rows.js        |  13 ++-
 .../server/src/api/controllers/row/index.ts   |   5 +-
 .../src/api/controllers/view/exporters.ts     |  18 +++-
 packages/server/src/sdk/app/rows/search.ts    |   2 +
 .../src/sdk/app/rows/search/external.ts       |  21 +++-
 .../src/sdk/app/rows/search/internal.ts       |  21 +++-
 packages/server/src/sdk/app/rows/utils.ts     |  19 +++-
 packages/types/src/api/web/app/rows.ts        |   2 +
 12 files changed, 174 insertions(+), 43 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index ab324e35d8..de6d44c372 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit ab324e35d855012bd0f49caa53c6dd765223c6fa
+Subproject commit de6d44c372a7f48ca0ce8c6c0c19311d4bc21646
diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
index f6c8479b4e..5955cc762d 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
@@ -1,9 +1,9 @@
 <script>
-  import { Label, Select, Body, Multiselect } from "@budibase/bbui"
-  import { findAllMatchingComponents, findComponent } from "helpers/components"
-  import { selectedScreen } from "stores/builder"
+  import { Label, Select, Body } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
+  import ColumnEditor from "../../ColumnEditor/ColumnEditor.svelte"
+  import { findAllMatchingComponents } from "helpers/components"
+  import { selectedScreen } from "stores/builder"
 
   export let parameters
 
@@ -18,37 +18,65 @@
     },
   ]
 
+  const DELIMITERS = [
+    {
+      label: ",",
+      value: ",",
+    },
+    {
+      label: ";",
+      value: ";",
+    },
+    {
+      label: ":",
+      value: ":",
+    },
+    {
+      label: "|",
+      value: "|",
+    },
+    {
+      label: "~",
+      value: "~",
+    },
+    {
+      label: "[tab]",
+      value: "\t",
+    },
+    {
+      label: "[space]",
+      value: " ",
+    },
+  ]
+
   $: tables = findAllMatchingComponents($selectedScreen?.props, component =>
     component._component.endsWith("table")
-  ).map(table => ({
-    label: table._instanceName,
-    value: table._id,
-  }))
+  )
   $: tableBlocks = findAllMatchingComponents(
     $selectedScreen?.props,
     component => component._component.endsWith("tableblock")
-  ).map(block => ({
-    label: block._instanceName,
-    value: `${block._id}-table`,
+  )
+  $: components = tables.concat(tableBlocks)
+  $: componentOptions = components.map(table => ({
+    label: table._instanceName,
+    value: table._component.includes("tableblock")
+      ? `${table._id}-table`
+      : table._id,
   }))
-  $: componentOptions = tables.concat(tableBlocks)
-  $: columnOptions = getColumnOptions(parameters.tableComponentId)
-
-  const getColumnOptions = tableId => {
-    // Strip block suffix if block component
-    if (tableId?.includes("-")) {
-      tableId = tableId.split("-")[0]
-    }
-    const selectedTable = findComponent($selectedScreen?.props, tableId)
-    const datasource = getDatasourceForProvider($selectedScreen, selectedTable)
-    const { schema } = getSchemaForDatasource($selectedScreen, datasource)
-    return Object.keys(schema || {})
-  }
+  $: selectedTableId = parameters.tableComponentId?.includes("-")
+    ? parameters.tableComponentId.split("-")[0]
+    : parameters.tableComponentId
+  $: selectedTable = components.find(
+    component => component._id === selectedTableId
+  )
 
   onMount(() => {
     if (!parameters.type) {
       parameters.type = "csv"
     }
+    if (!parameters.delimiter) {
+      parameters.delimiter = ","
+    }
   })
 </script>
 
@@ -67,13 +95,29 @@
       options={componentOptions}
       on:change={() => (parameters.columns = [])}
     />
+    <span />
     <Label small>Export as</Label>
     <Select bind:value={parameters.type} options={FORMATS} />
+    <Select
+      bind:value={parameters.delimiter}
+      placeholder={null}
+      options={DELIMITERS}
+      disabled={parameters.type !== "csv"}
+    />
     <Label small>Export columns</Label>
-    <Multiselect
-      placeholder="All columns"
+    <ColumnEditor
       bind:value={parameters.columns}
-      options={columnOptions}
+      allowCellEditing={false}
+      componentInstance={selectedTable}
+      on:change={e => {
+        const columns = e.detail
+        parameters.customHeaders = columns.reduce((headerMap, column) => {
+          return {
+            [column.name]: column.displayName,
+            ...headerMap,
+          }
+        }, {})
+      }}
     />
   </div>
 </div>
@@ -97,8 +141,8 @@
   .params {
     display: grid;
     column-gap: var(--spacing-xs);
-    row-gap: var(--spacing-s);
-    grid-template-columns: 90px 1fr;
+    row-gap: var(--spacing-m);
+    grid-template-columns: 90px 1fr 90px;
     align-items: center;
   }
 </style>
diff --git a/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte b/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte
index 2b9fa573c2..742ab785a1 100644
--- a/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte
+++ b/packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte
@@ -29,6 +29,12 @@
     allowLinks: true,
   })
 
+  $: {
+    value = (value || []).filter(
+      column => (schema || {})[column.name || column] !== undefined
+    )
+  }
+
   const getText = value => {
     if (!value?.length) {
       return "All columns"
diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js
index b2068ad152..68478b76ac 100644
--- a/packages/client/src/utils/buttonActions.js
+++ b/packages/client/src/utils/buttonActions.js
@@ -341,7 +341,11 @@ const exportDataHandler = async action => {
         tableId: selection.tableId,
         rows: selection.selectedRows,
         format: action.parameters.type,
-        columns: action.parameters.columns,
+        columns: action.parameters.columns?.map(
+          column => column.name || column
+        ),
+        delimiter: action.parameters.delimiter,
+        customHeaders: action.parameters.customHeaders,
       })
       download(
         new Blob([data], { type: "text/plain" }),
diff --git a/packages/frontend-core/src/api/rows.js b/packages/frontend-core/src/api/rows.js
index 79f837e864..0a0d48da43 100644
--- a/packages/frontend-core/src/api/rows.js
+++ b/packages/frontend-core/src/api/rows.js
@@ -89,13 +89,24 @@ export const buildRowEndpoints = API => ({
    * @param rows the array of rows to export
    * @param format the format to export (csv or json)
    * @param columns which columns to export (all if undefined)
+   * @param delimiter how values should be separated in a CSV (default is comma)
    */
-  exportRows: async ({ tableId, rows, format, columns, search }) => {
+  exportRows: async ({
+    tableId,
+    rows,
+    format,
+    columns,
+    search,
+    delimiter,
+    customHeaders,
+  }) => {
     return await API.post({
       url: `/api/${tableId}/rows/exportRows?format=${format}`,
       body: {
         rows,
         columns,
+        delimiter,
+        customHeaders,
         ...search,
       },
       parseResponse: async response => {
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 1ad8a2a695..ec56919d12 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -223,7 +223,8 @@ export const exportRows = async (
 
   const format = ctx.query.format
 
-  const { rows, columns, query, sort, sortOrder } = ctx.request.body
+  const { rows, columns, query, sort, sortOrder, delimiter, customHeaders } =
+    ctx.request.body
   if (typeof format !== "string" || !exporters.isFormat(format)) {
     ctx.throw(
       400,
@@ -241,6 +242,8 @@ export const exportRows = async (
     query,
     sort,
     sortOrder,
+    delimiter,
+    customHeaders,
   })
   ctx.attachment(fileName)
   ctx.body = apiFileReturn(content)
diff --git a/packages/server/src/api/controllers/view/exporters.ts b/packages/server/src/api/controllers/view/exporters.ts
index d6caff6035..3b5f951dca 100644
--- a/packages/server/src/api/controllers/view/exporters.ts
+++ b/packages/server/src/api/controllers/view/exporters.ts
@@ -1,7 +1,19 @@
 import { Row, TableSchema } from "@budibase/types"
 
-export function csv(headers: string[], rows: Row[]) {
-  let csv = headers.map(key => `"${key}"`).join(",")
+function getHeaders(
+  headers: string[],
+  customHeaders: { [key: string]: string }
+) {
+  return headers.map(header => `"${customHeaders[header] || header}"`)
+}
+
+export function csv(
+  headers: string[],
+  rows: Row[],
+  delimiter: string = ",",
+  customHeaders: { [key: string]: string } = {}
+) {
+  let csv = getHeaders(headers, customHeaders).join(delimiter)
 
   for (let row of rows) {
     csv = `${csv}\n${headers
@@ -15,7 +27,7 @@ export function csv(headers: string[], rows: Row[]) {
             : ""
         return val.trim()
       })
-      .join(",")}`
+      .join(delimiter)}`
   }
   return csv
 }
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 4b71179839..8b24f9bc5f 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -36,11 +36,13 @@ export async function search(options: SearchParams): Promise<{
 export interface ExportRowsParams {
   tableId: string
   format: Format
+  delimiter?: string
   rowIds?: string[]
   columns?: string[]
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  customHeaders?: { [key: string]: string }
 }
 
 export interface ExportRowsResult {
diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts
index 8465f997e3..e2d1a1b32c 100644
--- a/packages/server/src/sdk/app/rows/search/external.ts
+++ b/packages/server/src/sdk/app/rows/search/external.ts
@@ -101,7 +101,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, columns, rowIds, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    columns,
+    rowIds,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
   let requestQuery: SearchFilters = {}
@@ -153,12 +163,17 @@ export async function exportRows(
     rows = result.rows
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
 
   let content: string
   switch (format) {
     case exporters.Format.CSV:
-      content = exporters.csv(headers ?? Object.keys(schema), exportRows)
+      content = exporters.csv(
+        headers ?? Object.keys(schema),
+        exportRows,
+        delimiter,
+        customHeaders
+      )
       break
     case exporters.Format.JSON:
       content = exporters.json(exportRows)
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 22cb3985b7..2d3c32e02e 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -84,7 +84,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, rowIds, columns, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    rowIds,
+    columns,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
 
@@ -124,11 +134,16 @@ export async function exportRows(
     rows = result
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
   if (format === Format.CSV) {
     return {
       fileName: "export.csv",
-      content: csv(headers ?? Object.keys(rows[0]), exportRows),
+      content: csv(
+        headers ?? Object.keys(rows[0]),
+        exportRows,
+        delimiter,
+        customHeaders
+      ),
     }
   } else if (format === Format.JSON) {
     return {
diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts
index 14868a4013..0ff85f40ac 100644
--- a/packages/server/src/sdk/app/rows/utils.ts
+++ b/packages/server/src/sdk/app/rows/utils.ts
@@ -16,7 +16,8 @@ export function cleanExportRows(
   rows: any[],
   schema: TableSchema,
   format: string,
-  columns?: string[]
+  columns?: string[],
+  customHeaders: { [key: string]: string } = {}
 ) {
   let cleanRows = [...rows]
 
@@ -44,11 +45,27 @@ export function cleanExportRows(
         }
       }
     }
+  } else if (format === Format.JSON) {
+    // Replace row keys with custom headers
+    for (let row of cleanRows) {
+      renameKeys(customHeaders, row)
+    }
   }
 
   return cleanRows
 }
 
+function renameKeys(keysMap: { [key: string]: any }, row: any) {
+  for (const key in keysMap) {
+    Object.defineProperty(
+      row,
+      keysMap[key],
+      Object.getOwnPropertyDescriptor(row, key) || {}
+    )
+    delete row[key]
+  }
+}
+
 function isForeignKey(key: string, table: Table) {
   const relationships = Object.values(table.schema).filter(isRelationshipColumn)
   return relationships.some(
diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts
index dad3286754..14e28e4a01 100644
--- a/packages/types/src/api/web/app/rows.ts
+++ b/packages/types/src/api/web/app/rows.ts
@@ -37,6 +37,8 @@ export interface ExportRowsRequest {
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  delimiter?: string
+  customHeaders?: { [key: string]: string }
 }
 
 export type ExportRowsResponse = ReadStream

From b1dd8999cb9ff7d542277fe02d33586064abfdf1 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 27 Feb 2024 09:33:44 +0000
Subject: [PATCH 30/47] Bump version to 2.20.11

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 54e106cd5a..623fbf6d43 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.20.10",
+  "version": "2.20.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From c11527d5399fb63f62ed7f45869c42d37efbb856 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Feb 2024 09:56:56 +0000
Subject: [PATCH 31/47] Adding in char sequence.

---
 .../server/src/api/controllers/row/alias.ts   | 38 ++++++++++++++-----
 .../src/integrations/tests/sqlAlias.spec.ts   |  2 +-
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 747166a7ba..280d50de9b 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -8,33 +8,51 @@ import {
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
 import { cloneDeep } from "lodash"
 
+class CharSequence {
+  static alphabet = "abcdefghijklmnopqrstuvwxyz"
+  counters: number[]
+
+  constructor() {
+    this.counters = [0]
+  }
+
+  get character() {
+    return this.counters.map(i => CharSequence.alphabet[i]).join("")
+  }
+
+  next() {
+    for (let i = this.counters.length - 1; i >= 0; i--) {
+      if (this.counters[i] < CharSequence.alphabet.length - 1) {
+        this.counters[i]++
+        return
+      }
+      this.counters[i] = 0
+    }
+    this.counters.unshift(0)
+  }
+}
+
 export default class AliasTables {
-  character: string
   aliases: Record<string, string>
   tableAliases: Record<string, string>
   tableNames: string[]
+  charSeq: CharSequence
 
   constructor(tableNames: string[]) {
     this.tableNames = tableNames
-    this.character = "a"
     this.aliases = {}
     this.tableAliases = {}
+    this.charSeq = new CharSequence()
   }
 
   getAlias(tableName: string) {
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
     }
-    const char = this.character
+    const char = this.charSeq.character
+    this.charSeq.next()
     this.aliases[tableName] = char
     this.tableAliases[char] = tableName
-    this.character =
-      char.substring(0, char.length - 1) +
-      String.fromCharCode(char.charCodeAt(char.length - 1) + 1)
-    // reached end of characters, extend number of characters used
-    if (this.character.charAt(this.character.length - 1) === "z") {
-      this.character = new Array(this.character.length + 1).fill("a").join("")
-    }
     return char
   }
 
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index c35c2b45b6..6e8bd0ea72 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -171,7 +171,7 @@ describe("Captures of real examples", () => {
       for (let table of tableNames) {
         alias = aliasing.getAlias(table)
       }
-      expect(alias).toEqual("aaay")
+      expect(alias).toEqual("cu")
     })
   })
 })

From c0bb03e9382ac16c14c0fd29537d443e8126dd94 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Feb 2024 09:58:06 +0000
Subject: [PATCH 32/47] Adding length check back for get row.

---
 packages/server/src/api/controllers/row/ExternalRequest.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index dceadb3cd4..165a1847b5 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -346,7 +346,7 @@ export class ExternalRequest<T extends Operation> {
       endpoint: getEndpoint(table._id!, Operation.READ),
       filters: buildFilters(rowId, {}, table),
     })
-    if (Array.isArray(response)) {
+    if (Array.isArray(response) && response.length > 0) {
       return response[0]
     } else {
       throw new Error(`Cannot fetch row by ID "${rowId}"`)

From 3d48a8f7f3d463ae735bc177af700c879e34e554 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Tue, 27 Feb 2024 11:26:45 +0000
Subject: [PATCH 33/47] Don't encode twice (#13138)

---
 packages/builder/src/helpers/data/utils.js           | 6 +++++-
 packages/builder/src/helpers/tests/dataUtils.test.js | 7 +++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/helpers/data/utils.js b/packages/builder/src/helpers/data/utils.js
index a29ce8db6d..a592b57a26 100644
--- a/packages/builder/src/helpers/data/utils.js
+++ b/packages/builder/src/helpers/data/utils.js
@@ -17,6 +17,10 @@ export function breakQueryString(qs) {
   return paramObj
 }
 
+function isEncoded(str) {
+  return typeof str == "string" && decodeURIComponent(str) !== str
+}
+
 export function buildQueryString(obj) {
   let str = ""
   if (obj) {
@@ -35,7 +39,7 @@ export function buildQueryString(obj) {
         value = value.replace(binding, marker)
         bindingMarkers[marker] = binding
       })
-      let encoded = encodeURIComponent(value || "")
+      let encoded = isEncoded(value) ? value : encodeURIComponent(value || "")
       Object.entries(bindingMarkers).forEach(([marker, binding]) => {
         encoded = encoded.replace(marker, binding)
       })
diff --git a/packages/builder/src/helpers/tests/dataUtils.test.js b/packages/builder/src/helpers/tests/dataUtils.test.js
index 8fc2d706d7..bd207ea339 100644
--- a/packages/builder/src/helpers/tests/dataUtils.test.js
+++ b/packages/builder/src/helpers/tests/dataUtils.test.js
@@ -39,4 +39,11 @@ describe("check query string utils", () => {
     expect(broken.key1).toBe(obj2.key1)
     expect(broken.key2).toBe(obj2.key2)
   })
+
+  it("should not encode a URL more than once when building the query string", () => {
+    const queryString = buildQueryString({
+      values: "a%2Cb%2Cc",
+    })
+    expect(queryString).toBe("values=a%2Cb%2Cc")
+  })
 })

From 4c203ea35d57ce9248ae31786a58356e3d2719b4 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Tue, 27 Feb 2024 12:49:11 +0000
Subject: [PATCH 34/47] Budi 6149 export data make csv delimiter configurable
 (#13140)

* Add delimiter option

* Add custom delimiter

* external export delimiter

* Custom headers for row export

* External export rows custom headers

* Support custom JSON export labels

* Handle export table source switch

* update account portal

* Add space as delimiter

* Refactor

* update account portal

* Update columns from ColumnEditor

* Revert "Update columns from ColumnEditor"

This reverts commit d23293cd8b5d007b13a8825bc80e161d60f691c0.

* Revert "Revert "Update columns from ColumnEditor""

This reverts commit 0f16bddff7a745f7593e331ceedcec5511bbe4ba.
---
 .../controls/ButtonActionEditor/actions/ExportData.svelte      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
index 5955cc762d..096341783d 100644
--- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
+++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte
@@ -106,11 +106,12 @@
     />
     <Label small>Export columns</Label>
     <ColumnEditor
-      bind:value={parameters.columns}
+      value={parameters.columns}
       allowCellEditing={false}
       componentInstance={selectedTable}
       on:change={e => {
         const columns = e.detail
+        parameters.columns = columns
         parameters.customHeaders = columns.reduce((headerMap, column) => {
           return {
             [column.name]: column.displayName,

From 2a8bb872adc87d01827eabe6de0535859c5fe6da Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 27 Feb 2024 12:49:37 +0000
Subject: [PATCH 35/47] Bump version to 2.20.12

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 623fbf6d43..e1a469adf1 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.20.11",
+  "version": "2.20.12",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

From fac9f18bc2214f6f62249a372c88803ea19a4a7e Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Feb 2024 13:40:31 +0000
Subject: [PATCH 36/47] PR comments.

---
 .../api/controllers/row/ExternalRequest.ts    | 31 ++++++++++++++-----
 .../server/src/api/controllers/row/alias.ts   | 13 +++-----
 .../src/integrations/tests/sqlAlias.spec.ts   |  4 +--
 3 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index 165a1847b5..685af4e98e 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -640,14 +640,26 @@ export class ExternalRequest<T extends Operation> {
       ) {
         continue
       }
-      const isMany = field.relationshipType === RelationshipType.MANY_TO_MANY
-      const tableId = isMany ? field.through! : field.tableId!
+      let tableId: string | undefined,
+        lookupField: string | undefined,
+        fieldName: string | undefined
+      if (isManyToMany(field)) {
+        tableId = field.through
+        lookupField = primaryKey
+        fieldName = field.throughTo || primaryKey
+      } else if (isManyToOne(field)) {
+        tableId = field.tableId
+        lookupField = field.foreignKey
+        fieldName = field.fieldName
+      }
+      if (!tableId || !lookupField || !fieldName) {
+        throw new Error(
+          "Unable to lookup relationships - undefined column properties."
+        )
+      }
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore
       const linkPrimaryKey = this.tables[relatedTableName].primary[0]
-
-      const lookupField = isMany ? primaryKey : field.foreignKey
-      const fieldName = isMany ? field.throughTo || primaryKey : field.fieldName
       if (!lookupField || !row[lookupField]) {
         continue
       }
@@ -660,9 +672,12 @@ export class ExternalRequest<T extends Operation> {
         },
       })
       // this is the response from knex if no rows found
-      const rows: Row[] = response?.[0].read ? [] : (response as Row[])
-      const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
-      related[storeTo] = { rows, isMany, tableId }
+      const rows: Row[] =
+        !Array.isArray(response) || response?.[0].read ? [] : response
+      const storeTo = isManyToMany(field)
+        ? field.throughFrom || linkPrimaryKey
+        : fieldName
+      related[storeTo] = { rows, isMany: isManyToMany(field), tableId }
     }
     return related
   }
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 280d50de9b..9d54bbff8e 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -16,19 +16,17 @@ class CharSequence {
     this.counters = [0]
   }
 
-  get character() {
-    return this.counters.map(i => CharSequence.alphabet[i]).join("")
-  }
-
-  next() {
+  getCharacter(): string {
+    const char = this.counters.map(i => CharSequence.alphabet[i]).join("")
     for (let i = this.counters.length - 1; i >= 0; i--) {
       if (this.counters[i] < CharSequence.alphabet.length - 1) {
         this.counters[i]++
-        return
+        return char
       }
       this.counters[i] = 0
     }
     this.counters.unshift(0)
+    return char
   }
 }
 
@@ -49,8 +47,7 @@ export default class AliasTables {
     if (this.aliases[tableName]) {
       return this.aliases[tableName]
     }
-    const char = this.charSeq.character
-    this.charSeq.next()
+    const char = this.charSeq.getCharacter()
     this.aliases[tableName] = char
     this.tableAliases[char] = tableName
     return char
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 6e8bd0ea72..61dbc11840 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -164,14 +164,14 @@ describe("Captures of real examples", () => {
     it("should handle over 'z' max character alias", () => {
       const tableNames = []
       for (let i = 0; i < 100; i++) {
-        tableNames.push(generator.word())
+        tableNames.push(generator.guid())
       }
       const aliasing = new AliasTables(tableNames)
       let alias: string = ""
       for (let table of tableNames) {
         alias = aliasing.getAlias(table)
       }
-      expect(alias).toEqual("cu")
+      expect(alias).toEqual("cv")
     })
   })
 })

From 9a9b74579529cd061735c371b73ec862ea1856cc Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Tue, 27 Feb 2024 15:29:35 +0000
Subject: [PATCH 37/47] App Preview Skeleton (#12898)

* wip

* wip

* wip

* wip

* wip

* wip

* builder design skeleton

* pre doc loading skeleton for portal preview

* feature complete?

* wip

* linting

* linting

* linting

* fix ts error?

* ts types fix

* linting

* remove unnecessary build stuff

* include skeleton in server app directly

* linting

* linting

* fix builder preview aspect ratio for skeleton

* remove dev tools skeleton

* linting
---
 .../[screenId]/_components/AppPreview.svelte  |  43 ++-
 .../builder/portal/apps/[appId]/index.svelte  |  51 +++-
 .../client/src/components/ClientApp.svelte    | 258 ++++++++++--------
 .../client/src/components/FreeFooter.svelte   |   1 +
 .../src/components/ClientAppSkeleton.svelte   | 244 +++++++++++++++++
 .../frontend-core/src/components/index.js     |   1 +
 .../frontend-core/src/themes/midnight.css     |   3 +
 packages/frontend-core/src/themes/nord.css    |   3 +
 packages/server/package.json                  |   1 +
 .../src/api/controllers/static/index.ts       |  79 +++++-
 .../static/templates/BudibaseApp.svelte       |   7 +
 .../api/controllers/static/templates/app.hbs  |   8 +-
 packages/server/src/api/routes/static.ts      |   2 +-
 scripts/build.js                              |  14 +-
 14 files changed, 563 insertions(+), 152 deletions(-)
 create mode 100644 packages/frontend-core/src/components/ClientAppSkeleton.svelte

diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
index fa126bbc99..4bd62c0049 100644
--- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte
@@ -12,17 +12,11 @@
     hoverStore,
   } from "stores/builder"
   import ConfirmDialog from "components/common/ConfirmDialog.svelte"
-  import {
-    ProgressCircle,
-    Layout,
-    Heading,
-    Body,
-    Icon,
-    notifications,
-  } from "@budibase/bbui"
+  import { Layout, Heading, Body, Icon, notifications } from "@budibase/bbui"
   import ErrorSVG from "@budibase/frontend-core/assets/error.svg?raw"
   import { findComponent, findComponentPath } from "helpers/components"
   import { isActive, goto } from "@roxi/routify"
+  import { ClientAppSkeleton } from "@budibase/frontend-core"
 
   let iframe
   let layout
@@ -240,8 +234,16 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <div class="component-container">
   {#if loading}
-    <div class="center">
-      <ProgressCircle />
+    <div
+      class={`loading ${$builderStore.theme}`}
+      class:tablet={$previewStore.previewDevice === "tablet"}
+      class:mobile={$previewStore.previewDevice === "mobile"}
+    >
+      <ClientAppSkeleton
+        sideNav={$builderStore.navigation?.navigation === "Left"}
+        hideFooter
+        hideDevTools
+      />
     </div>
   {:else if error}
     <div class="center error">
@@ -258,8 +260,6 @@
     bind:this={iframe}
     src="/app/preview"
     class:hidden={loading || error}
-    class:tablet={$previewStore.previewDevice === "tablet"}
-    class:mobile={$previewStore.previewDevice === "mobile"}
   />
   <div
     class="add-component"
@@ -279,6 +279,25 @@
 />
 
 <style>
+  .loading {
+    position: absolute;
+    container-type: inline-size;
+    width: 100%;
+    height: 100%;
+    border: 2px solid transparent;
+    box-sizing: border-box;
+  }
+
+  .loading.tablet {
+    width: calc(1024px + 6px);
+    max-height: calc(768px + 6px);
+  }
+
+  .loading.mobile {
+    width: calc(390px + 6px);
+    max-height: calc(844px + 6px);
+  }
+
   .component-container {
     grid-row-start: middle;
     grid-column-start: middle;
diff --git a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
index e87c1104dd..5af416e654 100644
--- a/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
+++ b/packages/builder/src/pages/builder/portal/apps/[appId]/index.svelte
@@ -1,16 +1,22 @@
 <script>
+  import { onMount, onDestroy } from "svelte"
   import { params, goto } from "@roxi/routify"
-  import { apps, auth, sideBarCollapsed } from "stores/portal"
+  import { licensing, apps, auth, sideBarCollapsed } from "stores/portal"
   import { Link, Body, ActionButton } from "@budibase/bbui"
   import { sdk } from "@budibase/shared-core"
   import { API } from "api"
   import ErrorSVG from "./ErrorSVG.svelte"
+  import { ClientAppSkeleton } from "@budibase/frontend-core"
 
   $: app = $apps.find(app => app.appId === $params.appId)
   $: iframeUrl = getIframeURL(app)
   $: isBuilder = sdk.users.isBuilder($auth.user, app?.devId)
 
+  let loading = true
+
   const getIframeURL = app => {
+    loading = true
+
     if (app.status === "published") {
       return `/app${app.url}`
     }
@@ -28,6 +34,20 @@
   }
 
   $: fetchScreens(app?.devId)
+
+  const receiveMessage = async message => {
+    if (message.data.type === "docLoaded") {
+      loading = false
+    }
+  }
+
+  onMount(() => {
+    window.addEventListener("message", receiveMessage)
+  })
+
+  onDestroy(() => {
+    window.removeEventListener("message", receiveMessage)
+  })
 </script>
 
 <div class="container">
@@ -78,7 +98,17 @@
       </Body>
     </div>
   {:else}
-    <iframe src={iframeUrl} title={app.name} />
+    <div class:hide={!loading} class="loading">
+      <div class={`loadingThemeWrapper ${app.theme}`}>
+        <ClientAppSkeleton
+          noAnimation
+          hideDevTools={app?.status === "published"}
+          sideNav={app?.navigation.navigation === "Left"}
+          hideFooter={$licensing.brandingEnabled}
+        />
+      </div>
+    </div>
+    <iframe class:hide={loading} src={iframeUrl} title={app.name} />
   {/if}
 </div>
 
@@ -100,6 +130,23 @@
     flex: 0 0 50px;
   }
 
+  .loading {
+    height: 100%;
+    border: 1px solid var(--spectrum-global-color-gray-300);
+    border-radius: var(--spacing-s);
+    overflow: hidden;
+  }
+  .loadingThemeWrapper {
+    height: 100%;
+    container-type: inline-size;
+  }
+
+  .hide {
+    visibility: hidden;
+    height: 0;
+    border: none;
+  }
+
   iframe {
     flex: 1 1 auto;
     border-radius: var(--spacing-s);
diff --git a/packages/client/src/components/ClientApp.svelte b/packages/client/src/components/ClientApp.svelte
index 4efa8af4e6..a70710ec38 100644
--- a/packages/client/src/components/ClientApp.svelte
+++ b/packages/client/src/components/ClientApp.svelte
@@ -80,11 +80,18 @@
     }
   }
 
+  let fontsLoaded = false
+
   // Load app config
   onMount(async () => {
+    document.fonts.ready.then(() => {
+      fontsLoaded = true
+    })
+
     await initialise()
     await authStore.actions.fetchUser()
     dataLoaded = true
+
     if (get(builderStore).inBuilder) {
       builderStore.actions.notifyLoaded()
     } else {
@@ -93,6 +100,12 @@
       })
     }
   })
+
+  $: {
+    if (dataLoaded && fontsLoaded) {
+      document.getElementById("clientAppSkeletonLoader")?.remove()
+    }
+  }
 </script>
 
 <svelte:head>
@@ -103,140 +116,140 @@
   {/if}
 </svelte:head>
 
-{#if dataLoaded}
-  <div
-    id="spectrum-root"
-    lang="en"
-    dir="ltr"
-    class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
-    class:builder={$builderStore.inBuilder}
-  >
-    <DeviceBindingsProvider>
-      <UserBindingsProvider>
-        <StateBindingsProvider>
-          <RowSelectionProvider>
-            <QueryParamsProvider>
-              <!-- Settings bar can be rendered outside of device preview -->
-              <!-- Key block needs to be outside the if statement or it breaks -->
-              {#key $builderStore.selectedComponentId}
-                {#if $builderStore.inBuilder}
-                  <SettingsBar />
-                {/if}
-              {/key}
+<div
+  id="spectrum-root"
+  lang="en"
+  dir="ltr"
+  class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
+  class:builder={$builderStore.inBuilder}
+  class:show={fontsLoaded && dataLoaded}
+>
+  <DeviceBindingsProvider>
+    <UserBindingsProvider>
+      <StateBindingsProvider>
+        <RowSelectionProvider>
+          <QueryParamsProvider>
+            <!-- Settings bar can be rendered outside of device preview -->
+            <!-- Key block needs to be outside the if statement or it breaks -->
+            {#key $builderStore.selectedComponentId}
+              {#if $builderStore.inBuilder}
+                <SettingsBar />
+              {/if}
+            {/key}
 
-              <!-- Clip boundary for selection indicators -->
-              <div
-                id="clip-root"
-                class:preview={$builderStore.inBuilder}
-                class:tablet-preview={$builderStore.previewDevice === "tablet"}
-                class:mobile-preview={$builderStore.previewDevice === "mobile"}
-              >
-                <!-- Actual app -->
-                <div id="app-root">
-                  {#if showDevTools}
-                    <DevToolsHeader />
+            <!-- Clip boundary for selection indicators -->
+            <div
+              id="clip-root"
+              class:preview={$builderStore.inBuilder}
+              class:tablet-preview={$builderStore.previewDevice === "tablet"}
+              class:mobile-preview={$builderStore.previewDevice === "mobile"}
+            >
+              <!-- Actual app -->
+              <div id="app-root">
+                {#if showDevTools}
+                  <DevToolsHeader />
+                {/if}
+
+                <div id="app-body">
+                  {#if permissionError}
+                    <div class="error">
+                      <Layout justifyItems="center" gap="S">
+                        <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                        {@html ErrorSVG}
+                        <Heading size="L">
+                          You don't have permission to use this app
+                        </Heading>
+                        <Body size="S">
+                          Ask your administrator to grant you access
+                        </Body>
+                      </Layout>
+                    </div>
+                  {:else if !$screenStore.activeLayout}
+                    <div class="error">
+                      <Layout justifyItems="center" gap="S">
+                        <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                        {@html ErrorSVG}
+                        <Heading size="L">
+                          Something went wrong rendering your app
+                        </Heading>
+                        <Body size="S">
+                          Get in touch with support if this issue persists
+                        </Body>
+                      </Layout>
+                    </div>
+                  {:else if embedNoScreens}
+                    <div class="error">
+                      <Layout justifyItems="center" gap="S">
+                        <!-- eslint-disable-next-line svelte/no-at-html-tags -->
+                        {@html ErrorSVG}
+                        <Heading size="L">
+                          This Budibase app is not publicly accessible
+                        </Heading>
+                      </Layout>
+                    </div>
+                  {:else}
+                    <CustomThemeWrapper>
+                      {#key $screenStore.activeLayout._id}
+                        <Component
+                          isLayout
+                          instance={$screenStore.activeLayout.props}
+                        />
+                      {/key}
+
+                      <!--
+                        Flatpickr needs to be inside the theme wrapper.
+                        It also needs its own container because otherwise it hijacks
+                        key events on the whole page. It is painful to work with.
+                      -->
+                      <div id="flatpickr-root" />
+
+                      <!-- Modal container to ensure they sit on top -->
+                      <div class="modal-container" />
+
+                      <!-- Layers on top of app -->
+                      <NotificationDisplay />
+                      <ConfirmationDisplay />
+                      <PeekScreenDisplay />
+                    </CustomThemeWrapper>
                   {/if}
 
-                  <div id="app-body">
-                    {#if permissionError}
-                      <div class="error">
-                        <Layout justifyItems="center" gap="S">
-                          <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                          {@html ErrorSVG}
-                          <Heading size="L">
-                            You don't have permission to use this app
-                          </Heading>
-                          <Body size="S">
-                            Ask your administrator to grant you access
-                          </Body>
-                        </Layout>
-                      </div>
-                    {:else if !$screenStore.activeLayout}
-                      <div class="error">
-                        <Layout justifyItems="center" gap="S">
-                          <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                          {@html ErrorSVG}
-                          <Heading size="L">
-                            Something went wrong rendering your app
-                          </Heading>
-                          <Body size="S">
-                            Get in touch with support if this issue persists
-                          </Body>
-                        </Layout>
-                      </div>
-                    {:else if embedNoScreens}
-                      <div class="error">
-                        <Layout justifyItems="center" gap="S">
-                          <!-- eslint-disable-next-line svelte/no-at-html-tags -->
-                          {@html ErrorSVG}
-                          <Heading size="L">
-                            This Budibase app is not publicly accessible
-                          </Heading>
-                        </Layout>
-                      </div>
-                    {:else}
-                      <CustomThemeWrapper>
-                        {#key $screenStore.activeLayout._id}
-                          <Component
-                            isLayout
-                            instance={$screenStore.activeLayout.props}
-                          />
-                        {/key}
-
-                        <!--
-                          Flatpickr needs to be inside the theme wrapper.
-                          It also needs its own container because otherwise it hijacks
-                          key events on the whole page. It is painful to work with.
-                        -->
-                        <div id="flatpickr-root" />
-
-                        <!-- Modal container to ensure they sit on top -->
-                        <div class="modal-container" />
-
-                        <!-- Layers on top of app -->
-                        <NotificationDisplay />
-                        <ConfirmationDisplay />
-                        <PeekScreenDisplay />
-                      </CustomThemeWrapper>
-                    {/if}
-
-                    {#if showDevTools}
-                      <DevTools />
-                    {/if}
-                  </div>
-
-                  {#if !$builderStore.inBuilder && licensing.logoEnabled()}
-                    <FreeFooter />
+                  {#if showDevTools}
+                    <DevTools />
                   {/if}
                 </div>
 
-                <!-- Preview and dev tools utilities  -->
-                {#if $appStore.isDevApp}
-                  <SelectionIndicator />
-                {/if}
-                {#if $builderStore.inBuilder || $devToolsStore.allowSelection}
-                  <HoverIndicator />
-                {/if}
-                {#if $builderStore.inBuilder}
-                  <DNDHandler />
-                  <GridDNDHandler />
+                {#if !$builderStore.inBuilder && licensing.logoEnabled()}
+                  <FreeFooter />
                 {/if}
               </div>
-            </QueryParamsProvider>
-          </RowSelectionProvider>
-        </StateBindingsProvider>
-      </UserBindingsProvider>
-    </DeviceBindingsProvider>
-  </div>
-  <KeyboardManager />
-{/if}
+
+              <!-- Preview and dev tools utilities  -->
+              {#if $appStore.isDevApp}
+                <SelectionIndicator />
+              {/if}
+              {#if $builderStore.inBuilder || $devToolsStore.allowSelection}
+                <HoverIndicator />
+              {/if}
+              {#if $builderStore.inBuilder}
+                <DNDHandler />
+                <GridDNDHandler />
+              {/if}
+            </div>
+          </QueryParamsProvider>
+        </RowSelectionProvider>
+      </StateBindingsProvider>
+    </UserBindingsProvider>
+  </DeviceBindingsProvider>
+</div>
+<KeyboardManager />
 
 <style>
   #spectrum-root {
+    height: 0;
+    visibility: hidden;
     padding: 0;
     margin: 0;
     overflow: hidden;
-    height: 100%;
     width: 100%;
     display: flex;
     flex-direction: row;
@@ -257,6 +270,11 @@
     background-color: transparent;
   }
 
+  #spectrum-root.show {
+    height: 100%;
+    visibility: visible;
+  }
+
   #app-root {
     overflow: hidden;
     height: 100%;
diff --git a/packages/client/src/components/FreeFooter.svelte b/packages/client/src/components/FreeFooter.svelte
index 3ba2f1bfdb..868f5377c7 100644
--- a/packages/client/src/components/FreeFooter.svelte
+++ b/packages/client/src/components/FreeFooter.svelte
@@ -13,6 +13,7 @@
 
 <style>
   .free-footer {
+    min-height: 51px;
     flex: 0 0 auto;
     padding: 16px 20px;
     border-top: 1px solid var(--spectrum-global-color-gray-300);
diff --git a/packages/frontend-core/src/components/ClientAppSkeleton.svelte b/packages/frontend-core/src/components/ClientAppSkeleton.svelte
new file mode 100644
index 0000000000..a1c90d2db7
--- /dev/null
+++ b/packages/frontend-core/src/components/ClientAppSkeleton.svelte
@@ -0,0 +1,244 @@
+<script>
+  export let sideNav = false
+  export let hideDevTools = false
+  export let hideFooter = false
+  export let noAnimation = false
+</script>
+
+<div class:sideNav id="clientAppSkeletonLoader" class="skeleton">
+  <div class="animation" class:noAnimation />
+
+  {#if !hideDevTools}
+    <div class="devTools" />
+  {/if}
+  <div class="main">
+    <div class="nav" />
+    <div class="body">
+      <div class="bodyVerticalPadding" />
+      <div class="bodyHorizontal">
+        <div class="bodyHorizontalPadding" />
+        <svg
+          class="svg"
+          xmlns="http://www.w3.org/2000/svg"
+          xmlns:xlink="http://www.w3.org/1999/xlink"
+          width="240"
+          height="256"
+        >
+          <mask id="mask">
+            <rect x="0" y="0" width="240" height="256" fill="white" />
+            <rect x="0" y="0" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="56" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="112" width="240" height="32" rx="6" fill="black" />
+            <rect x="0" y="168" width="240" height="32" rx="6" fill="black" />
+            <rect x="71" y="224" width="98" height="32" rx="6" fill="black" />
+          </mask>
+
+          <rect
+            x="0"
+            y="0"
+            width="240"
+            height="256"
+            fill="black"
+            mask="url(#mask)"
+          />
+        </svg>
+        <div class="bodyHorizontalPadding" />
+      </div>
+      <div class="bodyVerticalPadding" />
+    </div>
+  </div>
+  {#if !hideFooter}
+    <div class="footer" />
+  {/if}
+</div>
+
+<style>
+  .skeleton {
+    position: relative;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+    border-radius: 4px;
+    overflow: hidden;
+    background-color: var(--spectrum-global-color-gray-200);
+  }
+
+  .animation {
+    position: absolute;
+    height: 100%;
+    width: 100%;
+    background: linear-gradient(
+      to right,
+      transparent 0%,
+      var(--spectrum-global-color-gray-300) 20%,
+      transparent 40%,
+      transparent 100%
+    );
+    animation-duration: 1.3s;
+    animation-fill-mode: forwards;
+    animation-iteration-count: infinite;
+    animation-name: shimmer;
+    animation-timing-function: linear;
+  }
+
+  .noAnimation {
+    animation-name: none;
+    background: transparent;
+  }
+
+  .devTools {
+    display: flex;
+    box-sizing: border-box;
+    background-color: black;
+    height: 60px;
+    padding: 1px 24px 1px 20px;
+    display: flex;
+    align-items: center;
+    z-index: 1;
+    flex-shrink: 0;
+
+    color: white;
+    mix-blend-mode: multiply;
+    background: rgb(0 0 0);
+    font-size: 30px;
+    font-family: Source Sans Pro;
+    -webkit-font-smoothing: antialiased;
+  }
+
+  .main {
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .main {
+      flex-direction: column;
+      width: initial;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .main {
+      flex-direction: column;
+      width: initial;
+    }
+  }
+
+  .sideNav .main {
+    flex-direction: row;
+    width: 100%;
+  }
+
+  .nav {
+    flex-shrink: 0;
+    width: 100%;
+    height: 141px;
+    background-color: transparent;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .nav {
+      height: 61px;
+      width: initial;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .nav {
+      height: 61px;
+      width: initial;
+    }
+  }
+
+  .sideNav .nav {
+    height: 100%;
+    width: 251px;
+  }
+
+  .body {
+    z-index: 2;
+    display: flex;
+    flex-direction: column;
+    height: 100%;
+    position: relative;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .body {
+      width: initial;
+      height: 100%;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .body {
+      width: initial;
+      height: 100%;
+    }
+  }
+
+  .sideNav .body {
+    width: 100%;
+    height: initial;
+  }
+
+  .body :global(svg > rect) {
+    fill: var(--spectrum-alias-background-color-primary);
+  }
+
+  .body :global(svg) {
+    flex-shrink: 0;
+  }
+
+  .bodyHorizontal {
+    display: flex;
+    flex-shrink: 0;
+  }
+
+  .bodyHorizontalPadding {
+    height: 100%;
+    flex-grow: 1;
+    background-color: var(--spectrum-alias-background-color-primary);
+  }
+
+  .bodyVerticalPadding {
+    width: 100%;
+    flex-grow: 1;
+    background-color: var(--spectrum-alias-background-color-primary);
+  }
+
+  .footer {
+    flex-shrink: 0;
+    box-sizing: border-box;
+    z-index: 1;
+    height: 52px;
+    width: 100%;
+  }
+
+  @media (max-width: 720px) {
+    #clientAppSkeletonLoader .footer {
+      border-top: none;
+    }
+  }
+
+  @container (max-width: 720px) {
+    #clientAppSkeletonLoader .footer {
+      border-top: none;
+    }
+  }
+
+  .sideNav .footer {
+    border-top: 3px solid var(--spectrum-alias-background-color-primary);
+  }
+
+  @keyframes shimmer {
+    0% {
+      left: -170%;
+    }
+
+    100% {
+      left: 170%;
+    }
+  }
+</style>
diff --git a/packages/frontend-core/src/components/index.js b/packages/frontend-core/src/components/index.js
index f724e1e4d9..f71420b12b 100644
--- a/packages/frontend-core/src/components/index.js
+++ b/packages/frontend-core/src/components/index.js
@@ -5,3 +5,4 @@ export { default as UserAvatar } from "./UserAvatar.svelte"
 export { default as UserAvatars } from "./UserAvatars.svelte"
 export { default as Updating } from "./Updating.svelte"
 export { Grid } from "./grid"
+export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte"
diff --git a/packages/frontend-core/src/themes/midnight.css b/packages/frontend-core/src/themes/midnight.css
index e311452262..cf6a4fbd13 100644
--- a/packages/frontend-core/src/themes/midnight.css
+++ b/packages/frontend-core/src/themes/midnight.css
@@ -17,5 +17,8 @@
     --modal-background: var(--spectrum-global-color-gray-50);
     --drop-shadow: rgba(0, 0, 0, 0.25) !important;
     --spectrum-global-color-blue-100: rgba(35, 40, 50) !important;
+
+    --spectrum-alias-background-color-secondary: var(--spectrum-global-color-gray-75);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
 }
 
diff --git a/packages/frontend-core/src/themes/nord.css b/packages/frontend-core/src/themes/nord.css
index d47dbe8aa8..bc142db0fd 100644
--- a/packages/frontend-core/src/themes/nord.css
+++ b/packages/frontend-core/src/themes/nord.css
@@ -50,4 +50,7 @@
   --modal-background: var(--spectrum-global-color-gray-50);
   --drop-shadow: rgba(0, 0, 0, 0.15) !important;
   --spectrum-global-color-blue-100: rgb(56, 65, 84) !important;
+
+  --spectrum-alias-background-color-secondary: var(--spectrum-global-color-gray-75);
+  --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
 }
diff --git a/packages/server/package.json b/packages/server/package.json
index 45980a4be6..4f1a9fb3cc 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -52,6 +52,7 @@
     "@budibase/pro": "0.0.0",
     "@budibase/shared-core": "0.0.0",
     "@budibase/string-templates": "0.0.0",
+    "@budibase/frontend-core": "0.0.0",
     "@budibase/types": "0.0.0",
     "@bull-board/api": "5.10.2",
     "@bull-board/koa": "5.10.2",
diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts
index 5a3803e6d5..367934445a 100644
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@@ -1,7 +1,5 @@
 import { InvalidFileExtensions } from "@budibase/shared-core"
-
 import AppComponent from "./templates/BudibaseApp.svelte"
-
 import { join } from "../../../utilities/centralPath"
 import * as uuid from "uuid"
 import { ObjectStoreBuckets } from "../../../constants"
@@ -24,7 +22,13 @@ import AWS from "aws-sdk"
 import fs from "fs"
 import sdk from "../../../sdk"
 import * as pro from "@budibase/pro"
-import { App, Ctx, ProcessAttachmentResponse } from "@budibase/types"
+import {
+  UserCtx,
+  App,
+  Ctx,
+  ProcessAttachmentResponse,
+  Feature,
+} from "@budibase/types"
 import {
   getAppMigrationVersion,
   getLatestMigrationId,
@@ -32,6 +36,61 @@ import {
 
 import send from "koa-send"
 
+const getThemeVariables = (theme: string) => {
+  if (theme === "spectrum--lightest") {
+    return `
+      --spectrum-global-color-gray-50: rgb(255, 255, 255);
+      --spectrum-global-color-gray-200: rgb(244, 244, 244);
+      --spectrum-global-color-gray-300: rgb(234, 234, 234);
+      --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
+    `
+  }
+  if (theme === "spectrum--light") {
+    return `
+    --spectrum-global-color-gray-50: rgb(255, 255, 255);
+    --spectrum-global-color-gray-200: rgb(234, 234, 234);
+    --spectrum-global-color-gray-300: rgb(225, 225, 225);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-50);
+
+    `
+  }
+  if (theme === "spectrum--dark") {
+    return `
+    --spectrum-global-color-gray-100: rgb(50, 50, 50);
+    --spectrum-global-color-gray-200: rgb(62, 62, 62);
+    --spectrum-global-color-gray-300: rgb(74, 74, 74);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--darkest") {
+    return `
+  --spectrum-global-color-gray-100: rgb(30, 30, 30);
+  --spectrum-global-color-gray-200: rgb(44, 44, 44);
+  --spectrum-global-color-gray-300: rgb(57, 57, 57);
+  --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--nord") {
+    return `
+    --spectrum-global-color-gray-100: #3b4252;
+
+  --spectrum-global-color-gray-200: #424a5c;
+  --spectrum-global-color-gray-300: #4c566a;
+  --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+  if (theme === "spectrum--midnight") {
+    return `
+    --hue: 220;
+    --sat: 10%;
+    --spectrum-global-color-gray-100: hsl(var(--hue), var(--sat), 17%);
+    --spectrum-global-color-gray-200: hsl(var(--hue), var(--sat), 20%);
+    --spectrum-global-color-gray-300: hsl(var(--hue), var(--sat), 24%);
+    --spectrum-alias-background-color-primary: var(--spectrum-global-color-gray-100);
+    `
+  }
+}
+
 export const toggleBetaUiFeature = async function (ctx: Ctx) {
   const cookieName = `beta:${ctx.params.feature}`
 
@@ -146,7 +205,7 @@ const requiresMigration = async (ctx: Ctx) => {
   return requiresMigrations
 }
 
-export const serveApp = async function (ctx: Ctx) {
+export const serveApp = async function (ctx: UserCtx) {
   const needMigrations = await requiresMigration(ctx)
 
   const bbHeaderEmbed =
@@ -167,9 +226,19 @@ export const serveApp = async function (ctx: Ctx) {
     const appInfo = await db.get<any>(DocumentType.APP_METADATA)
     let appId = context.getAppId()
 
+    const hideDevTools = !!ctx.params.appUrl
+    const sideNav = appInfo.navigation.navigation === "Left"
+    const hideFooter =
+      ctx?.user?.license?.features?.includes(Feature.BRANDING) || false
+    const themeVariables = getThemeVariables(appInfo?.theme)
+
     if (!env.isJest()) {
       const plugins = objectStore.enrichPluginURLs(appInfo.usedPlugins)
+
       const { head, html, css } = AppComponent.render({
+        hideDevTools,
+        sideNav,
+        hideFooter,
         metaImage:
           branding?.metaImageUrl ||
           "https://res.cloudinary.com/daog6scxm/image/upload/v1698759482/meta-images/plain-branded-meta-image-coral_ocxmgu.png",
@@ -194,7 +263,7 @@ export const serveApp = async function (ctx: Ctx) {
       ctx.body = await processString(appHbs, {
         head,
         body: html,
-        style: css.code,
+        css: `:root{${themeVariables}} ${css.code}`,
         appId,
         embedded: bbHeaderEmbed,
       })
diff --git a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
index 7819368fc0..63b293b4ca 100644
--- a/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
+++ b/packages/server/src/api/controllers/static/templates/BudibaseApp.svelte
@@ -1,4 +1,6 @@
 <script>
+  import ClientAppSkeleton from "@budibase/frontend-core/src/components/ClientAppSkeleton.svelte"
+
   export let title = ""
   export let favicon = ""
 
@@ -9,6 +11,10 @@
   export let clientLibPath
   export let usedPlugins
   export let appMigrating
+
+  export let hideDevTools
+  export let sideNav
+  export let hideFooter
 </script>
 
 <svelte:head>
@@ -96,6 +102,7 @@
 </svelte:head>
 
 <body id="app">
+  <ClientAppSkeleton {hideDevTools} {sideNav} {hideFooter} />
   <div id="error">
     {#if clientLibPath}
       <h1>There was an error loading your app</h1>
diff --git a/packages/server/src/api/controllers/static/templates/app.hbs b/packages/server/src/api/controllers/static/templates/app.hbs
index 8c445158a0..b01b723c3e 100644
--- a/packages/server/src/api/controllers/static/templates/app.hbs
+++ b/packages/server/src/api/controllers/static/templates/app.hbs
@@ -1,8 +1,12 @@
 <html>
-
+  <script>
+    document.fonts.ready.then(() => {
+      window.parent.postMessage({ type: "docLoaded" });
+    })
+  </script>
   <head>
     {{{head}}}
-    <style>{{{style}}}</style>
+    <style>{{{css}}}</style>
   </head>
 
   <script>
diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts
index 4e9e10fd56..a5c421b2e6 100644
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@@ -51,8 +51,8 @@ router
     controller.deleteObjects
   )
   .get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
-  .get("/:appId/:path*", controller.serveApp)
   .get("/app/:appUrl/:path*", controller.serveApp)
+  .get("/:appId/:path*", controller.serveApp)
   .post(
     "/api/attachments/:datasourceId/url",
     authorized(PermissionType.TABLE, PermissionLevel.READ),
diff --git a/scripts/build.js b/scripts/build.js
index c6b72a80bd..1443614024 100755
--- a/scripts/build.js
+++ b/scripts/build.js
@@ -17,12 +17,6 @@ const { nodeExternalsPlugin } = require("esbuild-node-externals")
 const svelteCompilePlugin = {
   name: 'svelteCompile',
   setup(build) {
-    // This resolve handler is necessary to bundle the Svelte runtime into the the final output,
-    // otherwise the bundled script will attempt to resolve it at runtime
-    build.onResolve({ filter: /svelte\/internal/ }, async () => {
-      return { path: `${process.cwd()}/../../node_modules/svelte/src/runtime/internal/ssr.js` }
-    })
-
     // Compiles `.svelte` files into JS classes so that they can be directly imported into our
     // Typescript packages
     build.onLoad({ filter: /\.svelte$/ }, async (args) => {
@@ -37,7 +31,7 @@ const svelteCompilePlugin = {
           contents: js.code,
           // The loader this is passed to, basically how the above provided content is "treated",
           // the contents provided above will be transpiled and bundled like any other JS file.
-          loader: 'js', 
+          loader: 'js',
           // Where to resolve any imports present in the loaded file
           resolveDir: dir
         }
@@ -80,11 +74,11 @@ async function runBuild(entry, outfile) {
     plugins: [
       svelteCompilePlugin,
       TsconfigPathsPlugin({ tsconfig: tsconfigPathPluginContent }),
-      nodeExternalsPlugin(),
+      nodeExternalsPlugin({
+        allowList: ["@budibase/frontend-core", "svelte"]
+      }),
     ],
     preserveSymlinks: true,
-    loader: {
-    },
     metafile: true,
     external: [
       "deasync",

From 35c1f5bbce2f8d99bebcf56e7f815df0d4e227a8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Feb 2024 17:15:20 +0000
Subject: [PATCH 38/47] Adding handling for columns with dots in them.

---
 packages/server/scripts/integrations/postgres/init.sql  | 4 ++++
 packages/server/src/api/controllers/row/alias.ts        | 9 ++++-----
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 9 +++++++++
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql
index b7ce1b7d5b..2d51a2b5cb 100644
--- a/packages/server/scripts/integrations/postgres/init.sql
+++ b/packages/server/scripts/integrations/postgres/init.sql
@@ -2,6 +2,10 @@ SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE SCHEMA "test-1";
 CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support');
+CREATE TABLE "Bad.Table" (
+    BadID SERIAL PRIMARY KEY,
+    "Bad.Column" text
+);
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index 9d54bbff8e..cb4d4f1453 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -55,11 +55,10 @@ export default class AliasTables {
 
   aliasField(field: string) {
     const tableNames = this.tableNames
-    if (field.includes(".")) {
-      const [tableName, column] = field.split(".")
-      if (tableNames.includes(tableName)) {
-        return `${this.getAlias(tableName)}.${column}`
-      }
+    const foundTable = tableNames.find(name => field.includes(name))
+    if (foundTable) {
+      const aliasedTable = this.getAlias(foundTable)
+      return field.replace(foundTable, aliasedTable)
     }
     return field
   }
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 61dbc11840..acf0f9898f 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -174,4 +174,13 @@ describe("Captures of real examples", () => {
       expect(alias).toEqual("cv")
     })
   })
+
+  describe("check some edge cases", () => {
+    it("should handle table names/columns with dots in them", () => {
+      const tableNames = ["hello.world", "foo.bar.baz"]
+      const aliasing = new AliasTables(tableNames)
+      const aliased = aliasing.aliasField("hello.world.field")
+      expect(aliased).toEqual("a.field")
+    })
+  })
 })

From 4068df602564f889da33df8f409bbad3ef9a9878 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 27 Feb 2024 17:46:02 +0000
Subject: [PATCH 39/47] Adding error handling for table names or columns which
 contain dots (invalid).

---
 .../scripts/integrations/postgres/init.sql    |  4 ---
 .../server/src/sdk/app/datasources/plus.ts    | 36 +++++++++++++++----
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql
index 2d51a2b5cb..b7ce1b7d5b 100644
--- a/packages/server/scripts/integrations/postgres/init.sql
+++ b/packages/server/scripts/integrations/postgres/init.sql
@@ -2,10 +2,6 @@ SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE SCHEMA "test-1";
 CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support');
-CREATE TABLE "Bad.Table" (
-    BadID SERIAL PRIMARY KEY,
-    "Bad.Column" text
-);
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
diff --git a/packages/server/src/sdk/app/datasources/plus.ts b/packages/server/src/sdk/app/datasources/plus.ts
index 04cd508863..31ec51c728 100644
--- a/packages/server/src/sdk/app/datasources/plus.ts
+++ b/packages/server/src/sdk/app/datasources/plus.ts
@@ -3,12 +3,33 @@ import {
   DatasourcePlus,
   IntegrationBase,
   Schema,
+  Table,
 } from "@budibase/types"
 import * as datasources from "./datasources"
 import tableSdk from "../tables"
 import { getIntegration } from "../../../integrations"
 import { context } from "@budibase/backend-core"
 
+function checkForSchemaErrors(schema: Record<string, Table>) {
+  const errors: Record<string, string> = {}
+  for (let [tableName, table] of Object.entries(schema)) {
+    if (tableName.includes(".")) {
+      errors[tableName] = "Table names containing dots are not supported."
+    } else {
+      const columnNames = Object.keys(table.schema)
+      const invalidColumnName = columnNames.find(columnName =>
+        columnName.includes(".")
+      )
+      if (invalidColumnName) {
+        errors[
+          tableName
+        ] = `Column '${invalidColumnName}' is not supported as it contains a dot.`
+      }
+    }
+  }
+  return errors
+}
+
 export async function buildFilteredSchema(
   datasource: Datasource,
   filter?: string[]
@@ -30,16 +51,19 @@ export async function buildFilteredSchema(
       filteredSchema.errors[key] = schema.errors[key]
     }
   }
-  return filteredSchema
+
+  return {
+    ...filteredSchema,
+    errors: {
+      ...filteredSchema.errors,
+      ...checkForSchemaErrors(filteredSchema.tables),
+    },
+  }
 }
 
 async function buildSchemaHelper(datasource: Datasource): Promise<Schema> {
   const connector = (await getConnector(datasource)) as DatasourcePlus
-  const externalSchema = await connector.buildSchema(
-    datasource._id!,
-    datasource.entities!
-  )
-  return externalSchema
+  return await connector.buildSchema(datasource._id!, datasource.entities!)
 }
 
 export async function getConnector(

From 0c51a60e9ab16765ca1c5def7d5b1584331124c7 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 10:16:21 +0000
Subject: [PATCH 40/47] Updating test cases as per PR comments.

---
 .../src/integrations/tests/sqlAlias.spec.ts       | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index acf0f9898f..0672b421f2 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -176,11 +176,18 @@ describe("Captures of real examples", () => {
   })
 
   describe("check some edge cases", () => {
+    const dotTableNames = ["hello.world", "foo.bar.baz"]
+
     it("should handle table names/columns with dots in them", () => {
-      const tableNames = ["hello.world", "foo.bar.baz"]
-      const aliasing = new AliasTables(tableNames)
-      const aliased = aliasing.aliasField("hello.world.field")
-      expect(aliased).toEqual("a.field")
+      const aliasing = new AliasTables(dotTableNames)
+      const aliased = aliasing.aliasField(`"hello.world"."field"`)
+      expect(aliased).toEqual(`"a"."field"`)
+    })
+
+    it("should confirm table with dots in them works with grave accents", () => {
+      const aliasing = new AliasTables(dotTableNames)
+      const aliased = aliasing.aliasField("`hello.world`.`field`")
+      expect(aliased).toEqual("`a`.`field`")
     })
   })
 })

From a2df1deb8542a602572c93647245f06c891461ae Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 11:08:35 +0000
Subject: [PATCH 41/47] Fixing issue found by test cases, column names
 containing table names.

---
 packages/server/src/api/controllers/row/alias.ts        | 5 +++--
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 7 +++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index cb4d4f1453..ec96bb4603 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -55,10 +55,11 @@ export default class AliasTables {
 
   aliasField(field: string) {
     const tableNames = this.tableNames
-    const foundTable = tableNames.find(name => field.includes(name))
+    const possibleTableName = field.substring(0, field.lastIndexOf("."))
+    const foundTable = tableNames.find(name => possibleTableName.includes(name))
     if (foundTable) {
       const aliasedTable = this.getAlias(foundTable)
-      return field.replace(foundTable, aliasedTable)
+      field = field.replace(foundTable, aliasedTable)
     }
     return field
   }
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 0672b421f2..6812d358b7 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -189,5 +189,12 @@ describe("Captures of real examples", () => {
       const aliased = aliasing.aliasField("`hello.world`.`field`")
       expect(aliased).toEqual("`a`.`field`")
     })
+
+    it("should handle if a table name is used in a column", () => {
+      const tableNames = ["hello", "world"]
+      const aliasing = new AliasTables(tableNames)
+      const aliased = aliasing.aliasField(`"hello"."world_relation"`)
+      expect(aliased).toEqual(`"a"."world_relation"`)
+    })
   })
 })

From cd81a83b949589f3413e589032247f8f99755335 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 13:37:11 +0000
Subject: [PATCH 42/47] Reverting changes to handle full stops.

---
 .../server/src/api/controllers/row/alias.ts   | 15 ++++++++-----
 .../src/integrations/tests/sqlAlias.spec.ts   | 21 +++++++------------
 2 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index ec96bb4603..d2d41e085f 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -55,11 +55,16 @@ export default class AliasTables {
 
   aliasField(field: string) {
     const tableNames = this.tableNames
-    const possibleTableName = field.substring(0, field.lastIndexOf("."))
-    const foundTable = tableNames.find(name => possibleTableName.includes(name))
-    if (foundTable) {
-      const aliasedTable = this.getAlias(foundTable)
-      field = field.replace(foundTable, aliasedTable)
+    if (field.includes(".")) {
+      const [tableName, column] = field.split(".")
+      const foundTableName = tableNames.find(name => tableName.includes(name))
+      if (foundTableName) {
+        const aliasedTableName = tableName.replace(
+          foundTableName,
+          this.getAlias(foundTableName)
+        )
+        return `${aliasedTableName}.${column}`
+      }
     }
     return field
   }
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 6812d358b7..3fb90a3b4d 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -176,25 +176,18 @@ describe("Captures of real examples", () => {
   })
 
   describe("check some edge cases", () => {
-    const dotTableNames = ["hello.world", "foo.bar.baz"]
+    const tableNames = ["hello", "world"]
 
-    it("should handle table names/columns with dots in them", () => {
-      const aliasing = new AliasTables(dotTableNames)
-      const aliased = aliasing.aliasField(`"hello.world"."field"`)
+    it("should quoted table names", () => {
+      const aliasing = new AliasTables(tableNames)
+      const aliased = aliasing.aliasField(`"hello"."field"`)
       expect(aliased).toEqual(`"a"."field"`)
     })
 
-    it("should confirm table with dots in them works with grave accents", () => {
-      const aliasing = new AliasTables(dotTableNames)
-      const aliased = aliasing.aliasField("`hello.world`.`field`")
-      expect(aliased).toEqual("`a`.`field`")
-    })
-
-    it("should handle if a table name is used in a column", () => {
-      const tableNames = ["hello", "world"]
+    it("should handle quoted table names with graves", () => {
       const aliasing = new AliasTables(tableNames)
-      const aliased = aliasing.aliasField(`"hello"."world_relation"`)
-      expect(aliased).toEqual(`"a"."world_relation"`)
+      const aliased = aliasing.aliasField("`hello`.`world`")
+      expect(aliased).toEqual("`a`.`world`")
     })
   })
 })

From aa6fa7661824186674096ca8cfc7dc019c547bea Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 13:37:47 +0000
Subject: [PATCH 43/47] Missing word in test case.

---
 packages/server/src/integrations/tests/sqlAlias.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 3fb90a3b4d..497b981f68 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -178,7 +178,7 @@ describe("Captures of real examples", () => {
   describe("check some edge cases", () => {
     const tableNames = ["hello", "world"]
 
-    it("should quoted table names", () => {
+    it("should handle quoted table names", () => {
       const aliasing = new AliasTables(tableNames)
       const aliased = aliasing.aliasField(`"hello"."field"`)
       expect(aliased).toEqual(`"a"."field"`)

From 039df5d47615442b4f7ed84b21a0a89cbad8f265 Mon Sep 17 00:00:00 2001
From: melohagan <101575380+melohagan@users.noreply.github.com>
Date: Wed, 28 Feb 2024 13:44:52 +0000
Subject: [PATCH 44/47] FIX: cannot delete a renamed column with a mysql
 datasource (#13147)

* Support MySQL column rename in Budibase

* Delete NOT ignored types

* unit tests

* update account portal
---
 packages/account-portal                       |   2 +-
 packages/server/src/integrations/base/sql.ts  |  16 +-
 .../server/src/integrations/base/sqlTable.ts  |  26 +++-
 packages/server/src/integrations/mysql.ts     |   1 -
 packages/server/src/integrations/oracle.ts    |   2 +-
 packages/server/src/integrations/postgres.ts  |   2 +-
 .../server/src/integrations/tests/sql.spec.ts | 138 ++++++++++++++++--
 7 files changed, 155 insertions(+), 32 deletions(-)

diff --git a/packages/account-portal b/packages/account-portal
index de6d44c372..19f7a5829f 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit de6d44c372a7f48ca0ce8c6c0c19311d4bc21646
+Subproject commit 19f7a5829f4d23cbc694136e45d94482a59a475a
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index 29c8416b34..e52e9dd2ae 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -400,7 +400,7 @@ class InternalBuilder {
     return query.limit(BASE_LIMIT)
   }
 
-  create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
     if (endpoint.schema) {
@@ -422,7 +422,7 @@ class InternalBuilder {
     }
   }
 
-  bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
+  bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder {
     const { endpoint, body } = json
     let query: KnexQuery = knex(endpoint.entityId)
     if (endpoint.schema) {
@@ -491,7 +491,7 @@ class InternalBuilder {
     return this.addFilters(query, filters, { relationship: true })
   }
 
-  update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     if (endpoint.schema) {
@@ -507,7 +507,7 @@ class InternalBuilder {
     }
   }
 
-  delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     if (endpoint.schema) {
@@ -537,17 +537,17 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
    * which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes.
    * @return the query ready to be passed to the driver.
    */
-  _query(json: QueryJson, opts: QueryOptions = {}) {
+  _query(json: QueryJson, opts: QueryOptions = {}): Knex.SqlNative | Knex.Sql {
     const sqlClient = this.getSqlClient()
     const client = knex({ client: sqlClient })
-    let query
+    let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {
       case Operation.CREATE:
         query = builder.create(client, json, opts)
         break
       case Operation.READ:
-        query = builder.read(client, json, this.limit)
+        query = builder.read(client, json, this.limit) as Knex.QueryBuilder
         break
       case Operation.UPDATE:
         query = builder.update(client, json, opts)
@@ -565,8 +565,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
       default:
         throw `Operation type is not supported by SQL query builder`
     }
-
-    // @ts-ignore
     return query.toSQL().toNative()
   }
 
diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts
index f553690b23..f3560791e6 100644
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@@ -9,7 +9,7 @@ import {
   Table,
   FieldType,
 } from "@budibase/types"
-import { breakExternalTableId } from "../utils"
+import { breakExternalTableId, SqlClient } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
 import { utils } from "@budibase/shared-core"
@@ -135,7 +135,8 @@ function generateSchema(
   // need to check if any columns have been deleted
   if (oldTable) {
     const deletedColumns = Object.entries(oldTable.schema).filter(
-      ([key, column]) => isIgnoredType(column.type) && table.schema[key] == null
+      ([key, column]) =>
+        !isIgnoredType(column.type) && table.schema[key] == null
     )
     deletedColumns.forEach(([key, column]) => {
       if (renamed?.old === key || isIgnoredType(column.type)) {
@@ -197,13 +198,14 @@ class SqlTableQueryBuilder {
     return json.endpoint.operation
   }
 
-  _tableQuery(json: QueryJson): any {
+  _tableQuery(json: QueryJson): Knex.Sql | Knex.SqlNative {
     let client = knex({ client: this.sqlClient }).schema
-    if (json?.endpoint?.schema) {
-      client = client.withSchema(json.endpoint.schema)
+    let schemaName = json?.endpoint?.schema
+    if (schemaName) {
+      client = client.withSchema(schemaName)
     }
 
-    let query
+    let query: Knex.SchemaBuilder
     if (!json.table || !json.meta || !json.meta.tables) {
       throw "Cannot execute without table being specified"
     }
@@ -215,6 +217,18 @@ class SqlTableQueryBuilder {
         if (!json.meta || !json.meta.table) {
           throw "Must specify old table for update"
         }
+        // renameColumn does not work for MySQL, so return a raw query
+        if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
+          const updatedColumn = json.meta.renamed.updated
+          const tableName = schemaName
+            ? `\`${schemaName}\`.\`${json.table.name}\``
+            : `\`${json.table.name}\``
+          const externalType = json.table.schema[updatedColumn].externalType!
+          return {
+            sql: `alter table ${tableName} change column \`${json.meta.renamed.old}\` \`${updatedColumn}\` ${externalType};`,
+            bindings: [],
+          }
+        }
         query = buildUpdateTable(
           client,
           json.table,
diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index 8ec73307f4..5a206e1a7f 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -12,7 +12,6 @@ import {
   SourceName,
   Schema,
   TableSourceType,
-  FieldType,
 } from "@budibase/types"
 import {
   getSqlQuery,
diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts
index cf76622581..b3aefc578c 100644
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@@ -421,7 +421,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
 
   async query(json: QueryJson) {
     const operation = this._operation(json)
-    const input = this._query(json, { disableReturning: true })
+    const input = this._query(json, { disableReturning: true }) as SqlQuery
     if (Array.isArray(input)) {
       const responses = []
       for (let query of input) {
diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts
index 9949dee6bb..bea31d4031 100644
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@@ -419,7 +419,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
   async query(json: QueryJson) {
     const operation = this._operation(json).toLowerCase()
-    const input = this._query(json)
+    const input = this._query(json) as SqlQuery
     if (Array.isArray(input)) {
       const responses = []
       for (let query of input) {
diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts
index 5cc4849d03..fd705fc27c 100644
--- a/packages/server/src/integrations/tests/sql.spec.ts
+++ b/packages/server/src/integrations/tests/sql.spec.ts
@@ -1,3 +1,11 @@
+import {
+  Operation,
+  QueryJson,
+  TableSourceType,
+  Table,
+  FieldType,
+} from "@budibase/types"
+
 const Sql = require("../base/sql").default
 const { SqlClient } = require("../utils")
 
@@ -17,7 +25,7 @@ function generateReadJson({
   filters,
   sort,
   paginate,
-}: any = {}) {
+}: any = {}): QueryJson {
   return {
     endpoint: endpoint(table || TABLE_NAME, "READ"),
     resource: {
@@ -28,6 +36,10 @@ function generateReadJson({
     paginate: paginate || {},
     meta: {
       table: {
+        type: "table",
+        sourceType: TableSourceType.EXTERNAL,
+        sourceId: "SOURCE_ID",
+        schema: {},
         name: table || TABLE_NAME,
         primary: ["id"],
       },
@@ -35,34 +47,40 @@ function generateReadJson({
   }
 }
 
-function generateCreateJson(table = TABLE_NAME, body = {}) {
+function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
   return {
     endpoint: endpoint(table, "CREATE"),
     body,
   }
 }
 
-function generateUpdateJson(table = TABLE_NAME, body = {}, filters = {}) {
+function generateUpdateJson({
+  table = TABLE_NAME,
+  body = {},
+  filters = {},
+  meta = {},
+}): QueryJson {
   return {
     endpoint: endpoint(table, "UPDATE"),
     filters,
     body,
+    meta,
   }
 }
 
-function generateDeleteJson(table = TABLE_NAME, filters = {}) {
+function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
   return {
     endpoint: endpoint(table, "DELETE"),
     filters,
   }
 }
 
-function generateRelationshipJson(config: { schema?: string } = {}) {
+function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
   return {
     endpoint: {
       datasourceId: "Postgres",
       entityId: "brands",
-      operation: "READ",
+      operation: Operation.READ,
       schema: config.schema,
     },
     resource: {
@@ -76,7 +94,6 @@ function generateRelationshipJson(config: { schema?: string } = {}) {
     },
     filters: {},
     sort: {},
-    paginate: {},
     relationships: [
       {
         from: "brand_id",
@@ -240,17 +257,17 @@ describe("SQL query builder", () => {
 
   it("should test an update statement", () => {
     const query = sql._query(
-      generateUpdateJson(
-        TABLE_NAME,
-        {
+      generateUpdateJson({
+        table: TABLE_NAME,
+        body: {
           name: "John",
         },
-        {
+        filters: {
           equal: {
             id: 1001,
           },
-        }
-      )
+        },
+      })
     )
     expect(query).toEqual({
       bindings: ["John", 1001],
@@ -682,4 +699,99 @@ describe("SQL query builder", () => {
       sql: `insert into \"test\" (\"name\") values ($1) returning *`,
     })
   })
+
+  it("should be able to rename column for MySQL", () => {
+    const table: Table = {
+      type: "table",
+      sourceType: TableSourceType.EXTERNAL,
+      name: TABLE_NAME,
+      schema: {
+        first_name: {
+          type: FieldType.STRING,
+          name: "first_name",
+          externalType: "varchar(45)",
+        },
+      },
+      sourceId: "SOURCE_ID",
+    }
+    const oldTable: Table = {
+      ...table,
+      schema: {
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          externalType: "varchar(45)",
+        },
+      },
+    }
+    const query = new Sql(SqlClient.MY_SQL, limit)._query({
+      table,
+      endpoint: {
+        datasourceId: "MySQL",
+        operation: Operation.UPDATE_TABLE,
+        entityId: TABLE_NAME,
+      },
+      meta: {
+        table: oldTable,
+        tables: [oldTable],
+        renamed: {
+          old: "name",
+          updated: "first_name",
+        },
+      },
+    })
+    expect(query).toEqual({
+      bindings: [],
+      sql: `alter table \`${TABLE_NAME}\` change column \`name\` \`first_name\` varchar(45);`,
+    })
+  })
+
+  it("should be able to delete a column", () => {
+    const table: Table = {
+      type: "table",
+      sourceType: TableSourceType.EXTERNAL,
+      name: TABLE_NAME,
+      schema: {
+        first_name: {
+          type: FieldType.STRING,
+          name: "first_name",
+          externalType: "varchar(45)",
+        },
+      },
+      sourceId: "SOURCE_ID",
+    }
+    const oldTable: Table = {
+      ...table,
+      schema: {
+        first_name: {
+          type: FieldType.STRING,
+          name: "first_name",
+          externalType: "varchar(45)",
+        },
+        last_name: {
+          type: FieldType.STRING,
+          name: "last_name",
+          externalType: "varchar(45)",
+        },
+      },
+    }
+    const query = sql._query({
+      table,
+      endpoint: {
+        datasourceId: "Postgres",
+        operation: Operation.UPDATE_TABLE,
+        entityId: TABLE_NAME,
+      },
+      meta: {
+        table: oldTable,
+        tables: [oldTable],
+      },
+    })
+    expect(query).toEqual([
+      {
+        bindings: [],
+        sql: `alter table "${TABLE_NAME}" drop column "last_name"`,
+      },
+    ])
+  })
 })

From 58e42714402c4f90c10820f9ff5b80c23e5b7e64 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 14:38:43 +0000
Subject: [PATCH 45/47] Fixing issue with quotes/graves.

---
 packages/server/src/api/controllers/row/alias.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index d2d41e085f..c27bed6a4b 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -57,13 +57,15 @@ export default class AliasTables {
     const tableNames = this.tableNames
     if (field.includes(".")) {
       const [tableName, column] = field.split(".")
-      const foundTableName = tableNames.find(name => tableName.includes(name))
+      const foundTableName = tableNames.find(
+        name => tableName.includes(name) && tableName.indexOf(name) <= 1
+      )
       if (foundTableName) {
         const aliasedTableName = tableName.replace(
           foundTableName,
           this.getAlias(foundTableName)
         )
-        return `${aliasedTableName}.${column}`
+        field = `${aliasedTableName}.${column}`
       }
     }
     return field

From 56b1855f6ad3174ea804d0698ded893a015c4f61 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 15:18:07 +0000
Subject: [PATCH 46/47] Adding test cases for table names in table names.

---
 .../server/src/api/controllers/row/alias.ts   |   10 +-
 .../server/src/api/routes/tests/row.spec.ts   | 3939 +++++++++--------
 packages/server/src/integrations/base/sql.ts  |   25 +-
 .../src/integrations/tests/sqlAlias.spec.ts   |   11 +
 4 files changed, 2008 insertions(+), 1977 deletions(-)

diff --git a/packages/server/src/api/controllers/row/alias.ts b/packages/server/src/api/controllers/row/alias.ts
index c27bed6a4b..9658a0d638 100644
--- a/packages/server/src/api/controllers/row/alias.ts
+++ b/packages/server/src/api/controllers/row/alias.ts
@@ -57,9 +57,13 @@ export default class AliasTables {
     const tableNames = this.tableNames
     if (field.includes(".")) {
       const [tableName, column] = field.split(".")
-      const foundTableName = tableNames.find(
-        name => tableName.includes(name) && tableName.indexOf(name) <= 1
-      )
+      const foundTableName = tableNames.find(name => {
+        const idx = tableName.indexOf(name)
+        if (idx === -1 || idx > 1) {
+          return
+        }
+        return Math.abs(tableName.length - name.length) <= 2
+      })
       if (foundTableName) {
         const aliasedTableName = tableName.replace(
           foundTableName,
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 239da36351..05c6b92bb0 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -39,907 +39,41 @@ tk.freeze(timestamp)
 
 const { basicRow } = setup.structures
 
-describe.each([
-  ["internal", undefined],
-  ["postgres", databaseTestProviders.postgres],
-])("/rows (%s)", (__, dsProvider) => {
-  const isInternal = !dsProvider
+describe.each([["postgres", databaseTestProviders.postgres]])(
+  "/rows (%s)",
+  (__, dsProvider) => {
+    const isInternal = !dsProvider
 
-  const request = setup.getRequest()
-  const config = setup.getConfig()
-  let table: Table
-  let tableId: string
+    const request = setup.getRequest()
+    const config = setup.getConfig()
+    let table: Table
+    let tableId: string
 
-  afterAll(async () => {
-    if (dsProvider) {
-      await dsProvider.stop()
-    }
-    setup.afterAll()
-  })
-
-  beforeAll(async () => {
-    await config.init()
-
-    if (dsProvider) {
-      await config.createDatasource({
-        datasource: await dsProvider.datasource(),
-      })
-    }
-  })
-
-  const generateTableConfig: () => SaveTableRequest = () => {
-    return {
-      name: uuid.v4(),
-      type: "table",
-      primary: ["id"],
-      primaryDisplay: "name",
-      sourceType: TableSourceType.INTERNAL,
-      sourceId: INTERNAL_TABLE_SOURCE_ID,
-      schema: {
-        id: {
-          type: FieldType.AUTO,
-          name: "id",
-          autocolumn: true,
-          constraints: {
-            presence: true,
-          },
-        },
-        name: {
-          type: FieldType.STRING,
-          name: "name",
-          constraints: {
-            type: "string",
-          },
-        },
-        description: {
-          type: FieldType.STRING,
-          name: "description",
-          constraints: {
-            type: "string",
-          },
-        },
-      },
-    }
-  }
-
-  beforeEach(async () => {
-    mocks.licenses.useCloudFree()
-  })
-
-  const loadRow = (id: string, tbl_Id: string, status = 200) =>
-    config.api.row.get(tbl_Id, id, { expectStatus: status })
-
-  const getRowUsage = async () => {
-    const { total } = await config.doInContext(null, () =>
-      quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
-    )
-    return total
-  }
-
-  const assertRowUsage = async (expected: number) => {
-    const usage = await getRowUsage()
-    expect(usage).toBe(expected)
-  }
-
-  const defaultRowFields = isInternal
-    ? {
-        type: "row",
-        createdAt: timestamp,
-        updatedAt: timestamp,
+    afterAll(async () => {
+      if (dsProvider) {
+        await dsProvider.stop()
       }
-    : undefined
-
-  async function createTable(
-    cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
-    opts?: { skipReassigning: boolean }
-  ) {
-    let table
-    if (dsProvider) {
-      table = await config.createExternalTable(cfg, opts)
-    } else {
-      table = await config.createTable(cfg, opts)
-    }
-    return table
-  }
-
-  beforeAll(async () => {
-    const tableConfig = generateTableConfig()
-    let table = await createTable(tableConfig)
-    tableId = table._id!
-  })
-
-  describe("save, load, update", () => {
-    it("returns a success message when the row is created", async () => {
-      const rowUsage = await getRowUsage()
-
-      const res = await request
-        .post(`/api/${tableId}/rows`)
-        .send(basicRow(tableId))
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect((res as any).res.statusMessage).toEqual(
-        `${config.table!.name} saved successfully`
-      )
-      expect(res.body.name).toEqual("Test Contact")
-      expect(res.body._rev).toBeDefined()
-      await assertRowUsage(rowUsage + 1)
+      setup.afterAll()
     })
 
-    it("Increment row autoId per create row request", async () => {
-      const rowUsage = await getRowUsage()
+    beforeAll(async () => {
+      await config.init()
 
-      const tableConfig = generateTableConfig()
-      const newTable = await createTable(
-        {
-          ...tableConfig,
-          name: "TestTableAuto",
-          schema: {
-            ...tableConfig.schema,
-            "Row ID": {
-              name: "Row ID",
-              type: FieldType.NUMBER,
-              subtype: AutoFieldSubType.AUTO_ID,
-              icon: "ri-magic-line",
-              autocolumn: true,
-              constraints: {
-                type: "number",
-                presence: true,
-                numericality: {
-                  greaterThanOrEqualTo: "",
-                  lessThanOrEqualTo: "",
-                },
-              },
-            },
-          },
-        },
-        { skipReassigning: true }
-      )
-
-      const ids = [1, 2, 3]
-
-      // Performing several create row requests should increment the autoID fields accordingly
-      const createRow = async (id: number) => {
-        const res = await config.api.row.save(newTable._id!, {
-          name: "row_" + id,
+      if (dsProvider) {
+        await config.createDatasource({
+          datasource: await dsProvider.datasource(),
         })
-        expect(res.name).toEqual("row_" + id)
-        expect(res._rev).toBeDefined()
-        expect(res["Row ID"]).toEqual(id)
       }
-
-      for (let i = 0; i < ids.length; i++) {
-        await createRow(ids[i])
-      }
-
-      await assertRowUsage(rowUsage + ids.length)
     })
 
-    it("updates a row successfully", async () => {
-      const existing = await config.createRow()
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.save(tableId, {
-        _id: existing._id,
-        _rev: existing._rev,
-        tableId,
-        name: "Updated Name",
-      })
-
-      expect(res.name).toEqual("Updated Name")
-      await assertRowUsage(rowUsage)
-    })
-
-    it("should load a row", async () => {
-      const existing = await config.createRow()
-
-      const res = await config.api.row.get(tableId, existing._id!)
-
-      expect(res.body).toEqual({
-        ...existing,
-        ...defaultRowFields,
-      })
-    })
-
-    it("should list all rows for given tableId", async () => {
-      const table = await createTable(generateTableConfig(), {
-        skipReassigning: true,
-      })
-      const tableId = table._id!
-      const newRow = {
-        tableId,
-        name: "Second Contact",
-        description: "new",
-      }
-      const firstRow = await config.createRow({ tableId })
-      await config.createRow(newRow)
-
-      const res = await config.api.row.fetch(tableId)
-
-      expect(res.length).toBe(2)
-      expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
-    })
-
-    it("load should return 404 when row does not exist", async () => {
-      await config.createRow()
-
-      await config.api.row.get(tableId, "1234567", {
-        expectStatus: 404,
-      })
-    })
-
-    isInternal &&
-      it("row values are coerced", async () => {
-        const str: FieldSchema = {
-          type: FieldType.STRING,
-          name: "str",
-          constraints: { type: "string", presence: false },
-        }
-        const attachment: FieldSchema = {
-          type: FieldType.ATTACHMENT,
-          name: "attachment",
-          constraints: { type: "array", presence: false },
-        }
-        const bool: FieldSchema = {
-          type: FieldType.BOOLEAN,
-          name: "boolean",
-          constraints: { type: "boolean", presence: false },
-        }
-        const number: FieldSchema = {
-          type: FieldType.NUMBER,
-          name: "str",
-          constraints: { type: "number", presence: false },
-        }
-        const datetime: FieldSchema = {
-          type: FieldType.DATETIME,
-          name: "datetime",
-          constraints: {
-            type: "string",
-            presence: false,
-            datetime: { earliest: "", latest: "" },
-          },
-        }
-        const arrayField: FieldSchema = {
-          type: FieldType.ARRAY,
-          constraints: {
-            type: "array",
-            presence: false,
-            inclusion: ["One", "Two", "Three"],
-          },
-          name: "Sample Tags",
-          sortable: false,
-        }
-        const optsField: FieldSchema = {
-          name: "Sample Opts",
-          type: FieldType.OPTIONS,
-          constraints: {
-            type: "string",
-            presence: false,
-            inclusion: ["Alpha", "Beta", "Gamma"],
-          },
-        }
-        const table = await createTable({
-          name: "TestTable2",
-          type: "table",
-          schema: {
-            name: str,
-            stringUndefined: str,
-            stringNull: str,
-            stringString: str,
-            numberEmptyString: number,
-            numberNull: number,
-            numberUndefined: number,
-            numberString: number,
-            numberNumber: number,
-            datetimeEmptyString: datetime,
-            datetimeNull: datetime,
-            datetimeUndefined: datetime,
-            datetimeString: datetime,
-            datetimeDate: datetime,
-            boolNull: bool,
-            boolEmpty: bool,
-            boolUndefined: bool,
-            boolString: bool,
-            boolBool: bool,
-            attachmentNull: attachment,
-            attachmentUndefined: attachment,
-            attachmentEmpty: attachment,
-            attachmentEmptyArrayStr: attachment,
-            arrayFieldEmptyArrayStr: arrayField,
-            arrayFieldArrayStrKnown: arrayField,
-            arrayFieldNull: arrayField,
-            arrayFieldUndefined: arrayField,
-            optsFieldEmptyStr: optsField,
-            optsFieldUndefined: optsField,
-            optsFieldNull: optsField,
-            optsFieldStrKnown: optsField,
-          },
-        })
-
-        const row = {
-          name: "Test Row",
-          stringUndefined: undefined,
-          stringNull: null,
-          stringString: "i am a string",
-          numberEmptyString: "",
-          numberNull: null,
-          numberUndefined: undefined,
-          numberString: "123",
-          numberNumber: 123,
-          datetimeEmptyString: "",
-          datetimeNull: null,
-          datetimeUndefined: undefined,
-          datetimeString: "1984-04-20T00:00:00.000Z",
-          datetimeDate: new Date("1984-04-20"),
-          boolNull: null,
-          boolEmpty: "",
-          boolUndefined: undefined,
-          boolString: "true",
-          boolBool: true,
-          tableId: table._id,
-          attachmentNull: null,
-          attachmentUndefined: undefined,
-          attachmentEmpty: "",
-          attachmentEmptyArrayStr: "[]",
-          arrayFieldEmptyArrayStr: "[]",
-          arrayFieldUndefined: undefined,
-          arrayFieldNull: null,
-          arrayFieldArrayStrKnown: "['One']",
-          optsFieldEmptyStr: "",
-          optsFieldUndefined: undefined,
-          optsFieldNull: null,
-          optsFieldStrKnown: "Alpha",
-        }
-
-        const createdRow = await config.createRow(row)
-        const id = createdRow._id!
-
-        const saved = (await loadRow(id, table._id!)).body
-
-        expect(saved.stringUndefined).toBe(undefined)
-        expect(saved.stringNull).toBe(null)
-        expect(saved.stringString).toBe("i am a string")
-        expect(saved.numberEmptyString).toBe(null)
-        expect(saved.numberNull).toBe(null)
-        expect(saved.numberUndefined).toBe(undefined)
-        expect(saved.numberString).toBe(123)
-        expect(saved.numberNumber).toBe(123)
-        expect(saved.datetimeEmptyString).toBe(null)
-        expect(saved.datetimeNull).toBe(null)
-        expect(saved.datetimeUndefined).toBe(undefined)
-        expect(saved.datetimeString).toBe(
-          new Date(row.datetimeString).toISOString()
-        )
-        expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
-        expect(saved.boolNull).toBe(null)
-        expect(saved.boolEmpty).toBe(null)
-        expect(saved.boolUndefined).toBe(undefined)
-        expect(saved.boolString).toBe(true)
-        expect(saved.boolBool).toBe(true)
-        expect(saved.attachmentNull).toEqual([])
-        expect(saved.attachmentUndefined).toBe(undefined)
-        expect(saved.attachmentEmpty).toEqual([])
-        expect(saved.attachmentEmptyArrayStr).toEqual([])
-        expect(saved.arrayFieldEmptyArrayStr).toEqual([])
-        expect(saved.arrayFieldNull).toEqual([])
-        expect(saved.arrayFieldUndefined).toEqual(undefined)
-        expect(saved.optsFieldEmptyStr).toEqual(null)
-        expect(saved.optsFieldUndefined).toEqual(undefined)
-        expect(saved.optsFieldNull).toEqual(null)
-        expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-        expect(saved.optsFieldStrKnown).toEqual("Alpha")
-      })
-  })
-
-  describe("view save", () => {
-    it("views have extra data trimmed", async () => {
-      const table = await createTable({
-        type: "table",
-        name: "orders",
-        primary: ["OrderID"],
-        schema: {
-          Country: {
-            type: FieldType.STRING,
-            name: "Country",
-          },
-          OrderID: {
-            type: FieldType.NUMBER,
-            name: "OrderID",
-          },
-          Story: {
-            type: FieldType.STRING,
-            name: "Story",
-          },
-        },
-      })
-
-      const createViewResponse = await config.createView({
-        name: uuid.v4(),
-        schema: {
-          Country: {
-            visible: true,
-          },
-          OrderID: {
-            visible: true,
-          },
-        },
-      })
-
-      const createRowResponse = await config.api.row.save(
-        createViewResponse.id,
-        {
-          OrderID: "1111",
-          Country: "Aussy",
-          Story: "aaaaa",
-        }
-      )
-
-      const row = await config.api.row.get(table._id!, createRowResponse._id!)
-      expect(row.body.Story).toBeUndefined()
-      expect(row.body).toEqual({
-        ...defaultRowFields,
-        OrderID: 1111,
-        Country: "Aussy",
-        _id: createRowResponse._id,
-        _rev: createRowResponse._rev,
-        tableId: table._id,
-      })
-    })
-  })
-
-  describe("patch", () => {
-    let otherTable: Table
-
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-      const otherTableConfig = generateTableConfig()
-      // need a short name of table here - for relationship tests
-      otherTableConfig.name = "a"
-      otherTableConfig.schema.relationship = {
-        name: "relationship",
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        type: FieldType.LINK,
-        tableId: table._id!,
-        fieldName: "relationship",
-      }
-      otherTable = await createTable(otherTableConfig)
-      // need to set the config back to the original table
-      config.table = table
-    })
-
-    it("should update only the fields that are supplied", async () => {
-      const existing = await config.createRow()
-
-      const rowUsage = await getRowUsage()
-
-      const row = await config.api.row.patch(table._id!, {
-        _id: existing._id!,
-        _rev: existing._rev!,
-        tableId: table._id!,
-        name: "Updated Name",
-      })
-
-      expect(row.name).toEqual("Updated Name")
-      expect(row.description).toEqual(existing.description)
-
-      const savedRow = await loadRow(row._id!, table._id!)
-
-      expect(savedRow.body.description).toEqual(existing.description)
-      expect(savedRow.body.name).toEqual("Updated Name")
-      await assertRowUsage(rowUsage)
-    })
-
-    it("should throw an error when given improper types", async () => {
-      const existing = await config.createRow()
-      const rowUsage = await getRowUsage()
-
-      await config.api.row.patch(
-        table._id!,
-        {
-          _id: existing._id!,
-          _rev: existing._rev!,
-          tableId: table._id!,
-          name: 1,
-        },
-        { expectStatus: 400 }
-      )
-
-      await assertRowUsage(rowUsage)
-    })
-
-    it("should not overwrite links if those links are not set", async () => {
-      let linkField: FieldSchema = {
-        type: FieldType.LINK,
-        name: "",
-        fieldName: "",
-        constraints: {
-          type: "array",
-          presence: false,
-        },
-        relationshipType: RelationshipType.ONE_TO_MANY,
-        tableId: InternalTable.USER_METADATA,
-      }
-
-      let table = await config.api.table.save({
-        name: "TestTable",
-        type: "table",
-        sourceType: TableSourceType.INTERNAL,
-        sourceId: INTERNAL_TABLE_SOURCE_ID,
-        schema: {
-          user1: { ...linkField, name: "user1", fieldName: "user1" },
-          user2: { ...linkField, name: "user2", fieldName: "user2" },
-        },
-      })
-
-      let user1 = await config.createUser()
-      let user2 = await config.createUser()
-
-      let row = await config.api.row.save(table._id!, {
-        user1: [{ _id: user1._id }],
-        user2: [{ _id: user2._id }],
-      })
-
-      let getResp = await config.api.row.get(table._id!, row._id!)
-      expect(getResp.body.user1[0]._id).toEqual(user1._id)
-      expect(getResp.body.user2[0]._id).toEqual(user2._id)
-
-      let patchResp = await config.api.row.patch(table._id!, {
-        _id: row._id!,
-        _rev: row._rev!,
-        tableId: table._id!,
-        user1: [{ _id: user2._id }],
-      })
-      expect(patchResp.user1[0]._id).toEqual(user2._id)
-      expect(patchResp.user2[0]._id).toEqual(user2._id)
-
-      getResp = await config.api.row.get(table._id!, row._id!)
-      expect(getResp.body.user1[0]._id).toEqual(user2._id)
-      expect(getResp.body.user2[0]._id).toEqual(user2._id)
-    })
-
-    it("should be able to update relationships when both columns are same name", async () => {
-      let row = await config.api.row.save(table._id!, {
-        name: "test",
-        description: "test",
-      })
-      let row2 = await config.api.row.save(otherTable._id!, {
-        name: "test",
-        description: "test",
-        relationship: [row._id],
-      })
-      row = (await config.api.row.get(table._id!, row._id!)).body
-      expect(row.relationship.length).toBe(1)
-      const resp = await config.api.row.patch(table._id!, {
-        _id: row._id!,
-        _rev: row._rev!,
-        tableId: row.tableId!,
-        name: "test2",
-        relationship: [row2._id],
-      })
-      expect(resp.relationship.length).toBe(1)
-    })
-  })
-
-  describe("destroy", () => {
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-    })
-
-    it("should be able to delete a row", async () => {
-      const createdRow = await config.createRow()
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.delete(table._id!, [createdRow])
-      expect(res.body[0]._id).toEqual(createdRow._id)
-      await assertRowUsage(rowUsage - 1)
-    })
-  })
-
-  describe("validate", () => {
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-    })
-
-    it("should return no errors on valid row", async () => {
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.validate(table._id!, { name: "ivan" })
-
-      expect(res.valid).toBe(true)
-      expect(Object.keys(res.errors)).toEqual([])
-      await assertRowUsage(rowUsage)
-    })
-
-    it("should errors on invalid row", async () => {
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.validate(table._id!, { name: 1 })
-
-      if (isInternal) {
-        expect(res.valid).toBe(false)
-        expect(Object.keys(res.errors)).toEqual(["name"])
-      } else {
-        // Validation for external is not implemented, so it will always return valid
-        expect(res.valid).toBe(true)
-        expect(Object.keys(res.errors)).toEqual([])
-      }
-      await assertRowUsage(rowUsage)
-    })
-  })
-
-  describe("bulkDelete", () => {
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-    })
-
-    it("should be able to delete a bulk set of rows", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.delete(table._id!, [row1, row2])
-
-      expect(res.body.length).toEqual(2)
-      await loadRow(row1._id!, table._id!, 404)
-      await assertRowUsage(rowUsage - 2)
-    })
-
-    it("should be able to delete a variety of row set types", async () => {
-      const [row1, row2, row3] = await Promise.all([
-        config.createRow(),
-        config.createRow(),
-        config.createRow(),
-      ])
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.delete(table._id!, [
-        row1,
-        row2._id,
-        { _id: row3._id },
-      ])
-
-      expect(res.body.length).toEqual(3)
-      await loadRow(row1._id!, table._id!, 404)
-      await assertRowUsage(rowUsage - 3)
-    })
-
-    it("should accept a valid row object and delete the row", async () => {
-      const row1 = await config.createRow()
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.delete(table._id!, row1)
-
-      expect(res.body.id).toEqual(row1._id)
-      await loadRow(row1._id!, table._id!, 404)
-      await assertRowUsage(rowUsage - 1)
-    })
-
-    it("Should ignore malformed/invalid delete requests", async () => {
-      const rowUsage = await getRowUsage()
-
-      const res = await config.api.row.delete(
-        table._id!,
-        { not: "valid" },
-        { expectStatus: 400 }
-      )
-      expect(res.body.message).toEqual("Invalid delete rows request")
-
-      const res2 = await config.api.row.delete(
-        table._id!,
-        { rows: 123 },
-        { expectStatus: 400 }
-      )
-      expect(res2.body.message).toEqual("Invalid delete rows request")
-
-      const res3 = await config.api.row.delete(table._id!, "invalid", {
-        expectStatus: 400,
-      })
-      expect(res3.body.message).toEqual("Invalid delete rows request")
-
-      await assertRowUsage(rowUsage)
-    })
-  })
-
-  // Legacy views are not available for external
-  isInternal &&
-    describe("fetchView", () => {
-      beforeEach(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
-      it("should be able to fetch tables contents via 'view'", async () => {
-        const row = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.legacyView.get(table._id!)
-        expect(res.body.length).toEqual(1)
-        expect(res.body[0]._id).toEqual(row._id)
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should throw an error if view doesn't exist", async () => {
-        const rowUsage = await getRowUsage()
-
-        await config.api.legacyView.get("derp", { expectStatus: 404 })
-
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should be able to run on a view", async () => {
-        const view = await config.createLegacyView({
-          tableId: table._id!,
-          name: "ViewTest",
-          filters: [],
-          schema: {},
-        })
-        const row = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.legacyView.get(view.name)
-        expect(res.body.length).toEqual(1)
-        expect(res.body[0]._id).toEqual(row._id)
-
-        await assertRowUsage(rowUsage)
-      })
-    })
-
-  describe("fetchEnrichedRows", () => {
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-    })
-
-    it("should allow enriching some linked rows", async () => {
-      const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
-        config.getTenantId(),
-        async () => {
-          const linkedTable = await config.createLinkedTable(
-            RelationshipType.ONE_TO_MANY,
-            ["link"],
-            {
-              // Making sure that the combined table name + column name is within postgres limits
-              name: uuid.v4().replace(/-/g, "").substring(0, 16),
-              type: "table",
-              primary: ["id"],
-              primaryDisplay: "id",
-              schema: {
-                id: {
-                  type: FieldType.AUTO,
-                  name: "id",
-                  autocolumn: true,
-                  constraints: {
-                    presence: true,
-                  },
-                },
-              },
-            }
-          )
-          const firstRow = await config.createRow({
-            name: "Test Contact",
-            description: "original description",
-            tableId: table._id,
-          })
-          const secondRow = await config.createRow({
-            name: "Test 2",
-            description: "og desc",
-            link: [{ _id: firstRow._id }],
-            tableId: linkedTable._id,
-          })
-          return { linkedTable, firstRow, secondRow }
-        }
-      )
-      const rowUsage = await getRowUsage()
-
-      // test basic enrichment
-      const resBasic = await config.api.row.get(
-        linkedTable._id!,
-        secondRow._id!
-      )
-      expect(resBasic.body.link.length).toBe(1)
-      expect(resBasic.body.link[0]).toEqual({
-        _id: firstRow._id,
-        primaryDisplay: firstRow.name,
-      })
-
-      // test full enrichment
-      const resEnriched = await config.api.row.getEnriched(
-        linkedTable._id!,
-        secondRow._id!
-      )
-      expect(resEnriched.body.link.length).toBe(1)
-      expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
-      expect(resEnriched.body.link[0].name).toBe("Test Contact")
-      expect(resEnriched.body.link[0].description).toBe("original description")
-      await assertRowUsage(rowUsage)
-    })
-  })
-
-  isInternal &&
-    describe("attachments", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
-      it("should allow enriching attachment rows", async () => {
-        const table = await config.createAttachmentTable()
-        const attachmentId = `${structures.uuid()}.csv`
-        const row = await config.createRow({
-          name: "test",
-          description: "test",
-          attachment: [
-            {
-              key: `${config.getAppId()}/attachments/${attachmentId}`,
-            },
-          ],
-          tableId: table._id,
-        })
-        // the environment needs configured for this
-        await setup.switchToSelfHosted(async () => {
-          return context.doInAppContext(config.getAppId(), async () => {
-            const enriched = await outputProcessing(table, [row])
-            expect((enriched as Row[])[0].attachment[0].url).toBe(
-              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
-            )
-          })
-        })
-      })
-    })
-
-  describe("exportData", () => {
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      table = await createTable(tableConfig)
-    })
-
-    it("should allow exporting all columns", async () => {
-      const existing = await config.createRow()
-      const res = await config.api.row.exportRows(table._id!, {
-        rows: [existing._id!],
-      })
-      const results = JSON.parse(res.text)
-      expect(results.length).toEqual(1)
-      const row = results[0]
-
-      // Ensure all original columns were exported
-      expect(Object.keys(row).length).toBeGreaterThanOrEqual(
-        Object.keys(existing).length
-      )
-      Object.keys(existing).forEach(key => {
-        expect(row[key]).toEqual(existing[key])
-      })
-    })
-
-    it("should allow exporting only certain columns", async () => {
-      const existing = await config.createRow()
-      const res = await config.api.row.exportRows(table._id!, {
-        rows: [existing._id!],
-        columns: ["_id"],
-      })
-      const results = JSON.parse(res.text)
-      expect(results.length).toEqual(1)
-      const row = results[0]
-
-      // Ensure only the _id column was exported
-      expect(Object.keys(row).length).toEqual(1)
-      expect(row._id).toEqual(existing._id)
-    })
-  })
-
-  describe("view 2.0", () => {
-    async function userTable(): Promise<Table> {
+    const generateTableConfig: () => SaveTableRequest = () => {
       return {
-        name: `users_${uuid.v4()}`,
-        sourceId: INTERNAL_TABLE_SOURCE_ID,
-        sourceType: TableSourceType.INTERNAL,
+        name: uuid.v4(),
         type: "table",
         primary: ["id"],
+        primaryDisplay: "name",
+        sourceType: TableSourceType.INTERNAL,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
         schema: {
           id: {
             type: FieldType.AUTO,
@@ -952,167 +86,857 @@ describe.each([
           name: {
             type: FieldType.STRING,
             name: "name",
+            constraints: {
+              type: "string",
+            },
           },
-          surname: {
+          description: {
             type: FieldType.STRING,
-            name: "surname",
-          },
-          age: {
-            type: FieldType.NUMBER,
-            name: "age",
-          },
-          address: {
-            type: FieldType.STRING,
-            name: "address",
-          },
-          jobTitle: {
-            type: FieldType.STRING,
-            name: "jobTitle",
+            name: "description",
+            constraints: {
+              type: "string",
+            },
           },
         },
       }
     }
 
-    const randomRowData = () => ({
-      name: generator.first(),
-      surname: generator.last(),
-      age: generator.age(),
-      address: generator.address(),
-      jobTitle: generator.word(),
+    beforeEach(async () => {
+      mocks.licenses.useCloudFree()
     })
 
-    describe("create", () => {
-      it("should persist a new row with only the provided view fields", async () => {
-        const table = await createTable(await userTable())
-        const view = await config.createView({
-          schema: {
-            name: { visible: true },
-            surname: { visible: true },
-            address: { visible: true },
-          },
-        })
+    const loadRow = (id: string, tbl_Id: string, status = 200) =>
+      config.api.row.get(tbl_Id, id, { expectStatus: status })
 
-        const data = randomRowData()
-        const newRow = await config.api.row.save(view.id, {
-          tableId: table!._id,
-          _viewId: view.id,
-          ...data,
-        })
+    const getRowUsage = async () => {
+      const { total } = await config.doInContext(null, () =>
+        quotas.getCurrentUsageValues(
+          QuotaUsageType.STATIC,
+          StaticQuotaName.ROWS
+        )
+      )
+      return total
+    }
 
-        const row = await config.api.row.get(table._id!, newRow._id!)
-        expect(row.body).toEqual({
-          name: data.name,
-          surname: data.surname,
-          address: data.address,
-          tableId: table!._id,
-          _id: newRow._id,
-          _rev: newRow._rev,
-          id: newRow.id,
-          ...defaultRowFields,
-        })
-        expect(row.body._viewId).toBeUndefined()
-        expect(row.body.age).toBeUndefined()
-        expect(row.body.jobTitle).toBeUndefined()
-      })
+    const assertRowUsage = async (expected: number) => {
+      const usage = await getRowUsage()
+      expect(usage).toBe(expected)
+    }
+
+    const defaultRowFields = isInternal
+      ? {
+          type: "row",
+          createdAt: timestamp,
+          updatedAt: timestamp,
+        }
+      : undefined
+
+    async function createTable(
+      cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
+      opts?: { skipReassigning: boolean }
+    ) {
+      let table
+      if (dsProvider) {
+        table = await config.createExternalTable(cfg, opts)
+      } else {
+        table = await config.createTable(cfg, opts)
+      }
+      return table
+    }
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      let table = await createTable(tableConfig)
+      tableId = table._id!
     })
 
-    describe("patch", () => {
-      it("should update only the view fields for a row", async () => {
-        const table = await createTable(await userTable())
-        const tableId = table._id!
-        const view = await config.createView({
-          schema: {
-            name: { visible: true },
-            address: { visible: true },
-          },
-        })
-
-        const newRow = await config.api.row.save(view.id, {
-          tableId,
-          _viewId: view.id,
-          ...randomRowData(),
-        })
-        const newData = randomRowData()
-        await config.api.row.patch(view.id, {
-          tableId,
-          _viewId: view.id,
-          _id: newRow._id!,
-          _rev: newRow._rev!,
-          ...newData,
-        })
-
-        const row = await config.api.row.get(tableId, newRow._id!)
-        expect(row.body).toEqual({
-          ...newRow,
-          name: newData.name,
-          address: newData.address,
-          _id: newRow._id,
-          _rev: expect.any(String),
-          id: newRow.id,
-          ...defaultRowFields,
-        })
-        expect(row.body._viewId).toBeUndefined()
-        expect(row.body.age).toBeUndefined()
-        expect(row.body.jobTitle).toBeUndefined()
-      })
-    })
-
-    describe("destroy", () => {
-      it("should be able to delete a row", async () => {
-        const table = await createTable(await userTable())
-        const tableId = table._id!
-        const view = await config.createView({
-          schema: {
-            name: { visible: true },
-            address: { visible: true },
-          },
-        })
-
-        const createdRow = await config.createRow()
+    describe("save, load, update", () => {
+      it("returns a success message when the row is created", async () => {
         const rowUsage = await getRowUsage()
 
-        await config.api.row.delete(view.id, [createdRow])
+        const res = await request
+          .post(`/api/${tableId}/rows`)
+          .send(basicRow(tableId))
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect((res as any).res.statusMessage).toEqual(
+          `${config.table!.name} saved successfully`
+        )
+        expect(res.body.name).toEqual("Test Contact")
+        expect(res.body._rev).toBeDefined()
+        await assertRowUsage(rowUsage + 1)
+      })
 
-        await assertRowUsage(rowUsage - 1)
+      it("Increment row autoId per create row request", async () => {
+        const rowUsage = await getRowUsage()
 
-        await config.api.row.get(tableId, createdRow._id!, {
+        const tableConfig = generateTableConfig()
+        const newTable = await createTable(
+          {
+            ...tableConfig,
+            name: "TestTableAuto",
+            schema: {
+              ...tableConfig.schema,
+              "Row ID": {
+                name: "Row ID",
+                type: FieldType.NUMBER,
+                subtype: AutoFieldSubType.AUTO_ID,
+                icon: "ri-magic-line",
+                autocolumn: true,
+                constraints: {
+                  type: "number",
+                  presence: true,
+                  numericality: {
+                    greaterThanOrEqualTo: "",
+                    lessThanOrEqualTo: "",
+                  },
+                },
+              },
+            },
+          },
+          { skipReassigning: true }
+        )
+
+        const ids = [1, 2, 3]
+
+        // Performing several create row requests should increment the autoID fields accordingly
+        const createRow = async (id: number) => {
+          const res = await config.api.row.save(newTable._id!, {
+            name: "row_" + id,
+          })
+          expect(res.name).toEqual("row_" + id)
+          expect(res._rev).toBeDefined()
+          expect(res["Row ID"]).toEqual(id)
+        }
+
+        for (let i = 0; i < ids.length; i++) {
+          await createRow(ids[i])
+        }
+
+        await assertRowUsage(rowUsage + ids.length)
+      })
+
+      it("updates a row successfully", async () => {
+        const existing = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.save(tableId, {
+          _id: existing._id,
+          _rev: existing._rev,
+          tableId,
+          name: "Updated Name",
+        })
+
+        expect(res.name).toEqual("Updated Name")
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should load a row", async () => {
+        const existing = await config.createRow()
+
+        const res = await config.api.row.get(tableId, existing._id!)
+
+        expect(res.body).toEqual({
+          ...existing,
+          ...defaultRowFields,
+        })
+      })
+
+      it("should list all rows for given tableId", async () => {
+        const table = await createTable(generateTableConfig(), {
+          skipReassigning: true,
+        })
+        const tableId = table._id!
+        const newRow = {
+          tableId,
+          name: "Second Contact",
+          description: "new",
+        }
+        const firstRow = await config.createRow({ tableId })
+        await config.createRow(newRow)
+
+        const res = await config.api.row.fetch(tableId)
+
+        expect(res.length).toBe(2)
+        expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
+        expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
+      })
+
+      it("load should return 404 when row does not exist", async () => {
+        await config.createRow()
+
+        await config.api.row.get(tableId, "1234567", {
           expectStatus: 404,
         })
       })
 
-      it("should be able to delete multiple rows", async () => {
-        const table = await createTable(await userTable())
-        const tableId = table._id!
-        const view = await config.createView({
+      isInternal &&
+        it("row values are coerced", async () => {
+          const str: FieldSchema = {
+            type: FieldType.STRING,
+            name: "str",
+            constraints: { type: "string", presence: false },
+          }
+          const attachment: FieldSchema = {
+            type: FieldType.ATTACHMENT,
+            name: "attachment",
+            constraints: { type: "array", presence: false },
+          }
+          const bool: FieldSchema = {
+            type: FieldType.BOOLEAN,
+            name: "boolean",
+            constraints: { type: "boolean", presence: false },
+          }
+          const number: FieldSchema = {
+            type: FieldType.NUMBER,
+            name: "str",
+            constraints: { type: "number", presence: false },
+          }
+          const datetime: FieldSchema = {
+            type: FieldType.DATETIME,
+            name: "datetime",
+            constraints: {
+              type: "string",
+              presence: false,
+              datetime: { earliest: "", latest: "" },
+            },
+          }
+          const arrayField: FieldSchema = {
+            type: FieldType.ARRAY,
+            constraints: {
+              type: "array",
+              presence: false,
+              inclusion: ["One", "Two", "Three"],
+            },
+            name: "Sample Tags",
+            sortable: false,
+          }
+          const optsField: FieldSchema = {
+            name: "Sample Opts",
+            type: FieldType.OPTIONS,
+            constraints: {
+              type: "string",
+              presence: false,
+              inclusion: ["Alpha", "Beta", "Gamma"],
+            },
+          }
+          const table = await createTable({
+            name: "TestTable2",
+            type: "table",
+            schema: {
+              name: str,
+              stringUndefined: str,
+              stringNull: str,
+              stringString: str,
+              numberEmptyString: number,
+              numberNull: number,
+              numberUndefined: number,
+              numberString: number,
+              numberNumber: number,
+              datetimeEmptyString: datetime,
+              datetimeNull: datetime,
+              datetimeUndefined: datetime,
+              datetimeString: datetime,
+              datetimeDate: datetime,
+              boolNull: bool,
+              boolEmpty: bool,
+              boolUndefined: bool,
+              boolString: bool,
+              boolBool: bool,
+              attachmentNull: attachment,
+              attachmentUndefined: attachment,
+              attachmentEmpty: attachment,
+              attachmentEmptyArrayStr: attachment,
+              arrayFieldEmptyArrayStr: arrayField,
+              arrayFieldArrayStrKnown: arrayField,
+              arrayFieldNull: arrayField,
+              arrayFieldUndefined: arrayField,
+              optsFieldEmptyStr: optsField,
+              optsFieldUndefined: optsField,
+              optsFieldNull: optsField,
+              optsFieldStrKnown: optsField,
+            },
+          })
+
+          const row = {
+            name: "Test Row",
+            stringUndefined: undefined,
+            stringNull: null,
+            stringString: "i am a string",
+            numberEmptyString: "",
+            numberNull: null,
+            numberUndefined: undefined,
+            numberString: "123",
+            numberNumber: 123,
+            datetimeEmptyString: "",
+            datetimeNull: null,
+            datetimeUndefined: undefined,
+            datetimeString: "1984-04-20T00:00:00.000Z",
+            datetimeDate: new Date("1984-04-20"),
+            boolNull: null,
+            boolEmpty: "",
+            boolUndefined: undefined,
+            boolString: "true",
+            boolBool: true,
+            tableId: table._id,
+            attachmentNull: null,
+            attachmentUndefined: undefined,
+            attachmentEmpty: "",
+            attachmentEmptyArrayStr: "[]",
+            arrayFieldEmptyArrayStr: "[]",
+            arrayFieldUndefined: undefined,
+            arrayFieldNull: null,
+            arrayFieldArrayStrKnown: "['One']",
+            optsFieldEmptyStr: "",
+            optsFieldUndefined: undefined,
+            optsFieldNull: null,
+            optsFieldStrKnown: "Alpha",
+          }
+
+          const createdRow = await config.createRow(row)
+          const id = createdRow._id!
+
+          const saved = (await loadRow(id, table._id!)).body
+
+          expect(saved.stringUndefined).toBe(undefined)
+          expect(saved.stringNull).toBe(null)
+          expect(saved.stringString).toBe("i am a string")
+          expect(saved.numberEmptyString).toBe(null)
+          expect(saved.numberNull).toBe(null)
+          expect(saved.numberUndefined).toBe(undefined)
+          expect(saved.numberString).toBe(123)
+          expect(saved.numberNumber).toBe(123)
+          expect(saved.datetimeEmptyString).toBe(null)
+          expect(saved.datetimeNull).toBe(null)
+          expect(saved.datetimeUndefined).toBe(undefined)
+          expect(saved.datetimeString).toBe(
+            new Date(row.datetimeString).toISOString()
+          )
+          expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
+          expect(saved.boolNull).toBe(null)
+          expect(saved.boolEmpty).toBe(null)
+          expect(saved.boolUndefined).toBe(undefined)
+          expect(saved.boolString).toBe(true)
+          expect(saved.boolBool).toBe(true)
+          expect(saved.attachmentNull).toEqual([])
+          expect(saved.attachmentUndefined).toBe(undefined)
+          expect(saved.attachmentEmpty).toEqual([])
+          expect(saved.attachmentEmptyArrayStr).toEqual([])
+          expect(saved.arrayFieldEmptyArrayStr).toEqual([])
+          expect(saved.arrayFieldNull).toEqual([])
+          expect(saved.arrayFieldUndefined).toEqual(undefined)
+          expect(saved.optsFieldEmptyStr).toEqual(null)
+          expect(saved.optsFieldUndefined).toEqual(undefined)
+          expect(saved.optsFieldNull).toEqual(null)
+          expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
+          expect(saved.optsFieldStrKnown).toEqual("Alpha")
+        })
+    })
+
+    describe("view save", () => {
+      it("views have extra data trimmed", async () => {
+        const table = await createTable({
+          type: "table",
+          name: "orders",
+          primary: ["OrderID"],
           schema: {
-            name: { visible: true },
-            address: { visible: true },
+            Country: {
+              type: FieldType.STRING,
+              name: "Country",
+            },
+            OrderID: {
+              type: FieldType.NUMBER,
+              name: "OrderID",
+            },
+            Story: {
+              type: FieldType.STRING,
+              name: "Story",
+            },
           },
         })
 
-        const rows = await Promise.all([
+        const createViewResponse = await config.createView({
+          name: uuid.v4(),
+          schema: {
+            Country: {
+              visible: true,
+            },
+            OrderID: {
+              visible: true,
+            },
+          },
+        })
+
+        const createRowResponse = await config.api.row.save(
+          createViewResponse.id,
+          {
+            OrderID: "1111",
+            Country: "Aussy",
+            Story: "aaaaa",
+          }
+        )
+
+        const row = await config.api.row.get(table._id!, createRowResponse._id!)
+        expect(row.body.Story).toBeUndefined()
+        expect(row.body).toEqual({
+          ...defaultRowFields,
+          OrderID: 1111,
+          Country: "Aussy",
+          _id: createRowResponse._id,
+          _rev: createRowResponse._rev,
+          tableId: table._id,
+        })
+      })
+    })
+
+    describe("patch", () => {
+      let otherTable: Table
+
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+        const otherTableConfig = generateTableConfig()
+        // need a short name of table here - for relationship tests
+        otherTableConfig.name = "a"
+        otherTableConfig.schema.relationship = {
+          name: "relationship",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: table._id!,
+          fieldName: "relationship",
+        }
+        otherTable = await createTable(otherTableConfig)
+        // need to set the config back to the original table
+        config.table = table
+      })
+
+      it("should update only the fields that are supplied", async () => {
+        const existing = await config.createRow()
+
+        const rowUsage = await getRowUsage()
+
+        const row = await config.api.row.patch(table._id!, {
+          _id: existing._id!,
+          _rev: existing._rev!,
+          tableId: table._id!,
+          name: "Updated Name",
+        })
+
+        expect(row.name).toEqual("Updated Name")
+        expect(row.description).toEqual(existing.description)
+
+        const savedRow = await loadRow(row._id!, table._id!)
+
+        expect(savedRow.body.description).toEqual(existing.description)
+        expect(savedRow.body.name).toEqual("Updated Name")
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should throw an error when given improper types", async () => {
+        const existing = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        await config.api.row.patch(
+          table._id!,
+          {
+            _id: existing._id!,
+            _rev: existing._rev!,
+            tableId: table._id!,
+            name: 1,
+          },
+          { expectStatus: 400 }
+        )
+
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should not overwrite links if those links are not set", async () => {
+        let linkField: FieldSchema = {
+          type: FieldType.LINK,
+          name: "",
+          fieldName: "",
+          constraints: {
+            type: "array",
+            presence: false,
+          },
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          tableId: InternalTable.USER_METADATA,
+        }
+
+        let table = await config.api.table.save({
+          name: "TestTable",
+          type: "table",
+          sourceType: TableSourceType.INTERNAL,
+          sourceId: INTERNAL_TABLE_SOURCE_ID,
+          schema: {
+            user1: { ...linkField, name: "user1", fieldName: "user1" },
+            user2: { ...linkField, name: "user2", fieldName: "user2" },
+          },
+        })
+
+        let user1 = await config.createUser()
+        let user2 = await config.createUser()
+
+        let row = await config.api.row.save(table._id!, {
+          user1: [{ _id: user1._id }],
+          user2: [{ _id: user2._id }],
+        })
+
+        let getResp = await config.api.row.get(table._id!, row._id!)
+        expect(getResp.body.user1[0]._id).toEqual(user1._id)
+        expect(getResp.body.user2[0]._id).toEqual(user2._id)
+
+        let patchResp = await config.api.row.patch(table._id!, {
+          _id: row._id!,
+          _rev: row._rev!,
+          tableId: table._id!,
+          user1: [{ _id: user2._id }],
+        })
+        expect(patchResp.user1[0]._id).toEqual(user2._id)
+        expect(patchResp.user2[0]._id).toEqual(user2._id)
+
+        getResp = await config.api.row.get(table._id!, row._id!)
+        expect(getResp.body.user1[0]._id).toEqual(user2._id)
+        expect(getResp.body.user2[0]._id).toEqual(user2._id)
+      })
+
+      it("should be able to update relationships when both columns are same name", async () => {
+        let row = await config.api.row.save(table._id!, {
+          name: "test",
+          description: "test",
+        })
+        let row2 = await config.api.row.save(otherTable._id!, {
+          name: "test",
+          description: "test",
+          relationship: [row._id],
+        })
+        row = (await config.api.row.get(table._id!, row._id!)).body
+        expect(row.relationship.length).toBe(1)
+        const resp = await config.api.row.patch(table._id!, {
+          _id: row._id!,
+          _rev: row._rev!,
+          tableId: row.tableId!,
+          name: "test2",
+          relationship: [row2._id],
+        })
+        expect(resp.relationship.length).toBe(1)
+      })
+    })
+
+    describe("destroy", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should be able to delete a row", async () => {
+        const createdRow = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.delete(table._id!, [createdRow])
+        expect(res.body[0]._id).toEqual(createdRow._id)
+        await assertRowUsage(rowUsage - 1)
+      })
+    })
+
+    describe("validate", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should return no errors on valid row", async () => {
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.validate(table._id!, { name: "ivan" })
+
+        expect(res.valid).toBe(true)
+        expect(Object.keys(res.errors)).toEqual([])
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should errors on invalid row", async () => {
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.validate(table._id!, { name: 1 })
+
+        if (isInternal) {
+          expect(res.valid).toBe(false)
+          expect(Object.keys(res.errors)).toEqual(["name"])
+        } else {
+          // Validation for external is not implemented, so it will always return valid
+          expect(res.valid).toBe(true)
+          expect(Object.keys(res.errors)).toEqual([])
+        }
+        await assertRowUsage(rowUsage)
+      })
+    })
+
+    describe("bulkDelete", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should be able to delete a bulk set of rows", async () => {
+        const row1 = await config.createRow()
+        const row2 = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.delete(table._id!, [row1, row2])
+
+        expect(res.body.length).toEqual(2)
+        await loadRow(row1._id!, table._id!, 404)
+        await assertRowUsage(rowUsage - 2)
+      })
+
+      it("should be able to delete a variety of row set types", async () => {
+        const [row1, row2, row3] = await Promise.all([
           config.createRow(),
           config.createRow(),
           config.createRow(),
         ])
         const rowUsage = await getRowUsage()
 
-        await config.api.row.delete(view.id, [rows[0], rows[2]])
+        const res = await config.api.row.delete(table._id!, [
+          row1,
+          row2._id,
+          { _id: row3._id },
+        ])
 
-        await assertRowUsage(rowUsage - 2)
+        expect(res.body.length).toEqual(3)
+        await loadRow(row1._id!, table._id!, 404)
+        await assertRowUsage(rowUsage - 3)
+      })
 
-        await config.api.row.get(tableId, rows[0]._id!, {
-          expectStatus: 404,
+      it("should accept a valid row object and delete the row", async () => {
+        const row1 = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.delete(table._id!, row1)
+
+        expect(res.body.id).toEqual(row1._id)
+        await loadRow(row1._id!, table._id!, 404)
+        await assertRowUsage(rowUsage - 1)
+      })
+
+      it("Should ignore malformed/invalid delete requests", async () => {
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.row.delete(
+          table._id!,
+          { not: "valid" },
+          { expectStatus: 400 }
+        )
+        expect(res.body.message).toEqual("Invalid delete rows request")
+
+        const res2 = await config.api.row.delete(
+          table._id!,
+          { rows: 123 },
+          { expectStatus: 400 }
+        )
+        expect(res2.body.message).toEqual("Invalid delete rows request")
+
+        const res3 = await config.api.row.delete(table._id!, "invalid", {
+          expectStatus: 400,
         })
-        await config.api.row.get(tableId, rows[2]._id!, {
-          expectStatus: 404,
-        })
-        await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
+        expect(res3.body.message).toEqual("Invalid delete rows request")
+
+        await assertRowUsage(rowUsage)
       })
     })
 
-    describe("view search", () => {
-      const viewSchema = { age: { visible: true }, name: { visible: true } }
+    // Legacy views are not available for external
+    isInternal &&
+      describe("fetchView", () => {
+        beforeEach(async () => {
+          const tableConfig = generateTableConfig()
+          table = await createTable(tableConfig)
+        })
+
+        it("should be able to fetch tables contents via 'view'", async () => {
+          const row = await config.createRow()
+          const rowUsage = await getRowUsage()
+
+          const res = await config.api.legacyView.get(table._id!)
+          expect(res.body.length).toEqual(1)
+          expect(res.body[0]._id).toEqual(row._id)
+          await assertRowUsage(rowUsage)
+        })
+
+        it("should throw an error if view doesn't exist", async () => {
+          const rowUsage = await getRowUsage()
+
+          await config.api.legacyView.get("derp", { expectStatus: 404 })
+
+          await assertRowUsage(rowUsage)
+        })
+
+        it("should be able to run on a view", async () => {
+          const view = await config.createLegacyView({
+            tableId: table._id!,
+            name: "ViewTest",
+            filters: [],
+            schema: {},
+          })
+          const row = await config.createRow()
+          const rowUsage = await getRowUsage()
+
+          const res = await config.api.legacyView.get(view.name)
+          expect(res.body.length).toEqual(1)
+          expect(res.body[0]._id).toEqual(row._id)
+
+          await assertRowUsage(rowUsage)
+        })
+      })
+
+    describe("fetchEnrichedRows", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should allow enriching some linked rows", async () => {
+        const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
+          config.getTenantId(),
+          async () => {
+            const linkedTable = await config.createLinkedTable(
+              RelationshipType.ONE_TO_MANY,
+              ["link"],
+              {
+                // Making sure that the combined table name + column name is within postgres limits
+                name: uuid.v4().replace(/-/g, "").substring(0, 16),
+                type: "table",
+                primary: ["id"],
+                primaryDisplay: "id",
+                schema: {
+                  id: {
+                    type: FieldType.AUTO,
+                    name: "id",
+                    autocolumn: true,
+                    constraints: {
+                      presence: true,
+                    },
+                  },
+                },
+              }
+            )
+            const firstRow = await config.createRow({
+              name: "Test Contact",
+              description: "original description",
+              tableId: table._id,
+            })
+            const secondRow = await config.createRow({
+              name: "Test 2",
+              description: "og desc",
+              link: [{ _id: firstRow._id }],
+              tableId: linkedTable._id,
+            })
+            return { linkedTable, firstRow, secondRow }
+          }
+        )
+        const rowUsage = await getRowUsage()
+
+        // test basic enrichment
+        const resBasic = await config.api.row.get(
+          linkedTable._id!,
+          secondRow._id!
+        )
+        expect(resBasic.body.link.length).toBe(1)
+        expect(resBasic.body.link[0]).toEqual({
+          _id: firstRow._id,
+          primaryDisplay: firstRow.name,
+        })
+
+        // test full enrichment
+        const resEnriched = await config.api.row.getEnriched(
+          linkedTable._id!,
+          secondRow._id!
+        )
+        expect(resEnriched.body.link.length).toBe(1)
+        expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
+        expect(resEnriched.body.link[0].name).toBe("Test Contact")
+        expect(resEnriched.body.link[0].description).toBe(
+          "original description"
+        )
+        await assertRowUsage(rowUsage)
+      })
+    })
+
+    isInternal &&
+      describe("attachments", () => {
+        beforeAll(async () => {
+          const tableConfig = generateTableConfig()
+          table = await createTable(tableConfig)
+        })
+
+        it("should allow enriching attachment rows", async () => {
+          const table = await config.createAttachmentTable()
+          const attachmentId = `${structures.uuid()}.csv`
+          const row = await config.createRow({
+            name: "test",
+            description: "test",
+            attachment: [
+              {
+                key: `${config.getAppId()}/attachments/${attachmentId}`,
+              },
+            ],
+            tableId: table._id,
+          })
+          // the environment needs configured for this
+          await setup.switchToSelfHosted(async () => {
+            return context.doInAppContext(config.getAppId(), async () => {
+              const enriched = await outputProcessing(table, [row])
+              expect((enriched as Row[])[0].attachment[0].url).toBe(
+                `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+              )
+            })
+          })
+        })
+      })
+
+    describe("exportData", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should allow exporting all columns", async () => {
+        const existing = await config.createRow()
+        const res = await config.api.row.exportRows(table._id!, {
+          rows: [existing._id!],
+        })
+        const results = JSON.parse(res.text)
+        expect(results.length).toEqual(1)
+        const row = results[0]
+
+        // Ensure all original columns were exported
+        expect(Object.keys(row).length).toBeGreaterThanOrEqual(
+          Object.keys(existing).length
+        )
+        Object.keys(existing).forEach(key => {
+          expect(row[key]).toEqual(existing[key])
+        })
+      })
+
+      it("should allow exporting only certain columns", async () => {
+        const existing = await config.createRow()
+        const res = await config.api.row.exportRows(table._id!, {
+          rows: [existing._id!],
+          columns: ["_id"],
+        })
+        const results = JSON.parse(res.text)
+        expect(results.length).toEqual(1)
+        const row = results[0]
+
+        // Ensure only the _id column was exported
+        expect(Object.keys(row).length).toEqual(1)
+        expect(row._id).toEqual(existing._id)
+      })
+    })
+
+    describe("view 2.0", () => {
       async function userTable(): Promise<Table> {
         return {
           name: `users_${uuid.v4()}`,
@@ -1132,949 +956,1098 @@ describe.each([
             name: {
               type: FieldType.STRING,
               name: "name",
-              constraints: { type: "string" },
+            },
+            surname: {
+              type: FieldType.STRING,
+              name: "surname",
             },
             age: {
               type: FieldType.NUMBER,
               name: "age",
-              constraints: {},
+            },
+            address: {
+              type: FieldType.STRING,
+              name: "address",
+            },
+            jobTitle: {
+              type: FieldType.STRING,
+              name: "jobTitle",
             },
           },
         }
       }
 
-      it("returns empty rows from view when no schema is passed", async () => {
-        const table = await createTable(await userTable())
-        const rows = await Promise.all(
-          Array.from({ length: 10 }, () =>
-            config.api.row.save(table._id!, { tableId: table._id })
-          )
-        )
-
-        const createViewResponse = await config.createView()
-        const response = await config.api.viewV2.search(createViewResponse.id)
-
-        expect(response.body.rows).toHaveLength(10)
-        expect(response.body).toEqual({
-          rows: expect.arrayContaining(
-            rows.map(r => ({
-              _viewId: createViewResponse.id,
-              tableId: table._id,
-              _id: r._id,
-              _rev: r._rev,
-              ...defaultRowFields,
-            }))
-          ),
-          ...(isInternal
-            ? {}
-            : {
-                hasNextPage: false,
-                bookmark: null,
-              }),
-        })
+      const randomRowData = () => ({
+        name: generator.first(),
+        surname: generator.last(),
+        age: generator.age(),
+        address: generator.address(),
+        jobTitle: generator.word(),
       })
 
-      it("searching respects the view filters", async () => {
-        const table = await createTable(await userTable())
-
-        await Promise.all(
-          Array.from({ length: 10 }, () =>
-            config.api.row.save(table._id!, {
-              tableId: table._id,
-              name: generator.name(),
-              age: generator.integer({ min: 10, max: 30 }),
-            })
-          )
-        )
-
-        const expectedRows = await Promise.all(
-          Array.from({ length: 5 }, () =>
-            config.api.row.save(table._id!, {
-              tableId: table._id,
-              name: generator.name(),
-              age: 40,
-            })
-          )
-        )
-
-        const createViewResponse = await config.createView({
-          query: [
-            { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 },
-          ],
-          schema: viewSchema,
-        })
-
-        const response = await config.api.viewV2.search(createViewResponse.id)
-
-        expect(response.body.rows).toHaveLength(5)
-        expect(response.body).toEqual({
-          rows: expect.arrayContaining(
-            expectedRows.map(r => ({
-              _viewId: createViewResponse.id,
-              tableId: table._id,
-              name: r.name,
-              age: r.age,
-              _id: r._id,
-              _rev: r._rev,
-              ...defaultRowFields,
-            }))
-          ),
-          ...(isInternal
-            ? {}
-            : {
-                hasNextPage: false,
-                bookmark: null,
-              }),
-        })
-      })
-
-      const sortTestOptions: [
-        {
-          field: string
-          order?: SortOrder
-          type?: SortType
-        },
-        string[]
-      ][] = [
-        [
-          {
-            field: "name",
-            order: SortOrder.ASCENDING,
-            type: SortType.STRING,
-          },
-          ["Alice", "Bob", "Charly", "Danny"],
-        ],
-        [
-          {
-            field: "name",
-          },
-          ["Alice", "Bob", "Charly", "Danny"],
-        ],
-        [
-          {
-            field: "name",
-            order: SortOrder.DESCENDING,
-          },
-          ["Danny", "Charly", "Bob", "Alice"],
-        ],
-        [
-          {
-            field: "name",
-            order: SortOrder.DESCENDING,
-            type: SortType.STRING,
-          },
-          ["Danny", "Charly", "Bob", "Alice"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.ASCENDING,
-            type: SortType.number,
-          },
-          ["Danny", "Alice", "Charly", "Bob"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.ASCENDING,
-          },
-          ["Danny", "Alice", "Charly", "Bob"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.DESCENDING,
-          },
-          ["Bob", "Charly", "Alice", "Danny"],
-        ],
-        [
-          {
-            field: "age",
-            order: SortOrder.DESCENDING,
-            type: SortType.number,
-          },
-          ["Bob", "Charly", "Alice", "Danny"],
-        ],
-      ]
-
-      describe("sorting", () => {
-        beforeAll(async () => {
+      describe("create", () => {
+        it("should persist a new row with only the provided view fields", async () => {
           const table = await createTable(await userTable())
-          const users = [
-            { name: "Alice", age: 25 },
-            { name: "Bob", age: 30 },
-            { name: "Charly", age: 27 },
-            { name: "Danny", age: 15 },
-          ]
+          const view = await config.createView({
+            schema: {
+              name: { visible: true },
+              surname: { visible: true },
+              address: { visible: true },
+            },
+          })
+
+          const data = randomRowData()
+          const newRow = await config.api.row.save(view.id, {
+            tableId: table!._id,
+            _viewId: view.id,
+            ...data,
+          })
+
+          const row = await config.api.row.get(table._id!, newRow._id!)
+          expect(row.body).toEqual({
+            name: data.name,
+            surname: data.surname,
+            address: data.address,
+            tableId: table!._id,
+            _id: newRow._id,
+            _rev: newRow._rev,
+            id: newRow.id,
+            ...defaultRowFields,
+          })
+          expect(row.body._viewId).toBeUndefined()
+          expect(row.body.age).toBeUndefined()
+          expect(row.body.jobTitle).toBeUndefined()
+        })
+      })
+
+      describe("patch", () => {
+        it("should update only the view fields for a row", async () => {
+          const table = await createTable(await userTable())
+          const tableId = table._id!
+          const view = await config.createView({
+            schema: {
+              name: { visible: true },
+              address: { visible: true },
+            },
+          })
+
+          const newRow = await config.api.row.save(view.id, {
+            tableId,
+            _viewId: view.id,
+            ...randomRowData(),
+          })
+          const newData = randomRowData()
+          await config.api.row.patch(view.id, {
+            tableId,
+            _viewId: view.id,
+            _id: newRow._id!,
+            _rev: newRow._rev!,
+            ...newData,
+          })
+
+          const row = await config.api.row.get(tableId, newRow._id!)
+          expect(row.body).toEqual({
+            ...newRow,
+            name: newData.name,
+            address: newData.address,
+            _id: newRow._id,
+            _rev: expect.any(String),
+            id: newRow.id,
+            ...defaultRowFields,
+          })
+          expect(row.body._viewId).toBeUndefined()
+          expect(row.body.age).toBeUndefined()
+          expect(row.body.jobTitle).toBeUndefined()
+        })
+      })
+
+      describe("destroy", () => {
+        it("should be able to delete a row", async () => {
+          const table = await createTable(await userTable())
+          const tableId = table._id!
+          const view = await config.createView({
+            schema: {
+              name: { visible: true },
+              address: { visible: true },
+            },
+          })
+
+          const createdRow = await config.createRow()
+          const rowUsage = await getRowUsage()
+
+          await config.api.row.delete(view.id, [createdRow])
+
+          await assertRowUsage(rowUsage - 1)
+
+          await config.api.row.get(tableId, createdRow._id!, {
+            expectStatus: 404,
+          })
+        })
+
+        it("should be able to delete multiple rows", async () => {
+          const table = await createTable(await userTable())
+          const tableId = table._id!
+          const view = await config.createView({
+            schema: {
+              name: { visible: true },
+              address: { visible: true },
+            },
+          })
+
+          const rows = await Promise.all([
+            config.createRow(),
+            config.createRow(),
+            config.createRow(),
+          ])
+          const rowUsage = await getRowUsage()
+
+          await config.api.row.delete(view.id, [rows[0], rows[2]])
+
+          await assertRowUsage(rowUsage - 2)
+
+          await config.api.row.get(tableId, rows[0]._id!, {
+            expectStatus: 404,
+          })
+          await config.api.row.get(tableId, rows[2]._id!, {
+            expectStatus: 404,
+          })
+          await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
+        })
+      })
+
+      describe("view search", () => {
+        const viewSchema = { age: { visible: true }, name: { visible: true } }
+        async function userTable(): Promise<Table> {
+          return {
+            name: `users_${uuid.v4()}`,
+            sourceId: INTERNAL_TABLE_SOURCE_ID,
+            sourceType: TableSourceType.INTERNAL,
+            type: "table",
+            primary: ["id"],
+            schema: {
+              id: {
+                type: FieldType.AUTO,
+                name: "id",
+                autocolumn: true,
+                constraints: {
+                  presence: true,
+                },
+              },
+              name: {
+                type: FieldType.STRING,
+                name: "name",
+                constraints: { type: "string" },
+              },
+              age: {
+                type: FieldType.NUMBER,
+                name: "age",
+                constraints: {},
+              },
+            },
+          }
+        }
+
+        it("returns empty rows from view when no schema is passed", async () => {
+          const table = await createTable(await userTable())
+          const rows = await Promise.all(
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, { tableId: table._id })
+            )
+          )
+
+          const createViewResponse = await config.createView()
+          const response = await config.api.viewV2.search(createViewResponse.id)
+
+          expect(response.body.rows).toHaveLength(10)
+          expect(response.body).toEqual({
+            rows: expect.arrayContaining(
+              rows.map(r => ({
+                _viewId: createViewResponse.id,
+                tableId: table._id,
+                _id: r._id,
+                _rev: r._rev,
+                ...defaultRowFields,
+              }))
+            ),
+            ...(isInternal
+              ? {}
+              : {
+                  hasNextPage: false,
+                  bookmark: null,
+                }),
+          })
+        })
+
+        it("searching respects the view filters", async () => {
+          const table = await createTable(await userTable())
+
           await Promise.all(
-            users.map(u =>
+            Array.from({ length: 10 }, () =>
               config.api.row.save(table._id!, {
                 tableId: table._id,
-                ...u,
+                name: generator.name(),
+                age: generator.integer({ min: 10, max: 30 }),
               })
             )
           )
-        })
 
-        it.each(sortTestOptions)(
-          "allow sorting (%s)",
-          async (sortParams, expected) => {
-            const createViewResponse = await config.createView({
-              sort: sortParams,
-              schema: viewSchema,
-            })
-
-            const response = await config.api.viewV2.search(
-              createViewResponse.id
+          const expectedRows = await Promise.all(
+            Array.from({ length: 5 }, () =>
+              config.api.row.save(table._id!, {
+                tableId: table._id,
+                name: generator.name(),
+                age: 40,
+              })
             )
-
-            expect(response.body.rows).toHaveLength(4)
-            expect(response.body.rows).toEqual(
-              expected.map(name => expect.objectContaining({ name }))
-            )
-          }
-        )
-
-        it.each(sortTestOptions)(
-          "allow override the default view sorting (%s)",
-          async (sortParams, expected) => {
-            const createViewResponse = await config.createView({
-              sort: {
-                field: "name",
-                order: SortOrder.ASCENDING,
-                type: SortType.STRING,
-              },
-              schema: viewSchema,
-            })
-
-            const response = await config.api.viewV2.search(
-              createViewResponse.id,
-              {
-                sort: sortParams.field,
-                sortOrder: sortParams.order,
-                sortType: sortParams.type,
-                query: {},
-              }
-            )
-
-            expect(response.body.rows).toHaveLength(4)
-            expect(response.body.rows).toEqual(
-              expected.map(name => expect.objectContaining({ name }))
-            )
-          }
-        )
-      })
-
-      it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await createTable(await userTable())
-        const rows = await Promise.all(
-          Array.from({ length: 10 }, () =>
-            config.api.row.save(table._id!, {
-              tableId: table._id,
-              name: generator.name(),
-              age: generator.age(),
-            })
           )
-        )
 
-        const view = await config.createView({
-          schema: { name: { visible: true } },
-        })
-        const response = await config.api.viewV2.search(view.id)
+          const createViewResponse = await config.createView({
+            query: [
+              { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 },
+            ],
+            schema: viewSchema,
+          })
 
-        expect(response.body.rows).toHaveLength(10)
-        expect(response.body.rows).toEqual(
-          expect.arrayContaining(
-            rows.map(r => ({
-              ...(isInternal
-                ? expectAnyInternalColsAttributes
-                : expectAnyExternalColsAttributes),
-              _viewId: view.id,
-              name: r.name,
-            }))
-          )
-        )
-      })
+          const response = await config.api.viewV2.search(createViewResponse.id)
 
-      it("views without data can be returned", async () => {
-        const table = await createTable(await userTable())
-
-        const createViewResponse = await config.createView()
-        const response = await config.api.viewV2.search(createViewResponse.id)
-
-        expect(response.body.rows).toHaveLength(0)
-      })
-
-      it("respects the limit parameter", async () => {
-        await createTable(await userTable())
-        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
-
-        const limit = generator.integer({ min: 1, max: 8 })
-
-        const createViewResponse = await config.createView()
-        const response = await config.api.viewV2.search(createViewResponse.id, {
-          limit,
-          query: {},
+          expect(response.body.rows).toHaveLength(5)
+          expect(response.body).toEqual({
+            rows: expect.arrayContaining(
+              expectedRows.map(r => ({
+                _viewId: createViewResponse.id,
+                tableId: table._id,
+                name: r.name,
+                age: r.age,
+                _id: r._id,
+                _rev: r._rev,
+                ...defaultRowFields,
+              }))
+            ),
+            ...(isInternal
+              ? {}
+              : {
+                  hasNextPage: false,
+                  bookmark: null,
+                }),
+          })
         })
 
-        expect(response.body.rows).toHaveLength(limit)
-      })
-
-      it("can handle pagination", async () => {
-        await createTable(await userTable())
-        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
-
-        const createViewResponse = await config.createView()
-        const allRows = (await config.api.viewV2.search(createViewResponse.id))
-          .body.rows
-
-        const firstPageResponse = await config.api.viewV2.search(
-          createViewResponse.id,
+        const sortTestOptions: [
           {
-            paginate: true,
-            limit: 4,
-            query: {},
-          }
-        )
-        expect(firstPageResponse.body).toEqual({
-          rows: expect.arrayContaining(allRows.slice(0, 4)),
-          totalRows: isInternal ? 10 : undefined,
-          hasNextPage: true,
-          bookmark: expect.anything(),
-        })
-
-        const secondPageResponse = await config.api.viewV2.search(
-          createViewResponse.id,
-          {
-            paginate: true,
-            limit: 4,
-            bookmark: firstPageResponse.body.bookmark,
-
-            query: {},
-          }
-        )
-        expect(secondPageResponse.body).toEqual({
-          rows: expect.arrayContaining(allRows.slice(4, 8)),
-          totalRows: isInternal ? 10 : undefined,
-          hasNextPage: true,
-          bookmark: expect.anything(),
-        })
-
-        const lastPageResponse = await config.api.viewV2.search(
-          createViewResponse.id,
-          {
-            paginate: true,
-            limit: 4,
-            bookmark: secondPageResponse.body.bookmark,
-            query: {},
-          }
-        )
-        expect(lastPageResponse.body).toEqual({
-          rows: expect.arrayContaining(allRows.slice(8)),
-          totalRows: isInternal ? 10 : undefined,
-          hasNextPage: false,
-          bookmark: expect.anything(),
-        })
-      })
-
-      isInternal &&
-        it("doesn't allow creating in user table", async () => {
-          const userTableId = InternalTable.USER_METADATA
-          const response = await config.api.row.save(
-            userTableId,
+            field: string
+            order?: SortOrder
+            type?: SortType
+          },
+          string[]
+        ][] = [
+          [
             {
-              tableId: userTableId,
-              firstName: "Joe",
-              lastName: "Joe",
-              email: "joe@joe.com",
-              roles: {},
+              field: "name",
+              order: SortOrder.ASCENDING,
+              type: SortType.STRING,
             },
-            { expectStatus: 400 }
+            ["Alice", "Bob", "Charly", "Danny"],
+          ],
+          [
+            {
+              field: "name",
+            },
+            ["Alice", "Bob", "Charly", "Danny"],
+          ],
+          [
+            {
+              field: "name",
+              order: SortOrder.DESCENDING,
+            },
+            ["Danny", "Charly", "Bob", "Alice"],
+          ],
+          [
+            {
+              field: "name",
+              order: SortOrder.DESCENDING,
+              type: SortType.STRING,
+            },
+            ["Danny", "Charly", "Bob", "Alice"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.ASCENDING,
+              type: SortType.number,
+            },
+            ["Danny", "Alice", "Charly", "Bob"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.ASCENDING,
+            },
+            ["Danny", "Alice", "Charly", "Bob"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.DESCENDING,
+            },
+            ["Bob", "Charly", "Alice", "Danny"],
+          ],
+          [
+            {
+              field: "age",
+              order: SortOrder.DESCENDING,
+              type: SortType.number,
+            },
+            ["Bob", "Charly", "Alice", "Danny"],
+          ],
+        ]
+
+        describe("sorting", () => {
+          beforeAll(async () => {
+            const table = await createTable(await userTable())
+            const users = [
+              { name: "Alice", age: 25 },
+              { name: "Bob", age: 30 },
+              { name: "Charly", age: 27 },
+              { name: "Danny", age: 15 },
+            ]
+            await Promise.all(
+              users.map(u =>
+                config.api.row.save(table._id!, {
+                  tableId: table._id,
+                  ...u,
+                })
+              )
+            )
+          })
+
+          it.each(sortTestOptions)(
+            "allow sorting (%s)",
+            async (sortParams, expected) => {
+              const createViewResponse = await config.createView({
+                sort: sortParams,
+                schema: viewSchema,
+              })
+
+              const response = await config.api.viewV2.search(
+                createViewResponse.id
+              )
+
+              expect(response.body.rows).toHaveLength(4)
+              expect(response.body.rows).toEqual(
+                expected.map(name => expect.objectContaining({ name }))
+              )
+            }
+          )
+
+          it.each(sortTestOptions)(
+            "allow override the default view sorting (%s)",
+            async (sortParams, expected) => {
+              const createViewResponse = await config.createView({
+                sort: {
+                  field: "name",
+                  order: SortOrder.ASCENDING,
+                  type: SortType.STRING,
+                },
+                schema: viewSchema,
+              })
+
+              const response = await config.api.viewV2.search(
+                createViewResponse.id,
+                {
+                  sort: sortParams.field,
+                  sortOrder: sortParams.order,
+                  sortType: sortParams.type,
+                  query: {},
+                }
+              )
+
+              expect(response.body.rows).toHaveLength(4)
+              expect(response.body.rows).toEqual(
+                expected.map(name => expect.objectContaining({ name }))
+              )
+            }
           )
-          expect(response.message).toBe("Cannot create new user entry.")
         })
 
-      describe("permissions", () => {
-        let viewId: string
-        let tableId: string
+        it("when schema is defined, defined columns and row attributes are returned", async () => {
+          const table = await createTable(await userTable())
+          const rows = await Promise.all(
+            Array.from({ length: 10 }, () =>
+              config.api.row.save(table._id!, {
+                tableId: table._id,
+                name: generator.name(),
+                age: generator.age(),
+              })
+            )
+          )
 
-        beforeAll(async () => {
+          const view = await config.createView({
+            schema: { name: { visible: true } },
+          })
+          const response = await config.api.viewV2.search(view.id)
+
+          expect(response.body.rows).toHaveLength(10)
+          expect(response.body.rows).toEqual(
+            expect.arrayContaining(
+              rows.map(r => ({
+                ...(isInternal
+                  ? expectAnyInternalColsAttributes
+                  : expectAnyExternalColsAttributes),
+                _viewId: view.id,
+                name: r.name,
+              }))
+            )
+          )
+        })
+
+        it("views without data can be returned", async () => {
+          const table = await createTable(await userTable())
+
+          const createViewResponse = await config.createView()
+          const response = await config.api.viewV2.search(createViewResponse.id)
+
+          expect(response.body.rows).toHaveLength(0)
+        })
+
+        it("respects the limit parameter", async () => {
+          await createTable(await userTable())
+          await Promise.all(
+            Array.from({ length: 10 }, () => config.createRow())
+          )
+
+          const limit = generator.integer({ min: 1, max: 8 })
+
+          const createViewResponse = await config.createView()
+          const response = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              limit,
+              query: {},
+            }
+          )
+
+          expect(response.body.rows).toHaveLength(limit)
+        })
+
+        it("can handle pagination", async () => {
           await createTable(await userTable())
           await Promise.all(
             Array.from({ length: 10 }, () => config.createRow())
           )
 
           const createViewResponse = await config.createView()
+          const allRows = (
+            await config.api.viewV2.search(createViewResponse.id)
+          ).body.rows
 
-          tableId = table._id!
-          viewId = createViewResponse.id
-        })
+          const firstPageResponse = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              paginate: true,
+              limit: 4,
+              query: {},
+            }
+          )
+          expect(firstPageResponse.body).toEqual({
+            rows: expect.arrayContaining(allRows.slice(0, 4)),
+            totalRows: isInternal ? 10 : undefined,
+            hasNextPage: true,
+            bookmark: expect.anything(),
+          })
 
-        beforeEach(() => {
-          mocks.licenses.useViewPermissions()
-        })
+          const secondPageResponse = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              paginate: true,
+              limit: 4,
+              bookmark: firstPageResponse.body.bookmark,
 
-        it("does not allow public users to fetch by default", async () => {
-          await config.publish()
-          await config.api.viewV2.search(viewId, undefined, {
-            expectStatus: 403,
-            usePublicUser: true,
+              query: {},
+            }
+          )
+          expect(secondPageResponse.body).toEqual({
+            rows: expect.arrayContaining(allRows.slice(4, 8)),
+            totalRows: isInternal ? 10 : undefined,
+            hasNextPage: true,
+            bookmark: expect.anything(),
+          })
+
+          const lastPageResponse = await config.api.viewV2.search(
+            createViewResponse.id,
+            {
+              paginate: true,
+              limit: 4,
+              bookmark: secondPageResponse.body.bookmark,
+              query: {},
+            }
+          )
+          expect(lastPageResponse.body).toEqual({
+            rows: expect.arrayContaining(allRows.slice(8)),
+            totalRows: isInternal ? 10 : undefined,
+            hasNextPage: false,
+            bookmark: expect.anything(),
           })
         })
 
-        it("allow public users to fetch when permissions are explicit", async () => {
-          await config.api.permission.set({
-            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-            level: PermissionLevel.READ,
-            resourceId: viewId,
-          })
-          await config.publish()
-
-          const response = await config.api.viewV2.search(viewId, undefined, {
-            usePublicUser: true,
+        isInternal &&
+          it("doesn't allow creating in user table", async () => {
+            const userTableId = InternalTable.USER_METADATA
+            const response = await config.api.row.save(
+              userTableId,
+              {
+                tableId: userTableId,
+                firstName: "Joe",
+                lastName: "Joe",
+                email: "joe@joe.com",
+                roles: {},
+              },
+              { expectStatus: 400 }
+            )
+            expect(response.message).toBe("Cannot create new user entry.")
           })
 
-          expect(response.body.rows).toHaveLength(10)
-        })
+        describe("permissions", () => {
+          let viewId: string
+          let tableId: string
 
-        it("allow public users to fetch when permissions are inherited", async () => {
-          await config.api.permission.set({
-            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-            level: PermissionLevel.READ,
-            resourceId: tableId,
-          })
-          await config.publish()
+          beforeAll(async () => {
+            await createTable(await userTable())
+            await Promise.all(
+              Array.from({ length: 10 }, () => config.createRow())
+            )
 
-          const response = await config.api.viewV2.search(viewId, undefined, {
-            usePublicUser: true,
+            const createViewResponse = await config.createView()
+
+            tableId = table._id!
+            viewId = createViewResponse.id
           })
 
-          expect(response.body.rows).toHaveLength(10)
-        })
-
-        it("respects inherited permissions, not allowing not public views from public tables", async () => {
-          await config.api.permission.set({
-            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-            level: PermissionLevel.READ,
-            resourceId: tableId,
+          beforeEach(() => {
+            mocks.licenses.useViewPermissions()
           })
-          await config.api.permission.set({
-            roleId: roles.BUILTIN_ROLE_IDS.POWER,
-            level: PermissionLevel.READ,
-            resourceId: viewId,
-          })
-          await config.publish()
 
-          await config.api.viewV2.search(viewId, undefined, {
-            usePublicUser: true,
-            expectStatus: 403,
+          it("does not allow public users to fetch by default", async () => {
+            await config.publish()
+            await config.api.viewV2.search(viewId, undefined, {
+              expectStatus: 403,
+              usePublicUser: true,
+            })
+          })
+
+          it("allow public users to fetch when permissions are explicit", async () => {
+            await config.api.permission.set({
+              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+              level: PermissionLevel.READ,
+              resourceId: viewId,
+            })
+            await config.publish()
+
+            const response = await config.api.viewV2.search(viewId, undefined, {
+              usePublicUser: true,
+            })
+
+            expect(response.body.rows).toHaveLength(10)
+          })
+
+          it("allow public users to fetch when permissions are inherited", async () => {
+            await config.api.permission.set({
+              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+              level: PermissionLevel.READ,
+              resourceId: tableId,
+            })
+            await config.publish()
+
+            const response = await config.api.viewV2.search(viewId, undefined, {
+              usePublicUser: true,
+            })
+
+            expect(response.body.rows).toHaveLength(10)
+          })
+
+          it("respects inherited permissions, not allowing not public views from public tables", async () => {
+            await config.api.permission.set({
+              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+              level: PermissionLevel.READ,
+              resourceId: tableId,
+            })
+            await config.api.permission.set({
+              roleId: roles.BUILTIN_ROLE_IDS.POWER,
+              level: PermissionLevel.READ,
+              resourceId: viewId,
+            })
+            await config.publish()
+
+            await config.api.viewV2.search(viewId, undefined, {
+              usePublicUser: true,
+              expectStatus: 403,
+            })
           })
         })
       })
     })
-  })
 
-  let o2mTable: Table
-  let m2mTable: Table
-  beforeAll(async () => {
-    o2mTable = await createTable(
-      { ...generateTableConfig(), name: "o2m" },
-      {
-        skipReassigning: true,
-      }
-    )
-    m2mTable = await createTable(
-      { ...generateTableConfig(), name: "m2m" },
-      {
-        skipReassigning: true,
-      }
-    )
-  })
+    let o2mTable: Table
+    let m2mTable: Table
+    beforeAll(async () => {
+      o2mTable = await createTable(
+        { ...generateTableConfig(), name: "o2m" },
+        {
+          skipReassigning: true,
+        }
+      )
+      m2mTable = await createTable(
+        { ...generateTableConfig(), name: "m2m" },
+        {
+          skipReassigning: true,
+        }
+      )
+    })
 
-  describe.each([
-    [
-      "relationship fields",
-      (): Record<string, FieldSchema> => ({
-        user: {
-          name: "user",
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          type: FieldType.LINK,
-          tableId: o2mTable._id!,
-          fieldName: "fk_o2m",
-        },
-        users: {
-          name: "users",
-          relationshipType: RelationshipType.MANY_TO_MANY,
-          type: FieldType.LINK,
-          tableId: m2mTable._id!,
-          fieldName: "fk_m2m",
-        },
-      }),
-      (tableId: string) =>
-        config.api.row.save(tableId, {
-          name: uuid.v4(),
+    describe.each([
+      [
+        "relationship fields",
+        (): Record<string, FieldSchema> => ({
+          user: {
+            name: "user",
+            relationshipType: RelationshipType.ONE_TO_MANY,
+            type: FieldType.LINK,
+            tableId: o2mTable._id!,
+            fieldName: "fk_o2m",
+          },
+          users: {
+            name: "users",
+            relationshipType: RelationshipType.MANY_TO_MANY,
+            type: FieldType.LINK,
+            tableId: m2mTable._id!,
+            fieldName: "fk_m2m",
+          },
+        }),
+        (tableId: string) =>
+          config.api.row.save(tableId, {
+            name: uuid.v4(),
+            description: generator.paragraph(),
+            tableId,
+          }),
+        (row: Row) => ({
+          _id: row._id,
+          primaryDisplay: row.name,
+        }),
+      ],
+      [
+        "bb reference fields",
+        (): Record<string, FieldSchema> => ({
+          user: {
+            name: "user",
+            type: FieldType.BB_REFERENCE,
+            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+          },
+          users: {
+            name: "users",
+            type: FieldType.BB_REFERENCE,
+            subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
+          },
+        }),
+        () => config.createUser(),
+        (row: Row) => ({
+          _id: row._id,
+          primaryDisplay: row.email,
+          email: row.email,
+          firstName: row.firstName,
+          lastName: row.lastName,
+        }),
+      ],
+    ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
+      let tableId: string
+      let o2mData: Row[]
+      let m2mData: Row[]
+
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+
+        if (config.datasource) {
+          tableConfig.sourceId = config.datasource._id!
+          if (config.datasource.plus) {
+            tableConfig.sourceType = TableSourceType.EXTERNAL
+          }
+        }
+        const table = await config.api.table.save({
+          ...tableConfig,
+          schema: {
+            ...tableConfig.schema,
+            ...relSchema(),
+          },
+        })
+        tableId = table._id!
+
+        o2mData = [
+          await dataGenerator(o2mTable._id!),
+          await dataGenerator(o2mTable._id!),
+          await dataGenerator(o2mTable._id!),
+          await dataGenerator(o2mTable._id!),
+        ]
+
+        m2mData = [
+          await dataGenerator(m2mTable._id!),
+          await dataGenerator(m2mTable._id!),
+          await dataGenerator(m2mTable._id!),
+          await dataGenerator(m2mTable._id!),
+        ]
+      })
+
+      it("can save a row when relationship fields are empty", async () => {
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        expect(row).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          _id: expect.any(String),
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          type: isInternal ? "row" : undefined,
+        })
+      })
+
+      it("can save a row with a single relationship field", async () => {
+        const user = _.sample(o2mData)!
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user],
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        expect(row).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          user: [user].map(u => resultMapper(u)),
+          _id: expect.any(String),
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          type: isInternal ? "row" : undefined,
+          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+        })
+      })
+
+      it("can save a row with a multiple relationship field", async () => {
+        const selectedUsers = _.sampleSize(m2mData, 2)
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: selectedUsers,
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        expect(row).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          users: expect.arrayContaining(
+            selectedUsers.map(u => resultMapper(u))
+          ),
+          _id: expect.any(String),
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          type: isInternal ? "row" : undefined,
+        })
+      })
+
+      it("can retrieve rows with no populated relationships", async () => {
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+        expect(retrieved).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          user: undefined,
+          users: undefined,
+          _id: row._id,
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          ...defaultRowFields,
+        })
+      })
+
+      it("can retrieve rows with populated relationships", async () => {
+        const user1 = _.sample(o2mData)!
+        const [user2, user3] = _.sampleSize(m2mData, 2)
+
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user2, user3],
+          user: [user1],
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+        expect(retrieved).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          user: expect.arrayContaining([user1].map(u => resultMapper(u))),
+          users: expect.arrayContaining(
+            [user2, user3].map(u => resultMapper(u))
+          ),
+          _id: row._id,
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
+          ...defaultRowFields,
+        })
+      })
+
+      it("can update an existing populated row", async () => {
+        const user = _.sample(o2mData)!
+        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        const updatedRow = await config.api.row.save(tableId, {
+          ...row,
+          user: [user],
+          users: [users3, users1],
+        })
+        expect(updatedRow).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          user: expect.arrayContaining([user].map(u => resultMapper(u))),
+          users: expect.arrayContaining(
+            [users3, users1].map(u => resultMapper(u))
+          ),
+          _id: row._id,
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          type: isInternal ? "row" : undefined,
+          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+        })
+      })
+
+      it("can wipe an existing populated relationships in row", async () => {
+        const [user1, user2] = _.sampleSize(m2mData, 2)
+
+        const rowData = {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [user1, user2],
+        }
+        const row = await config.api.row.save(tableId, rowData)
+
+        const updatedRow = await config.api.row.save(tableId, {
+          ...row,
+          user: null,
+          users: null,
+        })
+        expect(updatedRow).toEqual({
+          name: rowData.name,
+          description: rowData.description,
+          tableId,
+          _id: row._id,
+          _rev: expect.any(String),
+          id: isInternal ? undefined : expect.any(Number),
+          type: isInternal ? "row" : undefined,
+        })
+      })
+
+      it("fetch all will populate the relationships", async () => {
+        const [user1] = _.sampleSize(o2mData, 1)
+        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+        const rows: {
+          name: string
+          description: string
+          user?: Row[]
+          users?: Row[]
+          tableId: string
+        }[] = [
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            users: [users1, users2],
+          },
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            user: [user1],
+            users: [users1, users3],
+          },
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            users: [users3],
+          },
+        ]
+
+        await config.api.row.save(tableId, rows[0])
+        await config.api.row.save(tableId, rows[1])
+        await config.api.row.save(tableId, rows[2])
+
+        const res = await config.api.row.fetch(tableId)
+
+        expect(res).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              name: r.name,
+              description: r.description,
+              tableId,
+              user: r.user?.map(u => resultMapper(u)),
+              users: r.users?.length
+                ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+                : undefined,
+              _id: expect.any(String),
+              _rev: expect.any(String),
+              id: isInternal ? undefined : expect.any(Number),
+              [`fk_${o2mTable.name}_fk_o2m`]:
+                isInternal || !r.user?.length ? undefined : r.user[0].id,
+              ...defaultRowFields,
+            }))
+          )
+        )
+      })
+
+      it("search all will populate the relationships", async () => {
+        const [user1] = _.sampleSize(o2mData, 1)
+        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+        const rows: {
+          name: string
+          description: string
+          user?: Row[]
+          users?: Row[]
+          tableId: string
+        }[] = [
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            users: [users1, users2],
+          },
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            user: [user1],
+            users: [users1, users3],
+          },
+          {
+            ...basicRow(tableId),
+            name: generator.name(),
+            description: generator.name(),
+            users: [users3],
+          },
+        ]
+
+        await config.api.row.save(tableId, rows[0])
+        await config.api.row.save(tableId, rows[1])
+        await config.api.row.save(tableId, rows[2])
+
+        const res = await config.api.row.search(tableId)
+
+        expect(res).toEqual({
+          rows: expect.arrayContaining(
+            rows.map(r => ({
+              name: r.name,
+              description: r.description,
+              tableId,
+              user: r.user?.map(u => resultMapper(u)),
+              users: r.users?.length
+                ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+                : undefined,
+              _id: expect.any(String),
+              _rev: expect.any(String),
+              id: isInternal ? undefined : expect.any(Number),
+              [`fk_${o2mTable.name}_fk_o2m`]:
+                isInternal || !r.user?.length ? undefined : r.user[0].id,
+              ...defaultRowFields,
+            }))
+          ),
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
+        })
+      })
+    })
+
+    describe("Formula fields", () => {
+      let relationshipTable: Table, tableId: string, relatedRow: Row
+
+      beforeAll(async () => {
+        const otherTableId = config.table!._id!
+        const cfg = generateTableConfig()
+        relationshipTable = await config.createLinkedTable(
+          RelationshipType.ONE_TO_MANY,
+          ["links"],
+          {
+            ...cfg,
+            // needs to be a short name
+            name: "b",
+            schema: {
+              ...cfg.schema,
+              formula: {
+                name: "formula",
+                type: FieldType.FORMULA,
+                formula: "{{ links.0.name }}",
+                formulaType: FormulaType.DYNAMIC,
+              },
+            },
+          }
+        )
+
+        tableId = relationshipTable._id!
+
+        relatedRow = await config.api.row.save(otherTableId, {
+          name: generator.word(),
+          description: generator.paragraph(),
+        })
+        await config.api.row.save(tableId, {
+          name: generator.word(),
           description: generator.paragraph(),
           tableId,
-        }),
-      (row: Row) => ({
-        _id: row._id,
-        primaryDisplay: row.name,
-      }),
-    ],
-    [
-      "bb reference fields",
-      (): Record<string, FieldSchema> => ({
-        user: {
-          name: "user",
-          type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-        },
-        users: {
-          name: "users",
-          type: FieldType.BB_REFERENCE,
-          subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
-        },
-      }),
-      () => config.createUser(),
-      (row: Row) => ({
-        _id: row._id,
-        primaryDisplay: row.email,
-        email: row.email,
-        firstName: row.firstName,
-        lastName: row.lastName,
-      }),
-    ],
-  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
-    let tableId: string
-    let o2mData: Row[]
-    let m2mData: Row[]
-
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-
-      if (config.datasource) {
-        tableConfig.sourceId = config.datasource._id!
-        if (config.datasource.plus) {
-          tableConfig.sourceType = TableSourceType.EXTERNAL
-        }
-      }
-      const table = await config.api.table.save({
-        ...tableConfig,
-        schema: {
-          ...tableConfig.schema,
-          ...relSchema(),
-        },
-      })
-      tableId = table._id!
-
-      o2mData = [
-        await dataGenerator(o2mTable._id!),
-        await dataGenerator(o2mTable._id!),
-        await dataGenerator(o2mTable._id!),
-        await dataGenerator(o2mTable._id!),
-      ]
-
-      m2mData = [
-        await dataGenerator(m2mTable._id!),
-        await dataGenerator(m2mTable._id!),
-        await dataGenerator(m2mTable._id!),
-        await dataGenerator(m2mTable._id!),
-      ]
-    })
-
-    it("can save a row when relationship fields are empty", async () => {
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      expect(row).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        _id: expect.any(String),
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        type: isInternal ? "row" : undefined,
-      })
-    })
-
-    it("can save a row with a single relationship field", async () => {
-      const user = _.sample(o2mData)!
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-        user: [user],
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      expect(row).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        user: [user].map(u => resultMapper(u)),
-        _id: expect.any(String),
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        type: isInternal ? "row" : undefined,
-        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
-      })
-    })
-
-    it("can save a row with a multiple relationship field", async () => {
-      const selectedUsers = _.sampleSize(m2mData, 2)
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-        users: selectedUsers,
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      expect(row).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
-        _id: expect.any(String),
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        type: isInternal ? "row" : undefined,
-      })
-    })
-
-    it("can retrieve rows with no populated relationships", async () => {
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
-      expect(retrieved).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        user: undefined,
-        users: undefined,
-        _id: row._id,
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        ...defaultRowFields,
-      })
-    })
-
-    it("can retrieve rows with populated relationships", async () => {
-      const user1 = _.sample(o2mData)!
-      const [user2, user3] = _.sampleSize(m2mData, 2)
-
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-        users: [user2, user3],
-        user: [user1],
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
-      expect(retrieved).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
-        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
-        _id: row._id,
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
-        ...defaultRowFields,
-      })
-    })
-
-    it("can update an existing populated row", async () => {
-      const user = _.sample(o2mData)!
-      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
-
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-        users: [users1, users2],
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      const updatedRow = await config.api.row.save(tableId, {
-        ...row,
-        user: [user],
-        users: [users3, users1],
-      })
-      expect(updatedRow).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        user: expect.arrayContaining([user].map(u => resultMapper(u))),
-        users: expect.arrayContaining(
-          [users3, users1].map(u => resultMapper(u))
-        ),
-        _id: row._id,
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        type: isInternal ? "row" : undefined,
-        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
-      })
-    })
-
-    it("can wipe an existing populated relationships in row", async () => {
-      const [user1, user2] = _.sampleSize(m2mData, 2)
-
-      const rowData = {
-        ...basicRow(tableId),
-        name: generator.name(),
-        description: generator.name(),
-        users: [user1, user2],
-      }
-      const row = await config.api.row.save(tableId, rowData)
-
-      const updatedRow = await config.api.row.save(tableId, {
-        ...row,
-        user: null,
-        users: null,
-      })
-      expect(updatedRow).toEqual({
-        name: rowData.name,
-        description: rowData.description,
-        tableId,
-        _id: row._id,
-        _rev: expect.any(String),
-        id: isInternal ? undefined : expect.any(Number),
-        type: isInternal ? "row" : undefined,
-      })
-    })
-
-    it("fetch all will populate the relationships", async () => {
-      const [user1] = _.sampleSize(o2mData, 1)
-      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
-
-      const rows: {
-        name: string
-        description: string
-        user?: Row[]
-        users?: Row[]
-        tableId: string
-      }[] = [
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [users1, users2],
-        },
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          user: [user1],
-          users: [users1, users3],
-        },
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [users3],
-        },
-      ]
-
-      await config.api.row.save(tableId, rows[0])
-      await config.api.row.save(tableId, rows[1])
-      await config.api.row.save(tableId, rows[2])
-
-      const res = await config.api.row.fetch(tableId)
-
-      expect(res).toEqual(
-        expect.arrayContaining(
-          rows.map(r => ({
-            name: r.name,
-            description: r.description,
-            tableId,
-            user: r.user?.map(u => resultMapper(u)),
-            users: r.users?.length
-              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              : undefined,
-            _id: expect.any(String),
-            _rev: expect.any(String),
-            id: isInternal ? undefined : expect.any(Number),
-            [`fk_${o2mTable.name}_fk_o2m`]:
-              isInternal || !r.user?.length ? undefined : r.user[0].id,
-            ...defaultRowFields,
-          }))
-        )
-      )
-    })
-
-    it("search all will populate the relationships", async () => {
-      const [user1] = _.sampleSize(o2mData, 1)
-      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
-
-      const rows: {
-        name: string
-        description: string
-        user?: Row[]
-        users?: Row[]
-        tableId: string
-      }[] = [
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [users1, users2],
-        },
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          user: [user1],
-          users: [users1, users3],
-        },
-        {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [users3],
-        },
-      ]
-
-      await config.api.row.save(tableId, rows[0])
-      await config.api.row.save(tableId, rows[1])
-      await config.api.row.save(tableId, rows[2])
-
-      const res = await config.api.row.search(tableId)
-
-      expect(res).toEqual({
-        rows: expect.arrayContaining(
-          rows.map(r => ({
-            name: r.name,
-            description: r.description,
-            tableId,
-            user: r.user?.map(u => resultMapper(u)),
-            users: r.users?.length
-              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-              : undefined,
-            _id: expect.any(String),
-            _rev: expect.any(String),
-            id: isInternal ? undefined : expect.any(Number),
-            [`fk_${o2mTable.name}_fk_o2m`]:
-              isInternal || !r.user?.length ? undefined : r.user[0].id,
-            ...defaultRowFields,
-          }))
-        ),
-        ...(isInternal
-          ? {}
-          : {
-              hasNextPage: false,
-              bookmark: null,
-            }),
-      })
-    })
-  })
-
-  describe("Formula fields", () => {
-    let relationshipTable: Table, tableId: string, relatedRow: Row
-
-    beforeAll(async () => {
-      const otherTableId = config.table!._id!
-      const cfg = generateTableConfig()
-      relationshipTable = await config.createLinkedTable(
-        RelationshipType.ONE_TO_MANY,
-        ["links"],
-        {
-          ...cfg,
-          // needs to be a short name
-          name: "b",
-          schema: {
-            ...cfg.schema,
-            formula: {
-              name: "formula",
-              type: FieldType.FORMULA,
-              formula: "{{ links.0.name }}",
-              formulaType: FormulaType.DYNAMIC,
-            },
-          },
-        }
-      )
-
-      tableId = relationshipTable._id!
-
-      relatedRow = await config.api.row.save(otherTableId, {
-        name: generator.word(),
-        description: generator.paragraph(),
-      })
-      await config.api.row.save(tableId, {
-        name: generator.word(),
-        description: generator.paragraph(),
-        tableId,
-        links: [relatedRow._id],
-      })
-    })
-
-    it("should be able to search for rows containing formulas", async () => {
-      const { rows } = await config.api.row.search(tableId)
-      expect(rows.length).toBe(1)
-      expect(rows[0].links.length).toBe(1)
-      const row = rows[0]
-      expect(row.formula).toBe(relatedRow.name)
-    })
-  })
-
-  describe("Formula JS protection", () => {
-    it("should time out JS execution if a single cell takes too long", async () => {
-      await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
-        const js = Buffer.from(
-          `
-              let i = 0;
-              while (true) {
-                i++;
-              }
-              return i;
-            `
-        ).toString("base64")
-
-        const table = await config.createTable({
-          name: "table",
-          type: "table",
-          schema: {
-            text: {
-              name: "text",
-              type: FieldType.STRING,
-            },
-            formula: {
-              name: "formula",
-              type: FieldType.FORMULA,
-              formula: `{{ js "${js}"}}`,
-              formulaType: FormulaType.DYNAMIC,
-            },
-          },
+          links: [relatedRow._id],
         })
+      })
 
-        await config.api.row.save(table._id!, { text: "foo" })
-        const { rows } = await config.api.row.search(table._id!)
-        expect(rows).toHaveLength(1)
+      it("should be able to search for rows containing formulas", async () => {
+        const { rows } = await config.api.row.search(tableId)
+        expect(rows.length).toBe(1)
+        expect(rows[0].links.length).toBe(1)
         const row = rows[0]
-        expect(row.text).toBe("foo")
-        expect(row.formula).toBe("Timed out while executing JS")
+        expect(row.formula).toBe(relatedRow.name)
       })
     })
 
-    it("should time out JS execution if a multiple cells take too long", async () => {
-      await config.withEnv(
-        {
-          JS_PER_INVOCATION_TIMEOUT_MS: 20,
-          JS_PER_REQUEST_TIMEOUT_MS: 40,
-        },
-        async () => {
+    describe("Formula JS protection", () => {
+      it("should time out JS execution if a single cell takes too long", async () => {
+        await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
           const js = Buffer.from(
             `
               let i = 0;
@@ -2102,81 +2075,125 @@ describe.each([
             },
           })
 
-          for (let i = 0; i < 10; i++) {
-            await config.api.row.save(table._id!, { text: "foo" })
-          }
-
-          // Run this test 3 times to make sure that there's no cross-request
-          // pollution of the execution time tracking.
-          for (let reqs = 0; reqs < 3; reqs++) {
-            const { rows } = await config.api.row.search(table._id!)
-            expect(rows).toHaveLength(10)
-
-            let i = 0
-            for (; i < 10; i++) {
-              const row = rows[i]
-              if (row.formula !== "Timed out while executing JS") {
-                break
-              }
-            }
-
-            // Given the execution times are not deterministic, we can't be sure
-            // of the exact number of rows that were executed before the timeout
-            // but it should absolutely be at least 1.
-            expect(i).toBeGreaterThan(0)
-            expect(i).toBeLessThan(5)
-
-            for (; i < 10; i++) {
-              const row = rows[i]
-              expect(row.text).toBe("foo")
-              expect(row.formula).toBe("Request JS execution limit hit")
-            }
-          }
-        }
-      )
-    })
-
-    it("should not carry over context between formulas", async () => {
-      const js = Buffer.from(`return $("[text]");`).toString("base64")
-      const table = await config.createTable({
-        name: "table",
-        type: "table",
-        schema: {
-          text: {
-            name: "text",
-            type: FieldType.STRING,
-          },
-          formula: {
-            name: "formula",
-            type: FieldType.FORMULA,
-            formula: `{{ js "${js}"}}`,
-            formulaType: FormulaType.DYNAMIC,
-          },
-        },
+          await config.api.row.save(table._id!, { text: "foo" })
+          const { rows } = await config.api.row.search(table._id!)
+          expect(rows).toHaveLength(1)
+          const row = rows[0]
+          expect(row.text).toBe("foo")
+          expect(row.formula).toBe("Timed out while executing JS")
+        })
       })
 
-      for (let i = 0; i < 10; i++) {
-        await config.api.row.save(table._id!, { text: `foo${i}` })
-      }
+      it("should time out JS execution if a multiple cells take too long", async () => {
+        await config.withEnv(
+          {
+            JS_PER_INVOCATION_TIMEOUT_MS: 20,
+            JS_PER_REQUEST_TIMEOUT_MS: 40,
+          },
+          async () => {
+            const js = Buffer.from(
+              `
+              let i = 0;
+              while (true) {
+                i++;
+              }
+              return i;
+            `
+            ).toString("base64")
 
-      const { rows } = await config.api.row.search(table._id!)
-      expect(rows).toHaveLength(10)
+            const table = await config.createTable({
+              name: "table",
+              type: "table",
+              schema: {
+                text: {
+                  name: "text",
+                  type: FieldType.STRING,
+                },
+                formula: {
+                  name: "formula",
+                  type: FieldType.FORMULA,
+                  formula: `{{ js "${js}"}}`,
+                  formulaType: FormulaType.DYNAMIC,
+                },
+              },
+            })
 
-      const formulaValues = rows.map(r => r.formula)
-      expect(formulaValues).toEqual(
-        expect.arrayContaining([
-          "foo0",
-          "foo1",
-          "foo2",
-          "foo3",
-          "foo4",
-          "foo5",
-          "foo6",
-          "foo7",
-          "foo8",
-          "foo9",
-        ])
-      )
+            for (let i = 0; i < 10; i++) {
+              await config.api.row.save(table._id!, { text: "foo" })
+            }
+
+            // Run this test 3 times to make sure that there's no cross-request
+            // pollution of the execution time tracking.
+            for (let reqs = 0; reqs < 3; reqs++) {
+              const { rows } = await config.api.row.search(table._id!)
+              expect(rows).toHaveLength(10)
+
+              let i = 0
+              for (; i < 10; i++) {
+                const row = rows[i]
+                if (row.formula !== "Timed out while executing JS") {
+                  break
+                }
+              }
+
+              // Given the execution times are not deterministic, we can't be sure
+              // of the exact number of rows that were executed before the timeout
+              // but it should absolutely be at least 1.
+              expect(i).toBeGreaterThan(0)
+              expect(i).toBeLessThan(5)
+
+              for (; i < 10; i++) {
+                const row = rows[i]
+                expect(row.text).toBe("foo")
+                expect(row.formula).toBe("Request JS execution limit hit")
+              }
+            }
+          }
+        )
+      })
+
+      it("should not carry over context between formulas", async () => {
+        const js = Buffer.from(`return $("[text]");`).toString("base64")
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            text: {
+              name: "text",
+              type: FieldType.STRING,
+            },
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: `{{ js "${js}"}}`,
+              formulaType: FormulaType.DYNAMIC,
+            },
+          },
+        })
+
+        for (let i = 0; i < 10; i++) {
+          await config.api.row.save(table._id!, { text: `foo${i}` })
+        }
+
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows).toHaveLength(10)
+
+        const formulaValues = rows.map(r => r.formula)
+        expect(formulaValues).toEqual(
+          expect.arrayContaining([
+            "foo0",
+            "foo1",
+            "foo2",
+            "foo3",
+            "foo4",
+            "foo5",
+            "foo6",
+            "foo7",
+            "foo8",
+            "foo9",
+          ])
+        )
+      })
     })
-  })
-})
+  }
+)
diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts
index fdca32189c..6605052598 100644
--- a/packages/server/src/integrations/base/sql.ts
+++ b/packages/server/src/integrations/base/sql.ts
@@ -17,7 +17,6 @@ const envLimit = environment.SQL_MAX_ROWS
   : null
 const BASE_LIMIT = envLimit || 5000
 
-type KnexQuery = Knex.QueryBuilder
 // these are invalid dates sent by the client, need to convert them to a real max date
 const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
 const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
@@ -127,11 +126,11 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
-    query: KnexQuery,
+    query: Knex.QueryBuilder,
     filters: SearchFilters | undefined,
     tableName: string,
     opts: { aliases?: Record<string, string>; relationship?: boolean }
-  ): KnexQuery {
+  ): Knex.QueryBuilder {
     function getTableName(name: string) {
       const alias = opts.aliases?.[name]
       return alias || name
@@ -320,7 +319,7 @@ class InternalBuilder {
     return query
   }
 
-  addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
+  addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
     let { sort, paginate } = json
     const table = json.meta?.table
     if (sort && Object.keys(sort || {}).length > 0) {
@@ -348,12 +347,12 @@ class InternalBuilder {
   }
 
   addRelationships(
-    query: KnexQuery,
+    query: Knex.QueryBuilder,
     fromTable: string,
     relationships: RelationshipsJson[] | undefined,
     schema: string | undefined,
     aliases?: Record<string, string>
-  ): KnexQuery {
+  ): Knex.QueryBuilder {
     if (!relationships) {
       return query
     }
@@ -429,7 +428,7 @@ class InternalBuilder {
     knex: Knex,
     endpoint: QueryJson["endpoint"],
     aliases?: QueryJson["tableAliases"]
-  ): KnexQuery {
+  ): Knex.QueryBuilder {
     const tableName = endpoint.entityId
     const tableAliased = aliases?.[tableName]
       ? `${tableName} as ${aliases?.[tableName]}`
@@ -441,7 +440,7 @@ class InternalBuilder {
     return query
   }
 
-  create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body } = json
     let query = this.knexWithAlias(knex, endpoint)
     const parsedBody = parseBody(body)
@@ -460,7 +459,7 @@ class InternalBuilder {
     }
   }
 
-  bulkCreate(knex: Knex, json: QueryJson): KnexQuery {
+  bulkCreate(knex: Knex, json: QueryJson): Knex.QueryBuilder {
     const { endpoint, body } = json
     let query = this.knexWithAlias(knex, endpoint)
     if (!Array.isArray(body)) {
@@ -470,7 +469,7 @@ class InternalBuilder {
     return query.insert(parsedBody)
   }
 
-  read(knex: Knex, json: QueryJson, limit: number): KnexQuery {
+  read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
     let { endpoint, resource, filters, paginate, relationships, tableAliases } =
       json
 
@@ -531,7 +530,7 @@ class InternalBuilder {
     })
   }
 
-  update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, body, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     const parsedBody = parseBody(body)
@@ -546,7 +545,7 @@ class InternalBuilder {
     }
   }
 
-  delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
+  delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
     const { endpoint, filters, tableAliases } = json
     let query = this.knexWithAlias(knex, endpoint, tableAliases)
     query = this.addFilters(query, filters, endpoint.entityId, {
@@ -578,7 +577,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
   _query(json: QueryJson, opts: QueryOptions = {}): Knex.SqlNative | Knex.Sql {
     const sqlClient = this.getSqlClient()
     const client = knex({ client: sqlClient })
-    let query: KnexQuery
+    let query: Knex.QueryBuilder
     const builder = new InternalBuilder(sqlClient)
     switch (this._operation(json)) {
       case Operation.CREATE:
diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts
index 497b981f68..9b3f6a1b38 100644
--- a/packages/server/src/integrations/tests/sqlAlias.spec.ts
+++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts
@@ -189,5 +189,16 @@ describe("Captures of real examples", () => {
       const aliased = aliasing.aliasField("`hello`.`world`")
       expect(aliased).toEqual("`a`.`world`")
     })
+
+    it("should handle table names in table names correctly", () => {
+      const tableNames = ["he", "hell", "hello"]
+      const aliasing = new AliasTables(tableNames)
+      const aliased1 = aliasing.aliasField("`he`.`world`")
+      const aliased2 = aliasing.aliasField("`hell`.`world`")
+      const aliased3 = aliasing.aliasField("`hello`.`world`")
+      expect(aliased1).toEqual("`a`.`world`")
+      expect(aliased2).toEqual("`b`.`world`")
+      expect(aliased3).toEqual("`c`.`world`")
+    })
   })
 })

From d471a2bf0f2760d5bea11ddf32b58eb8db92961b Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 28 Feb 2024 15:19:55 +0000
Subject: [PATCH 47/47] Undoing change.

---
 .../server/src/api/routes/tests/row.spec.ts   | 3961 ++++++++---------
 1 file changed, 1972 insertions(+), 1989 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 05c6b92bb0..239da36351 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -39,41 +39,907 @@ tk.freeze(timestamp)
 
 const { basicRow } = setup.structures
 
-describe.each([["postgres", databaseTestProviders.postgres]])(
-  "/rows (%s)",
-  (__, dsProvider) => {
-    const isInternal = !dsProvider
+describe.each([
+  ["internal", undefined],
+  ["postgres", databaseTestProviders.postgres],
+])("/rows (%s)", (__, dsProvider) => {
+  const isInternal = !dsProvider
 
-    const request = setup.getRequest()
-    const config = setup.getConfig()
-    let table: Table
-    let tableId: string
+  const request = setup.getRequest()
+  const config = setup.getConfig()
+  let table: Table
+  let tableId: string
 
-    afterAll(async () => {
-      if (dsProvider) {
-        await dsProvider.stop()
+  afterAll(async () => {
+    if (dsProvider) {
+      await dsProvider.stop()
+    }
+    setup.afterAll()
+  })
+
+  beforeAll(async () => {
+    await config.init()
+
+    if (dsProvider) {
+      await config.createDatasource({
+        datasource: await dsProvider.datasource(),
+      })
+    }
+  })
+
+  const generateTableConfig: () => SaveTableRequest = () => {
+    return {
+      name: uuid.v4(),
+      type: "table",
+      primary: ["id"],
+      primaryDisplay: "name",
+      sourceType: TableSourceType.INTERNAL,
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      schema: {
+        id: {
+          type: FieldType.AUTO,
+          name: "id",
+          autocolumn: true,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            type: "string",
+          },
+        },
+        description: {
+          type: FieldType.STRING,
+          name: "description",
+          constraints: {
+            type: "string",
+          },
+        },
+      },
+    }
+  }
+
+  beforeEach(async () => {
+    mocks.licenses.useCloudFree()
+  })
+
+  const loadRow = (id: string, tbl_Id: string, status = 200) =>
+    config.api.row.get(tbl_Id, id, { expectStatus: status })
+
+  const getRowUsage = async () => {
+    const { total } = await config.doInContext(null, () =>
+      quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
+    )
+    return total
+  }
+
+  const assertRowUsage = async (expected: number) => {
+    const usage = await getRowUsage()
+    expect(usage).toBe(expected)
+  }
+
+  const defaultRowFields = isInternal
+    ? {
+        type: "row",
+        createdAt: timestamp,
+        updatedAt: timestamp,
       }
-      setup.afterAll()
+    : undefined
+
+  async function createTable(
+    cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
+    opts?: { skipReassigning: boolean }
+  ) {
+    let table
+    if (dsProvider) {
+      table = await config.createExternalTable(cfg, opts)
+    } else {
+      table = await config.createTable(cfg, opts)
+    }
+    return table
+  }
+
+  beforeAll(async () => {
+    const tableConfig = generateTableConfig()
+    let table = await createTable(tableConfig)
+    tableId = table._id!
+  })
+
+  describe("save, load, update", () => {
+    it("returns a success message when the row is created", async () => {
+      const rowUsage = await getRowUsage()
+
+      const res = await request
+        .post(`/api/${tableId}/rows`)
+        .send(basicRow(tableId))
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect((res as any).res.statusMessage).toEqual(
+        `${config.table!.name} saved successfully`
+      )
+      expect(res.body.name).toEqual("Test Contact")
+      expect(res.body._rev).toBeDefined()
+      await assertRowUsage(rowUsage + 1)
     })
 
+    it("Increment row autoId per create row request", async () => {
+      const rowUsage = await getRowUsage()
+
+      const tableConfig = generateTableConfig()
+      const newTable = await createTable(
+        {
+          ...tableConfig,
+          name: "TestTableAuto",
+          schema: {
+            ...tableConfig.schema,
+            "Row ID": {
+              name: "Row ID",
+              type: FieldType.NUMBER,
+              subtype: AutoFieldSubType.AUTO_ID,
+              icon: "ri-magic-line",
+              autocolumn: true,
+              constraints: {
+                type: "number",
+                presence: true,
+                numericality: {
+                  greaterThanOrEqualTo: "",
+                  lessThanOrEqualTo: "",
+                },
+              },
+            },
+          },
+        },
+        { skipReassigning: true }
+      )
+
+      const ids = [1, 2, 3]
+
+      // Performing several create row requests should increment the autoID fields accordingly
+      const createRow = async (id: number) => {
+        const res = await config.api.row.save(newTable._id!, {
+          name: "row_" + id,
+        })
+        expect(res.name).toEqual("row_" + id)
+        expect(res._rev).toBeDefined()
+        expect(res["Row ID"]).toEqual(id)
+      }
+
+      for (let i = 0; i < ids.length; i++) {
+        await createRow(ids[i])
+      }
+
+      await assertRowUsage(rowUsage + ids.length)
+    })
+
+    it("updates a row successfully", async () => {
+      const existing = await config.createRow()
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.save(tableId, {
+        _id: existing._id,
+        _rev: existing._rev,
+        tableId,
+        name: "Updated Name",
+      })
+
+      expect(res.name).toEqual("Updated Name")
+      await assertRowUsage(rowUsage)
+    })
+
+    it("should load a row", async () => {
+      const existing = await config.createRow()
+
+      const res = await config.api.row.get(tableId, existing._id!)
+
+      expect(res.body).toEqual({
+        ...existing,
+        ...defaultRowFields,
+      })
+    })
+
+    it("should list all rows for given tableId", async () => {
+      const table = await createTable(generateTableConfig(), {
+        skipReassigning: true,
+      })
+      const tableId = table._id!
+      const newRow = {
+        tableId,
+        name: "Second Contact",
+        description: "new",
+      }
+      const firstRow = await config.createRow({ tableId })
+      await config.createRow(newRow)
+
+      const res = await config.api.row.fetch(tableId)
+
+      expect(res.length).toBe(2)
+      expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
+      expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
+    })
+
+    it("load should return 404 when row does not exist", async () => {
+      await config.createRow()
+
+      await config.api.row.get(tableId, "1234567", {
+        expectStatus: 404,
+      })
+    })
+
+    isInternal &&
+      it("row values are coerced", async () => {
+        const str: FieldSchema = {
+          type: FieldType.STRING,
+          name: "str",
+          constraints: { type: "string", presence: false },
+        }
+        const attachment: FieldSchema = {
+          type: FieldType.ATTACHMENT,
+          name: "attachment",
+          constraints: { type: "array", presence: false },
+        }
+        const bool: FieldSchema = {
+          type: FieldType.BOOLEAN,
+          name: "boolean",
+          constraints: { type: "boolean", presence: false },
+        }
+        const number: FieldSchema = {
+          type: FieldType.NUMBER,
+          name: "str",
+          constraints: { type: "number", presence: false },
+        }
+        const datetime: FieldSchema = {
+          type: FieldType.DATETIME,
+          name: "datetime",
+          constraints: {
+            type: "string",
+            presence: false,
+            datetime: { earliest: "", latest: "" },
+          },
+        }
+        const arrayField: FieldSchema = {
+          type: FieldType.ARRAY,
+          constraints: {
+            type: "array",
+            presence: false,
+            inclusion: ["One", "Two", "Three"],
+          },
+          name: "Sample Tags",
+          sortable: false,
+        }
+        const optsField: FieldSchema = {
+          name: "Sample Opts",
+          type: FieldType.OPTIONS,
+          constraints: {
+            type: "string",
+            presence: false,
+            inclusion: ["Alpha", "Beta", "Gamma"],
+          },
+        }
+        const table = await createTable({
+          name: "TestTable2",
+          type: "table",
+          schema: {
+            name: str,
+            stringUndefined: str,
+            stringNull: str,
+            stringString: str,
+            numberEmptyString: number,
+            numberNull: number,
+            numberUndefined: number,
+            numberString: number,
+            numberNumber: number,
+            datetimeEmptyString: datetime,
+            datetimeNull: datetime,
+            datetimeUndefined: datetime,
+            datetimeString: datetime,
+            datetimeDate: datetime,
+            boolNull: bool,
+            boolEmpty: bool,
+            boolUndefined: bool,
+            boolString: bool,
+            boolBool: bool,
+            attachmentNull: attachment,
+            attachmentUndefined: attachment,
+            attachmentEmpty: attachment,
+            attachmentEmptyArrayStr: attachment,
+            arrayFieldEmptyArrayStr: arrayField,
+            arrayFieldArrayStrKnown: arrayField,
+            arrayFieldNull: arrayField,
+            arrayFieldUndefined: arrayField,
+            optsFieldEmptyStr: optsField,
+            optsFieldUndefined: optsField,
+            optsFieldNull: optsField,
+            optsFieldStrKnown: optsField,
+          },
+        })
+
+        const row = {
+          name: "Test Row",
+          stringUndefined: undefined,
+          stringNull: null,
+          stringString: "i am a string",
+          numberEmptyString: "",
+          numberNull: null,
+          numberUndefined: undefined,
+          numberString: "123",
+          numberNumber: 123,
+          datetimeEmptyString: "",
+          datetimeNull: null,
+          datetimeUndefined: undefined,
+          datetimeString: "1984-04-20T00:00:00.000Z",
+          datetimeDate: new Date("1984-04-20"),
+          boolNull: null,
+          boolEmpty: "",
+          boolUndefined: undefined,
+          boolString: "true",
+          boolBool: true,
+          tableId: table._id,
+          attachmentNull: null,
+          attachmentUndefined: undefined,
+          attachmentEmpty: "",
+          attachmentEmptyArrayStr: "[]",
+          arrayFieldEmptyArrayStr: "[]",
+          arrayFieldUndefined: undefined,
+          arrayFieldNull: null,
+          arrayFieldArrayStrKnown: "['One']",
+          optsFieldEmptyStr: "",
+          optsFieldUndefined: undefined,
+          optsFieldNull: null,
+          optsFieldStrKnown: "Alpha",
+        }
+
+        const createdRow = await config.createRow(row)
+        const id = createdRow._id!
+
+        const saved = (await loadRow(id, table._id!)).body
+
+        expect(saved.stringUndefined).toBe(undefined)
+        expect(saved.stringNull).toBe(null)
+        expect(saved.stringString).toBe("i am a string")
+        expect(saved.numberEmptyString).toBe(null)
+        expect(saved.numberNull).toBe(null)
+        expect(saved.numberUndefined).toBe(undefined)
+        expect(saved.numberString).toBe(123)
+        expect(saved.numberNumber).toBe(123)
+        expect(saved.datetimeEmptyString).toBe(null)
+        expect(saved.datetimeNull).toBe(null)
+        expect(saved.datetimeUndefined).toBe(undefined)
+        expect(saved.datetimeString).toBe(
+          new Date(row.datetimeString).toISOString()
+        )
+        expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
+        expect(saved.boolNull).toBe(null)
+        expect(saved.boolEmpty).toBe(null)
+        expect(saved.boolUndefined).toBe(undefined)
+        expect(saved.boolString).toBe(true)
+        expect(saved.boolBool).toBe(true)
+        expect(saved.attachmentNull).toEqual([])
+        expect(saved.attachmentUndefined).toBe(undefined)
+        expect(saved.attachmentEmpty).toEqual([])
+        expect(saved.attachmentEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldNull).toEqual([])
+        expect(saved.arrayFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldEmptyStr).toEqual(null)
+        expect(saved.optsFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldNull).toEqual(null)
+        expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
+        expect(saved.optsFieldStrKnown).toEqual("Alpha")
+      })
+  })
+
+  describe("view save", () => {
+    it("views have extra data trimmed", async () => {
+      const table = await createTable({
+        type: "table",
+        name: "orders",
+        primary: ["OrderID"],
+        schema: {
+          Country: {
+            type: FieldType.STRING,
+            name: "Country",
+          },
+          OrderID: {
+            type: FieldType.NUMBER,
+            name: "OrderID",
+          },
+          Story: {
+            type: FieldType.STRING,
+            name: "Story",
+          },
+        },
+      })
+
+      const createViewResponse = await config.createView({
+        name: uuid.v4(),
+        schema: {
+          Country: {
+            visible: true,
+          },
+          OrderID: {
+            visible: true,
+          },
+        },
+      })
+
+      const createRowResponse = await config.api.row.save(
+        createViewResponse.id,
+        {
+          OrderID: "1111",
+          Country: "Aussy",
+          Story: "aaaaa",
+        }
+      )
+
+      const row = await config.api.row.get(table._id!, createRowResponse._id!)
+      expect(row.body.Story).toBeUndefined()
+      expect(row.body).toEqual({
+        ...defaultRowFields,
+        OrderID: 1111,
+        Country: "Aussy",
+        _id: createRowResponse._id,
+        _rev: createRowResponse._rev,
+        tableId: table._id,
+      })
+    })
+  })
+
+  describe("patch", () => {
+    let otherTable: Table
+
     beforeAll(async () => {
-      await config.init()
-
-      if (dsProvider) {
-        await config.createDatasource({
-          datasource: await dsProvider.datasource(),
-        })
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+      const otherTableConfig = generateTableConfig()
+      // need a short name of table here - for relationship tests
+      otherTableConfig.name = "a"
+      otherTableConfig.schema.relationship = {
+        name: "relationship",
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        type: FieldType.LINK,
+        tableId: table._id!,
+        fieldName: "relationship",
       }
+      otherTable = await createTable(otherTableConfig)
+      // need to set the config back to the original table
+      config.table = table
     })
 
-    const generateTableConfig: () => SaveTableRequest = () => {
-      return {
-        name: uuid.v4(),
+    it("should update only the fields that are supplied", async () => {
+      const existing = await config.createRow()
+
+      const rowUsage = await getRowUsage()
+
+      const row = await config.api.row.patch(table._id!, {
+        _id: existing._id!,
+        _rev: existing._rev!,
+        tableId: table._id!,
+        name: "Updated Name",
+      })
+
+      expect(row.name).toEqual("Updated Name")
+      expect(row.description).toEqual(existing.description)
+
+      const savedRow = await loadRow(row._id!, table._id!)
+
+      expect(savedRow.body.description).toEqual(existing.description)
+      expect(savedRow.body.name).toEqual("Updated Name")
+      await assertRowUsage(rowUsage)
+    })
+
+    it("should throw an error when given improper types", async () => {
+      const existing = await config.createRow()
+      const rowUsage = await getRowUsage()
+
+      await config.api.row.patch(
+        table._id!,
+        {
+          _id: existing._id!,
+          _rev: existing._rev!,
+          tableId: table._id!,
+          name: 1,
+        },
+        { expectStatus: 400 }
+      )
+
+      await assertRowUsage(rowUsage)
+    })
+
+    it("should not overwrite links if those links are not set", async () => {
+      let linkField: FieldSchema = {
+        type: FieldType.LINK,
+        name: "",
+        fieldName: "",
+        constraints: {
+          type: "array",
+          presence: false,
+        },
+        relationshipType: RelationshipType.ONE_TO_MANY,
+        tableId: InternalTable.USER_METADATA,
+      }
+
+      let table = await config.api.table.save({
+        name: "TestTable",
         type: "table",
-        primary: ["id"],
-        primaryDisplay: "name",
         sourceType: TableSourceType.INTERNAL,
         sourceId: INTERNAL_TABLE_SOURCE_ID,
+        schema: {
+          user1: { ...linkField, name: "user1", fieldName: "user1" },
+          user2: { ...linkField, name: "user2", fieldName: "user2" },
+        },
+      })
+
+      let user1 = await config.createUser()
+      let user2 = await config.createUser()
+
+      let row = await config.api.row.save(table._id!, {
+        user1: [{ _id: user1._id }],
+        user2: [{ _id: user2._id }],
+      })
+
+      let getResp = await config.api.row.get(table._id!, row._id!)
+      expect(getResp.body.user1[0]._id).toEqual(user1._id)
+      expect(getResp.body.user2[0]._id).toEqual(user2._id)
+
+      let patchResp = await config.api.row.patch(table._id!, {
+        _id: row._id!,
+        _rev: row._rev!,
+        tableId: table._id!,
+        user1: [{ _id: user2._id }],
+      })
+      expect(patchResp.user1[0]._id).toEqual(user2._id)
+      expect(patchResp.user2[0]._id).toEqual(user2._id)
+
+      getResp = await config.api.row.get(table._id!, row._id!)
+      expect(getResp.body.user1[0]._id).toEqual(user2._id)
+      expect(getResp.body.user2[0]._id).toEqual(user2._id)
+    })
+
+    it("should be able to update relationships when both columns are same name", async () => {
+      let row = await config.api.row.save(table._id!, {
+        name: "test",
+        description: "test",
+      })
+      let row2 = await config.api.row.save(otherTable._id!, {
+        name: "test",
+        description: "test",
+        relationship: [row._id],
+      })
+      row = (await config.api.row.get(table._id!, row._id!)).body
+      expect(row.relationship.length).toBe(1)
+      const resp = await config.api.row.patch(table._id!, {
+        _id: row._id!,
+        _rev: row._rev!,
+        tableId: row.tableId!,
+        name: "test2",
+        relationship: [row2._id],
+      })
+      expect(resp.relationship.length).toBe(1)
+    })
+  })
+
+  describe("destroy", () => {
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+    })
+
+    it("should be able to delete a row", async () => {
+      const createdRow = await config.createRow()
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.delete(table._id!, [createdRow])
+      expect(res.body[0]._id).toEqual(createdRow._id)
+      await assertRowUsage(rowUsage - 1)
+    })
+  })
+
+  describe("validate", () => {
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+    })
+
+    it("should return no errors on valid row", async () => {
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.validate(table._id!, { name: "ivan" })
+
+      expect(res.valid).toBe(true)
+      expect(Object.keys(res.errors)).toEqual([])
+      await assertRowUsage(rowUsage)
+    })
+
+    it("should errors on invalid row", async () => {
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.validate(table._id!, { name: 1 })
+
+      if (isInternal) {
+        expect(res.valid).toBe(false)
+        expect(Object.keys(res.errors)).toEqual(["name"])
+      } else {
+        // Validation for external is not implemented, so it will always return valid
+        expect(res.valid).toBe(true)
+        expect(Object.keys(res.errors)).toEqual([])
+      }
+      await assertRowUsage(rowUsage)
+    })
+  })
+
+  describe("bulkDelete", () => {
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+    })
+
+    it("should be able to delete a bulk set of rows", async () => {
+      const row1 = await config.createRow()
+      const row2 = await config.createRow()
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.delete(table._id!, [row1, row2])
+
+      expect(res.body.length).toEqual(2)
+      await loadRow(row1._id!, table._id!, 404)
+      await assertRowUsage(rowUsage - 2)
+    })
+
+    it("should be able to delete a variety of row set types", async () => {
+      const [row1, row2, row3] = await Promise.all([
+        config.createRow(),
+        config.createRow(),
+        config.createRow(),
+      ])
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.delete(table._id!, [
+        row1,
+        row2._id,
+        { _id: row3._id },
+      ])
+
+      expect(res.body.length).toEqual(3)
+      await loadRow(row1._id!, table._id!, 404)
+      await assertRowUsage(rowUsage - 3)
+    })
+
+    it("should accept a valid row object and delete the row", async () => {
+      const row1 = await config.createRow()
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.delete(table._id!, row1)
+
+      expect(res.body.id).toEqual(row1._id)
+      await loadRow(row1._id!, table._id!, 404)
+      await assertRowUsage(rowUsage - 1)
+    })
+
+    it("Should ignore malformed/invalid delete requests", async () => {
+      const rowUsage = await getRowUsage()
+
+      const res = await config.api.row.delete(
+        table._id!,
+        { not: "valid" },
+        { expectStatus: 400 }
+      )
+      expect(res.body.message).toEqual("Invalid delete rows request")
+
+      const res2 = await config.api.row.delete(
+        table._id!,
+        { rows: 123 },
+        { expectStatus: 400 }
+      )
+      expect(res2.body.message).toEqual("Invalid delete rows request")
+
+      const res3 = await config.api.row.delete(table._id!, "invalid", {
+        expectStatus: 400,
+      })
+      expect(res3.body.message).toEqual("Invalid delete rows request")
+
+      await assertRowUsage(rowUsage)
+    })
+  })
+
+  // Legacy views are not available for external
+  isInternal &&
+    describe("fetchView", () => {
+      beforeEach(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should be able to fetch tables contents via 'view'", async () => {
+        const row = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.legacyView.get(table._id!)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should throw an error if view doesn't exist", async () => {
+        const rowUsage = await getRowUsage()
+
+        await config.api.legacyView.get("derp", { expectStatus: 404 })
+
+        await assertRowUsage(rowUsage)
+      })
+
+      it("should be able to run on a view", async () => {
+        const view = await config.createLegacyView({
+          tableId: table._id!,
+          name: "ViewTest",
+          filters: [],
+          schema: {},
+        })
+        const row = await config.createRow()
+        const rowUsage = await getRowUsage()
+
+        const res = await config.api.legacyView.get(view.name)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+
+        await assertRowUsage(rowUsage)
+      })
+    })
+
+  describe("fetchEnrichedRows", () => {
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+    })
+
+    it("should allow enriching some linked rows", async () => {
+      const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
+        config.getTenantId(),
+        async () => {
+          const linkedTable = await config.createLinkedTable(
+            RelationshipType.ONE_TO_MANY,
+            ["link"],
+            {
+              // Making sure that the combined table name + column name is within postgres limits
+              name: uuid.v4().replace(/-/g, "").substring(0, 16),
+              type: "table",
+              primary: ["id"],
+              primaryDisplay: "id",
+              schema: {
+                id: {
+                  type: FieldType.AUTO,
+                  name: "id",
+                  autocolumn: true,
+                  constraints: {
+                    presence: true,
+                  },
+                },
+              },
+            }
+          )
+          const firstRow = await config.createRow({
+            name: "Test Contact",
+            description: "original description",
+            tableId: table._id,
+          })
+          const secondRow = await config.createRow({
+            name: "Test 2",
+            description: "og desc",
+            link: [{ _id: firstRow._id }],
+            tableId: linkedTable._id,
+          })
+          return { linkedTable, firstRow, secondRow }
+        }
+      )
+      const rowUsage = await getRowUsage()
+
+      // test basic enrichment
+      const resBasic = await config.api.row.get(
+        linkedTable._id!,
+        secondRow._id!
+      )
+      expect(resBasic.body.link.length).toBe(1)
+      expect(resBasic.body.link[0]).toEqual({
+        _id: firstRow._id,
+        primaryDisplay: firstRow.name,
+      })
+
+      // test full enrichment
+      const resEnriched = await config.api.row.getEnriched(
+        linkedTable._id!,
+        secondRow._id!
+      )
+      expect(resEnriched.body.link.length).toBe(1)
+      expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
+      expect(resEnriched.body.link[0].name).toBe("Test Contact")
+      expect(resEnriched.body.link[0].description).toBe("original description")
+      await assertRowUsage(rowUsage)
+    })
+  })
+
+  isInternal &&
+    describe("attachments", () => {
+      beforeAll(async () => {
+        const tableConfig = generateTableConfig()
+        table = await createTable(tableConfig)
+      })
+
+      it("should allow enriching attachment rows", async () => {
+        const table = await config.createAttachmentTable()
+        const attachmentId = `${structures.uuid()}.csv`
+        const row = await config.createRow({
+          name: "test",
+          description: "test",
+          attachment: [
+            {
+              key: `${config.getAppId()}/attachments/${attachmentId}`,
+            },
+          ],
+          tableId: table._id,
+        })
+        // the environment needs configured for this
+        await setup.switchToSelfHosted(async () => {
+          return context.doInAppContext(config.getAppId(), async () => {
+            const enriched = await outputProcessing(table, [row])
+            expect((enriched as Row[])[0].attachment[0].url).toBe(
+              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+            )
+          })
+        })
+      })
+    })
+
+  describe("exportData", () => {
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      table = await createTable(tableConfig)
+    })
+
+    it("should allow exporting all columns", async () => {
+      const existing = await config.createRow()
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+      })
+      const results = JSON.parse(res.text)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      // Ensure all original columns were exported
+      expect(Object.keys(row).length).toBeGreaterThanOrEqual(
+        Object.keys(existing).length
+      )
+      Object.keys(existing).forEach(key => {
+        expect(row[key]).toEqual(existing[key])
+      })
+    })
+
+    it("should allow exporting only certain columns", async () => {
+      const existing = await config.createRow()
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+        columns: ["_id"],
+      })
+      const results = JSON.parse(res.text)
+      expect(results.length).toEqual(1)
+      const row = results[0]
+
+      // Ensure only the _id column was exported
+      expect(Object.keys(row).length).toEqual(1)
+      expect(row._id).toEqual(existing._id)
+    })
+  })
+
+  describe("view 2.0", () => {
+    async function userTable(): Promise<Table> {
+      return {
+        name: `users_${uuid.v4()}`,
+        sourceId: INTERNAL_TABLE_SOURCE_ID,
+        sourceType: TableSourceType.INTERNAL,
+        type: "table",
+        primary: ["id"],
         schema: {
           id: {
             type: FieldType.AUTO,
@@ -86,857 +952,167 @@ describe.each([["postgres", databaseTestProviders.postgres]])(
           name: {
             type: FieldType.STRING,
             name: "name",
-            constraints: {
-              type: "string",
-            },
           },
-          description: {
+          surname: {
             type: FieldType.STRING,
-            name: "description",
-            constraints: {
-              type: "string",
-            },
+            name: "surname",
+          },
+          age: {
+            type: FieldType.NUMBER,
+            name: "age",
+          },
+          address: {
+            type: FieldType.STRING,
+            name: "address",
+          },
+          jobTitle: {
+            type: FieldType.STRING,
+            name: "jobTitle",
           },
         },
       }
     }
 
-    beforeEach(async () => {
-      mocks.licenses.useCloudFree()
+    const randomRowData = () => ({
+      name: generator.first(),
+      surname: generator.last(),
+      age: generator.age(),
+      address: generator.address(),
+      jobTitle: generator.word(),
     })
 
-    const loadRow = (id: string, tbl_Id: string, status = 200) =>
-      config.api.row.get(tbl_Id, id, { expectStatus: status })
-
-    const getRowUsage = async () => {
-      const { total } = await config.doInContext(null, () =>
-        quotas.getCurrentUsageValues(
-          QuotaUsageType.STATIC,
-          StaticQuotaName.ROWS
-        )
-      )
-      return total
-    }
-
-    const assertRowUsage = async (expected: number) => {
-      const usage = await getRowUsage()
-      expect(usage).toBe(expected)
-    }
-
-    const defaultRowFields = isInternal
-      ? {
-          type: "row",
-          createdAt: timestamp,
-          updatedAt: timestamp,
-        }
-      : undefined
-
-    async function createTable(
-      cfg: Omit<SaveTableRequest, "sourceId" | "sourceType">,
-      opts?: { skipReassigning: boolean }
-    ) {
-      let table
-      if (dsProvider) {
-        table = await config.createExternalTable(cfg, opts)
-      } else {
-        table = await config.createTable(cfg, opts)
-      }
-      return table
-    }
-
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      let table = await createTable(tableConfig)
-      tableId = table._id!
-    })
-
-    describe("save, load, update", () => {
-      it("returns a success message when the row is created", async () => {
-        const rowUsage = await getRowUsage()
-
-        const res = await request
-          .post(`/api/${tableId}/rows`)
-          .send(basicRow(tableId))
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect((res as any).res.statusMessage).toEqual(
-          `${config.table!.name} saved successfully`
-        )
-        expect(res.body.name).toEqual("Test Contact")
-        expect(res.body._rev).toBeDefined()
-        await assertRowUsage(rowUsage + 1)
-      })
-
-      it("Increment row autoId per create row request", async () => {
-        const rowUsage = await getRowUsage()
-
-        const tableConfig = generateTableConfig()
-        const newTable = await createTable(
-          {
-            ...tableConfig,
-            name: "TestTableAuto",
-            schema: {
-              ...tableConfig.schema,
-              "Row ID": {
-                name: "Row ID",
-                type: FieldType.NUMBER,
-                subtype: AutoFieldSubType.AUTO_ID,
-                icon: "ri-magic-line",
-                autocolumn: true,
-                constraints: {
-                  type: "number",
-                  presence: true,
-                  numericality: {
-                    greaterThanOrEqualTo: "",
-                    lessThanOrEqualTo: "",
-                  },
-                },
-              },
-            },
-          },
-          { skipReassigning: true }
-        )
-
-        const ids = [1, 2, 3]
-
-        // Performing several create row requests should increment the autoID fields accordingly
-        const createRow = async (id: number) => {
-          const res = await config.api.row.save(newTable._id!, {
-            name: "row_" + id,
-          })
-          expect(res.name).toEqual("row_" + id)
-          expect(res._rev).toBeDefined()
-          expect(res["Row ID"]).toEqual(id)
-        }
-
-        for (let i = 0; i < ids.length; i++) {
-          await createRow(ids[i])
-        }
-
-        await assertRowUsage(rowUsage + ids.length)
-      })
-
-      it("updates a row successfully", async () => {
-        const existing = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.save(tableId, {
-          _id: existing._id,
-          _rev: existing._rev,
-          tableId,
-          name: "Updated Name",
-        })
-
-        expect(res.name).toEqual("Updated Name")
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should load a row", async () => {
-        const existing = await config.createRow()
-
-        const res = await config.api.row.get(tableId, existing._id!)
-
-        expect(res.body).toEqual({
-          ...existing,
-          ...defaultRowFields,
-        })
-      })
-
-      it("should list all rows for given tableId", async () => {
-        const table = await createTable(generateTableConfig(), {
-          skipReassigning: true,
-        })
-        const tableId = table._id!
-        const newRow = {
-          tableId,
-          name: "Second Contact",
-          description: "new",
-        }
-        const firstRow = await config.createRow({ tableId })
-        await config.createRow(newRow)
-
-        const res = await config.api.row.fetch(tableId)
-
-        expect(res.length).toBe(2)
-        expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
-        expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
-      })
-
-      it("load should return 404 when row does not exist", async () => {
-        await config.createRow()
-
-        await config.api.row.get(tableId, "1234567", {
-          expectStatus: 404,
-        })
-      })
-
-      isInternal &&
-        it("row values are coerced", async () => {
-          const str: FieldSchema = {
-            type: FieldType.STRING,
-            name: "str",
-            constraints: { type: "string", presence: false },
-          }
-          const attachment: FieldSchema = {
-            type: FieldType.ATTACHMENT,
-            name: "attachment",
-            constraints: { type: "array", presence: false },
-          }
-          const bool: FieldSchema = {
-            type: FieldType.BOOLEAN,
-            name: "boolean",
-            constraints: { type: "boolean", presence: false },
-          }
-          const number: FieldSchema = {
-            type: FieldType.NUMBER,
-            name: "str",
-            constraints: { type: "number", presence: false },
-          }
-          const datetime: FieldSchema = {
-            type: FieldType.DATETIME,
-            name: "datetime",
-            constraints: {
-              type: "string",
-              presence: false,
-              datetime: { earliest: "", latest: "" },
-            },
-          }
-          const arrayField: FieldSchema = {
-            type: FieldType.ARRAY,
-            constraints: {
-              type: "array",
-              presence: false,
-              inclusion: ["One", "Two", "Three"],
-            },
-            name: "Sample Tags",
-            sortable: false,
-          }
-          const optsField: FieldSchema = {
-            name: "Sample Opts",
-            type: FieldType.OPTIONS,
-            constraints: {
-              type: "string",
-              presence: false,
-              inclusion: ["Alpha", "Beta", "Gamma"],
-            },
-          }
-          const table = await createTable({
-            name: "TestTable2",
-            type: "table",
-            schema: {
-              name: str,
-              stringUndefined: str,
-              stringNull: str,
-              stringString: str,
-              numberEmptyString: number,
-              numberNull: number,
-              numberUndefined: number,
-              numberString: number,
-              numberNumber: number,
-              datetimeEmptyString: datetime,
-              datetimeNull: datetime,
-              datetimeUndefined: datetime,
-              datetimeString: datetime,
-              datetimeDate: datetime,
-              boolNull: bool,
-              boolEmpty: bool,
-              boolUndefined: bool,
-              boolString: bool,
-              boolBool: bool,
-              attachmentNull: attachment,
-              attachmentUndefined: attachment,
-              attachmentEmpty: attachment,
-              attachmentEmptyArrayStr: attachment,
-              arrayFieldEmptyArrayStr: arrayField,
-              arrayFieldArrayStrKnown: arrayField,
-              arrayFieldNull: arrayField,
-              arrayFieldUndefined: arrayField,
-              optsFieldEmptyStr: optsField,
-              optsFieldUndefined: optsField,
-              optsFieldNull: optsField,
-              optsFieldStrKnown: optsField,
-            },
-          })
-
-          const row = {
-            name: "Test Row",
-            stringUndefined: undefined,
-            stringNull: null,
-            stringString: "i am a string",
-            numberEmptyString: "",
-            numberNull: null,
-            numberUndefined: undefined,
-            numberString: "123",
-            numberNumber: 123,
-            datetimeEmptyString: "",
-            datetimeNull: null,
-            datetimeUndefined: undefined,
-            datetimeString: "1984-04-20T00:00:00.000Z",
-            datetimeDate: new Date("1984-04-20"),
-            boolNull: null,
-            boolEmpty: "",
-            boolUndefined: undefined,
-            boolString: "true",
-            boolBool: true,
-            tableId: table._id,
-            attachmentNull: null,
-            attachmentUndefined: undefined,
-            attachmentEmpty: "",
-            attachmentEmptyArrayStr: "[]",
-            arrayFieldEmptyArrayStr: "[]",
-            arrayFieldUndefined: undefined,
-            arrayFieldNull: null,
-            arrayFieldArrayStrKnown: "['One']",
-            optsFieldEmptyStr: "",
-            optsFieldUndefined: undefined,
-            optsFieldNull: null,
-            optsFieldStrKnown: "Alpha",
-          }
-
-          const createdRow = await config.createRow(row)
-          const id = createdRow._id!
-
-          const saved = (await loadRow(id, table._id!)).body
-
-          expect(saved.stringUndefined).toBe(undefined)
-          expect(saved.stringNull).toBe(null)
-          expect(saved.stringString).toBe("i am a string")
-          expect(saved.numberEmptyString).toBe(null)
-          expect(saved.numberNull).toBe(null)
-          expect(saved.numberUndefined).toBe(undefined)
-          expect(saved.numberString).toBe(123)
-          expect(saved.numberNumber).toBe(123)
-          expect(saved.datetimeEmptyString).toBe(null)
-          expect(saved.datetimeNull).toBe(null)
-          expect(saved.datetimeUndefined).toBe(undefined)
-          expect(saved.datetimeString).toBe(
-            new Date(row.datetimeString).toISOString()
-          )
-          expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
-          expect(saved.boolNull).toBe(null)
-          expect(saved.boolEmpty).toBe(null)
-          expect(saved.boolUndefined).toBe(undefined)
-          expect(saved.boolString).toBe(true)
-          expect(saved.boolBool).toBe(true)
-          expect(saved.attachmentNull).toEqual([])
-          expect(saved.attachmentUndefined).toBe(undefined)
-          expect(saved.attachmentEmpty).toEqual([])
-          expect(saved.attachmentEmptyArrayStr).toEqual([])
-          expect(saved.arrayFieldEmptyArrayStr).toEqual([])
-          expect(saved.arrayFieldNull).toEqual([])
-          expect(saved.arrayFieldUndefined).toEqual(undefined)
-          expect(saved.optsFieldEmptyStr).toEqual(null)
-          expect(saved.optsFieldUndefined).toEqual(undefined)
-          expect(saved.optsFieldNull).toEqual(null)
-          expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-          expect(saved.optsFieldStrKnown).toEqual("Alpha")
-        })
-    })
-
-    describe("view save", () => {
-      it("views have extra data trimmed", async () => {
-        const table = await createTable({
-          type: "table",
-          name: "orders",
-          primary: ["OrderID"],
+    describe("create", () => {
+      it("should persist a new row with only the provided view fields", async () => {
+        const table = await createTable(await userTable())
+        const view = await config.createView({
           schema: {
-            Country: {
-              type: FieldType.STRING,
-              name: "Country",
-            },
-            OrderID: {
-              type: FieldType.NUMBER,
-              name: "OrderID",
-            },
-            Story: {
-              type: FieldType.STRING,
-              name: "Story",
-            },
+            name: { visible: true },
+            surname: { visible: true },
+            address: { visible: true },
           },
         })
 
-        const createViewResponse = await config.createView({
-          name: uuid.v4(),
-          schema: {
-            Country: {
-              visible: true,
-            },
-            OrderID: {
-              visible: true,
-            },
-          },
+        const data = randomRowData()
+        const newRow = await config.api.row.save(view.id, {
+          tableId: table!._id,
+          _viewId: view.id,
+          ...data,
         })
 
-        const createRowResponse = await config.api.row.save(
-          createViewResponse.id,
-          {
-            OrderID: "1111",
-            Country: "Aussy",
-            Story: "aaaaa",
-          }
-        )
-
-        const row = await config.api.row.get(table._id!, createRowResponse._id!)
-        expect(row.body.Story).toBeUndefined()
+        const row = await config.api.row.get(table._id!, newRow._id!)
         expect(row.body).toEqual({
+          name: data.name,
+          surname: data.surname,
+          address: data.address,
+          tableId: table!._id,
+          _id: newRow._id,
+          _rev: newRow._rev,
+          id: newRow.id,
           ...defaultRowFields,
-          OrderID: 1111,
-          Country: "Aussy",
-          _id: createRowResponse._id,
-          _rev: createRowResponse._rev,
-          tableId: table._id,
         })
+        expect(row.body._viewId).toBeUndefined()
+        expect(row.body.age).toBeUndefined()
+        expect(row.body.jobTitle).toBeUndefined()
       })
     })
 
     describe("patch", () => {
-      let otherTable: Table
-
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-        const otherTableConfig = generateTableConfig()
-        // need a short name of table here - for relationship tests
-        otherTableConfig.name = "a"
-        otherTableConfig.schema.relationship = {
-          name: "relationship",
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          type: FieldType.LINK,
-          tableId: table._id!,
-          fieldName: "relationship",
-        }
-        otherTable = await createTable(otherTableConfig)
-        // need to set the config back to the original table
-        config.table = table
-      })
-
-      it("should update only the fields that are supplied", async () => {
-        const existing = await config.createRow()
-
-        const rowUsage = await getRowUsage()
-
-        const row = await config.api.row.patch(table._id!, {
-          _id: existing._id!,
-          _rev: existing._rev!,
-          tableId: table._id!,
-          name: "Updated Name",
-        })
-
-        expect(row.name).toEqual("Updated Name")
-        expect(row.description).toEqual(existing.description)
-
-        const savedRow = await loadRow(row._id!, table._id!)
-
-        expect(savedRow.body.description).toEqual(existing.description)
-        expect(savedRow.body.name).toEqual("Updated Name")
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should throw an error when given improper types", async () => {
-        const existing = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        await config.api.row.patch(
-          table._id!,
-          {
-            _id: existing._id!,
-            _rev: existing._rev!,
-            tableId: table._id!,
-            name: 1,
-          },
-          { expectStatus: 400 }
-        )
-
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should not overwrite links if those links are not set", async () => {
-        let linkField: FieldSchema = {
-          type: FieldType.LINK,
-          name: "",
-          fieldName: "",
-          constraints: {
-            type: "array",
-            presence: false,
-          },
-          relationshipType: RelationshipType.ONE_TO_MANY,
-          tableId: InternalTable.USER_METADATA,
-        }
-
-        let table = await config.api.table.save({
-          name: "TestTable",
-          type: "table",
-          sourceType: TableSourceType.INTERNAL,
-          sourceId: INTERNAL_TABLE_SOURCE_ID,
+      it("should update only the view fields for a row", async () => {
+        const table = await createTable(await userTable())
+        const tableId = table._id!
+        const view = await config.createView({
           schema: {
-            user1: { ...linkField, name: "user1", fieldName: "user1" },
-            user2: { ...linkField, name: "user2", fieldName: "user2" },
+            name: { visible: true },
+            address: { visible: true },
           },
         })
 
-        let user1 = await config.createUser()
-        let user2 = await config.createUser()
-
-        let row = await config.api.row.save(table._id!, {
-          user1: [{ _id: user1._id }],
-          user2: [{ _id: user2._id }],
+        const newRow = await config.api.row.save(view.id, {
+          tableId,
+          _viewId: view.id,
+          ...randomRowData(),
+        })
+        const newData = randomRowData()
+        await config.api.row.patch(view.id, {
+          tableId,
+          _viewId: view.id,
+          _id: newRow._id!,
+          _rev: newRow._rev!,
+          ...newData,
         })
 
-        let getResp = await config.api.row.get(table._id!, row._id!)
-        expect(getResp.body.user1[0]._id).toEqual(user1._id)
-        expect(getResp.body.user2[0]._id).toEqual(user2._id)
-
-        let patchResp = await config.api.row.patch(table._id!, {
-          _id: row._id!,
-          _rev: row._rev!,
-          tableId: table._id!,
-          user1: [{ _id: user2._id }],
+        const row = await config.api.row.get(tableId, newRow._id!)
+        expect(row.body).toEqual({
+          ...newRow,
+          name: newData.name,
+          address: newData.address,
+          _id: newRow._id,
+          _rev: expect.any(String),
+          id: newRow.id,
+          ...defaultRowFields,
         })
-        expect(patchResp.user1[0]._id).toEqual(user2._id)
-        expect(patchResp.user2[0]._id).toEqual(user2._id)
-
-        getResp = await config.api.row.get(table._id!, row._id!)
-        expect(getResp.body.user1[0]._id).toEqual(user2._id)
-        expect(getResp.body.user2[0]._id).toEqual(user2._id)
-      })
-
-      it("should be able to update relationships when both columns are same name", async () => {
-        let row = await config.api.row.save(table._id!, {
-          name: "test",
-          description: "test",
-        })
-        let row2 = await config.api.row.save(otherTable._id!, {
-          name: "test",
-          description: "test",
-          relationship: [row._id],
-        })
-        row = (await config.api.row.get(table._id!, row._id!)).body
-        expect(row.relationship.length).toBe(1)
-        const resp = await config.api.row.patch(table._id!, {
-          _id: row._id!,
-          _rev: row._rev!,
-          tableId: row.tableId!,
-          name: "test2",
-          relationship: [row2._id],
-        })
-        expect(resp.relationship.length).toBe(1)
+        expect(row.body._viewId).toBeUndefined()
+        expect(row.body.age).toBeUndefined()
+        expect(row.body.jobTitle).toBeUndefined()
       })
     })
 
     describe("destroy", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
       it("should be able to delete a row", async () => {
+        const table = await createTable(await userTable())
+        const tableId = table._id!
+        const view = await config.createView({
+          schema: {
+            name: { visible: true },
+            address: { visible: true },
+          },
+        })
+
         const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
 
-        const res = await config.api.row.delete(table._id!, [createdRow])
-        expect(res.body[0]._id).toEqual(createdRow._id)
+        await config.api.row.delete(view.id, [createdRow])
+
         await assertRowUsage(rowUsage - 1)
-      })
-    })
 
-    describe("validate", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
+        await config.api.row.get(tableId, createdRow._id!, {
+          expectStatus: 404,
+        })
       })
 
-      it("should return no errors on valid row", async () => {
+      it("should be able to delete multiple rows", async () => {
+        const table = await createTable(await userTable())
+        const tableId = table._id!
+        const view = await config.createView({
+          schema: {
+            name: { visible: true },
+            address: { visible: true },
+          },
+        })
+
+        const rows = await Promise.all([
+          config.createRow(),
+          config.createRow(),
+          config.createRow(),
+        ])
         const rowUsage = await getRowUsage()
 
-        const res = await config.api.row.validate(table._id!, { name: "ivan" })
+        await config.api.row.delete(view.id, [rows[0], rows[2]])
 
-        expect(res.valid).toBe(true)
-        expect(Object.keys(res.errors)).toEqual([])
-        await assertRowUsage(rowUsage)
-      })
-
-      it("should errors on invalid row", async () => {
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.validate(table._id!, { name: 1 })
-
-        if (isInternal) {
-          expect(res.valid).toBe(false)
-          expect(Object.keys(res.errors)).toEqual(["name"])
-        } else {
-          // Validation for external is not implemented, so it will always return valid
-          expect(res.valid).toBe(true)
-          expect(Object.keys(res.errors)).toEqual([])
-        }
-        await assertRowUsage(rowUsage)
-      })
-    })
-
-    describe("bulkDelete", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
-      it("should be able to delete a bulk set of rows", async () => {
-        const row1 = await config.createRow()
-        const row2 = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.delete(table._id!, [row1, row2])
-
-        expect(res.body.length).toEqual(2)
-        await loadRow(row1._id!, table._id!, 404)
         await assertRowUsage(rowUsage - 2)
-      })
 
-      it("should be able to delete a variety of row set types", async () => {
-        const [row1, row2, row3] = await Promise.all([
-          config.createRow(),
-          config.createRow(),
-          config.createRow(),
-        ])
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.delete(table._id!, [
-          row1,
-          row2._id,
-          { _id: row3._id },
-        ])
-
-        expect(res.body.length).toEqual(3)
-        await loadRow(row1._id!, table._id!, 404)
-        await assertRowUsage(rowUsage - 3)
-      })
-
-      it("should accept a valid row object and delete the row", async () => {
-        const row1 = await config.createRow()
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.delete(table._id!, row1)
-
-        expect(res.body.id).toEqual(row1._id)
-        await loadRow(row1._id!, table._id!, 404)
-        await assertRowUsage(rowUsage - 1)
-      })
-
-      it("Should ignore malformed/invalid delete requests", async () => {
-        const rowUsage = await getRowUsage()
-
-        const res = await config.api.row.delete(
-          table._id!,
-          { not: "valid" },
-          { expectStatus: 400 }
-        )
-        expect(res.body.message).toEqual("Invalid delete rows request")
-
-        const res2 = await config.api.row.delete(
-          table._id!,
-          { rows: 123 },
-          { expectStatus: 400 }
-        )
-        expect(res2.body.message).toEqual("Invalid delete rows request")
-
-        const res3 = await config.api.row.delete(table._id!, "invalid", {
-          expectStatus: 400,
+        await config.api.row.get(tableId, rows[0]._id!, {
+          expectStatus: 404,
         })
-        expect(res3.body.message).toEqual("Invalid delete rows request")
-
-        await assertRowUsage(rowUsage)
+        await config.api.row.get(tableId, rows[2]._id!, {
+          expectStatus: 404,
+        })
+        await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
       })
     })
 
-    // Legacy views are not available for external
-    isInternal &&
-      describe("fetchView", () => {
-        beforeEach(async () => {
-          const tableConfig = generateTableConfig()
-          table = await createTable(tableConfig)
-        })
-
-        it("should be able to fetch tables contents via 'view'", async () => {
-          const row = await config.createRow()
-          const rowUsage = await getRowUsage()
-
-          const res = await config.api.legacyView.get(table._id!)
-          expect(res.body.length).toEqual(1)
-          expect(res.body[0]._id).toEqual(row._id)
-          await assertRowUsage(rowUsage)
-        })
-
-        it("should throw an error if view doesn't exist", async () => {
-          const rowUsage = await getRowUsage()
-
-          await config.api.legacyView.get("derp", { expectStatus: 404 })
-
-          await assertRowUsage(rowUsage)
-        })
-
-        it("should be able to run on a view", async () => {
-          const view = await config.createLegacyView({
-            tableId: table._id!,
-            name: "ViewTest",
-            filters: [],
-            schema: {},
-          })
-          const row = await config.createRow()
-          const rowUsage = await getRowUsage()
-
-          const res = await config.api.legacyView.get(view.name)
-          expect(res.body.length).toEqual(1)
-          expect(res.body[0]._id).toEqual(row._id)
-
-          await assertRowUsage(rowUsage)
-        })
-      })
-
-    describe("fetchEnrichedRows", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
-      it("should allow enriching some linked rows", async () => {
-        const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
-          config.getTenantId(),
-          async () => {
-            const linkedTable = await config.createLinkedTable(
-              RelationshipType.ONE_TO_MANY,
-              ["link"],
-              {
-                // Making sure that the combined table name + column name is within postgres limits
-                name: uuid.v4().replace(/-/g, "").substring(0, 16),
-                type: "table",
-                primary: ["id"],
-                primaryDisplay: "id",
-                schema: {
-                  id: {
-                    type: FieldType.AUTO,
-                    name: "id",
-                    autocolumn: true,
-                    constraints: {
-                      presence: true,
-                    },
-                  },
-                },
-              }
-            )
-            const firstRow = await config.createRow({
-              name: "Test Contact",
-              description: "original description",
-              tableId: table._id,
-            })
-            const secondRow = await config.createRow({
-              name: "Test 2",
-              description: "og desc",
-              link: [{ _id: firstRow._id }],
-              tableId: linkedTable._id,
-            })
-            return { linkedTable, firstRow, secondRow }
-          }
-        )
-        const rowUsage = await getRowUsage()
-
-        // test basic enrichment
-        const resBasic = await config.api.row.get(
-          linkedTable._id!,
-          secondRow._id!
-        )
-        expect(resBasic.body.link.length).toBe(1)
-        expect(resBasic.body.link[0]).toEqual({
-          _id: firstRow._id,
-          primaryDisplay: firstRow.name,
-        })
-
-        // test full enrichment
-        const resEnriched = await config.api.row.getEnriched(
-          linkedTable._id!,
-          secondRow._id!
-        )
-        expect(resEnriched.body.link.length).toBe(1)
-        expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
-        expect(resEnriched.body.link[0].name).toBe("Test Contact")
-        expect(resEnriched.body.link[0].description).toBe(
-          "original description"
-        )
-        await assertRowUsage(rowUsage)
-      })
-    })
-
-    isInternal &&
-      describe("attachments", () => {
-        beforeAll(async () => {
-          const tableConfig = generateTableConfig()
-          table = await createTable(tableConfig)
-        })
-
-        it("should allow enriching attachment rows", async () => {
-          const table = await config.createAttachmentTable()
-          const attachmentId = `${structures.uuid()}.csv`
-          const row = await config.createRow({
-            name: "test",
-            description: "test",
-            attachment: [
-              {
-                key: `${config.getAppId()}/attachments/${attachmentId}`,
-              },
-            ],
-            tableId: table._id,
-          })
-          // the environment needs configured for this
-          await setup.switchToSelfHosted(async () => {
-            return context.doInAppContext(config.getAppId(), async () => {
-              const enriched = await outputProcessing(table, [row])
-              expect((enriched as Row[])[0].attachment[0].url).toBe(
-                `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
-              )
-            })
-          })
-        })
-      })
-
-    describe("exportData", () => {
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-        table = await createTable(tableConfig)
-      })
-
-      it("should allow exporting all columns", async () => {
-        const existing = await config.createRow()
-        const res = await config.api.row.exportRows(table._id!, {
-          rows: [existing._id!],
-        })
-        const results = JSON.parse(res.text)
-        expect(results.length).toEqual(1)
-        const row = results[0]
-
-        // Ensure all original columns were exported
-        expect(Object.keys(row).length).toBeGreaterThanOrEqual(
-          Object.keys(existing).length
-        )
-        Object.keys(existing).forEach(key => {
-          expect(row[key]).toEqual(existing[key])
-        })
-      })
-
-      it("should allow exporting only certain columns", async () => {
-        const existing = await config.createRow()
-        const res = await config.api.row.exportRows(table._id!, {
-          rows: [existing._id!],
-          columns: ["_id"],
-        })
-        const results = JSON.parse(res.text)
-        expect(results.length).toEqual(1)
-        const row = results[0]
-
-        // Ensure only the _id column was exported
-        expect(Object.keys(row).length).toEqual(1)
-        expect(row._id).toEqual(existing._id)
-      })
-    })
-
-    describe("view 2.0", () => {
+    describe("view search", () => {
+      const viewSchema = { age: { visible: true }, name: { visible: true } }
       async function userTable(): Promise<Table> {
         return {
           name: `users_${uuid.v4()}`,
@@ -956,1034 +1132,36 @@ describe.each([["postgres", databaseTestProviders.postgres]])(
             name: {
               type: FieldType.STRING,
               name: "name",
-            },
-            surname: {
-              type: FieldType.STRING,
-              name: "surname",
+              constraints: { type: "string" },
             },
             age: {
               type: FieldType.NUMBER,
               name: "age",
-            },
-            address: {
-              type: FieldType.STRING,
-              name: "address",
-            },
-            jobTitle: {
-              type: FieldType.STRING,
-              name: "jobTitle",
+              constraints: {},
             },
           },
         }
       }
 
-      const randomRowData = () => ({
-        name: generator.first(),
-        surname: generator.last(),
-        age: generator.age(),
-        address: generator.address(),
-        jobTitle: generator.word(),
-      })
-
-      describe("create", () => {
-        it("should persist a new row with only the provided view fields", async () => {
-          const table = await createTable(await userTable())
-          const view = await config.createView({
-            schema: {
-              name: { visible: true },
-              surname: { visible: true },
-              address: { visible: true },
-            },
-          })
-
-          const data = randomRowData()
-          const newRow = await config.api.row.save(view.id, {
-            tableId: table!._id,
-            _viewId: view.id,
-            ...data,
-          })
-
-          const row = await config.api.row.get(table._id!, newRow._id!)
-          expect(row.body).toEqual({
-            name: data.name,
-            surname: data.surname,
-            address: data.address,
-            tableId: table!._id,
-            _id: newRow._id,
-            _rev: newRow._rev,
-            id: newRow.id,
-            ...defaultRowFields,
-          })
-          expect(row.body._viewId).toBeUndefined()
-          expect(row.body.age).toBeUndefined()
-          expect(row.body.jobTitle).toBeUndefined()
-        })
-      })
-
-      describe("patch", () => {
-        it("should update only the view fields for a row", async () => {
-          const table = await createTable(await userTable())
-          const tableId = table._id!
-          const view = await config.createView({
-            schema: {
-              name: { visible: true },
-              address: { visible: true },
-            },
-          })
-
-          const newRow = await config.api.row.save(view.id, {
-            tableId,
-            _viewId: view.id,
-            ...randomRowData(),
-          })
-          const newData = randomRowData()
-          await config.api.row.patch(view.id, {
-            tableId,
-            _viewId: view.id,
-            _id: newRow._id!,
-            _rev: newRow._rev!,
-            ...newData,
-          })
-
-          const row = await config.api.row.get(tableId, newRow._id!)
-          expect(row.body).toEqual({
-            ...newRow,
-            name: newData.name,
-            address: newData.address,
-            _id: newRow._id,
-            _rev: expect.any(String),
-            id: newRow.id,
-            ...defaultRowFields,
-          })
-          expect(row.body._viewId).toBeUndefined()
-          expect(row.body.age).toBeUndefined()
-          expect(row.body.jobTitle).toBeUndefined()
-        })
-      })
-
-      describe("destroy", () => {
-        it("should be able to delete a row", async () => {
-          const table = await createTable(await userTable())
-          const tableId = table._id!
-          const view = await config.createView({
-            schema: {
-              name: { visible: true },
-              address: { visible: true },
-            },
-          })
-
-          const createdRow = await config.createRow()
-          const rowUsage = await getRowUsage()
-
-          await config.api.row.delete(view.id, [createdRow])
-
-          await assertRowUsage(rowUsage - 1)
-
-          await config.api.row.get(tableId, createdRow._id!, {
-            expectStatus: 404,
-          })
-        })
-
-        it("should be able to delete multiple rows", async () => {
-          const table = await createTable(await userTable())
-          const tableId = table._id!
-          const view = await config.createView({
-            schema: {
-              name: { visible: true },
-              address: { visible: true },
-            },
-          })
-
-          const rows = await Promise.all([
-            config.createRow(),
-            config.createRow(),
-            config.createRow(),
-          ])
-          const rowUsage = await getRowUsage()
-
-          await config.api.row.delete(view.id, [rows[0], rows[2]])
-
-          await assertRowUsage(rowUsage - 2)
-
-          await config.api.row.get(tableId, rows[0]._id!, {
-            expectStatus: 404,
-          })
-          await config.api.row.get(tableId, rows[2]._id!, {
-            expectStatus: 404,
-          })
-          await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
-        })
-      })
-
-      describe("view search", () => {
-        const viewSchema = { age: { visible: true }, name: { visible: true } }
-        async function userTable(): Promise<Table> {
-          return {
-            name: `users_${uuid.v4()}`,
-            sourceId: INTERNAL_TABLE_SOURCE_ID,
-            sourceType: TableSourceType.INTERNAL,
-            type: "table",
-            primary: ["id"],
-            schema: {
-              id: {
-                type: FieldType.AUTO,
-                name: "id",
-                autocolumn: true,
-                constraints: {
-                  presence: true,
-                },
-              },
-              name: {
-                type: FieldType.STRING,
-                name: "name",
-                constraints: { type: "string" },
-              },
-              age: {
-                type: FieldType.NUMBER,
-                name: "age",
-                constraints: {},
-              },
-            },
-          }
-        }
-
-        it("returns empty rows from view when no schema is passed", async () => {
-          const table = await createTable(await userTable())
-          const rows = await Promise.all(
-            Array.from({ length: 10 }, () =>
-              config.api.row.save(table._id!, { tableId: table._id })
-            )
-          )
-
-          const createViewResponse = await config.createView()
-          const response = await config.api.viewV2.search(createViewResponse.id)
-
-          expect(response.body.rows).toHaveLength(10)
-          expect(response.body).toEqual({
-            rows: expect.arrayContaining(
-              rows.map(r => ({
-                _viewId: createViewResponse.id,
-                tableId: table._id,
-                _id: r._id,
-                _rev: r._rev,
-                ...defaultRowFields,
-              }))
-            ),
-            ...(isInternal
-              ? {}
-              : {
-                  hasNextPage: false,
-                  bookmark: null,
-                }),
-          })
-        })
-
-        it("searching respects the view filters", async () => {
-          const table = await createTable(await userTable())
-
-          await Promise.all(
-            Array.from({ length: 10 }, () =>
-              config.api.row.save(table._id!, {
-                tableId: table._id,
-                name: generator.name(),
-                age: generator.integer({ min: 10, max: 30 }),
-              })
-            )
-          )
-
-          const expectedRows = await Promise.all(
-            Array.from({ length: 5 }, () =>
-              config.api.row.save(table._id!, {
-                tableId: table._id,
-                name: generator.name(),
-                age: 40,
-              })
-            )
-          )
-
-          const createViewResponse = await config.createView({
-            query: [
-              { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 },
-            ],
-            schema: viewSchema,
-          })
-
-          const response = await config.api.viewV2.search(createViewResponse.id)
-
-          expect(response.body.rows).toHaveLength(5)
-          expect(response.body).toEqual({
-            rows: expect.arrayContaining(
-              expectedRows.map(r => ({
-                _viewId: createViewResponse.id,
-                tableId: table._id,
-                name: r.name,
-                age: r.age,
-                _id: r._id,
-                _rev: r._rev,
-                ...defaultRowFields,
-              }))
-            ),
-            ...(isInternal
-              ? {}
-              : {
-                  hasNextPage: false,
-                  bookmark: null,
-                }),
-          })
-        })
-
-        const sortTestOptions: [
-          {
-            field: string
-            order?: SortOrder
-            type?: SortType
-          },
-          string[]
-        ][] = [
-          [
-            {
-              field: "name",
-              order: SortOrder.ASCENDING,
-              type: SortType.STRING,
-            },
-            ["Alice", "Bob", "Charly", "Danny"],
-          ],
-          [
-            {
-              field: "name",
-            },
-            ["Alice", "Bob", "Charly", "Danny"],
-          ],
-          [
-            {
-              field: "name",
-              order: SortOrder.DESCENDING,
-            },
-            ["Danny", "Charly", "Bob", "Alice"],
-          ],
-          [
-            {
-              field: "name",
-              order: SortOrder.DESCENDING,
-              type: SortType.STRING,
-            },
-            ["Danny", "Charly", "Bob", "Alice"],
-          ],
-          [
-            {
-              field: "age",
-              order: SortOrder.ASCENDING,
-              type: SortType.number,
-            },
-            ["Danny", "Alice", "Charly", "Bob"],
-          ],
-          [
-            {
-              field: "age",
-              order: SortOrder.ASCENDING,
-            },
-            ["Danny", "Alice", "Charly", "Bob"],
-          ],
-          [
-            {
-              field: "age",
-              order: SortOrder.DESCENDING,
-            },
-            ["Bob", "Charly", "Alice", "Danny"],
-          ],
-          [
-            {
-              field: "age",
-              order: SortOrder.DESCENDING,
-              type: SortType.number,
-            },
-            ["Bob", "Charly", "Alice", "Danny"],
-          ],
-        ]
-
-        describe("sorting", () => {
-          beforeAll(async () => {
-            const table = await createTable(await userTable())
-            const users = [
-              { name: "Alice", age: 25 },
-              { name: "Bob", age: 30 },
-              { name: "Charly", age: 27 },
-              { name: "Danny", age: 15 },
-            ]
-            await Promise.all(
-              users.map(u =>
-                config.api.row.save(table._id!, {
-                  tableId: table._id,
-                  ...u,
-                })
-              )
-            )
-          })
-
-          it.each(sortTestOptions)(
-            "allow sorting (%s)",
-            async (sortParams, expected) => {
-              const createViewResponse = await config.createView({
-                sort: sortParams,
-                schema: viewSchema,
-              })
-
-              const response = await config.api.viewV2.search(
-                createViewResponse.id
-              )
-
-              expect(response.body.rows).toHaveLength(4)
-              expect(response.body.rows).toEqual(
-                expected.map(name => expect.objectContaining({ name }))
-              )
-            }
-          )
-
-          it.each(sortTestOptions)(
-            "allow override the default view sorting (%s)",
-            async (sortParams, expected) => {
-              const createViewResponse = await config.createView({
-                sort: {
-                  field: "name",
-                  order: SortOrder.ASCENDING,
-                  type: SortType.STRING,
-                },
-                schema: viewSchema,
-              })
-
-              const response = await config.api.viewV2.search(
-                createViewResponse.id,
-                {
-                  sort: sortParams.field,
-                  sortOrder: sortParams.order,
-                  sortType: sortParams.type,
-                  query: {},
-                }
-              )
-
-              expect(response.body.rows).toHaveLength(4)
-              expect(response.body.rows).toEqual(
-                expected.map(name => expect.objectContaining({ name }))
-              )
-            }
-          )
-        })
-
-        it("when schema is defined, defined columns and row attributes are returned", async () => {
-          const table = await createTable(await userTable())
-          const rows = await Promise.all(
-            Array.from({ length: 10 }, () =>
-              config.api.row.save(table._id!, {
-                tableId: table._id,
-                name: generator.name(),
-                age: generator.age(),
-              })
-            )
-          )
-
-          const view = await config.createView({
-            schema: { name: { visible: true } },
-          })
-          const response = await config.api.viewV2.search(view.id)
-
-          expect(response.body.rows).toHaveLength(10)
-          expect(response.body.rows).toEqual(
-            expect.arrayContaining(
-              rows.map(r => ({
-                ...(isInternal
-                  ? expectAnyInternalColsAttributes
-                  : expectAnyExternalColsAttributes),
-                _viewId: view.id,
-                name: r.name,
-              }))
-            )
-          )
-        })
-
-        it("views without data can be returned", async () => {
-          const table = await createTable(await userTable())
-
-          const createViewResponse = await config.createView()
-          const response = await config.api.viewV2.search(createViewResponse.id)
-
-          expect(response.body.rows).toHaveLength(0)
-        })
-
-        it("respects the limit parameter", async () => {
-          await createTable(await userTable())
-          await Promise.all(
-            Array.from({ length: 10 }, () => config.createRow())
-          )
-
-          const limit = generator.integer({ min: 1, max: 8 })
-
-          const createViewResponse = await config.createView()
-          const response = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              limit,
-              query: {},
-            }
-          )
-
-          expect(response.body.rows).toHaveLength(limit)
-        })
-
-        it("can handle pagination", async () => {
-          await createTable(await userTable())
-          await Promise.all(
-            Array.from({ length: 10 }, () => config.createRow())
-          )
-
-          const createViewResponse = await config.createView()
-          const allRows = (
-            await config.api.viewV2.search(createViewResponse.id)
-          ).body.rows
-
-          const firstPageResponse = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              paginate: true,
-              limit: 4,
-              query: {},
-            }
-          )
-          expect(firstPageResponse.body).toEqual({
-            rows: expect.arrayContaining(allRows.slice(0, 4)),
-            totalRows: isInternal ? 10 : undefined,
-            hasNextPage: true,
-            bookmark: expect.anything(),
-          })
-
-          const secondPageResponse = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              paginate: true,
-              limit: 4,
-              bookmark: firstPageResponse.body.bookmark,
-
-              query: {},
-            }
-          )
-          expect(secondPageResponse.body).toEqual({
-            rows: expect.arrayContaining(allRows.slice(4, 8)),
-            totalRows: isInternal ? 10 : undefined,
-            hasNextPage: true,
-            bookmark: expect.anything(),
-          })
-
-          const lastPageResponse = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              paginate: true,
-              limit: 4,
-              bookmark: secondPageResponse.body.bookmark,
-              query: {},
-            }
-          )
-          expect(lastPageResponse.body).toEqual({
-            rows: expect.arrayContaining(allRows.slice(8)),
-            totalRows: isInternal ? 10 : undefined,
-            hasNextPage: false,
-            bookmark: expect.anything(),
-          })
-        })
-
-        isInternal &&
-          it("doesn't allow creating in user table", async () => {
-            const userTableId = InternalTable.USER_METADATA
-            const response = await config.api.row.save(
-              userTableId,
-              {
-                tableId: userTableId,
-                firstName: "Joe",
-                lastName: "Joe",
-                email: "joe@joe.com",
-                roles: {},
-              },
-              { expectStatus: 400 }
-            )
-            expect(response.message).toBe("Cannot create new user entry.")
-          })
-
-        describe("permissions", () => {
-          let viewId: string
-          let tableId: string
-
-          beforeAll(async () => {
-            await createTable(await userTable())
-            await Promise.all(
-              Array.from({ length: 10 }, () => config.createRow())
-            )
-
-            const createViewResponse = await config.createView()
-
-            tableId = table._id!
-            viewId = createViewResponse.id
-          })
-
-          beforeEach(() => {
-            mocks.licenses.useViewPermissions()
-          })
-
-          it("does not allow public users to fetch by default", async () => {
-            await config.publish()
-            await config.api.viewV2.search(viewId, undefined, {
-              expectStatus: 403,
-              usePublicUser: true,
-            })
-          })
-
-          it("allow public users to fetch when permissions are explicit", async () => {
-            await config.api.permission.set({
-              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-              level: PermissionLevel.READ,
-              resourceId: viewId,
-            })
-            await config.publish()
-
-            const response = await config.api.viewV2.search(viewId, undefined, {
-              usePublicUser: true,
-            })
-
-            expect(response.body.rows).toHaveLength(10)
-          })
-
-          it("allow public users to fetch when permissions are inherited", async () => {
-            await config.api.permission.set({
-              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-              level: PermissionLevel.READ,
-              resourceId: tableId,
-            })
-            await config.publish()
-
-            const response = await config.api.viewV2.search(viewId, undefined, {
-              usePublicUser: true,
-            })
-
-            expect(response.body.rows).toHaveLength(10)
-          })
-
-          it("respects inherited permissions, not allowing not public views from public tables", async () => {
-            await config.api.permission.set({
-              roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
-              level: PermissionLevel.READ,
-              resourceId: tableId,
-            })
-            await config.api.permission.set({
-              roleId: roles.BUILTIN_ROLE_IDS.POWER,
-              level: PermissionLevel.READ,
-              resourceId: viewId,
-            })
-            await config.publish()
-
-            await config.api.viewV2.search(viewId, undefined, {
-              usePublicUser: true,
-              expectStatus: 403,
-            })
-          })
-        })
-      })
-    })
-
-    let o2mTable: Table
-    let m2mTable: Table
-    beforeAll(async () => {
-      o2mTable = await createTable(
-        { ...generateTableConfig(), name: "o2m" },
-        {
-          skipReassigning: true,
-        }
-      )
-      m2mTable = await createTable(
-        { ...generateTableConfig(), name: "m2m" },
-        {
-          skipReassigning: true,
-        }
-      )
-    })
-
-    describe.each([
-      [
-        "relationship fields",
-        (): Record<string, FieldSchema> => ({
-          user: {
-            name: "user",
-            relationshipType: RelationshipType.ONE_TO_MANY,
-            type: FieldType.LINK,
-            tableId: o2mTable._id!,
-            fieldName: "fk_o2m",
-          },
-          users: {
-            name: "users",
-            relationshipType: RelationshipType.MANY_TO_MANY,
-            type: FieldType.LINK,
-            tableId: m2mTable._id!,
-            fieldName: "fk_m2m",
-          },
-        }),
-        (tableId: string) =>
-          config.api.row.save(tableId, {
-            name: uuid.v4(),
-            description: generator.paragraph(),
-            tableId,
-          }),
-        (row: Row) => ({
-          _id: row._id,
-          primaryDisplay: row.name,
-        }),
-      ],
-      [
-        "bb reference fields",
-        (): Record<string, FieldSchema> => ({
-          user: {
-            name: "user",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
-          },
-          users: {
-            name: "users",
-            type: FieldType.BB_REFERENCE,
-            subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
-          },
-        }),
-        () => config.createUser(),
-        (row: Row) => ({
-          _id: row._id,
-          primaryDisplay: row.email,
-          email: row.email,
-          firstName: row.firstName,
-          lastName: row.lastName,
-        }),
-      ],
-    ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
-      let tableId: string
-      let o2mData: Row[]
-      let m2mData: Row[]
-
-      beforeAll(async () => {
-        const tableConfig = generateTableConfig()
-
-        if (config.datasource) {
-          tableConfig.sourceId = config.datasource._id!
-          if (config.datasource.plus) {
-            tableConfig.sourceType = TableSourceType.EXTERNAL
-          }
-        }
-        const table = await config.api.table.save({
-          ...tableConfig,
-          schema: {
-            ...tableConfig.schema,
-            ...relSchema(),
-          },
-        })
-        tableId = table._id!
-
-        o2mData = [
-          await dataGenerator(o2mTable._id!),
-          await dataGenerator(o2mTable._id!),
-          await dataGenerator(o2mTable._id!),
-          await dataGenerator(o2mTable._id!),
-        ]
-
-        m2mData = [
-          await dataGenerator(m2mTable._id!),
-          await dataGenerator(m2mTable._id!),
-          await dataGenerator(m2mTable._id!),
-          await dataGenerator(m2mTable._id!),
-        ]
-      })
-
-      it("can save a row when relationship fields are empty", async () => {
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        expect(row).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          _id: expect.any(String),
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          type: isInternal ? "row" : undefined,
-        })
-      })
-
-      it("can save a row with a single relationship field", async () => {
-        const user = _.sample(o2mData)!
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          user: [user],
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        expect(row).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          user: [user].map(u => resultMapper(u)),
-          _id: expect.any(String),
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          type: isInternal ? "row" : undefined,
-          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
-        })
-      })
-
-      it("can save a row with a multiple relationship field", async () => {
-        const selectedUsers = _.sampleSize(m2mData, 2)
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: selectedUsers,
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        expect(row).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          users: expect.arrayContaining(
-            selectedUsers.map(u => resultMapper(u))
-          ),
-          _id: expect.any(String),
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          type: isInternal ? "row" : undefined,
-        })
-      })
-
-      it("can retrieve rows with no populated relationships", async () => {
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        const { body: retrieved } = await config.api.row.get(tableId, row._id!)
-        expect(retrieved).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          user: undefined,
-          users: undefined,
-          _id: row._id,
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          ...defaultRowFields,
-        })
-      })
-
-      it("can retrieve rows with populated relationships", async () => {
-        const user1 = _.sample(o2mData)!
-        const [user2, user3] = _.sampleSize(m2mData, 2)
-
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [user2, user3],
-          user: [user1],
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        const { body: retrieved } = await config.api.row.get(tableId, row._id!)
-        expect(retrieved).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          user: expect.arrayContaining([user1].map(u => resultMapper(u))),
-          users: expect.arrayContaining(
-            [user2, user3].map(u => resultMapper(u))
-          ),
-          _id: row._id,
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
-          ...defaultRowFields,
-        })
-      })
-
-      it("can update an existing populated row", async () => {
-        const user = _.sample(o2mData)!
-        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
-
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [users1, users2],
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        const updatedRow = await config.api.row.save(tableId, {
-          ...row,
-          user: [user],
-          users: [users3, users1],
-        })
-        expect(updatedRow).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          user: expect.arrayContaining([user].map(u => resultMapper(u))),
-          users: expect.arrayContaining(
-            [users3, users1].map(u => resultMapper(u))
-          ),
-          _id: row._id,
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          type: isInternal ? "row" : undefined,
-          [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
-        })
-      })
-
-      it("can wipe an existing populated relationships in row", async () => {
-        const [user1, user2] = _.sampleSize(m2mData, 2)
-
-        const rowData = {
-          ...basicRow(tableId),
-          name: generator.name(),
-          description: generator.name(),
-          users: [user1, user2],
-        }
-        const row = await config.api.row.save(tableId, rowData)
-
-        const updatedRow = await config.api.row.save(tableId, {
-          ...row,
-          user: null,
-          users: null,
-        })
-        expect(updatedRow).toEqual({
-          name: rowData.name,
-          description: rowData.description,
-          tableId,
-          _id: row._id,
-          _rev: expect.any(String),
-          id: isInternal ? undefined : expect.any(Number),
-          type: isInternal ? "row" : undefined,
-        })
-      })
-
-      it("fetch all will populate the relationships", async () => {
-        const [user1] = _.sampleSize(o2mData, 1)
-        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
-
-        const rows: {
-          name: string
-          description: string
-          user?: Row[]
-          users?: Row[]
-          tableId: string
-        }[] = [
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            users: [users1, users2],
-          },
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            user: [user1],
-            users: [users1, users3],
-          },
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            users: [users3],
-          },
-        ]
-
-        await config.api.row.save(tableId, rows[0])
-        await config.api.row.save(tableId, rows[1])
-        await config.api.row.save(tableId, rows[2])
-
-        const res = await config.api.row.fetch(tableId)
-
-        expect(res).toEqual(
-          expect.arrayContaining(
-            rows.map(r => ({
-              name: r.name,
-              description: r.description,
-              tableId,
-              user: r.user?.map(u => resultMapper(u)),
-              users: r.users?.length
-                ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-                : undefined,
-              _id: expect.any(String),
-              _rev: expect.any(String),
-              id: isInternal ? undefined : expect.any(Number),
-              [`fk_${o2mTable.name}_fk_o2m`]:
-                isInternal || !r.user?.length ? undefined : r.user[0].id,
-              ...defaultRowFields,
-            }))
+      it("returns empty rows from view when no schema is passed", async () => {
+        const table = await createTable(await userTable())
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, { tableId: table._id })
           )
         )
-      })
 
-      it("search all will populate the relationships", async () => {
-        const [user1] = _.sampleSize(o2mData, 1)
-        const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+        const createViewResponse = await config.createView()
+        const response = await config.api.viewV2.search(createViewResponse.id)
 
-        const rows: {
-          name: string
-          description: string
-          user?: Row[]
-          users?: Row[]
-          tableId: string
-        }[] = [
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            users: [users1, users2],
-          },
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            user: [user1],
-            users: [users1, users3],
-          },
-          {
-            ...basicRow(tableId),
-            name: generator.name(),
-            description: generator.name(),
-            users: [users3],
-          },
-        ]
-
-        await config.api.row.save(tableId, rows[0])
-        await config.api.row.save(tableId, rows[1])
-        await config.api.row.save(tableId, rows[2])
-
-        const res = await config.api.row.search(tableId)
-
-        expect(res).toEqual({
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body).toEqual({
           rows: expect.arrayContaining(
             rows.map(r => ({
-              name: r.name,
-              description: r.description,
-              tableId,
-              user: r.user?.map(u => resultMapper(u)),
-              users: r.users?.length
-                ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
-                : undefined,
-              _id: expect.any(String),
-              _rev: expect.any(String),
-              id: isInternal ? undefined : expect.any(Number),
-              [`fk_${o2mTable.name}_fk_o2m`]:
-                isInternal || !r.user?.length ? undefined : r.user[0].id,
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              _id: r._id,
+              _rev: r._rev,
               ...defaultRowFields,
             }))
           ),
@@ -1995,59 +1173,908 @@ describe.each([["postgres", databaseTestProviders.postgres]])(
               }),
         })
       })
-    })
 
-    describe("Formula fields", () => {
-      let relationshipTable: Table, tableId: string, relatedRow: Row
+      it("searching respects the view filters", async () => {
+        const table = await createTable(await userTable())
 
-      beforeAll(async () => {
-        const otherTableId = config.table!._id!
-        const cfg = generateTableConfig()
-        relationshipTable = await config.createLinkedTable(
-          RelationshipType.ONE_TO_MANY,
-          ["links"],
+        await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.integer({ min: 10, max: 30 }),
+            })
+          )
+        )
+
+        const expectedRows = await Promise.all(
+          Array.from({ length: 5 }, () =>
+            config.api.row.save(table._id!, {
+              tableId: table._id,
+              name: generator.name(),
+              age: 40,
+            })
+          )
+        )
+
+        const createViewResponse = await config.createView({
+          query: [
+            { operator: SearchQueryOperators.EQUAL, field: "age", value: 40 },
+          ],
+          schema: viewSchema,
+        })
+
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(5)
+        expect(response.body).toEqual({
+          rows: expect.arrayContaining(
+            expectedRows.map(r => ({
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              name: r.name,
+              age: r.age,
+              _id: r._id,
+              _rev: r._rev,
+              ...defaultRowFields,
+            }))
+          ),
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
+        })
+      })
+
+      const sortTestOptions: [
+        {
+          field: string
+          order?: SortOrder
+          type?: SortType
+        },
+        string[]
+      ][] = [
+        [
           {
-            ...cfg,
-            // needs to be a short name
-            name: "b",
-            schema: {
-              ...cfg.schema,
-              formula: {
-                name: "formula",
-                type: FieldType.FORMULA,
-                formula: "{{ links.0.name }}",
-                formulaType: FormulaType.DYNAMIC,
-              },
-            },
+            field: "name",
+            order: SortOrder.ASCENDING,
+            type: SortType.STRING,
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+          },
+          ["Alice", "Bob", "Charly", "Danny"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "name",
+            order: SortOrder.DESCENDING,
+            type: SortType.STRING,
+          },
+          ["Danny", "Charly", "Bob", "Alice"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+            type: SortType.number,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.ASCENDING,
+          },
+          ["Danny", "Alice", "Charly", "Bob"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+        [
+          {
+            field: "age",
+            order: SortOrder.DESCENDING,
+            type: SortType.number,
+          },
+          ["Bob", "Charly", "Alice", "Danny"],
+        ],
+      ]
+
+      describe("sorting", () => {
+        beforeAll(async () => {
+          const table = await createTable(await userTable())
+          const users = [
+            { name: "Alice", age: 25 },
+            { name: "Bob", age: 30 },
+            { name: "Charly", age: 27 },
+            { name: "Danny", age: 15 },
+          ]
+          await Promise.all(
+            users.map(u =>
+              config.api.row.save(table._id!, {
+                tableId: table._id,
+                ...u,
+              })
+            )
+          )
+        })
+
+        it.each(sortTestOptions)(
+          "allow sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: sortParams,
+              schema: viewSchema,
+            })
+
+            const response = await config.api.viewV2.search(
+              createViewResponse.id
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
           }
         )
 
-        tableId = relationshipTable._id!
+        it.each(sortTestOptions)(
+          "allow override the default view sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: {
+                field: "name",
+                order: SortOrder.ASCENDING,
+                type: SortType.STRING,
+              },
+              schema: viewSchema,
+            })
 
-        relatedRow = await config.api.row.save(otherTableId, {
-          name: generator.word(),
-          description: generator.paragraph(),
+            const response = await config.api.viewV2.search(
+              createViewResponse.id,
+              {
+                sort: sortParams.field,
+                sortOrder: sortParams.order,
+                sortType: sortParams.type,
+                query: {},
+              }
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
+          }
+        )
+      })
+
+      it("when schema is defined, defined columns and row attributes are returned", async () => {
+        const table = await createTable(await userTable())
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.age(),
+            })
+          )
+        )
+
+        const view = await config.createView({
+          schema: { name: { visible: true } },
         })
-        await config.api.row.save(tableId, {
-          name: generator.word(),
-          description: generator.paragraph(),
-          tableId,
-          links: [relatedRow._id],
+        const response = await config.api.viewV2.search(view.id)
+
+        expect(response.body.rows).toHaveLength(10)
+        expect(response.body.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...(isInternal
+                ? expectAnyInternalColsAttributes
+                : expectAnyExternalColsAttributes),
+              _viewId: view.id,
+              name: r.name,
+            }))
+          )
+        )
+      })
+
+      it("views without data can be returned", async () => {
+        const table = await createTable(await userTable())
+
+        const createViewResponse = await config.createView()
+        const response = await config.api.viewV2.search(createViewResponse.id)
+
+        expect(response.body.rows).toHaveLength(0)
+      })
+
+      it("respects the limit parameter", async () => {
+        await createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
+
+        const limit = generator.integer({ min: 1, max: 8 })
+
+        const createViewResponse = await config.createView()
+        const response = await config.api.viewV2.search(createViewResponse.id, {
+          limit,
+          query: {},
+        })
+
+        expect(response.body.rows).toHaveLength(limit)
+      })
+
+      it("can handle pagination", async () => {
+        await createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
+
+        const createViewResponse = await config.createView()
+        const allRows = (await config.api.viewV2.search(createViewResponse.id))
+          .body.rows
+
+        const firstPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            query: {},
+          }
+        )
+        expect(firstPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(0, 4)),
+          totalRows: isInternal ? 10 : undefined,
+          hasNextPage: true,
+          bookmark: expect.anything(),
+        })
+
+        const secondPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: firstPageResponse.body.bookmark,
+
+            query: {},
+          }
+        )
+        expect(secondPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(4, 8)),
+          totalRows: isInternal ? 10 : undefined,
+          hasNextPage: true,
+          bookmark: expect.anything(),
+        })
+
+        const lastPageResponse = await config.api.viewV2.search(
+          createViewResponse.id,
+          {
+            paginate: true,
+            limit: 4,
+            bookmark: secondPageResponse.body.bookmark,
+            query: {},
+          }
+        )
+        expect(lastPageResponse.body).toEqual({
+          rows: expect.arrayContaining(allRows.slice(8)),
+          totalRows: isInternal ? 10 : undefined,
+          hasNextPage: false,
+          bookmark: expect.anything(),
         })
       })
 
-      it("should be able to search for rows containing formulas", async () => {
-        const { rows } = await config.api.row.search(tableId)
-        expect(rows.length).toBe(1)
-        expect(rows[0].links.length).toBe(1)
-        const row = rows[0]
-        expect(row.formula).toBe(relatedRow.name)
+      isInternal &&
+        it("doesn't allow creating in user table", async () => {
+          const userTableId = InternalTable.USER_METADATA
+          const response = await config.api.row.save(
+            userTableId,
+            {
+              tableId: userTableId,
+              firstName: "Joe",
+              lastName: "Joe",
+              email: "joe@joe.com",
+              roles: {},
+            },
+            { expectStatus: 400 }
+          )
+          expect(response.message).toBe("Cannot create new user entry.")
+        })
+
+      describe("permissions", () => {
+        let viewId: string
+        let tableId: string
+
+        beforeAll(async () => {
+          await createTable(await userTable())
+          await Promise.all(
+            Array.from({ length: 10 }, () => config.createRow())
+          )
+
+          const createViewResponse = await config.createView()
+
+          tableId = table._id!
+          viewId = createViewResponse.id
+        })
+
+        beforeEach(() => {
+          mocks.licenses.useViewPermissions()
+        })
+
+        it("does not allow public users to fetch by default", async () => {
+          await config.publish()
+          await config.api.viewV2.search(viewId, undefined, {
+            expectStatus: 403,
+            usePublicUser: true,
+          })
+        })
+
+        it("allow public users to fetch when permissions are explicit", async () => {
+          await config.api.permission.set({
+            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+            level: PermissionLevel.READ,
+            resourceId: viewId,
+          })
+          await config.publish()
+
+          const response = await config.api.viewV2.search(viewId, undefined, {
+            usePublicUser: true,
+          })
+
+          expect(response.body.rows).toHaveLength(10)
+        })
+
+        it("allow public users to fetch when permissions are inherited", async () => {
+          await config.api.permission.set({
+            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+            level: PermissionLevel.READ,
+            resourceId: tableId,
+          })
+          await config.publish()
+
+          const response = await config.api.viewV2.search(viewId, undefined, {
+            usePublicUser: true,
+          })
+
+          expect(response.body.rows).toHaveLength(10)
+        })
+
+        it("respects inherited permissions, not allowing not public views from public tables", async () => {
+          await config.api.permission.set({
+            roleId: roles.BUILTIN_ROLE_IDS.PUBLIC,
+            level: PermissionLevel.READ,
+            resourceId: tableId,
+          })
+          await config.api.permission.set({
+            roleId: roles.BUILTIN_ROLE_IDS.POWER,
+            level: PermissionLevel.READ,
+            resourceId: viewId,
+          })
+          await config.publish()
+
+          await config.api.viewV2.search(viewId, undefined, {
+            usePublicUser: true,
+            expectStatus: 403,
+          })
+        })
+      })
+    })
+  })
+
+  let o2mTable: Table
+  let m2mTable: Table
+  beforeAll(async () => {
+    o2mTable = await createTable(
+      { ...generateTableConfig(), name: "o2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+    m2mTable = await createTable(
+      { ...generateTableConfig(), name: "m2m" },
+      {
+        skipReassigning: true,
+      }
+    )
+  })
+
+  describe.each([
+    [
+      "relationship fields",
+      (): Record<string, FieldSchema> => ({
+        user: {
+          name: "user",
+          relationshipType: RelationshipType.ONE_TO_MANY,
+          type: FieldType.LINK,
+          tableId: o2mTable._id!,
+          fieldName: "fk_o2m",
+        },
+        users: {
+          name: "users",
+          relationshipType: RelationshipType.MANY_TO_MANY,
+          type: FieldType.LINK,
+          tableId: m2mTable._id!,
+          fieldName: "fk_m2m",
+        },
+      }),
+      (tableId: string) =>
+        config.api.row.save(tableId, {
+          name: uuid.v4(),
+          description: generator.paragraph(),
+          tableId,
+        }),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.name,
+      }),
+    ],
+    [
+      "bb reference fields",
+      (): Record<string, FieldSchema> => ({
+        user: {
+          name: "user",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USER,
+        },
+        users: {
+          name: "users",
+          type: FieldType.BB_REFERENCE,
+          subtype: FieldTypeSubtypes.BB_REFERENCE.USERS,
+        },
+      }),
+      () => config.createUser(),
+      (row: Row) => ({
+        _id: row._id,
+        primaryDisplay: row.email,
+        email: row.email,
+        firstName: row.firstName,
+        lastName: row.lastName,
+      }),
+    ],
+  ])("links - %s", (__, relSchema, dataGenerator, resultMapper) => {
+    let tableId: string
+    let o2mData: Row[]
+    let m2mData: Row[]
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+
+      if (config.datasource) {
+        tableConfig.sourceId = config.datasource._id!
+        if (config.datasource.plus) {
+          tableConfig.sourceType = TableSourceType.EXTERNAL
+        }
+      }
+      const table = await config.api.table.save({
+        ...tableConfig,
+        schema: {
+          ...tableConfig.schema,
+          ...relSchema(),
+        },
+      })
+      tableId = table._id!
+
+      o2mData = [
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+        await dataGenerator(o2mTable._id!),
+      ]
+
+      m2mData = [
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+        await dataGenerator(m2mTable._id!),
+      ]
+    })
+
+    it("can save a row when relationship fields are empty", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
       })
     })
 
-    describe("Formula JS protection", () => {
-      it("should time out JS execution if a single cell takes too long", async () => {
-        await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
+    it("can save a row with a single relationship field", async () => {
+      const user = _.sample(o2mData)!
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        user: [user],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: [user].map(u => resultMapper(u)),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can save a row with a multiple relationship field", async () => {
+      const selectedUsers = _.sampleSize(m2mData, 2)
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: selectedUsers,
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      expect(row).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        users: expect.arrayContaining(selectedUsers.map(u => resultMapper(u))),
+        _id: expect.any(String),
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("can retrieve rows with no populated relationships", async () => {
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: undefined,
+        users: undefined,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        ...defaultRowFields,
+      })
+    })
+
+    it("can retrieve rows with populated relationships", async () => {
+      const user1 = _.sample(o2mData)!
+      const [user2, user3] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user2, user3],
+        user: [user1],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const { body: retrieved } = await config.api.row.get(tableId, row._id!)
+      expect(retrieved).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user1].map(u => resultMapper(u))),
+        users: expect.arrayContaining([user2, user3].map(u => resultMapper(u))),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user1.id,
+        ...defaultRowFields,
+      })
+    })
+
+    it("can update an existing populated row", async () => {
+      const user = _.sample(o2mData)!
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [users1, users2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: [user],
+        users: [users3, users1],
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        user: expect.arrayContaining([user].map(u => resultMapper(u))),
+        users: expect.arrayContaining(
+          [users3, users1].map(u => resultMapper(u))
+        ),
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+        [`fk_${o2mTable.name}_fk_o2m`]: isInternal ? undefined : user.id,
+      })
+    })
+
+    it("can wipe an existing populated relationships in row", async () => {
+      const [user1, user2] = _.sampleSize(m2mData, 2)
+
+      const rowData = {
+        ...basicRow(tableId),
+        name: generator.name(),
+        description: generator.name(),
+        users: [user1, user2],
+      }
+      const row = await config.api.row.save(tableId, rowData)
+
+      const updatedRow = await config.api.row.save(tableId, {
+        ...row,
+        user: null,
+        users: null,
+      })
+      expect(updatedRow).toEqual({
+        name: rowData.name,
+        description: rowData.description,
+        tableId,
+        _id: row._id,
+        _rev: expect.any(String),
+        id: isInternal ? undefined : expect.any(Number),
+        type: isInternal ? "row" : undefined,
+      })
+    })
+
+    it("fetch all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.fetch(tableId)
+
+      expect(res).toEqual(
+        expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        )
+      )
+    })
+
+    it("search all will populate the relationships", async () => {
+      const [user1] = _.sampleSize(o2mData, 1)
+      const [users1, users2, users3] = _.sampleSize(m2mData, 3)
+
+      const rows: {
+        name: string
+        description: string
+        user?: Row[]
+        users?: Row[]
+        tableId: string
+      }[] = [
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users1, users2],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          user: [user1],
+          users: [users1, users3],
+        },
+        {
+          ...basicRow(tableId),
+          name: generator.name(),
+          description: generator.name(),
+          users: [users3],
+        },
+      ]
+
+      await config.api.row.save(tableId, rows[0])
+      await config.api.row.save(tableId, rows[1])
+      await config.api.row.save(tableId, rows[2])
+
+      const res = await config.api.row.search(tableId)
+
+      expect(res).toEqual({
+        rows: expect.arrayContaining(
+          rows.map(r => ({
+            name: r.name,
+            description: r.description,
+            tableId,
+            user: r.user?.map(u => resultMapper(u)),
+            users: r.users?.length
+              ? expect.arrayContaining(r.users?.map(u => resultMapper(u)))
+              : undefined,
+            _id: expect.any(String),
+            _rev: expect.any(String),
+            id: isInternal ? undefined : expect.any(Number),
+            [`fk_${o2mTable.name}_fk_o2m`]:
+              isInternal || !r.user?.length ? undefined : r.user[0].id,
+            ...defaultRowFields,
+          }))
+        ),
+        ...(isInternal
+          ? {}
+          : {
+              hasNextPage: false,
+              bookmark: null,
+            }),
+      })
+    })
+  })
+
+  describe("Formula fields", () => {
+    let relationshipTable: Table, tableId: string, relatedRow: Row
+
+    beforeAll(async () => {
+      const otherTableId = config.table!._id!
+      const cfg = generateTableConfig()
+      relationshipTable = await config.createLinkedTable(
+        RelationshipType.ONE_TO_MANY,
+        ["links"],
+        {
+          ...cfg,
+          // needs to be a short name
+          name: "b",
+          schema: {
+            ...cfg.schema,
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: "{{ links.0.name }}",
+              formulaType: FormulaType.DYNAMIC,
+            },
+          },
+        }
+      )
+
+      tableId = relationshipTable._id!
+
+      relatedRow = await config.api.row.save(otherTableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+      })
+      await config.api.row.save(tableId, {
+        name: generator.word(),
+        description: generator.paragraph(),
+        tableId,
+        links: [relatedRow._id],
+      })
+    })
+
+    it("should be able to search for rows containing formulas", async () => {
+      const { rows } = await config.api.row.search(tableId)
+      expect(rows.length).toBe(1)
+      expect(rows[0].links.length).toBe(1)
+      const row = rows[0]
+      expect(row.formula).toBe(relatedRow.name)
+    })
+  })
+
+  describe("Formula JS protection", () => {
+    it("should time out JS execution if a single cell takes too long", async () => {
+      await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
+        const js = Buffer.from(
+          `
+              let i = 0;
+              while (true) {
+                i++;
+              }
+              return i;
+            `
+        ).toString("base64")
+
+        const table = await config.createTable({
+          name: "table",
+          type: "table",
+          schema: {
+            text: {
+              name: "text",
+              type: FieldType.STRING,
+            },
+            formula: {
+              name: "formula",
+              type: FieldType.FORMULA,
+              formula: `{{ js "${js}"}}`,
+              formulaType: FormulaType.DYNAMIC,
+            },
+          },
+        })
+
+        await config.api.row.save(table._id!, { text: "foo" })
+        const { rows } = await config.api.row.search(table._id!)
+        expect(rows).toHaveLength(1)
+        const row = rows[0]
+        expect(row.text).toBe("foo")
+        expect(row.formula).toBe("Timed out while executing JS")
+      })
+    })
+
+    it("should time out JS execution if a multiple cells take too long", async () => {
+      await config.withEnv(
+        {
+          JS_PER_INVOCATION_TIMEOUT_MS: 20,
+          JS_PER_REQUEST_TIMEOUT_MS: 40,
+        },
+        async () => {
           const js = Buffer.from(
             `
               let i = 0;
@@ -2075,125 +2102,81 @@ describe.each([["postgres", databaseTestProviders.postgres]])(
             },
           })
 
-          await config.api.row.save(table._id!, { text: "foo" })
-          const { rows } = await config.api.row.search(table._id!)
-          expect(rows).toHaveLength(1)
-          const row = rows[0]
-          expect(row.text).toBe("foo")
-          expect(row.formula).toBe("Timed out while executing JS")
-        })
-      })
+          for (let i = 0; i < 10; i++) {
+            await config.api.row.save(table._id!, { text: "foo" })
+          }
 
-      it("should time out JS execution if a multiple cells take too long", async () => {
-        await config.withEnv(
-          {
-            JS_PER_INVOCATION_TIMEOUT_MS: 20,
-            JS_PER_REQUEST_TIMEOUT_MS: 40,
-          },
-          async () => {
-            const js = Buffer.from(
-              `
-              let i = 0;
-              while (true) {
-                i++;
+          // Run this test 3 times to make sure that there's no cross-request
+          // pollution of the execution time tracking.
+          for (let reqs = 0; reqs < 3; reqs++) {
+            const { rows } = await config.api.row.search(table._id!)
+            expect(rows).toHaveLength(10)
+
+            let i = 0
+            for (; i < 10; i++) {
+              const row = rows[i]
+              if (row.formula !== "Timed out while executing JS") {
+                break
               }
-              return i;
-            `
-            ).toString("base64")
-
-            const table = await config.createTable({
-              name: "table",
-              type: "table",
-              schema: {
-                text: {
-                  name: "text",
-                  type: FieldType.STRING,
-                },
-                formula: {
-                  name: "formula",
-                  type: FieldType.FORMULA,
-                  formula: `{{ js "${js}"}}`,
-                  formulaType: FormulaType.DYNAMIC,
-                },
-              },
-            })
-
-            for (let i = 0; i < 10; i++) {
-              await config.api.row.save(table._id!, { text: "foo" })
             }
 
-            // Run this test 3 times to make sure that there's no cross-request
-            // pollution of the execution time tracking.
-            for (let reqs = 0; reqs < 3; reqs++) {
-              const { rows } = await config.api.row.search(table._id!)
-              expect(rows).toHaveLength(10)
+            // Given the execution times are not deterministic, we can't be sure
+            // of the exact number of rows that were executed before the timeout
+            // but it should absolutely be at least 1.
+            expect(i).toBeGreaterThan(0)
+            expect(i).toBeLessThan(5)
 
-              let i = 0
-              for (; i < 10; i++) {
-                const row = rows[i]
-                if (row.formula !== "Timed out while executing JS") {
-                  break
-                }
-              }
-
-              // Given the execution times are not deterministic, we can't be sure
-              // of the exact number of rows that were executed before the timeout
-              // but it should absolutely be at least 1.
-              expect(i).toBeGreaterThan(0)
-              expect(i).toBeLessThan(5)
-
-              for (; i < 10; i++) {
-                const row = rows[i]
-                expect(row.text).toBe("foo")
-                expect(row.formula).toBe("Request JS execution limit hit")
-              }
+            for (; i < 10; i++) {
+              const row = rows[i]
+              expect(row.text).toBe("foo")
+              expect(row.formula).toBe("Request JS execution limit hit")
             }
           }
-        )
-      })
-
-      it("should not carry over context between formulas", async () => {
-        const js = Buffer.from(`return $("[text]");`).toString("base64")
-        const table = await config.createTable({
-          name: "table",
-          type: "table",
-          schema: {
-            text: {
-              name: "text",
-              type: FieldType.STRING,
-            },
-            formula: {
-              name: "formula",
-              type: FieldType.FORMULA,
-              formula: `{{ js "${js}"}}`,
-              formulaType: FormulaType.DYNAMIC,
-            },
-          },
-        })
-
-        for (let i = 0; i < 10; i++) {
-          await config.api.row.save(table._id!, { text: `foo${i}` })
         }
-
-        const { rows } = await config.api.row.search(table._id!)
-        expect(rows).toHaveLength(10)
-
-        const formulaValues = rows.map(r => r.formula)
-        expect(formulaValues).toEqual(
-          expect.arrayContaining([
-            "foo0",
-            "foo1",
-            "foo2",
-            "foo3",
-            "foo4",
-            "foo5",
-            "foo6",
-            "foo7",
-            "foo8",
-            "foo9",
-          ])
-        )
-      })
+      )
     })
-  }
-)
+
+    it("should not carry over context between formulas", async () => {
+      const js = Buffer.from(`return $("[text]");`).toString("base64")
+      const table = await config.createTable({
+        name: "table",
+        type: "table",
+        schema: {
+          text: {
+            name: "text",
+            type: FieldType.STRING,
+          },
+          formula: {
+            name: "formula",
+            type: FieldType.FORMULA,
+            formula: `{{ js "${js}"}}`,
+            formulaType: FormulaType.DYNAMIC,
+          },
+        },
+      })
+
+      for (let i = 0; i < 10; i++) {
+        await config.api.row.save(table._id!, { text: `foo${i}` })
+      }
+
+      const { rows } = await config.api.row.search(table._id!)
+      expect(rows).toHaveLength(10)
+
+      const formulaValues = rows.map(r => r.formula)
+      expect(formulaValues).toEqual(
+        expect.arrayContaining([
+          "foo0",
+          "foo1",
+          "foo2",
+          "foo3",
+          "foo4",
+          "foo5",
+          "foo6",
+          "foo7",
+          "foo8",
+          "foo9",
+        ])
+      )
+    })
+  })
+})