From 9fb2d89a330fd99a990cd6b65ebcc7374cbabddc Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 31 Aug 2023 15:11:39 +0200
Subject: [PATCH 01/94] Fix daysjs import error

---
 .../frontend-core/src/components/grid/cells/DateCell.svelte     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
index 53b159ee30..9144f5b769 100644
--- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
@@ -1,5 +1,5 @@
 <script>
-  import dayjs from "dayjs"
+  import { dayjs } from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 

From 122cd843abc2da063eb0c5fa15f55e4f6241105b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 30 Aug 2023 17:42:26 +0200
Subject: [PATCH 02/94] Type ds in testconfiguration

---
 .../src/tests/utilities/TestConfiguration.ts    | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 81bfa0abbd..137828f188 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -78,7 +78,7 @@ class TestConfiguration {
   table?: Table
   linkedTable: any
   automation: any
-  datasource: any
+  datasource?: Datasource
   tenantId?: string
   defaultUserValues: DefaultUserValues
   api: API
@@ -528,7 +528,7 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: any,
+    config?: Table,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
@@ -543,6 +543,11 @@ class TestConfiguration {
     if (config != null && config._id) {
       delete config._id
     }
+    config = config || basicTable()
+    if (this.datasource && !config.sourceId) {
+      config.sourceId = this.datasource._id
+    }
+
     return this.updateTable(config, options)
   }
 
@@ -678,17 +683,17 @@ class TestConfiguration {
     config = config || basicDatasource()
     const response = await this._req(config, null, controllers.datasource.save)
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
-  async updateDatasource(datasource: any) {
+  async updateDatasource(datasource: Datasource): Promise<Datasource> {
     const response = await this._req(
       datasource,
       { datasourceId: datasource._id },
       controllers.datasource.update
     )
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
   async restDatasource(cfg?: any) {
@@ -772,7 +777,7 @@ class TestConfiguration {
     if (!this.datasource && !config) {
       throw "No datasource created for query."
     }
-    config = config || basicQuery(this.datasource._id)
+    config = config || basicQuery(this.datasource!._id!)
     return this._req(config, null, controllers.query.save)
   }
 

From cb6977a18b421d64db16ff22b304b16cad092fa9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 17:01:49 +0200
Subject: [PATCH 03/94] Create non-plus datasource

---
 packages/server/src/api/routes/tests/row.spec.ts         | 7 +++++++
 packages/server/src/tests/utilities/TestConfiguration.ts | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index f7878b2182..71c4325f92 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -18,6 +18,7 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
+import { basicDatasource } from "../../../tests/utilities/structures"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -37,6 +38,12 @@ describe("/rows", () => {
   })
 
   beforeEach(async () => {
+    await config.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        plus: false,
+      },
+    })
     table = await config.createTable()
     row = basicRow(table._id!)
   })
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 137828f188..5d5d6a18e5 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -546,6 +546,9 @@ class TestConfiguration {
     config = config || basicTable()
     if (this.datasource && !config.sourceId) {
       config.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        config.type = "external"
+      }
     }
 
     return this.updateTable(config, options)

From 8931309669d9011ae69940c517e05b92f6f5afe7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 18:53:32 +0200
Subject: [PATCH 04/94] Datasource test api

---
 .../src/tests/utilities/TestConfiguration.ts  |  6 ++++-
 .../src/tests/utilities/api/datasource.ts     | 26 +++++++++++++++++++
 .../server/src/tests/utilities/api/index.ts   |  3 +++
 3 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/tests/utilities/api/datasource.ts

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 5d5d6a18e5..d759431bcc 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -684,7 +684,11 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this._req(config, null, controllers.datasource.save)
+    const response = await this._req(
+      { ...config },
+      null,
+      controllers.datasource.save
+    )
     this.datasource = response.datasource
     return this.datasource!
   }
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
new file mode 100644
index 0000000000..c07fd3bc61
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -0,0 +1,26 @@
+import { CreateDatasourceRequest, Datasource } from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class DatasourceAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .post(`/api/datasources`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+}
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index 40995b62f2..0521f9b19f 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -3,17 +3,20 @@ import { PermissionAPI } from "./permission"
 import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
+import { DatasourceAPI } from "./datasource"
 
 export default class API {
   table: TableAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
+  datasource: DatasourceAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
+    this.datasource = new DatasourceAPI(config)
   }
 }

From e4479ee522b80cfce73f0575c8d90d96d3cf0ce5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:04 +0200
Subject: [PATCH 05/94] Use docker utils helpers

---
 .../src/tests/utilities/api/datasource.ts     | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index c07fd3bc61..3c85a1c332 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -1,4 +1,9 @@
-import { CreateDatasourceRequest, Datasource } from "@budibase/types"
+import {
+  CreateDatasourceRequest,
+  Datasource,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -23,4 +28,17 @@ export class DatasourceAPI extends TestAPI {
       .expect(expectStatus)
     return result.body.datasource as Datasource
   }
+
+  verify = async (
+    data: VerifyDatasourceRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const result = await this.request
+      .post(`/api/datasources/verify`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result
+  }
 }

From baab7d3fb501b96ada3859d17c92f56c0caa6466 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:11 +0200
Subject: [PATCH 06/94] Use docker utils helpers

---
 .../src/integrations/tests/utils/index.ts     |  5 +++
 .../src/integrations/tests/utils/postgres.ts  | 34 +++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 packages/server/src/integrations/tests/utils/index.ts
 create mode 100644 packages/server/src/integrations/tests/utils/postgres.ts

diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
new file mode 100644
index 0000000000..2c8c583fd4
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -0,0 +1,5 @@
+import * as postgres from "./postgres"
+
+export const testDatasourceConfig = {
+  postgres: postgres.getDatasourceConfig,
+}
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
new file mode 100644
index 0000000000..9f9e98fe17
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -0,0 +1,34 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+
+export async function getDatasourceConfig(): Promise<Datasource> {
+  const containerPostgres = await new GenericContainer("postgres")
+    .withExposedPorts(5432)
+    .withEnv("POSTGRES_PASSWORD", "password")
+    .withWaitStrategy(
+      Wait.forLogMessage(
+        "PostgreSQL init process complete; ready for start up."
+      )
+    )
+    .start()
+
+  const host = containerPostgres.getContainerIpAddress()
+  const port = containerPostgres.getMappedPort(5432)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.POSTGRES,
+    plus: true,
+    config: {
+      host,
+      port,
+      database: "postgres",
+      user: "postgres",
+      password: "password",
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
+  }
+}

From 57801b5bcc903324c023d3dea363860bfa1f2c96 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:18 +0200
Subject: [PATCH 07/94] Use docker utils helpers

---
 .../src/integration-test/postgres.spec.ts     | 78 ++++---------------
 1 file changed, 16 insertions(+), 62 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 6b0e7b4266..7dde9f2e0c 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -12,13 +12,12 @@ import {
   FieldType,
   RelationshipType,
   Row,
-  SourceName,
   Table,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { testDatasourceConfig } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -35,61 +34,19 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let host: string
-  let port: number
-  const containers: StartedTestContainer[] = []
+  let pgDatasourceConfig: Datasource
 
   beforeAll(async () => {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
-        )
-      )
-      .start()
-
-    host = containerPostgres.getContainerIpAddress()
-    port = containerPostgres.getMappedPort(5432)
+    pgDatasourceConfig = await testDatasourceConfig.postgres()
 
     await config.init()
     const apiKey = await config.generateApiKey()
 
-    containers.push(containerPostgres)
-
     makeRequest = generateMakeRequest(apiKey, true)
   })
 
-  afterAll(async () => {
-    for (let container of containers) {
-      await container.stop()
-    }
-  })
-
-  function pgDatasourceConfig() {
-    return {
-      datasource: {
-        type: "datasource",
-        source: SourceName.POSTGRES,
-        plus: true,
-        config: {
-          host,
-          port,
-          database: "postgres",
-          user: "postgres",
-          password: "password",
-          schema: "public",
-          ssl: false,
-          rejectUnauthorized: false,
-          ca: false,
-        },
-      },
-    }
-  }
-
   beforeEach(async () => {
-    postgresDatasource = await config.createDatasource(pgDatasourceConfig())
+    postgresDatasource = await config.api.datasource.create(pgDatasourceConfig)
 
     async function createAuxTable(prefix: string) {
       return await config.createTable({
@@ -357,9 +314,9 @@ describe("postgres integrations", () => {
       config: {
         ca: false,
         database: "postgres",
-        host,
+        host: postgresDatasource.config!.host,
         password: "--secret-value--",
-        port,
+        port: postgresDatasource.config!.port,
         rejectUnauthorized: false,
         schema: "public",
         ssl: false,
@@ -1046,24 +1003,21 @@ describe("postgres integrations", () => {
 
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
-      const config = pgDatasourceConfig()
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: pgDatasourceConfig,
+      })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const config = pgDatasourceConfig()
-      config.datasource.config.password = "wrongpassword"
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: {
+          ...pgDatasourceConfig,
+          config: { ...pgDatasourceConfig.config, password: "wrongpassword" },
+        },
+      })
+
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(false)
       expect(response.body.error).toBeDefined()

From b27899b1e6469cea37ac4e417cd4291ee2b62f8d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 10:47:59 +0200
Subject: [PATCH 08/94] Redo helpers

---
 .../src/integration-test/postgres.spec.ts     | 20 +++++---
 .../src/integrations/tests/utils/index.ts     |  7 +--
 .../src/integrations/tests/utils/postgres.ts  | 46 +++++++++++++++++++
 3 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 7dde9f2e0c..fd85393477 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -17,7 +17,7 @@ import {
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { testDatasourceConfig } from "../integrations/tests/utils"
+import { PostgresProvider } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -34,10 +34,10 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let pgDatasourceConfig: Datasource
+  let provider: PostgresProvider
 
   beforeAll(async () => {
-    pgDatasourceConfig = await testDatasourceConfig.postgres()
+    provider = await PostgresProvider.init()
 
     await config.init()
     const apiKey = await config.generateApiKey()
@@ -46,7 +46,9 @@ describe("postgres integrations", () => {
   })
 
   beforeEach(async () => {
-    postgresDatasource = await config.api.datasource.create(pgDatasourceConfig)
+    postgresDatasource = await config.api.datasource.create(
+      provider.getDsConfig()
+    )
 
     async function createAuxTable(prefix: string) {
       return await config.createTable({
@@ -1004,17 +1006,21 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: pgDatasourceConfig,
+        datasource: provider.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
+      const dbConfig = provider.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
-          ...pgDatasourceConfig,
-          config: { ...pgDatasourceConfig.config, password: "wrongpassword" },
+          ...dbConfig,
+          config: {
+            ...dbConfig.config,
+            password: "wrongpassword",
+          },
         },
       })
 
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 2c8c583fd4..e80dbed83a 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,5 +1,6 @@
-import * as postgres from "./postgres"
+import { Datasource } from "@budibase/types"
+export * from "./postgres"
 
-export const testDatasourceConfig = {
-  postgres: postgres.getDatasourceConfig,
+export interface DatabasePlusTestProvider {
+  getDsConfig(): Datasource
 }
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 9f9e98fe17..514605306e 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,5 +1,51 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
+import { DatabasePlusTestProvider } from "."
+
+export class PostgresProvider implements DatabasePlusTestProvider {
+  private host: string
+  private port: number
+
+  private constructor(host: string, port: number) {
+    this.host = host
+    this.port = port
+  }
+
+  static async init() {
+    const containerPostgres = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
+        )
+      )
+      .start()
+
+    const host = containerPostgres.getContainerIpAddress()
+    const port = containerPostgres.getMappedPort(5432)
+    return new PostgresProvider(host, port)
+  }
+
+  getDsConfig(): Datasource {
+    return {
+      type: "datasource_plus",
+      source: SourceName.POSTGRES,
+      plus: true,
+      config: {
+        host: this.host,
+        port: this.port,
+        database: "postgres",
+        user: "postgres",
+        password: "password",
+        schema: "public",
+        ssl: false,
+        rejectUnauthorized: false,
+        ca: false,
+      },
+    }
+  }
+}
 
 export async function getDatasourceConfig(): Promise<Datasource> {
   const containerPostgres = await new GenericContainer("postgres")

From 6240740a42854bd70ede3859ed676590582526ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:11:47 +0200
Subject: [PATCH 09/94] Simplify

---
 .../src/integration-test/postgres.spec.ts     | 12 +++---
 .../src/integrations/tests/utils/index.ts     |  2 +-
 .../src/integrations/tests/utils/postgres.ts  | 41 ++++++++-----------
 3 files changed, 23 insertions(+), 32 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index fd85393477..faaa22f2c8 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -26,6 +26,8 @@ jest.setTimeout(30000)
 jest.unmock("pg")
 jest.mock("../websockets")
 
+const provider = new PostgresProvider()
+
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
     postgresDatasource: Datasource,
@@ -34,11 +36,7 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let provider: PostgresProvider
-
   beforeAll(async () => {
-    provider = await PostgresProvider.init()
-
     await config.init()
     const apiKey = await config.generateApiKey()
 
@@ -47,7 +45,7 @@ describe("postgres integrations", () => {
 
   beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
-      provider.getDsConfig()
+      await provider.getDsConfig()
     )
 
     async function createAuxTable(prefix: string) {
@@ -1006,14 +1004,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: provider.getDsConfig(),
+        datasource: await provider.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = provider.getDsConfig()
+      const dbConfig = await provider.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index e80dbed83a..20b6f277dd 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -2,5 +2,5 @@ import { Datasource } from "@budibase/types"
 export * from "./postgres"
 
 export interface DatabasePlusTestProvider {
-  getDsConfig(): Datasource
+  getDsConfig(): Promise<Datasource>
 }
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 514605306e..8d79a7e762 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,40 +1,33 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait } from "testcontainers"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
 import { DatabasePlusTestProvider } from "."
 
 export class PostgresProvider implements DatabasePlusTestProvider {
-  private host: string
-  private port: number
+  private container?: StartedTestContainer
 
-  private constructor(host: string, port: number) {
-    this.host = host
-    this.port = port
-  }
-
-  static async init() {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
+  async getDsConfig(): Promise<Datasource> {
+    if (!this.container) {
+      this.container = await new GenericContainer("postgres")
+        .withExposedPorts(5432)
+        .withEnv("POSTGRES_PASSWORD", "password")
+        .withWaitStrategy(
+          Wait.forLogMessage(
+            "PostgreSQL init process complete; ready for start up."
+          )
         )
-      )
-      .start()
+        .start()
+    }
 
-    const host = containerPostgres.getContainerIpAddress()
-    const port = containerPostgres.getMappedPort(5432)
-    return new PostgresProvider(host, port)
-  }
+    const host = this.container.getContainerIpAddress()
+    const port = this.container.getMappedPort(5432)
 
-  getDsConfig(): Datasource {
     return {
       type: "datasource_plus",
       source: SourceName.POSTGRES,
       plus: true,
       config: {
-        host: this.host,
-        port: this.port,
+        host,
+        port,
         database: "postgres",
         user: "postgres",
         password: "password",

From 887ebb2eebe5ab75a3d73190c0d8a1fc4ad29799 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:16:37 +0200
Subject: [PATCH 10/94] Simplify

---
 .../src/integration-test/postgres.spec.ts     | 10 ++-
 .../src/integrations/tests/utils/index.ts     |  6 +-
 .../src/integrations/tests/utils/postgres.ts  | 61 ++++---------------
 3 files changed, 22 insertions(+), 55 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index faaa22f2c8..0a53326cf4 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -17,7 +17,7 @@ import {
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { PostgresProvider } from "../integrations/tests/utils"
+import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -26,8 +26,6 @@ jest.setTimeout(30000)
 jest.unmock("pg")
 jest.mock("../websockets")
 
-const provider = new PostgresProvider()
-
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
     postgresDatasource: Datasource,
@@ -45,7 +43,7 @@ describe("postgres integrations", () => {
 
   beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
-      await provider.getDsConfig()
+      await databaseTestProviders.postgres.getDsConfig()
     )
 
     async function createAuxTable(prefix: string) {
@@ -1004,14 +1002,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: await provider.getDsConfig(),
+        datasource: await databaseTestProviders.postgres.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await provider.getDsConfig()
+      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 20b6f277dd..8893ce67f3 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,6 +1,10 @@
 import { Datasource } from "@budibase/types"
-export * from "./postgres"
+import * as pg from "./postgres"
 
 export interface DatabasePlusTestProvider {
   getDsConfig(): Promise<Datasource>
 }
+
+export const databaseTestProviders = {
+  postgres: pg,
+}
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 8d79a7e762..036e81bbd8 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,58 +1,23 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-import { DatabasePlusTestProvider } from "."
 
-export class PostgresProvider implements DatabasePlusTestProvider {
-  private container?: StartedTestContainer
+let container: StartedTestContainer | undefined
 
-  async getDsConfig(): Promise<Datasource> {
-    if (!this.container) {
-      this.container = await new GenericContainer("postgres")
-        .withExposedPorts(5432)
-        .withEnv("POSTGRES_PASSWORD", "password")
-        .withWaitStrategy(
-          Wait.forLogMessage(
-            "PostgreSQL init process complete; ready for start up."
-          )
+export async function getDsConfig(): Promise<Datasource> {
+  if (!container) {
+    container = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
         )
-        .start()
-    }
-
-    const host = this.container.getContainerIpAddress()
-    const port = this.container.getMappedPort(5432)
-
-    return {
-      type: "datasource_plus",
-      source: SourceName.POSTGRES,
-      plus: true,
-      config: {
-        host,
-        port,
-        database: "postgres",
-        user: "postgres",
-        password: "password",
-        schema: "public",
-        ssl: false,
-        rejectUnauthorized: false,
-        ca: false,
-      },
-    }
-  }
-}
-
-export async function getDatasourceConfig(): Promise<Datasource> {
-  const containerPostgres = await new GenericContainer("postgres")
-    .withExposedPorts(5432)
-    .withEnv("POSTGRES_PASSWORD", "password")
-    .withWaitStrategy(
-      Wait.forLogMessage(
-        "PostgreSQL init process complete; ready for start up."
       )
-    )
-    .start()
+      .start()
+  }
 
-  const host = containerPostgres.getContainerIpAddress()
-  const port = containerPostgres.getMappedPort(5432)
+  const host = container.getContainerIpAddress()
+  const port = container.getMappedPort(5432)
 
   return {
     type: "datasource_plus",

From 14303c9eb0599ddf6714fdf73331312928096c77 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:41:30 +0200
Subject: [PATCH 11/94] Extend test create api

---
 packages/server/src/tests/utilities/api/table.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts
index 70f0869650..04432a788a 100644
--- a/packages/server/src/tests/utilities/api/table.ts
+++ b/packages/server/src/tests/utilities/api/table.ts
@@ -1,4 +1,4 @@
-import { Table } from "@budibase/types"
+import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -7,6 +7,19 @@ export class TableAPI extends TestAPI {
     super(config)
   }
 
+  create = async (
+    data: SaveTableRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<SaveTableResponse> => {
+    const res = await this.request
+      .post(`/api/tables`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return res.body
+  }
+
   fetch = async (
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Table[]> => {

From f476c84bc25401dff1f1c5a78aa04968e581996a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 12:09:11 +0200
Subject: [PATCH 12/94] Run row api for pg

---
 .../server/src/api/routes/tests/row.spec.ts   | 57 ++++++++++++++++---
 1 file changed, 49 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 71c4325f92..d9fa0938e1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -4,10 +4,12 @@ import * as setup from "./utilities"
 import { context, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  Datasource,
   FieldType,
   MonthlyQuotaName,
   QuotaUsageType,
   Row,
+  SaveTableRequest,
   SortOrder,
   SortType,
   StaticQuotaName,
@@ -18,18 +20,22 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import { basicDatasource } from "../../../tests/utilities/structures"
+import { databaseTestProviders } from "../../../integrations/tests/utils"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
 const { basicRow } = setup.structures
 
-describe("/rows", () => {
+describe.each([
+  ["internal", undefined],
+  ["postgres", databaseTestProviders.postgres],
+])("/rows (%s)", (_, dsProvider) => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
   let row: Row
+  let datasource: Datasource | undefined
 
   afterAll(setup.afterAll)
 
@@ -38,13 +44,48 @@ describe("/rows", () => {
   })
 
   beforeEach(async () => {
-    await config.createDatasource({
-      datasource: {
-        ...basicDatasource().datasource,
-        plus: false,
+    let tableConfig: SaveTableRequest = {
+      name: "TestTable",
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          type: FieldType.AUTO,
+          name: "id",
+          autocolumn: true,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            type: "string",
+          },
+        },
+        description: {
+          type: FieldType.STRING,
+          name: "description",
+          constraints: {
+            type: "string",
+          },
+        },
       },
-    })
-    table = await config.createTable()
+    }
+
+    if (dsProvider) {
+      datasource = await config.api.datasource.create(
+        await dsProvider.getDsConfig()
+      )
+
+      tableConfig.sourceId = datasource._id
+      if (datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
+    table = await config.api.table.create(tableConfig)
     row = basicRow(table._id!)
   })
 

From 3fe7a9f4cbd02de1eab0cf510a5001b89abc1379 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 6 Sep 2023 12:51:08 +0200
Subject: [PATCH 13/94] Unmock pg

---
 packages/server/src/api/routes/tests/row.spec.ts      | 3 ++-
 packages/server/src/integrations/tests/utils/index.ts | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d9fa0938e1..968f5ff6c3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1,3 +1,5 @@
+import { databaseTestProviders } from "../../../integrations/tests/utils"
+
 import tk from "timekeeper"
 import { outputProcessing } from "../../../utilities/rowProcessor"
 import * as setup from "./utilities"
@@ -20,7 +22,6 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import { databaseTestProviders } from "../../../integrations/tests/utils"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 8893ce67f3..c145a9c809 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,3 +1,5 @@
+jest.unmock("pg")
+
 import { Datasource } from "@budibase/types"
 import * as pg from "./postgres"
 

From 7a8eab863c03df98016293d0ca849bee7859bdea Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 6 Sep 2023 13:03:35 +0200
Subject: [PATCH 14/94] Fix tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 968f5ff6c3..7d23714ccd 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -46,7 +46,7 @@ describe.each([
 
   beforeEach(async () => {
     let tableConfig: SaveTableRequest = {
-      name: "TestTable",
+      name: generator.guid(),
       type: "table",
       primary: ["id"],
       schema: {
@@ -124,6 +124,8 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
+  const createRow = () => config.api.row.save(table._id!, basicRow(table._id!))
+
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
@@ -200,7 +202,7 @@ describe.each([
     })
 
     it("updates a row successfully", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 0bda4a19529f7aa460aa1a340f8b7e5f98fd706d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 6 Sep 2023 16:56:27 +0100
Subject: [PATCH 15/94] Add option to duplicate columns

---
 .../src/components/grid/cells/DataCell.svelte |  5 ++-
 .../components/grid/cells/HeaderCell.svelte   | 36 +++++++++++++++++++
 .../grid/overlays/KeyboardManager.svelte      |  2 ++
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index 19bdf1bd74..daf0a03276 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -56,7 +56,10 @@
     isActive: () => api?.isActive?.() ?? false,
     onKeyDown: (...params) => api?.onKeyDown(...params),
     isReadonly: () => readonly,
-    getType: () => column.schema.type,
+    getType: () => {
+      console.log("getType", column)
+      return column.schema.type
+    },
     getValue: () => row[column.name],
     setValue: (value, options = { save: true }) => {
       validation.actions.setError(cellId, null)
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index c5f1f7896a..a58b4c8fe4 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -20,6 +20,8 @@
     config,
     ui,
     columns,
+    definition,
+    datasource,
   } = getContext("grid")
 
   const bannedDisplayColumnTypes = [
@@ -118,6 +120,33 @@
     open = false
   }
 
+  const duplicateColumn = async () => {
+    open = false
+
+    // Generate new name
+    let newName = `${column.name} copy`
+    let attempts = 2
+    while ($definition.schema[newName]) {
+      newName = `${column.name} copy ${attempts++}`
+    }
+
+    // Save schema with new column
+    const existingColumnDefinition = $definition.schema[column.name]
+    await datasource.actions.saveDefinition({
+      ...$definition,
+      schema: {
+        ...$definition.schema,
+        [newName]: {
+          ...existingColumnDefinition,
+          name: newName,
+          schema: {
+            ...existingColumnDefinition.schema,
+          },
+        },
+      },
+    })
+  }
+
   onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
@@ -192,6 +221,13 @@
       >
         Edit column
       </MenuItem>
+      <MenuItem
+        icon="Duplicate"
+        on:click={duplicateColumn}
+        disabled={!$config.canEditColumns}
+      >
+        Duplicate column
+      </MenuItem>
       <MenuItem
         icon="Label"
         on:click={makeDisplayColumn}
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index cd23f154b5..3232f23e76 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -212,8 +212,10 @@
 
   // Focuses the cell and starts entering a new value
   const startEnteringValue = (key, keyCode) => {
+    console.log("start", $focusedCellAPI, $focusedCellAPI.isReadonly())
     if ($focusedCellAPI && !$focusedCellAPI.isReadonly()) {
       const type = $focusedCellAPI.getType()
+      console.log(type)
       if (type === "number" && keyCodeIsNumber(keyCode)) {
         // Update the value locally but don't save it yet
         $focusedCellAPI.setValue(parseInt(key), { save: false })

From 32108abbc0e6064176d7e3798a93041370f159a2 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 6 Sep 2023 16:59:30 +0100
Subject: [PATCH 16/94] Remove logs

---
 .../frontend-core/src/components/grid/cells/DataCell.svelte  | 5 +----
 .../src/components/grid/overlays/KeyboardManager.svelte      | 2 --
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index daf0a03276..19bdf1bd74 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -56,10 +56,7 @@
     isActive: () => api?.isActive?.() ?? false,
     onKeyDown: (...params) => api?.onKeyDown(...params),
     isReadonly: () => readonly,
-    getType: () => {
-      console.log("getType", column)
-      return column.schema.type
-    },
+    getType: () => column.schema.type,
     getValue: () => row[column.name],
     setValue: (value, options = { save: true }) => {
       validation.actions.setError(cellId, null)
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index 3232f23e76..cd23f154b5 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -212,10 +212,8 @@
 
   // Focuses the cell and starts entering a new value
   const startEnteringValue = (key, keyCode) => {
-    console.log("start", $focusedCellAPI, $focusedCellAPI.isReadonly())
     if ($focusedCellAPI && !$focusedCellAPI.isReadonly()) {
       const type = $focusedCellAPI.getType()
-      console.log(type)
       if (type === "number" && keyCodeIsNumber(keyCode)) {
         // Update the value locally but don't save it yet
         $focusedCellAPI.setValue(parseInt(key), { save: false })

From 3c46d66333713671c0d144b92ba908deb1bb3077 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 09:25:19 +0200
Subject: [PATCH 17/94] yarn.lock

---
 yarn.lock | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/yarn.lock b/yarn.lock
index ab86a87560..8c93661665 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,6 +6269,14 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
+"@types/node-fetch@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
+  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6290,6 +6298,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
+"@types/node@14.18.20":
+  version "14.18.20"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
+  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
+
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"

From c6621e8c495d518a1d3825ec302b62441c88d202 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:04:35 +0200
Subject: [PATCH 18/94] Type response

---
 packages/server/src/api/controllers/row/external.ts | 2 +-
 packages/server/src/api/controllers/row/index.ts    | 3 ++-
 packages/server/src/api/controllers/row/internal.ts | 2 +-
 packages/server/src/api/controllers/row/utils.ts    | 2 +-
 packages/types/src/api/web/app/row.ts               | 2 ++
 5 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 491cbfa8f9..5a78b55ca7 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -106,7 +106,7 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
   return sdk.rows.external.getRow(tableId, id)
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 88d3f50dbe..0bc3c4d3c7 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -14,6 +14,7 @@ import {
   SearchRowResponse,
   SearchRowRequest,
   SearchParams,
+  GetRowResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -111,7 +112,7 @@ export async function fetch(ctx: any) {
   })
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx<void, GetRowResponse>) {
   const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
     datasourceId: tableId,
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 4a412c42eb..b80bd339e6 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -131,7 +131,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
   const table = await sdk.tables.getTable(tableId)
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index e85ec4553c..c1df80600c 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -27,7 +27,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
-  let row
+  let row: Row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {
diff --git a/packages/types/src/api/web/app/row.ts b/packages/types/src/api/web/app/row.ts
index f9623a3daf..6e051facae 100644
--- a/packages/types/src/api/web/app/row.ts
+++ b/packages/types/src/api/web/app/row.ts
@@ -1,5 +1,7 @@
 import { Row } from "../../../documents/app/row"
 
+export interface GetRowResponse extends Row {}
+
 export interface DeleteRows {
   rows: (Row | string)[]
 }

From f8adbb86a0e19d228629d4e1ce231a15412e51e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:04:49 +0200
Subject: [PATCH 19/94] Fix tests

---
 .../src/api/controllers/row/external.ts       |  8 +++-
 .../server/src/api/routes/tests/row.spec.ts   | 37 ++++++++++++-------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 5a78b55ca7..5a1c7e04ff 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -109,7 +109,13 @@ export async function save(ctx: UserCtx) {
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  return sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id)
+
+  if (!row) {
+    ctx.throw(404)
+  }
+
+  return row
 }
 
 export async function destroy(ctx: UserCtx) {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9c150d8968..821842e1b4 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -12,6 +12,7 @@ import {
   PermissionLevel,
   QuotaUsageType,
   Row,
+  SaveRowRequest,
   SaveTableRequest,
   SortOrder,
   SortType,
@@ -34,6 +35,8 @@ describe.each([
   ["internal", undefined],
   ["postgres", databaseTestProviders.postgres],
 ])("/rows (%s)", (_, dsProvider) => {
+  const isInternal = !dsProvider
+
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
@@ -127,9 +130,20 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = () => config.api.row.save(table._id!, basicRow(table._id!))
+  const createRow = (row?: SaveRowRequest) =>
+    config.api.row.save(table._id!, row || basicRow(table._id!))
 
   describe("save, load, update", () => {
+    function getDefaultFields() {
+      if (isInternal) {
+        return {
+          type: "row",
+          createdAt: timestamp,
+          updatedAt: timestamp,
+        }
+      }
+    }
+
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -230,7 +244,7 @@ describe.each([
     })
 
     it("should load a row", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -243,9 +257,8 @@ describe.each([
         ...row,
         _id: existing._id,
         _rev: existing._rev,
-        type: "row",
-        createdAt: timestamp,
-        updatedAt: timestamp,
+        id: existing.id,
+        ...getDefaultFields(),
       })
       await assertQueryUsage(queryUsage + 1)
     })
@@ -256,8 +269,8 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      await config.createRow()
-      await config.createRow(newRow)
+      await createRow()
+      await createRow(newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -273,14 +286,12 @@ describe.each([
     })
 
     it("load should return 404 when row does not exist", async () => {
-      await config.createRow()
+      await createRow()
       const queryUsage = await getQueryUsage()
 
-      await request
-        .get(`/api/${table._id}/rows/not-a-valid-id`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(404)
+      await config.api.row.get(table._id!, "1234567", {
+        expectStatus: 404,
+      })
       await assertQueryUsage(queryUsage) // no change
     })
 

From 2c6df7475547a33677554dc93ab6e7de28c0a877 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:12:46 +0200
Subject: [PATCH 20/94] Fix more tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 821842e1b4..14b59e24c1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -503,7 +503,7 @@ describe.each([
 
   describe("patch", () => {
     it("should update only the fields that are supplied", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
 
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -530,7 +530,7 @@ describe.each([
     })
 
     it("should throw an error when given improper types", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -552,7 +552,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await config.createRow(row)
+      const createdRow = await createRow(row)
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 93124d804bed15b313a6c4563dda9c11f9348e8c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:17:22 +0200
Subject: [PATCH 21/94] Fix validate test

---
 packages/server/src/api/controllers/row/index.ts | 5 +++--
 packages/types/src/api/web/app/row.ts            | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 0bc3c4d3c7..e86b466601 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -15,6 +15,7 @@ import {
   SearchRowRequest,
   SearchParams,
   GetRowResponse,
+  ValidateResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -215,11 +216,11 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   })
 }
 
-export async function validate(ctx: Ctx) {
+export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
   if (isExternalTable(tableId)) {
-    ctx.body = { valid: true }
+    ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
diff --git a/packages/types/src/api/web/app/row.ts b/packages/types/src/api/web/app/row.ts
index 6e051facae..4ab4090461 100644
--- a/packages/types/src/api/web/app/row.ts
+++ b/packages/types/src/api/web/app/row.ts
@@ -11,3 +11,8 @@ export interface DeleteRow {
 }
 
 export type DeleteRowRequest = DeleteRows | DeleteRow
+
+export interface ValidateResponse {
+  valid: boolean
+  errors: Record<string, any>
+}

From 3912517f6733e27ba293a695b83f785c6caa411a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:22:24 +0200
Subject: [PATCH 22/94] Do not validate external

---
 packages/server/src/api/routes/tests/row.spec.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 14b59e24c1..ab61a480f3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -599,8 +599,14 @@ describe.each([
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.valid).toBe(false)
-      expect(Object.keys(res.body.errors)).toEqual(["name"])
+      if (isInternal) {
+        expect(res.body.valid).toBe(false)
+        expect(Object.keys(res.body.errors)).toEqual(["name"])
+      } else {
+        // Validation for external is not implemented, so it will always return valid
+        expect(res.body.valid).toBe(true)
+        expect(Object.keys(res.body.errors)).toEqual([])
+      }
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })

From 93e9b1b8b4d7f4e24df169b8ab33c2929dcfde52 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:34:45 +0200
Subject: [PATCH 23/94] Fix tests

---
 packages/server/src/api/controllers/row/external.ts |  2 +-
 packages/server/src/api/routes/tests/row.spec.ts    | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 5a1c7e04ff..44d50886d3 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -125,7 +125,7 @@ export async function destroy(ctx: UserCtx) {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
-  return { response: { ok: true }, row }
+  return { response: { ok: true, id: _id }, row }
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ab61a480f3..ae28c14279 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -614,8 +614,8 @@ describe.each([
 
   describe("bulkDelete", () => {
     it("should be able to delete a bulk set of rows", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
+      const row1 = await createRow()
+      const row2 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -635,9 +635,9 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
-      const row3 = await config.createRow()
+      const row1 = await createRow()
+      const row2 = await createRow()
+      const row3 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -657,7 +657,7 @@ describe.each([
     })
 
     it("should accept a valid row object and delete the row", async () => {
-      const row1 = await config.createRow()
+      const row1 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 9e799c6b933eb6e582a53b5e021498cf8c83f6ae Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:43:19 +0200
Subject: [PATCH 24/94] Legacy views tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 99 ++++++++++---------
 .../src/tests/utilities/TestConfiguration.ts  |  7 +-
 2 files changed, 57 insertions(+), 49 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ae28c14279..0fab8eb426 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -710,55 +710,62 @@ describe.each([
     })
   })
 
-  describe("fetchView", () => {
-    it("should be able to fetch tables contents via 'view'", async () => {
-      const row = await config.createRow()
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
+  // Legacy views are not available for external
+  isInternal &&
+    describe("fetchView", () => {
+      it("should be able to fetch tables contents via 'view'", async () => {
+        const row = await createRow()
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/views/${table._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 1)
+        const res = await request
+          .get(`/api/views/${table._id}`)
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage + 1)
+      })
+
+      it("should throw an error if view doesn't exist", async () => {
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
+
+        await request
+          .get(`/api/views/derp`)
+          .set(config.defaultHeaders())
+          .expect(404)
+
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage)
+      })
+
+      it("should be able to run on a view", async () => {
+        const view = await config.createLegacyView({
+          tableId: table._id!,
+          name: "ViewTest",
+          filters: [],
+          schema: {},
+        })
+        const row = await createRow()
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
+
+        const res = await request
+          .get(`/api/views/${view.name}`)
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage + 1)
+      })
     })
 
-    it("should throw an error if view doesn't exist", async () => {
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
-
-      await request
-        .get(`/api/views/derp`)
-        .set(config.defaultHeaders())
-        .expect(404)
-
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
-    })
-
-    it("should be able to run on a view", async () => {
-      const view = await config.createLegacyView()
-      const row = await config.createRow()
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
-
-      const res = await request
-        .get(`/api/views/${view.name}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
-
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 1)
-    })
-  })
-
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
       const { table, firstRow, secondRow } = await tenancy.doInTenant(
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index e835779f93..72a8b9a146 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -50,6 +50,7 @@ import {
   SearchFilters,
   UserRoles,
   Automation,
+  View,
 } from "@budibase/types"
 
 import API from "./api"
@@ -629,12 +630,12 @@ class TestConfiguration {
 
   // VIEW
 
-  async createLegacyView(config?: any) {
-    if (!this.table) {
+  async createLegacyView(config?: View) {
+    if (!this.table && !config) {
       throw "Test requires table to be configured."
     }
     const view = config || {
-      tableId: this.table._id,
+      tableId: this.table!._id,
       name: "ViewTest",
     }
     return this._req(view, null, controllers.view.v1.save)

From 5e2e43a7d7e342014d97010ee723d6b23848f6f7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 13:52:17 +0200
Subject: [PATCH 25/94] Clean configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 70 ++++++++++---------
 1 file changed, 37 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0fab8eb426..d572097469 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -49,37 +49,39 @@ describe.each([
     await config.init()
   })
 
-  beforeEach(async () => {
-    mocks.licenses.useCloudFree()
-    let tableConfig: SaveTableRequest = {
-      name: generator.guid(),
-      type: "table",
-      primary: ["id"],
-      schema: {
-        id: {
-          type: FieldType.AUTO,
-          name: "id",
-          autocolumn: true,
-          constraints: {
-            presence: true,
-          },
-        },
-        name: {
-          type: FieldType.STRING,
-          name: "name",
-          constraints: {
-            type: "string",
-          },
-        },
-        description: {
-          type: FieldType.STRING,
-          name: "description",
-          constraints: {
-            type: "string",
-          },
+  const generateTableConfig: () => SaveTableRequest = () => ({
+    name: generator.guid(),
+    type: "table",
+    primary: ["id"],
+    schema: {
+      id: {
+        type: FieldType.AUTO,
+        name: "id",
+        autocolumn: true,
+        constraints: {
+          presence: true,
         },
       },
-    }
+      name: {
+        type: FieldType.STRING,
+        name: "name",
+        constraints: {
+          type: "string",
+        },
+      },
+      description: {
+        type: FieldType.STRING,
+        name: "description",
+        constraints: {
+          type: "string",
+        },
+      },
+    },
+  })
+
+  beforeEach(async () => {
+    mocks.licenses.useCloudFree()
+    const tableConfig = generateTableConfig()
 
     if (dsProvider) {
       datasource = await config.api.datasource.create(
@@ -93,6 +95,8 @@ describe.each([
     }
 
     table = await config.api.table.create(tableConfig)
+    config.table = table
+    config.datasource = datasource
     row = basicRow(table._id!)
   })
 
@@ -130,8 +134,8 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = (row?: SaveRowRequest) =>
-    config.api.row.save(table._id!, row || basicRow(table._id!))
+  const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
+    config.api.row.save(tableId, row || basicRow(table._id!))
 
   describe("save, load, update", () => {
     function getDefaultFields() {
@@ -270,7 +274,7 @@ describe.each([
         status: "new",
       }
       await createRow()
-      await createRow(newRow)
+      await createRow(table._id, newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -552,7 +556,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await createRow(row)
+      const createdRow = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 9b783e0804763554f949da4ed4dac85a804395a5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:08:49 +0200
Subject: [PATCH 26/94] Fix cleanup relationships

---
 .../server/src/api/controllers/table/external.ts | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 327904666d..513b989a3a 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -67,13 +67,15 @@ function cleanupRelationships(
   tables: Record<string, Table>,
   oldTable?: Table
 ) {
-  const tableToIterate = oldTable ? oldTable : table
-  // clean up relationships in couch table schemas
-  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
-    if (
-      schema.type === FieldTypes.LINK &&
-      (!oldTable || table.schema[key] == null)
-    ) {
+  const isUpdate = !!oldTable
+  if (!isUpdate) {
+    return
+  }
+
+  // When updating a table, we want to detect if some relationships were removed.
+  // If so, we want to remove the relationship from the other end
+  for (let [key, schema] of Object.entries(oldTable.schema)) {
+    if (schema.type === FieldTypes.LINK && !table.schema[key]) {
       const relatedTable = Object.values(tables).find(
         table => table._id === schema.tableId
       )

From a5142088d9f0c25b08a7e09e656a238290dd4948 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:09:50 +0200
Subject: [PATCH 27/94] Clean tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d572097469..01a6175249 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -772,22 +772,36 @@ describe.each([
 
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
-      const { table, firstRow, secondRow } = await tenancy.doInTenant(
+      const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
         async () => {
-          const table = await config.createLinkedTable()
-          const firstRow = await config.createRow({
+          const linkedTable = await config.createLinkedTable({
+            name: generator.guid(),
+            type: "table",
+            primary: ["id"],
+            schema: {
+              id: {
+                type: FieldType.AUTO,
+                name: "id",
+                autocolumn: true,
+                constraints: {
+                  presence: true,
+                },
+              },
+            },
+          })
+          const firstRow = await createRow(table._id, {
             name: "Test Contact",
             description: "original description",
             tableId: table._id,
           })
-          const secondRow = await config.createRow({
+          const secondRow = await createRow(linkedTable._id, {
             name: "Test 2",
             description: "og desc",
             link: [{ _id: firstRow._id }],
-            tableId: table._id,
+            tableId: linkedTable._id,
           })
-          return { table, firstRow, secondRow }
+          return { linkedTable, firstRow, secondRow }
         }
       )
       const rowUsage = await getRowUsage()
@@ -795,16 +809,16 @@ describe.each([
 
       // test basic enrichment
       const resBasic = await request
-        .get(`/api/${table._id}/rows/${secondRow._id}`)
+        .get(`/api/${linkedTable._id}/rows/${secondRow._id}`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(resBasic.body.link[0]._id).toBe(firstRow._id)
-      expect(resBasic.body.link[0].primaryDisplay).toBe("Test Contact")
+      expect(resBasic.body.link.length).toBe(1)
+      expect(resBasic.body.link[0]).toEqual({ _id: firstRow._id })
 
       // test full enrichment
       const resEnriched = await request
-        .get(`/api/${table._id}/${secondRow._id}/enrich`)
+        .get(`/api/${linkedTable._id}/${secondRow._id}/enrich`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)

From 45ddd46f4cb79ff71a76a1ce1c45e887115a8073 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:29:05 +0200
Subject: [PATCH 28/94] Fix tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 49 ++++++++++---------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 01a6175249..9d9bcb302c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -137,17 +137,15 @@ describe.each([
   const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
     config.api.row.save(tableId, row || basicRow(table._id!))
 
-  describe("save, load, update", () => {
-    function getDefaultFields() {
-      if (isInternal) {
-        return {
-          type: "row",
-          createdAt: timestamp,
-          updatedAt: timestamp,
-        }
+  const defaultRowFields = isInternal
+    ? {
+        type: "row",
+        createdAt: timestamp,
+        updatedAt: timestamp,
       }
-    }
+    : undefined
 
+  describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -262,7 +260,7 @@ describe.each([
         _id: existing._id,
         _rev: existing._rev,
         id: existing.id,
-        ...getDefaultFields(),
+        ...defaultRowFields,
       })
       await assertQueryUsage(queryUsage + 1)
     })
@@ -835,7 +833,7 @@ describe.each([
     it("should allow enriching attachment rows", async () => {
       const table = await config.createAttachmentTable()
       const attachmentId = `${structures.uuid()}.csv`
-      const row = await config.createRow({
+      const row = await createRow(table._id, {
         name: "test",
         description: "test",
         attachment: [
@@ -906,15 +904,24 @@ describe.each([
     function userTable(): Table {
       return {
         name: "user",
-        type: "user",
+        type: "table",
+        primary: ["id"],
         schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+            autocolumn: true,
+            constraints: {
+              presence: true,
+            },
+          },
           name: {
             type: FieldType.STRING,
             name: "name",
           },
           surname: {
             type: FieldType.STRING,
-            name: "name",
+            name: "surname",
           },
           age: {
             type: FieldType.NUMBER,
@@ -965,11 +972,10 @@ describe.each([
           surname: data.surname,
           address: data.address,
           tableId: config.table!._id,
-          type: "row",
-          _id: expect.any(String),
-          _rev: expect.any(String),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
+          _id: newRow._id,
+          _rev: newRow._rev,
+          id: newRow.id,
+          ...defaultRowFields,
         })
         expect(row.body._viewId).toBeUndefined()
         expect(row.body.age).toBeUndefined()
@@ -1006,13 +1012,12 @@ describe.each([
         const row = await config.api.row.get(tableId, newRow._id!)
         expect(row.body).toEqual({
           ...newRow,
-          type: "row",
           name: newData.name,
           address: newData.address,
-          _id: expect.any(String),
+          _id: newRow._id,
           _rev: expect.any(String),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
+          id: newRow.id,
+          ...defaultRowFields,
         })
         expect(row.body._viewId).toBeUndefined()
         expect(row.body.age).toBeUndefined()

From c26a4c3a1135e42d321f88edb596db1a84293630 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:46:10 +0200
Subject: [PATCH 29/94] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 99 ++++++++++---------
 1 file changed, 53 insertions(+), 46 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9d9bcb302c..fa72776f30 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -49,51 +49,57 @@ describe.each([
     await config.init()
   })
 
-  const generateTableConfig: () => SaveTableRequest = () => ({
-    name: generator.guid(),
-    type: "table",
-    primary: ["id"],
-    schema: {
-      id: {
-        type: FieldType.AUTO,
-        name: "id",
-        autocolumn: true,
-        constraints: {
-          presence: true,
-        },
-      },
-      name: {
-        type: FieldType.STRING,
-        name: "name",
-        constraints: {
-          type: "string",
-        },
-      },
-      description: {
-        type: FieldType.STRING,
-        name: "description",
-        constraints: {
-          type: "string",
-        },
-      },
-    },
-  })
-
-  beforeEach(async () => {
-    mocks.licenses.useCloudFree()
-    const tableConfig = generateTableConfig()
-
+  const tableDatasourceConfig = async () => {
+    const result: Partial<SaveTableRequest> = {}
     if (dsProvider) {
       datasource = await config.api.datasource.create(
         await dsProvider.getDsConfig()
       )
 
-      tableConfig.sourceId = datasource._id
+      result.sourceId = datasource._id
       if (datasource.plus) {
-        tableConfig.type = "external"
+        result.type = "external"
       }
     }
+    return result
+  }
 
+  const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
+    return {
+      name: generator.guid(),
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          type: FieldType.AUTO,
+          name: "id",
+          autocolumn: true,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            type: "string",
+          },
+        },
+        description: {
+          type: FieldType.STRING,
+          name: "description",
+          constraints: {
+            type: "string",
+          },
+        },
+      },
+      ...(await tableDatasourceConfig()),
+    }
+  }
+
+  beforeEach(async () => {
+    mocks.licenses.useCloudFree()
+    const tableConfig = await generateTableConfig()
     table = await config.api.table.create(tableConfig)
     config.table = table
     config.datasource = datasource
@@ -901,9 +907,9 @@ describe.each([
   })
 
   describe("view 2.0", () => {
-    function userTable(): Table {
+    async function userTable(): Promise<Table> {
       return {
-        name: "user",
+        name: `users_${generator.guid()}`,
         type: "table",
         primary: ["id"],
         schema: {
@@ -936,6 +942,7 @@ describe.each([
             name: "jobTitle",
           },
         },
+        ...(await tableDatasourceConfig()),
       }
     }
 
@@ -949,7 +956,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -985,7 +992,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1027,7 +1034,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1037,7 +1044,7 @@ describe.each([
           },
         })
 
-        const createdRow = await config.createRow()
+        const createdRow = await createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1052,7 +1059,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1063,9 +1070,9 @@ describe.each([
         })
 
         const rows = [
-          await config.createRow(),
-          await config.createRow(),
-          await config.createRow(),
+          await createRow(tableId),
+          await createRow(tableId),
+          await createRow(tableId),
         ]
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()

From b63b61655b3b4e25e4466f0516f6dbfa9a1ab411 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:54:27 +0200
Subject: [PATCH 30/94] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 36 ++++++++++++-------
 .../src/tests/utilities/TestConfiguration.ts  | 28 ++++++++++-----
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fa72776f30..0193ff095d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1094,11 +1094,20 @@ describe.each([
 
     describe("view search", () => {
       const viewSchema = { age: { visible: true }, name: { visible: true } }
-      function userTable(): Table {
+      async function userTable(): Promise<Table> {
         return {
-          name: "user",
-          type: "user",
+          name: `users_${generator.guid()}`,
+          type: "table",
+          primary: ["id"],
           schema: {
+            id: {
+              type: FieldType.AUTO,
+              name: "id",
+              autocolumn: true,
+              constraints: {
+                presence: true,
+              },
+            },
             name: {
               type: FieldType.STRING,
               name: "name",
@@ -1110,11 +1119,12 @@ describe.each([
               constraints: {},
             },
           },
+          ...(await tableDatasourceConfig()),
         }
       }
 
-      it("returns table rows from view", async () => {
-        const table = await config.createTable(userTable())
+      it.only("returns table rows from view", async () => {
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1130,7 +1140,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
           await config.createRow({
@@ -1235,7 +1245,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(userTable())
+          await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1266,7 +1276,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(userTable())
+          await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1307,7 +1317,7 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1337,7 +1347,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        await config.createTable(userTable())
+        await config.createTable(await userTable())
 
         const createViewResponse = await config.api.viewV2.create()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1346,7 +1356,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1363,7 +1373,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1428,7 +1438,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(userTable())
+          const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
             rows.push(await config.createRow({ tableId: table._id }))
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 72a8b9a146..030a08cee7 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -51,6 +51,8 @@ import {
   UserRoles,
   Automation,
   View,
+  FieldType,
+  RelationshipType,
 } from "@budibase/types"
 
 import API from "./api"
@@ -76,7 +78,6 @@ class TestConfiguration {
   globalUserId: any
   userMetadataId: any
   table?: Table
-  linkedTable: any
   automation: any
   datasource?: Datasource
   tenantId?: string
@@ -559,25 +560,34 @@ class TestConfiguration {
     return this._req(null, { tableId }, controllers.table.find)
   }
 
-  async createLinkedTable(relationshipType?: string, links: any = ["link"]) {
+  async createLinkedTable(
+    config?: Table,
+    relationshipType = RelationshipType.ONE_TO_MANY,
+    links: any = ["link"]
+  ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
-    const tableConfig: any = basicTable()
+    const tableConfig = config || basicTable()
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
-        type: "link",
+        type: FieldType.LINK,
         fieldName: link,
         tableId: this.table._id,
         name: link,
-      }
-      if (relationshipType) {
-        tableConfig.schema[link].relationshipType = relationshipType
+        relationshipType,
       }
     }
-    const linkedTable = await this.createTable(tableConfig)
-    this.linkedTable = linkedTable
+
+    if (this.datasource && !tableConfig.sourceId) {
+      tableConfig.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
+    const linkedTable = await this.api.table.create(tableConfig)
     return linkedTable
   }
 

From 1a7a1cdd1bbcbe473987e17d14a0e53d9287b2e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:23:34 +0200
Subject: [PATCH 31/94] Fix view test

---
 packages/server/src/api/routes/tests/row.spec.ts   | 14 ++++++++++++--
 packages/server/src/sdk/app/rows/search.ts         |  2 +-
 .../server/src/sdk/app/rows/search/internal.ts     |  6 +++++-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0193ff095d..905304356c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1123,7 +1123,7 @@ describe.each([
         }
       }
 
-      it.only("returns table rows from view", async () => {
+      it("returns empty rows from view when no schema is passed", async () => {
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
@@ -1135,7 +1135,17 @@ describe.each([
 
         expect(response.body.rows).toHaveLength(10)
         expect(response.body).toEqual({
-          rows: expect.arrayContaining(rows.map(expect.objectContaining)),
+          rows: expect.arrayContaining(
+            rows.map(r => ({
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              _id: r._id,
+              _rev: r._rev,
+              ...defaultRowFields,
+            }))
+          ),
+          hasNextPage: false,
+          bookmark: null,
         })
       })
 
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 4861f473ea..afee18b57c 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -20,7 +20,7 @@ function pickApi(tableId: any) {
 export async function search(options: SearchParams): Promise<{
   rows: any[]
   hasNextPage?: boolean
-  bookmark?: number | null
+  bookmark?: number | string | null
 }> {
   return pickApi(options.tableId).search(options)
 }
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 4cdeca87f6..0a6fd69ff6 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -59,7 +59,11 @@ export async function search(options: SearchParams) {
   if (paginate) {
     response = await paginatedSearch(query, params)
   } else {
-    response = await fullSearch(query, params)
+    response = {
+      ...(await fullSearch(query, params)),
+      hasNextPage: false,
+      bookmark: null,
+    }
   }
 
   // Enrich search results with relationships

From 8858fe38873d1b9d4c128cfa4d8f8b2b537d4d5f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:31:47 +0200
Subject: [PATCH 32/94] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 29 ++++++++++++++-----
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 905304356c..94014d458d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -20,6 +20,7 @@ import {
   Table,
 } from "@budibase/types"
 import {
+  expectAnyExternalColsAttributes,
   expectAnyInternalColsAttributes,
   generator,
   mocks,
@@ -1178,8 +1179,18 @@ describe.each([
         expect(response.body.rows).toHaveLength(5)
         expect(response.body).toEqual({
           rows: expect.arrayContaining(
-            expectedRows.map(expect.objectContaining)
+            expectedRows.map(r => ({
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              name: r.name,
+              age: r.age,
+              _id: r._id,
+              _rev: r._rev,
+              ...defaultRowFields,
+            }))
           ),
+          hasNextPage: false,
+          bookmark: null,
         })
       })
 
@@ -1277,9 +1288,9 @@ describe.each([
           const response = await config.api.viewV2.search(createViewResponse.id)
 
           expect(response.body.rows).toHaveLength(4)
-          expect(response.body).toEqual({
-            rows: expected.map(name => expect.objectContaining({ name })),
-          })
+          expect(response.body.rows).toEqual(
+            expected.map(name => expect.objectContaining({ name }))
+          )
         }
       )
 
@@ -1320,9 +1331,9 @@ describe.each([
           )
 
           expect(response.body.rows).toHaveLength(4)
-          expect(response.body).toEqual({
-            rows: expected.map(name => expect.objectContaining({ name })),
-          })
+          expect(response.body.rows).toEqual(
+            expected.map(name => expect.objectContaining({ name }))
+          )
         }
       )
 
@@ -1348,7 +1359,9 @@ describe.each([
         expect(response.body.rows).toEqual(
           expect.arrayContaining(
             rows.map(r => ({
-              ...expectAnyInternalColsAttributes,
+              ...(isInternal
+                ? expectAnyInternalColsAttributes
+                : expectAnyExternalColsAttributes),
               _viewId: view.id,
               name: r.name,
             }))

From 2f5aadec4bdc09b71dc16f34e4c9b2339d27dad8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:39:13 +0200
Subject: [PATCH 33/94] More fixes

---
 packages/server/src/api/routes/tests/row.spec.ts | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 94014d458d..2373b8f9cc 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1401,7 +1401,6 @@ describe.each([
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
         }
-        // rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
 
         const createViewResponse = await config.api.viewV2.create()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
@@ -1417,9 +1416,9 @@ describe.each([
         )
         expect(firstPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(0, 4)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
 
         const secondPageResponse = await config.api.viewV2.search(
@@ -1434,9 +1433,9 @@ describe.each([
         )
         expect(secondPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(4, 8)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
 
         const lastPageResponse = await config.api.viewV2.search(
@@ -1450,9 +1449,9 @@ describe.each([
         )
         expect(lastPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(8)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: false,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
       })
 

From 4e69e51ccad85c9663f13d411ea6fd6f265cd1d8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 17:42:54 +0200
Subject: [PATCH 34/94] Fix autocolumns

---
 .../src/api/controllers/row/ExternalRequest.ts | 18 ++++++++++++------
 .../server/src/api/controllers/row/external.ts | 18 +++++++++++++++++-
 .../server/src/api/routes/tests/row.spec.ts    |  4 ++--
 packages/server/src/sdk/app/tables/index.ts    | 15 +++++++++++++--
 .../server/src/sdk/app/tables/validation.ts    |  7 -------
 5 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index b24b642035..2c4428599b 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -1,4 +1,5 @@
 import {
+  AutoReason,
   Datasource,
   FieldSchema,
   FieldType,
@@ -24,7 +25,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
-import { FieldTypes } from "../../../constants"
+import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
@@ -259,6 +260,15 @@ function isOneSide(field: FieldSchema) {
   )
 }
 
+function isEditableColumn(column: FieldSchema) {
+  const isExternalAutoColumn =
+    column.autocolumn &&
+    column.autoReason !== AutoReason.FOREIGN_KEY &&
+    column.subtype !== AutoFieldSubTypes.AUTO_ID
+  const isFormula = column.type === FieldTypes.FORMULA
+  return !(isExternalAutoColumn || isFormula)
+}
+
 export class ExternalRequest {
   private operation: Operation
   private tableId: string
@@ -295,11 +305,7 @@ export class ExternalRequest {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (
-        row[key] == null ||
-        newRow[key] ||
-        !sdk.tables.isEditableColumn(field)
-      ) {
+      if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
         continue
       }
       // if its an empty string then it means return the column to null (if possible)
diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 44d50886d3..acae075165 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -18,6 +18,8 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
+import { inputProcessing } from "../../../utilities/rowProcessor"
+import { cloneDeep, isEqual } from "lodash"
 
 export async function handleRequest(
   operation: Operation,
@@ -88,10 +90,24 @@ export async function save(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = inputProcessing(
+    ctx.user,
+    cloneDeep(table),
+    inputs
+  )
+
   const response = await handleRequest(Operation.CREATE, tableId, {
-    row: inputs,
+    row,
   })
+
   const responseRow = response as { row: Row }
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
   const rowId = responseRow.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 2373b8f9cc..63850602ba 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -177,8 +177,8 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const newTable = await config.createTable({
+        ...table,
         name: "TestTableAuto",
-        type: "table",
         schema: {
           ...table.schema,
           "Row ID": {
@@ -189,7 +189,7 @@ describe.each([
             autocolumn: true,
             constraints: {
               type: "number",
-              presence: false,
+              presence: true,
               numericality: {
                 greaterThanOrEqualTo: "",
                 lessThanOrEqualTo: "",
diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
index 1bf9117837..64fcde4bff 100644
--- a/packages/server/src/sdk/app/tables/index.ts
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -12,7 +12,7 @@ import {
   TableViewsResponse,
 } from "@budibase/types"
 import datasources from "../datasources"
-import { isEditableColumn, populateExternalTableSchemas } from "./validation"
+import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -73,12 +73,23 @@ function enrichViewSchemas(table: Table): TableResponse {
   }
 }
 
+async function saveTable(table: Table) {
+  const db = context.getAppDB()
+  if (isExternalTable(table._id!)) {
+    const datasource = await sdk.datasources.get(table.sourceId!)
+    datasource.entities![table.name] = table
+    await db.put(datasource)
+  } else {
+    await db.put(table)
+  }
+}
+
 export default {
   getAllInternalTables,
   getAllExternalTables,
   getExternalTable,
   getTable,
   populateExternalTableSchemas,
-  isEditableColumn,
   enrichViewSchemas,
+  saveTable,
 }
diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts
index 8dc41107d3..56f3e84c7a 100644
--- a/packages/server/src/sdk/app/tables/validation.ts
+++ b/packages/server/src/sdk/app/tables/validation.ts
@@ -55,13 +55,6 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   return datasource
 }
 
-export function isEditableColumn(column: FieldSchema) {
-  const isAutoColumn =
-    column.autocolumn && column.autoReason !== AutoReason.FOREIGN_KEY
-  const isFormula = column.type === FieldTypes.FORMULA
-  return !(isAutoColumn || isFormula)
-}
-
 export function populateExternalTableSchemas(datasource: Datasource) {
   return checkForeignKeysAreAutoColumns(datasource)
 }

From 33be2d6c969c52be48f833c4f8eadb9b6c1b7e2a Mon Sep 17 00:00:00 2001
From: FlaminWrap <97764630+FlaminWrap@users.noreply.github.com>
Date: Fri, 8 Sep 2023 21:28:25 +0100
Subject: [PATCH 35/94] Fix #11353

Removes the slice as the v prefix has been removed

Addresses:
#11353
---
 .../builder/src/pages/builder/portal/settings/version.svelte   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/version.svelte b/packages/builder/src/pages/builder/portal/settings/version.svelte
index fb9144c730..c3898b7861 100644
--- a/packages/builder/src/pages/builder/portal/settings/version.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/version.svelte
@@ -63,8 +63,7 @@
       )
       const githubResponse = await githubCheck.json()
 
-      //Get tag and remove the v infront of the tage name e.g. v1.0.0 is 1.0.0
-      githubVersion = githubResponse.tag_name.slice(1)
+      githubVersion = githubResponse.tag_name
 
       //Get the release date and output it in the local time format
       githubPublishedDate = new Date(githubResponse.published_at)

From ca743e77fbfc00a8c21003ad4cb8db0e3e3978f4 Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Mon, 11 Sep 2023 07:52:43 +0100
Subject: [PATCH 36/94] Fixed typo of label for number type column.

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index fd13767a63..44c37813d6 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -523,7 +523,7 @@
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
-        <Label size="M">Max Value</Label>
+        <Label size="M">Min Value</Label>
       </div>
       <div class="input-length">
         <Input

From 55514653c34d3cada6009ccf2d88b92fb72e8991 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:25:40 +0200
Subject: [PATCH 37/94] yarn.lock

---
 yarn.lock | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 8c93661665..ab86a87560 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,14 +6269,6 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
-"@types/node-fetch@2.6.1":
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
-  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^3.0.0"
-
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6298,11 +6290,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
-"@types/node@14.18.20":
-  version "14.18.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
-  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
-
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"

From c70c627fc9f0ae12ca8f047e24cc08e44126e391 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:52:46 +0200
Subject: [PATCH 38/94] Fix view test

---
 .../server/src/api/routes/tests/row.spec.ts   | 37 +++++++++++++++----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 63850602ba..d66a30684e 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -471,7 +471,12 @@ describe.each([
     function orderTable(): Table {
       return {
         name: "orders",
+        primary: ["id"],
         schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+          },
           Country: {
             type: FieldType.STRING,
             name: "Country",
@@ -494,19 +499,35 @@ describe.each([
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
         schema: {
-          Country: {},
-          OrderID: {},
+          Country: {
+            visible: true,
+          },
+          OrderID: {
+            visible: true,
+          },
         },
       })
 
-      const response = await config.api.row.save(createViewResponse.id, {
-        Country: "Aussy",
-        OrderID: "1111",
-        Story: "aaaaa",
-      })
+      const createRowResponse = await config.api.row.save(
+        createViewResponse.id,
+        {
+          OrderID: "1111",
+          Country: "Aussy",
+          Story: "aaaaa",
+        }
+      )
 
-      const row = await config.api.row.get(table._id!, response._id!)
+      const row = await config.api.row.get(table._id!, createRowResponse._id!)
       expect(row.body.Story).toBeUndefined()
+      expect(row.body).toEqual({
+        ...defaultRowFields,
+        OrderID: "1111",
+        Country: "Aussy",
+        id: 1,
+        _id: "%5B1%5D",
+        _rev: createRowResponse._rev,
+        tableId: table._id,
+      })
     })
   })
 

From 8ee23e6168f16456bafc06f464a8790dc3c4c212 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:57:47 +0200
Subject: [PATCH 39/94] Fix

---
 packages/server/src/api/routes/tests/row.spec.ts | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d66a30684e..bbf5a13574 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -471,18 +471,14 @@ describe.each([
     function orderTable(): Table {
       return {
         name: "orders",
-        primary: ["id"],
+        primary: ["OrderID"],
         schema: {
-          id: {
-            type: FieldType.AUTO,
-            name: "id",
-          },
           Country: {
             type: FieldType.STRING,
             name: "Country",
           },
           OrderID: {
-            type: FieldType.STRING,
+            type: FieldType.NUMBER,
             name: "OrderID",
           },
           Story: {
@@ -521,10 +517,9 @@ describe.each([
       expect(row.body.Story).toBeUndefined()
       expect(row.body).toEqual({
         ...defaultRowFields,
-        OrderID: "1111",
+        OrderID: 1111,
         Country: "Aussy",
-        id: 1,
-        _id: "%5B1%5D",
+        _id: createRowResponse._id,
         _rev: createRowResponse._rev,
         tableId: table._id,
       })

From 66524d998b746b1ad92e265bb5eb9436aee8446c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 11:02:44 +0200
Subject: [PATCH 40/94] Run coerced only for internal

---
 .../server/src/api/routes/tests/row.spec.ts   | 305 +++++++++---------
 1 file changed, 153 insertions(+), 152 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index bbf5a13574..5fa3745987 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -304,167 +304,168 @@ describe.each([
       await assertQueryUsage(queryUsage) // no change
     })
 
-    it("row values are coerced", async () => {
-      const str = {
-        type: FieldType.STRING,
-        name: "str",
-        constraints: { type: "string", presence: false },
-      }
-      const attachment = {
-        type: FieldType.ATTACHMENT,
-        name: "attachment",
-        constraints: { type: "array", presence: false },
-      }
-      const bool = {
-        type: FieldType.BOOLEAN,
-        name: "boolean",
-        constraints: { type: "boolean", presence: false },
-      }
-      const number = {
-        type: FieldType.NUMBER,
-        name: "str",
-        constraints: { type: "number", presence: false },
-      }
-      const datetime = {
-        type: FieldType.DATETIME,
-        name: "datetime",
-        constraints: {
-          type: "string",
-          presence: false,
-          datetime: { earliest: "", latest: "" },
-        },
-      }
-      const arrayField = {
-        type: FieldType.ARRAY,
-        constraints: {
-          type: "array",
-          presence: false,
-          inclusion: ["One", "Two", "Three"],
-        },
-        name: "Sample Tags",
-        sortable: false,
-      }
-      const optsField = {
-          fieldName: "Sample Opts",
-          name: "Sample Opts",
-          type: FieldType.OPTIONS,
+    isInternal &&
+      it("row values are coerced", async () => {
+        const str = {
+          type: FieldType.STRING,
+          name: "str",
+          constraints: { type: "string", presence: false },
+        }
+        const attachment = {
+          type: FieldType.ATTACHMENT,
+          name: "attachment",
+          constraints: { type: "array", presence: false },
+        }
+        const bool = {
+          type: FieldType.BOOLEAN,
+          name: "boolean",
+          constraints: { type: "boolean", presence: false },
+        }
+        const number = {
+          type: FieldType.NUMBER,
+          name: "str",
+          constraints: { type: "number", presence: false },
+        }
+        const datetime = {
+          type: FieldType.DATETIME,
+          name: "datetime",
           constraints: {
             type: "string",
             presence: false,
-            inclusion: ["Alpha", "Beta", "Gamma"],
+            datetime: { earliest: "", latest: "" },
           },
-        },
-        table = await config.createTable({
-          name: "TestTable2",
-          type: "table",
-          schema: {
-            name: str,
-            stringUndefined: str,
-            stringNull: str,
-            stringString: str,
-            numberEmptyString: number,
-            numberNull: number,
-            numberUndefined: number,
-            numberString: number,
-            numberNumber: number,
-            datetimeEmptyString: datetime,
-            datetimeNull: datetime,
-            datetimeUndefined: datetime,
-            datetimeString: datetime,
-            datetimeDate: datetime,
-            boolNull: bool,
-            boolEmpty: bool,
-            boolUndefined: bool,
-            boolString: bool,
-            boolBool: bool,
-            attachmentNull: attachment,
-            attachmentUndefined: attachment,
-            attachmentEmpty: attachment,
-            attachmentEmptyArrayStr: attachment,
-            arrayFieldEmptyArrayStr: arrayField,
-            arrayFieldArrayStrKnown: arrayField,
-            arrayFieldNull: arrayField,
-            arrayFieldUndefined: arrayField,
-            optsFieldEmptyStr: optsField,
-            optsFieldUndefined: optsField,
-            optsFieldNull: optsField,
-            optsFieldStrKnown: optsField,
+        }
+        const arrayField = {
+          type: FieldType.ARRAY,
+          constraints: {
+            type: "array",
+            presence: false,
+            inclusion: ["One", "Two", "Three"],
           },
-        })
+          name: "Sample Tags",
+          sortable: false,
+        }
+        const optsField = {
+            fieldName: "Sample Opts",
+            name: "Sample Opts",
+            type: FieldType.OPTIONS,
+            constraints: {
+              type: "string",
+              presence: false,
+              inclusion: ["Alpha", "Beta", "Gamma"],
+            },
+          },
+          table = await config.createTable({
+            name: "TestTable2",
+            type: "table",
+            schema: {
+              name: str,
+              stringUndefined: str,
+              stringNull: str,
+              stringString: str,
+              numberEmptyString: number,
+              numberNull: number,
+              numberUndefined: number,
+              numberString: number,
+              numberNumber: number,
+              datetimeEmptyString: datetime,
+              datetimeNull: datetime,
+              datetimeUndefined: datetime,
+              datetimeString: datetime,
+              datetimeDate: datetime,
+              boolNull: bool,
+              boolEmpty: bool,
+              boolUndefined: bool,
+              boolString: bool,
+              boolBool: bool,
+              attachmentNull: attachment,
+              attachmentUndefined: attachment,
+              attachmentEmpty: attachment,
+              attachmentEmptyArrayStr: attachment,
+              arrayFieldEmptyArrayStr: arrayField,
+              arrayFieldArrayStrKnown: arrayField,
+              arrayFieldNull: arrayField,
+              arrayFieldUndefined: arrayField,
+              optsFieldEmptyStr: optsField,
+              optsFieldUndefined: optsField,
+              optsFieldNull: optsField,
+              optsFieldStrKnown: optsField,
+            },
+          })
 
-      row = {
-        name: "Test Row",
-        stringUndefined: undefined,
-        stringNull: null,
-        stringString: "i am a string",
-        numberEmptyString: "",
-        numberNull: null,
-        numberUndefined: undefined,
-        numberString: "123",
-        numberNumber: 123,
-        datetimeEmptyString: "",
-        datetimeNull: null,
-        datetimeUndefined: undefined,
-        datetimeString: "1984-04-20T00:00:00.000Z",
-        datetimeDate: new Date("1984-04-20"),
-        boolNull: null,
-        boolEmpty: "",
-        boolUndefined: undefined,
-        boolString: "true",
-        boolBool: true,
-        tableId: table._id,
-        attachmentNull: null,
-        attachmentUndefined: undefined,
-        attachmentEmpty: "",
-        attachmentEmptyArrayStr: "[]",
-        arrayFieldEmptyArrayStr: "[]",
-        arrayFieldUndefined: undefined,
-        arrayFieldNull: null,
-        arrayFieldArrayStrKnown: "['One']",
-        optsFieldEmptyStr: "",
-        optsFieldUndefined: undefined,
-        optsFieldNull: null,
-        optsFieldStrKnown: "Alpha",
-      }
+        row = {
+          name: "Test Row",
+          stringUndefined: undefined,
+          stringNull: null,
+          stringString: "i am a string",
+          numberEmptyString: "",
+          numberNull: null,
+          numberUndefined: undefined,
+          numberString: "123",
+          numberNumber: 123,
+          datetimeEmptyString: "",
+          datetimeNull: null,
+          datetimeUndefined: undefined,
+          datetimeString: "1984-04-20T00:00:00.000Z",
+          datetimeDate: new Date("1984-04-20"),
+          boolNull: null,
+          boolEmpty: "",
+          boolUndefined: undefined,
+          boolString: "true",
+          boolBool: true,
+          tableId: table._id,
+          attachmentNull: null,
+          attachmentUndefined: undefined,
+          attachmentEmpty: "",
+          attachmentEmptyArrayStr: "[]",
+          arrayFieldEmptyArrayStr: "[]",
+          arrayFieldUndefined: undefined,
+          arrayFieldNull: null,
+          arrayFieldArrayStrKnown: "['One']",
+          optsFieldEmptyStr: "",
+          optsFieldUndefined: undefined,
+          optsFieldNull: null,
+          optsFieldStrKnown: "Alpha",
+        }
 
-      const createdRow = await config.createRow(row)
-      const id = createdRow._id!
+        const createdRow = await config.createRow(row)
+        const id = createdRow._id!
 
-      const saved = (await loadRow(id, table._id!)).body
+        const saved = (await loadRow(id, table._id!)).body
 
-      expect(saved.stringUndefined).toBe(undefined)
-      expect(saved.stringNull).toBe("")
-      expect(saved.stringString).toBe("i am a string")
-      expect(saved.numberEmptyString).toBe(null)
-      expect(saved.numberNull).toBe(null)
-      expect(saved.numberUndefined).toBe(undefined)
-      expect(saved.numberString).toBe(123)
-      expect(saved.numberNumber).toBe(123)
-      expect(saved.datetimeEmptyString).toBe(null)
-      expect(saved.datetimeNull).toBe(null)
-      expect(saved.datetimeUndefined).toBe(undefined)
-      expect(saved.datetimeString).toBe(
-        new Date(row.datetimeString).toISOString()
-      )
-      expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
-      expect(saved.boolNull).toBe(null)
-      expect(saved.boolEmpty).toBe(null)
-      expect(saved.boolUndefined).toBe(undefined)
-      expect(saved.boolString).toBe(true)
-      expect(saved.boolBool).toBe(true)
-      expect(saved.attachmentNull).toEqual([])
-      expect(saved.attachmentUndefined).toBe(undefined)
-      expect(saved.attachmentEmpty).toEqual([])
-      expect(saved.attachmentEmptyArrayStr).toEqual([])
-      expect(saved.arrayFieldEmptyArrayStr).toEqual([])
-      expect(saved.arrayFieldNull).toEqual([])
-      expect(saved.arrayFieldUndefined).toEqual(undefined)
-      expect(saved.optsFieldEmptyStr).toEqual(null)
-      expect(saved.optsFieldUndefined).toEqual(undefined)
-      expect(saved.optsFieldNull).toEqual(null)
-      expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-      expect(saved.optsFieldStrKnown).toEqual("Alpha")
-    })
+        expect(saved.stringUndefined).toBe(undefined)
+        expect(saved.stringNull).toBe("")
+        expect(saved.stringString).toBe("i am a string")
+        expect(saved.numberEmptyString).toBe(null)
+        expect(saved.numberNull).toBe(null)
+        expect(saved.numberUndefined).toBe(undefined)
+        expect(saved.numberString).toBe(123)
+        expect(saved.numberNumber).toBe(123)
+        expect(saved.datetimeEmptyString).toBe(null)
+        expect(saved.datetimeNull).toBe(null)
+        expect(saved.datetimeUndefined).toBe(undefined)
+        expect(saved.datetimeString).toBe(
+          new Date(row.datetimeString).toISOString()
+        )
+        expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
+        expect(saved.boolNull).toBe(null)
+        expect(saved.boolEmpty).toBe(null)
+        expect(saved.boolUndefined).toBe(undefined)
+        expect(saved.boolString).toBe(true)
+        expect(saved.boolBool).toBe(true)
+        expect(saved.attachmentNull).toEqual([])
+        expect(saved.attachmentUndefined).toBe(undefined)
+        expect(saved.attachmentEmpty).toEqual([])
+        expect(saved.attachmentEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldNull).toEqual([])
+        expect(saved.arrayFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldEmptyStr).toEqual(null)
+        expect(saved.optsFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldNull).toEqual(null)
+        expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
+        expect(saved.optsFieldStrKnown).toEqual("Alpha")
+      })
   })
 
   describe("view save", () => {

From 45605100697e6f0b9a3df63a357661261e546c9d Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Tue, 12 Sep 2023 12:16:13 +0100
Subject: [PATCH 41/94] Customize signup flow BB changes (#11706)

* wip

* wip

* wip

* wip
---
 packages/bbui/src/Banner/Banner.svelte        |  4 +++
 .../bbui/src/Notification/Notification.svelte |  6 ++++-
 .../Notification/NotificationDisplay.svelte   |  3 ++-
 packages/bbui/src/Stores/banner.js            |  1 +
 packages/bbui/src/Stores/notifications.js     |  6 ++++-
 .../builder/src/pages/builder/_layout.svelte  | 27 +++++++++++++++++++
 6 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/packages/bbui/src/Banner/Banner.svelte b/packages/bbui/src/Banner/Banner.svelte
index 3810021a61..a04d469cc7 100644
--- a/packages/bbui/src/Banner/Banner.svelte
+++ b/packages/bbui/src/Banner/Banner.svelte
@@ -66,6 +66,10 @@
     pointer-events: all;
     width: 100%;
   }
+
+  .spectrum-Toast--neutral {
+    background-color: var(--grey-2);
+  }
   .spectrum-Button {
     border: 1px solid rgba(255, 255, 255, 0.2);
   }
diff --git a/packages/bbui/src/Notification/Notification.svelte b/packages/bbui/src/Notification/Notification.svelte
index eb2922a5de..26e60ed366 100644
--- a/packages/bbui/src/Notification/Notification.svelte
+++ b/packages/bbui/src/Notification/Notification.svelte
@@ -27,7 +27,11 @@
   <div class="spectrum-Toast-body" class:actionBody={!!action}>
     <div class="wrap spectrum-Toast-content">{message || ""}</div>
     {#if action}
-      <ActionButton quiet emphasized on:click={action}>
+      <ActionButton
+        quiet
+        emphasized
+        on:click={() => action(() => dispatch("dismiss"))}
+      >
         <div style="color: white; font-weight: 600;">{actionMessage}</div>
       </ActionButton>
     {/if}
diff --git a/packages/bbui/src/Notification/NotificationDisplay.svelte b/packages/bbui/src/Notification/NotificationDisplay.svelte
index 0f7e93eb23..6b7e68cece 100644
--- a/packages/bbui/src/Notification/NotificationDisplay.svelte
+++ b/packages/bbui/src/Notification/NotificationDisplay.svelte
@@ -8,7 +8,7 @@
 
 <Portal target=".modal-container">
   <div class="notifications">
-    {#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)}
+    {#each $notifications as { type, icon, message, id, dismissable, action, actionMessage, wide } (id)}
       <div transition:fly={{ y: 30 }}>
         <Notification
           {type}
@@ -16,6 +16,7 @@
           {message}
           {dismissable}
           {action}
+          {actionMessage}
           {wide}
           on:dismiss={() => notifications.dismiss(id)}
         />
diff --git a/packages/bbui/src/Stores/banner.js b/packages/bbui/src/Stores/banner.js
index 1a0b2d9ecc..fc93e7be99 100644
--- a/packages/bbui/src/Stores/banner.js
+++ b/packages/bbui/src/Stores/banner.js
@@ -1,6 +1,7 @@
 import { writable } from "svelte/store"
 
 export const BANNER_TYPES = {
+  NEUTRAL: "neutral",
   INFO: "info",
   NEGATIVE: "negative",
   WARNING: "warning",
diff --git a/packages/bbui/src/Stores/notifications.js b/packages/bbui/src/Stores/notifications.js
index 449d282f24..28331fffd8 100644
--- a/packages/bbui/src/Stores/notifications.js
+++ b/packages/bbui/src/Stores/notifications.js
@@ -27,7 +27,9 @@ export const createNotificationStore = () => {
       icon = "",
       autoDismiss = true,
       action = null,
+      actionMessage = null,
       wide = false,
+      dismissTimeout = NOTIFICATION_TIMEOUT,
     }
   ) => {
     if (block) {
@@ -44,14 +46,16 @@ export const createNotificationStore = () => {
           icon,
           dismissable: !autoDismiss,
           action,
+          actionMessage,
           wide,
+          dismissTimeout,
         },
       ]
     })
     if (autoDismiss) {
       const timeoutId = setTimeout(() => {
         dismissNotification(_id)
-      }, NOTIFICATION_TIMEOUT)
+      }, dismissTimeout)
       timeoutIds.add(timeoutId)
     }
   }
diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index b216958045..960822a39f 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -3,6 +3,7 @@
   import { admin, auth, licensing } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
+  import { banner, BANNER_TYPES } from "@budibase/bbui"
   import { API } from "api"
   import Branding from "./Branding.svelte"
 
@@ -16,6 +17,32 @@
   $: user = $auth.user
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
+  let showVerificationPrompt = false
+
+  const checkVerification = user => {
+    if (!showVerificationPrompt && user?.account?.verified === false) {
+      showVerificationPrompt = true
+      banner.queue([
+        {
+          message: `Please verify your account. We've sent the verification link to ${user.email}`,
+          type: BANNER_TYPES.NEUTRAL,
+          showCloseButton: false,
+          extraButtonAction: () => {
+            fetch(`${$admin.accountPortalUrl}/api/auth/reset`, {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({ email: user.email }),
+            })
+          },
+          extraButtonText: "Resend email",
+        },
+      ])
+    }
+  }
+
+  $: checkVerification(user)
 
   const validateTenantId = async () => {
     const host = window.location.host

From 6e23d746117ce6b760a0c338ab79de85fd5a5da7 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 12 Sep 2023 11:16:28 +0000
Subject: [PATCH 42/94] Bump version to 2.10.3-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 1669e15246..ebdf2a4466 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.3-alpha.0",
+  "version": "2.10.3-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9a8b43b218ce7274c0d5b96d45e79da8dc7aab2b Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 12 Sep 2023 15:39:25 +0000
Subject: [PATCH 43/94] Bump version to 2.10.3-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index ebdf2a4466..e070f6579c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.3-alpha.1",
+  "version": "2.10.3-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 88c68cfe5858e758d80fbeac2df3eb3dd14901bf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:07:20 +0200
Subject: [PATCH 44/94] Fetch relationships on external row find

---
 packages/server/src/api/controllers/row/external.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index acae075165..3881d1dd08 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -125,7 +125,9 @@ export async function save(ctx: UserCtx) {
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  const row = await sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id, {
+    relationships: true,
+  })
 
   if (!row) {
     ctx.throw(404)

From a537b17b239d8c8b166ba0e1e0cbb898a044f870 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:07:50 +0200
Subject: [PATCH 45/94] Run attachments only on internal

---
 .../server/src/api/routes/tests/row.spec.ts   | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 5fa3745987..90eb4820b8 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -853,31 +853,32 @@ describe.each([
     })
   })
 
-  describe("attachments", () => {
-    it("should allow enriching attachment rows", async () => {
-      const table = await config.createAttachmentTable()
-      const attachmentId = `${structures.uuid()}.csv`
-      const row = await createRow(table._id, {
-        name: "test",
-        description: "test",
-        attachment: [
-          {
-            key: `${config.getAppId()}/attachments/${attachmentId}`,
-          },
-        ],
-        tableId: table._id,
-      })
-      // the environment needs configured for this
-      await setup.switchToSelfHosted(async () => {
-        return context.doInAppContext(config.getAppId(), async () => {
-          const enriched = await outputProcessing(table, [row])
-          expect((enriched as Row[])[0].attachment[0].url).toBe(
-            `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
-          )
+  isInternal &&
+    describe("attachments", () => {
+      it("should allow enriching attachment rows", async () => {
+        const table = await config.createAttachmentTable()
+        const attachmentId = `${structures.uuid()}.csv`
+        const row = await createRow(table._id, {
+          name: "test",
+          description: "test",
+          attachment: [
+            {
+              key: `${config.getAppId()}/attachments/${attachmentId}`,
+            },
+          ],
+          tableId: table._id,
+        })
+        // the environment needs configured for this
+        await setup.switchToSelfHosted(async () => {
+          return context.doInAppContext(config.getAppId(), async () => {
+            const enriched = await outputProcessing(table, [row])
+            expect((enriched as Row[])[0].attachment[0].url).toBe(
+              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+            )
+          })
         })
       })
     })
-  })
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {

From 8b644555e3e108bebdfa7f5bbcac835c0f1705a5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:09:09 +0200
Subject: [PATCH 46/94] Fetch primaryDisplay

---
 packages/server/src/api/routes/tests/row.spec.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 90eb4820b8..0bf678ef93 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -70,6 +70,7 @@ describe.each([
       name: generator.guid(),
       type: "table",
       primary: ["id"],
+      primaryDisplay: "name",
       schema: {
         id: {
           type: FieldType.AUTO,
@@ -801,6 +802,7 @@ describe.each([
             name: generator.guid(),
             type: "table",
             primary: ["id"],
+            primaryDisplay: "id",
             schema: {
               id: {
                 type: FieldType.AUTO,
@@ -836,7 +838,10 @@ describe.each([
         .expect("Content-Type", /json/)
         .expect(200)
       expect(resBasic.body.link.length).toBe(1)
-      expect(resBasic.body.link[0]).toEqual({ _id: firstRow._id })
+      expect(resBasic.body.link[0]).toEqual({
+        _id: firstRow._id,
+        primaryDisplay: firstRow.name,
+      })
 
       // test full enrichment
       const resEnriched = await request

From dd47c79ef92921c560dcd8949f0cdc26be73402d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:09:42 +0200
Subject: [PATCH 47/94] Replace guids by words

---
 packages/server/src/api/routes/tests/row.spec.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0bf678ef93..fdf1a861db 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -67,7 +67,7 @@ describe.each([
 
   const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
     return {
-      name: generator.guid(),
+      name: generator.word(),
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
@@ -799,7 +799,7 @@ describe.each([
         config.getTenantId(),
         async () => {
           const linkedTable = await config.createLinkedTable({
-            name: generator.guid(),
+            name: generator.word(),
             type: "table",
             primary: ["id"],
             primaryDisplay: "id",
@@ -933,7 +933,7 @@ describe.each([
   describe("view 2.0", () => {
     async function userTable(): Promise<Table> {
       return {
-        name: `users_${generator.guid()}`,
+        name: `users_${generator.word()}`,
         type: "table",
         primary: ["id"],
         schema: {
@@ -1120,7 +1120,7 @@ describe.each([
       const viewSchema = { age: { visible: true }, name: { visible: true } }
       async function userTable(): Promise<Table> {
         return {
-          name: `users_${generator.guid()}`,
+          name: `users_${generator.word()}`,
           type: "table",
           primary: ["id"],
           schema: {

From cb762cc336237e87a695700c4cecba8c83655ebe Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:03:24 +0200
Subject: [PATCH 48/94] Use api for testing

---
 .../server/src/api/routes/tests/row.spec.ts   | 95 +++++++++++--------
 .../server/src/tests/utilities/api/viewV2.ts  |  4 +-
 2 files changed, 59 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fdf1a861db..2f1647b994 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -177,11 +177,12 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
+      const tableConfig = await generateTableConfig()
       const newTable = await config.createTable({
-        ...table,
+        ...tableConfig,
         name: "TestTableAuto",
         schema: {
-          ...table.schema,
+          ...tableConfig.schema,
           "Row ID": {
             name: "Row ID",
             type: FieldType.NUMBER,
@@ -356,7 +357,7 @@ describe.each([
               inclusion: ["Alpha", "Beta", "Gamma"],
             },
           },
-          table = await config.createTable({
+          table = await config.api.table.create({
             name: "TestTable2",
             type: "table",
             schema: {
@@ -392,6 +393,7 @@ describe.each([
               optsFieldNull: optsField,
               optsFieldStrKnown: optsField,
             },
+            ...(await tableDatasourceConfig()),
           })
 
         row = {
@@ -470,7 +472,7 @@ describe.each([
   })
 
   describe("view save", () => {
-    function orderTable(): Table {
+    async function orderTable(): Promise<Table> {
       return {
         name: "orders",
         primary: ["OrderID"],
@@ -488,11 +490,12 @@ describe.each([
             name: "Story",
           },
         },
+        ...(await tableDatasourceConfig()),
       }
     }
 
     it("views have extra data trimmed", async () => {
-      const table = await config.createTable(orderTable())
+      const table = await config.api.table.create(await orderTable())
 
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
@@ -887,7 +890,7 @@ describe.each([
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -910,7 +913,7 @@ describe.each([
     })
 
     it("should allow exporting only certain columns", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -980,7 +983,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -992,7 +995,7 @@ describe.each([
 
         const data = randomRowData()
         const newRow = await config.api.row.save(view.id, {
-          tableId: config.table!._id,
+          tableId: table!._id,
           _viewId: view.id,
           ...data,
         })
@@ -1002,7 +1005,7 @@ describe.each([
           name: data.name,
           surname: data.surname,
           address: data.address,
-          tableId: config.table!._id,
+          tableId: table!._id,
           _id: newRow._id,
           _rev: newRow._rev,
           id: newRow.id,
@@ -1016,7 +1019,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1058,7 +1061,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1083,7 +1086,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1148,13 +1151,17 @@ describe.each([
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(
+            await config.api.row.save(table._id!, { tableId: table._id })
+          )
         }
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(10)
@@ -1174,10 +1181,10 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
-          await config.createRow({
+          await config.api.row.save(table._id!, {
             tableId: table._id,
             name: generator.name(),
             age: generator.integer({ min: 10, max: 30 }),
@@ -1185,7 +1192,7 @@ describe.each([
 
         for (let i = 0; i < 5; i++)
           expectedRows.push(
-            await config.createRow({
+            await config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: 40,
@@ -1193,6 +1200,7 @@ describe.each([
           )
 
         const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id!,
           query: [{ operator: "equal", field: "age", value: 40 }],
           schema: viewSchema,
         })
@@ -1289,7 +1297,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1297,13 +1305,14 @@ describe.each([
             { name: "Danny", age: 15 },
           ]
           for (const user of users) {
-            await config.createRow({
-              tableId: config.table!._id,
+            await config.api.row.save(table._id!, {
+              tableId: table._id,
               ...user,
             })
           }
 
           const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
             sort: sortParams,
             schema: viewSchema,
           })
@@ -1320,7 +1329,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1328,13 +1337,14 @@ describe.each([
             { name: "Danny", age: 15 },
           ]
           for (const user of users) {
-            await config.createRow({
-              tableId: config.table!._id,
+            await config.api.row.save(table._id!, {
+              tableId: table._id,
               ...user,
             })
           }
 
           const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
             sort: {
               field: "name",
               order: SortOrder.ASCENDING,
@@ -1361,11 +1371,11 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
-            await config.createRow({
+            await config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: generator.age(),
@@ -1374,6 +1384,7 @@ describe.each([
         }
 
         const view = await config.api.viewV2.create({
+          tableId: table._id,
           schema: { name: { visible: true } },
         })
         const response = await config.api.viewV2.search(view.id)
@@ -1393,23 +1404,27 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(0)
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(await createRow(table._id, { tableId: table._id }))
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id, {
           limit,
           query: {},
@@ -1419,13 +1434,15 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(await createRow(table._id, { tableId: table._id }))
         }
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
           .body.rows
 
@@ -1483,13 +1500,15 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
-            rows.push(await config.createRow({ tableId: table._id }))
+            rows.push(await createRow(table._id, { tableId: table._id }))
           }
 
-          const createViewResponse = await config.api.viewV2.create()
+          const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
+          })
 
           tableId = table._id!
           viewId = createViewResponse.id
diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts
index 0682361e16..92a6d394bf 100644
--- a/packages/server/src/tests/utilities/api/viewV2.ts
+++ b/packages/server/src/tests/utilities/api/viewV2.ts
@@ -23,8 +23,8 @@ export class ViewV2API extends TestAPI {
     if (!tableId && !this.config.table) {
       throw "Test requires table to be configured."
     }
-    const table = this.config.table
-    tableId = table!._id!
+
+    tableId = tableId || this.config.table!._id!
     const view = {
       tableId,
       name: generator.guid(),

From 9c37f2f05690e7d145ed6f475c0ce9a3b2f0f45a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:18:19 +0200
Subject: [PATCH 49/94] Use configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 84 +++++++------------
 1 file changed, 31 insertions(+), 53 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 2f1647b994..9282d5c83b 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -41,31 +41,20 @@ describe.each([
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
-  let row: Row
-  let datasource: Datasource | undefined
 
   afterAll(setup.afterAll)
 
   beforeAll(async () => {
     await config.init()
+
+    if (dsProvider) {
+      await config.createDatasource({
+        datasource: await dsProvider.getDsConfig(),
+      })
+    }
   })
 
-  const tableDatasourceConfig = async () => {
-    const result: Partial<SaveTableRequest> = {}
-    if (dsProvider) {
-      datasource = await config.api.datasource.create(
-        await dsProvider.getDsConfig()
-      )
-
-      result.sourceId = datasource._id
-      if (datasource.plus) {
-        result.type = "external"
-      }
-    }
-    return result
-  }
-
-  const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
+  const generateTableConfig: () => SaveTableRequest = () => {
     return {
       name: generator.word(),
       type: "table",
@@ -95,17 +84,13 @@ describe.each([
           },
         },
       },
-      ...(await tableDatasourceConfig()),
     }
   }
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    const tableConfig = await generateTableConfig()
-    table = await config.api.table.create(tableConfig)
-    config.table = table
-    config.datasource = datasource
-    row = basicRow(table._id!)
+    const tableConfig = generateTableConfig()
+    table = await config.createTable(tableConfig)
   })
 
   const loadRow = async (id: string, tbl_Id: string, status = 200) =>
@@ -159,8 +144,8 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const res = await request
-        .post(`/api/${row.tableId}/rows`)
-        .send(row)
+        .post(`/api/${config.table!._id}/rows`)
+        .send(basicRow(config.table!._id!))
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
@@ -177,7 +162,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const tableConfig = await generateTableConfig()
+      const tableConfig = generateTableConfig()
       const newTable = await config.createTable({
         ...tableConfig,
         name: "TestTableAuto",
@@ -265,10 +250,7 @@ describe.each([
         .expect(200)
 
       expect(res.body).toEqual({
-        ...row,
-        _id: existing._id,
-        _rev: existing._rev,
-        id: existing.id,
+        ...existing,
         ...defaultRowFields,
       })
       await assertQueryUsage(queryUsage + 1)
@@ -280,7 +262,7 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      await createRow()
+      const firstRow = await createRow()
       await createRow(table._id, newRow)
       const queryUsage = await getQueryUsage()
 
@@ -292,7 +274,7 @@ describe.each([
 
       expect(res.body.length).toBe(2)
       expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === firstRow.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -357,7 +339,7 @@ describe.each([
               inclusion: ["Alpha", "Beta", "Gamma"],
             },
           },
-          table = await config.api.table.create({
+          table = await config.createTable({
             name: "TestTable2",
             type: "table",
             schema: {
@@ -393,10 +375,9 @@ describe.each([
               optsFieldNull: optsField,
               optsFieldStrKnown: optsField,
             },
-            ...(await tableDatasourceConfig()),
           })
 
-        row = {
+        const row = {
           name: "Test Row",
           stringUndefined: undefined,
           stringNull: null,
@@ -490,12 +471,11 @@ describe.each([
             name: "Story",
           },
         },
-        ...(await tableDatasourceConfig()),
       }
     }
 
     it("views have extra data trimmed", async () => {
-      const table = await config.api.table.create(await orderTable())
+      const table = await config.createTable(await orderTable())
 
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
@@ -969,7 +949,6 @@ describe.each([
             name: "jobTitle",
           },
         },
-        ...(await tableDatasourceConfig()),
       }
     }
 
@@ -983,7 +962,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -1019,7 +998,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1061,7 +1040,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1086,7 +1065,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1146,12 +1125,11 @@ describe.each([
               constraints: {},
             },
           },
-          ...(await tableDatasourceConfig()),
         }
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1181,7 +1159,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
           await config.api.row.save(table._id!, {
@@ -1297,7 +1275,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1329,7 +1307,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1371,7 +1349,7 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1404,7 +1382,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
 
         const createViewResponse = await config.api.viewV2.create({
           tableId: table._id,
@@ -1415,7 +1393,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await createRow(table._id, { tableId: table._id }))
@@ -1434,7 +1412,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await createRow(table._id, { tableId: table._id }))
@@ -1500,7 +1478,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
             rows.push(await createRow(table._id, { tableId: table._id }))

From 3843581e56d7e71eeafb80c6ebf2e5b68ffcea76 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:31:58 +0200
Subject: [PATCH 50/94] Unify

---
 .../server/src/api/routes/tests/row.spec.ts   | 63 +++++++++----------
 .../src/tests/utilities/TestConfiguration.ts  | 10 +--
 2 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9282d5c83b..7488056bc3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -127,9 +127,6 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
-    config.api.row.save(tableId, row || basicRow(table._id!))
-
   const defaultRowFields = isInternal
     ? {
         type: "row",
@@ -215,7 +212,7 @@ describe.each([
     })
 
     it("updates a row successfully", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -240,7 +237,7 @@ describe.each([
     })
 
     it("should load a row", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -262,8 +259,8 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      const firstRow = await createRow()
-      await createRow(table._id, newRow)
+      const firstRow = await config.createRow()
+      await config.createRow(newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -279,7 +276,7 @@ describe.each([
     })
 
     it("load should return 404 when row does not exist", async () => {
-      await createRow()
+      await config.createRow()
       const queryUsage = await getQueryUsage()
 
       await config.api.row.get(table._id!, "1234567", {
@@ -513,7 +510,7 @@ describe.each([
 
   describe("patch", () => {
     it("should update only the fields that are supplied", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
 
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -540,7 +537,7 @@ describe.each([
     })
 
     it("should throw an error when given improper types", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -562,7 +559,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await createRow()
+      const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -624,8 +621,8 @@ describe.each([
 
   describe("bulkDelete", () => {
     it("should be able to delete a bulk set of rows", async () => {
-      const row1 = await createRow()
-      const row2 = await createRow()
+      const row1 = await config.createRow()
+      const row2 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -645,9 +642,9 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await createRow()
-      const row2 = await createRow()
-      const row3 = await createRow()
+      const row1 = await config.createRow()
+      const row2 = await config.createRow()
+      const row3 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -667,7 +664,7 @@ describe.each([
     })
 
     it("should accept a valid row object and delete the row", async () => {
-      const row1 = await createRow()
+      const row1 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -724,7 +721,7 @@ describe.each([
   isInternal &&
     describe("fetchView", () => {
       it("should be able to fetch tables contents via 'view'", async () => {
-        const row = await createRow()
+        const row = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -759,7 +756,7 @@ describe.each([
           filters: [],
           schema: {},
         })
-        const row = await createRow()
+        const row = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -797,12 +794,12 @@ describe.each([
               },
             },
           })
-          const firstRow = await createRow(table._id, {
+          const firstRow = await config.createRow({
             name: "Test Contact",
             description: "original description",
             tableId: table._id,
           })
-          const secondRow = await createRow(linkedTable._id, {
+          const secondRow = await config.createRow({
             name: "Test 2",
             description: "og desc",
             link: [{ _id: firstRow._id }],
@@ -846,7 +843,7 @@ describe.each([
       it("should allow enriching attachment rows", async () => {
         const table = await config.createAttachmentTable()
         const attachmentId = `${structures.uuid()}.csv`
-        const row = await createRow(table._id, {
+        const row = await config.createRow({
           name: "test",
           description: "test",
           attachment: [
@@ -870,7 +867,7 @@ describe.each([
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -893,7 +890,7 @@ describe.each([
     })
 
     it("should allow exporting only certain columns", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -1050,7 +1047,7 @@ describe.each([
           },
         })
 
-        const createdRow = await createRow()
+        const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1075,11 +1072,11 @@ describe.each([
           },
         })
 
-        const rows = [
-          await createRow(tableId),
-          await createRow(tableId),
-          await createRow(tableId),
-        ]
+        const rows = await Promise.all([
+          config.createRow(),
+          config.createRow(),
+          config.createRow(),
+        ])
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1396,7 +1393,7 @@ describe.each([
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await createRow(table._id, { tableId: table._id }))
+          rows.push(await config.createRow())
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
@@ -1415,7 +1412,7 @@ describe.each([
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await createRow(table._id, { tableId: table._id }))
+          rows.push(await config.createRow())
         }
 
         const createViewResponse = await config.api.viewV2.create({
@@ -1481,7 +1478,7 @@ describe.each([
           const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
-            rows.push(await createRow(table._id, { tableId: table._id }))
+            rows.push(await config.createRow())
           }
 
           const createViewResponse = await config.api.viewV2.create({
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 030a08cee7..119a926085 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -533,7 +533,7 @@ class TestConfiguration {
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    const response = await this._req(config, null, controllers.table.save)
+    const response = await this.api.table.create(config)
     if (!skipReassigning) {
       this.table = response
     }
@@ -694,12 +694,8 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this._req(
-      { ...config },
-      null,
-      controllers.datasource.save
-    )
-    this.datasource = response.datasource
+    const response = await this.api.datasource.create({ ...config.datasource })
+    this.datasource = response
     return this.datasource!
   }
 

From fccb2f625c902b5699d1f815a15c1fc517b988ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:39:40 +0200
Subject: [PATCH 51/94] Use configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 48 ++++++-------------
 .../src/tests/utilities/TestConfiguration.ts  | 21 ++++++++
 2 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 7488056bc3..ee3f12b449 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -474,8 +474,8 @@ describe.each([
     it("views have extra data trimmed", async () => {
       const table = await config.createTable(await orderTable())
 
-      const createViewResponse = await config.api.viewV2.create({
-        tableId: table._id,
+      const createViewResponse = await config.createView({
+        name: generator.word(),
         schema: {
           Country: {
             visible: true,
@@ -960,8 +960,7 @@ describe.each([
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
         const table = await config.createTable(await userTable())
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             surname: { visible: true },
@@ -997,8 +996,7 @@ describe.each([
       it("should update only the view fields for a row", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1039,8 +1037,7 @@ describe.each([
       it("should be able to delete a row", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1064,8 +1061,7 @@ describe.each([
       it("should be able to delete multiple rows", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1134,9 +1130,7 @@ describe.each([
           )
         }
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(10)
@@ -1174,8 +1168,7 @@ describe.each([
             })
           )
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id!,
+        const createViewResponse = await config.createView({
           query: [{ operator: "equal", field: "age", value: 40 }],
           schema: viewSchema,
         })
@@ -1286,8 +1279,7 @@ describe.each([
             })
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
+          const createViewResponse = await config.createView({
             sort: sortParams,
             schema: viewSchema,
           })
@@ -1318,8 +1310,7 @@ describe.each([
             })
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
+          const createViewResponse = await config.createView({
             sort: {
               field: "name",
               order: SortOrder.ASCENDING,
@@ -1358,8 +1349,7 @@ describe.each([
           )
         }
 
-        const view = await config.api.viewV2.create({
-          tableId: table._id,
+        const view = await config.createView({
           schema: { name: { visible: true } },
         })
         const response = await config.api.viewV2.search(view.id)
@@ -1381,9 +1371,7 @@ describe.each([
       it("views without data can be returned", async () => {
         const table = await config.createTable(await userTable())
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(0)
@@ -1397,9 +1385,7 @@ describe.each([
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id, {
           limit,
           query: {},
@@ -1415,9 +1401,7 @@ describe.each([
           rows.push(await config.createRow())
         }
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
           .body.rows
 
@@ -1481,9 +1465,7 @@ describe.each([
             rows.push(await config.createRow())
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
-          })
+          const createViewResponse = await config.createView()
 
           tableId = table._id!
           viewId = createViewResponse.id
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 119a926085..fbb6e05c57 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -53,6 +53,8 @@ import {
   View,
   FieldType,
   RelationshipType,
+  ViewV2,
+  CreateViewRequest,
 } from "@budibase/types"
 
 import API from "./api"
@@ -651,6 +653,25 @@ class TestConfiguration {
     return this._req(view, null, controllers.view.v1.save)
   }
 
+  async createView(
+    config?: Omit<CreateViewRequest, "tableId" | "name"> & {
+      name?: string
+      tableId?: string
+    }
+  ) {
+    if (!this.table && !config?.tableId) {
+      throw "Test requires table to be configured."
+    }
+
+    const view: CreateViewRequest = {
+      ...config,
+      tableId: config?.tableId || this.table!._id!,
+      name: config?.name || generator.word(),
+    }
+
+    return await this.api.viewV2.create(view)
+  }
+
   // AUTOMATION
 
   async createAutomation(config?: any) {

From f0872c1fa3135bd6b1cd6e976d8c0c9a1ec75790 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:59:37 +0200
Subject: [PATCH 52/94] Improve api

---
 packages/server/src/tests/utilities/api/row.ts | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index c6ef4606d2..2f6760ebeb 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -22,6 +22,21 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  getEnriched = async (
+    sourceId: string,
+    rowId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows/${rowId}/enrich`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+    if (expectStatus !== 404) {
+      request.expect("Content-Type", /json/)
+    }
+    return request
+  }
+
   save = async (
     sourceId: string,
     row: SaveRowRequest,
@@ -51,7 +66,7 @@ export class RowAPI extends TestAPI {
 
   delete = async (
     sourceId: string,
-    rows: Row[],
+    rows: Row | string | (Row | string)[],
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request

From 458de1282ec1ab6cf4b93f5269464013529fd2df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:17:21 +0200
Subject: [PATCH 53/94] Use api for testing

---
 .../server/src/api/routes/tests/row.spec.ts   |  2 --
 .../server/src/api/routes/tests/table.spec.ts | 12 ++++++-----
 .../src/api/routes/tests/viewV2.spec.ts       | 10 ++++++---
 .../src/integration-test/postgres.spec.ts     |  2 +-
 .../src/tests/utilities/TestConfiguration.ts  | 21 ++++---------------
 .../src/tests/utilities/api/datasource.ts     | 17 +++++++++++++++
 6 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ee3f12b449..7ec2ef8ef9 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -6,13 +6,11 @@ import * as setup from "./utilities"
 import { context, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
-  Datasource,
   FieldType,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
   Row,
-  SaveRowRequest,
   SaveTableRequest,
   SortOrder,
   SortType,
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 9914e6d66f..f56c6e4e44 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -1,6 +1,6 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table } from "@budibase/types"
+import { FieldType, Table, ViewCalculation } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -90,8 +90,10 @@ describe("/tables", () => {
       await config.createLegacyView({
         name: "TestView",
         field: "Price",
-        calculation: "stats",
-        tableId: testTable._id,
+        calculation: ViewCalculation.STATISTICS,
+        tableId: testTable._id!,
+        schema: {},
+        filters: [],
       })
 
       const testRow = await request
@@ -254,7 +256,7 @@ describe("/tables", () => {
       }))
 
       await config.api.viewV2.create({ tableId })
-      await config.createLegacyView({ tableId, name: generator.guid() })
+      await config.createLegacyView()
 
       const res = await config.api.table.fetch()
 
@@ -366,7 +368,7 @@ describe("/tables", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
-      const dependentTable = await config.getTable(linkedTable._id)
+      const dependentTable = await config.api.table.get(linkedTable._id!)
       expect(dependentTable.schema.TestTable).not.toBeDefined()
     })
 
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 5e8ae09e55..6d893c1c7f 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -6,6 +6,7 @@ import {
   SortOrder,
   SortType,
   Table,
+  UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
 } from "@budibase/types"
@@ -418,9 +419,12 @@ describe.each([
       const res = await config.api.viewV2.create(newView)
       const view = await config.api.viewV2.get(res.id)
       expect(view!.schema?.Price).toBeUndefined()
-      const updatedTable = await config.getTable(table._id!)
-      const viewSchema = updatedTable.views[view!.name!].schema
-      expect(viewSchema.Price.visible).toEqual(false)
+      const updatedTable = await config.api.table.get(table._id!)
+      const viewSchema = updatedTable.views![view!.name!].schema as Record<
+        string,
+        UIFieldMetadata
+      >
+      expect(viewSchema.Price?.visible).toEqual(false)
     })
   })
 })
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 0a53326cf4..b0f6f5bb04 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -397,7 +397,7 @@ describe("postgres integrations", () => {
         expect(res.status).toBe(200)
         expect(res.body).toEqual(updatedRow)
 
-        const persistedRow = await config.getRow(
+        const persistedRow = await config.api.row.get(
           primaryPostgresTable._id!,
           row.id
         )
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index fbb6e05c57..44867e6b01 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -557,11 +557,6 @@ class TestConfiguration {
     return this.updateTable(config, options)
   }
 
-  async getTable(tableId?: string) {
-    tableId = tableId || this.table?._id
-    return this._req(null, { tableId }, controllers.table.find)
-  }
-
   async createLinkedTable(
     config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
@@ -609,11 +604,7 @@ class TestConfiguration {
     }
     const tableId = (config && config.tableId) || this.table._id
     config = config || basicRow(tableId!)
-    return this._req(config, { tableId }, controllers.row.save)
-  }
-
-  async getRow(tableId: string, rowId: string): Promise<Row> {
-    return this._req(null, { tableId, rowId }, controllers.row.find)
+    return this.api.row.save(tableId!, config)
   }
 
   async getRows(tableId: string) {
@@ -648,7 +639,7 @@ class TestConfiguration {
     }
     const view = config || {
       tableId: this.table!._id,
-      name: "ViewTest",
+      name: generator.guid(),
     }
     return this._req(view, null, controllers.view.v1.save)
   }
@@ -721,12 +712,8 @@ class TestConfiguration {
   }
 
   async updateDatasource(datasource: Datasource): Promise<Datasource> {
-    const response = await this._req(
-      datasource,
-      { datasourceId: datasource._id },
-      controllers.datasource.update
-    )
-    this.datasource = response.datasource
+    const response = await this.api.datasource.update(datasource)
+    this.datasource = response
     return this.datasource!
   }
 
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 3c85a1c332..2ce09959fc 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -29,6 +29,23 @@ export class DatasourceAPI extends TestAPI {
     return result.body.datasource as Datasource
   }
 
+  update = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .put(`/api/datasources/${config._id}`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
   verify = async (
     data: VerifyDatasourceRequest,
     { expectStatus } = { expectStatus: 200 }

From b5e6b42db20dca220fc46ebd534377ec5852dab3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:40:00 +0200
Subject: [PATCH 54/94] Reuse tables

---
 .../server/src/api/routes/tests/row.spec.ts   | 196 +++++++++---------
 .../server/src/tests/utilities/api/row.ts     |  12 ++
 2 files changed, 109 insertions(+), 99 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 7ec2ef8ef9..b610e9689e 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -133,19 +133,26 @@ describe.each([
       }
     : undefined
 
-  describe("save, load, update", () => {
+    let tableId: string
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      const table = await config.createTable(tableConfig)
+      tableId = table._id!
+    })
+
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
       const res = await request
-        .post(`/api/${config.table!._id}/rows`)
-        .send(basicRow(config.table!._id!))
+        .post(`/api/${tableId}/rows`)
+        .send(basicRow(tableId))
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
       expect((res as any).res.statusMessage).toEqual(
-        `${table.name} saved successfully`
+        `${config.table!.name} saved successfully`
       )
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
@@ -158,28 +165,31 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
-      const newTable = await config.createTable({
-        ...tableConfig,
-        name: "TestTableAuto",
-        schema: {
-          ...tableConfig.schema,
-          "Row ID": {
-            name: "Row ID",
-            type: FieldType.NUMBER,
-            subtype: "autoID",
-            icon: "ri-magic-line",
-            autocolumn: true,
-            constraints: {
-              type: "number",
-              presence: true,
-              numericality: {
-                greaterThanOrEqualTo: "",
-                lessThanOrEqualTo: "",
+      const newTable = await config.createTable(
+        {
+          ...tableConfig,
+          name: "TestTableAuto",
+          schema: {
+            ...tableConfig.schema,
+            "Row ID": {
+              name: "Row ID",
+              type: FieldType.NUMBER,
+              subtype: "autoID",
+              icon: "ri-magic-line",
+              autocolumn: true,
+              constraints: {
+                type: "number",
+                presence: true,
+                numericality: {
+                  greaterThanOrEqualTo: "",
+                  lessThanOrEqualTo: "",
+                },
               },
             },
           },
         },
-      })
+        { skipReassigning: true }
+      )
 
       const ids = [1, 2, 3]
 
@@ -214,22 +224,14 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows`)
-        .send({
-          _id: existing._id,
-          _rev: existing._rev,
-          tableId: table._id,
-          name: "Updated Name",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.save(tableId, {
+        _id: existing._id,
+        _rev: existing._rev,
+        tableId,
+        name: "Updated Name",
+      })
 
-      expect((res as any).res.statusMessage).toEqual(
-        `${table.name} updated successfully.`
-      )
-      expect(res.body.name).toEqual("Updated Name")
+      expect(res.name).toEqual("Updated Name")
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage + 1)
     })
@@ -238,11 +240,7 @@ describe.each([
       const existing = await config.createRow()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/${table._id}/rows/${existing._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.get(tableId, existing._id!)
 
       expect(res.body).toEqual({
         ...existing,
@@ -252,24 +250,24 @@ describe.each([
     })
 
     it("should list all rows for given tableId", async () => {
+      const table = await config.createTable(generateTableConfig(), {
+        skipReassigning: true,
+      })
+      const tableId = table._id!
       const newRow = {
-        tableId: table._id,
+        tableId,
         name: "Second Contact",
-        status: "new",
+        description: "new",
       }
-      const firstRow = await config.createRow()
+      const firstRow = await config.createRow({ tableId })
       await config.createRow(newRow)
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/${table._id}/rows`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.fetch(tableId)
 
-      expect(res.body.length).toBe(2)
-      expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find((r: Row) => r.name === firstRow.name)).toBeDefined()
+      expect(res.length).toBe(2)
+      expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
+      expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -277,7 +275,7 @@ describe.each([
       await config.createRow()
       const queryUsage = await getQueryUsage()
 
-      await config.api.row.get(table._id!, "1234567", {
+      await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
       })
       await assertQueryUsage(queryUsage) // no change
@@ -325,52 +323,52 @@ describe.each([
           sortable: false,
         }
         const optsField = {
-            fieldName: "Sample Opts",
-            name: "Sample Opts",
-            type: FieldType.OPTIONS,
-            constraints: {
-              type: "string",
-              presence: false,
-              inclusion: ["Alpha", "Beta", "Gamma"],
-            },
+          fieldName: "Sample Opts",
+          name: "Sample Opts",
+          type: FieldType.OPTIONS,
+          constraints: {
+            type: "string",
+            presence: false,
+            inclusion: ["Alpha", "Beta", "Gamma"],
           },
-          table = await config.createTable({
-            name: "TestTable2",
-            type: "table",
-            schema: {
-              name: str,
-              stringUndefined: str,
-              stringNull: str,
-              stringString: str,
-              numberEmptyString: number,
-              numberNull: number,
-              numberUndefined: number,
-              numberString: number,
-              numberNumber: number,
-              datetimeEmptyString: datetime,
-              datetimeNull: datetime,
-              datetimeUndefined: datetime,
-              datetimeString: datetime,
-              datetimeDate: datetime,
-              boolNull: bool,
-              boolEmpty: bool,
-              boolUndefined: bool,
-              boolString: bool,
-              boolBool: bool,
-              attachmentNull: attachment,
-              attachmentUndefined: attachment,
-              attachmentEmpty: attachment,
-              attachmentEmptyArrayStr: attachment,
-              arrayFieldEmptyArrayStr: arrayField,
-              arrayFieldArrayStrKnown: arrayField,
-              arrayFieldNull: arrayField,
-              arrayFieldUndefined: arrayField,
-              optsFieldEmptyStr: optsField,
-              optsFieldUndefined: optsField,
-              optsFieldNull: optsField,
-              optsFieldStrKnown: optsField,
-            },
-          })
+        }
+        const table = await config.createTable({
+          name: "TestTable2",
+          type: "table",
+          schema: {
+            name: str,
+            stringUndefined: str,
+            stringNull: str,
+            stringString: str,
+            numberEmptyString: number,
+            numberNull: number,
+            numberUndefined: number,
+            numberString: number,
+            numberNumber: number,
+            datetimeEmptyString: datetime,
+            datetimeNull: datetime,
+            datetimeUndefined: datetime,
+            datetimeString: datetime,
+            datetimeDate: datetime,
+            boolNull: bool,
+            boolEmpty: bool,
+            boolUndefined: bool,
+            boolString: bool,
+            boolBool: bool,
+            attachmentNull: attachment,
+            attachmentUndefined: attachment,
+            attachmentEmpty: attachment,
+            attachmentEmptyArrayStr: attachment,
+            arrayFieldEmptyArrayStr: arrayField,
+            arrayFieldArrayStrKnown: arrayField,
+            arrayFieldNull: arrayField,
+            arrayFieldUndefined: arrayField,
+            optsFieldEmptyStr: optsField,
+            optsFieldUndefined: optsField,
+            optsFieldNull: optsField,
+            optsFieldStrKnown: optsField,
+          },
+        })
 
         const row = {
           name: "Test Row",
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 2f6760ebeb..3c266f07b5 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -76,4 +76,16 @@ export class RowAPI extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  fetch = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
 }

From d522ff70b4eb6ec77eb30a842945c303e48c22c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:47:06 +0200
Subject: [PATCH 55/94] Don't create tables for each test

---
 .../server/src/api/routes/tests/row.spec.ts   | 53 +++++++++++++++----
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index b610e9689e..dfb43779a8 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -87,8 +87,6 @@ describe.each([
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    const tableConfig = generateTableConfig()
-    table = await config.createTable(tableConfig)
   })
 
   const loadRow = async (id: string, tbl_Id: string, status = 200) =>
@@ -133,6 +131,7 @@ describe.each([
       }
     : undefined
 
+  describe("save, load, update", () => {
     let tableId: string
 
     beforeAll(async () => {
@@ -446,8 +445,8 @@ describe.each([
   })
 
   describe("view save", () => {
-    async function orderTable(): Promise<Table> {
-      return {
+    it("views have extra data trimmed", async () => {
+      const table = await config.createTable({
         name: "orders",
         primary: ["OrderID"],
         schema: {
@@ -464,11 +463,7 @@ describe.each([
             name: "Story",
           },
         },
-      }
-    }
-
-    it("views have extra data trimmed", async () => {
-      const table = await config.createTable(await orderTable())
+      })
 
       const createViewResponse = await config.createView({
         name: generator.word(),
@@ -505,6 +500,11 @@ describe.each([
   })
 
   describe("patch", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should update only the fields that are supplied", async () => {
       const existing = await config.createRow()
 
@@ -554,6 +554,11 @@ describe.each([
   })
 
   describe("destroy", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
@@ -574,6 +579,11 @@ describe.each([
   })
 
   describe("validate", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should return no errors on valid row", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -616,6 +626,11 @@ describe.each([
   })
 
   describe("bulkDelete", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should be able to delete a bulk set of rows", async () => {
       const row1 = await config.createRow()
       const row2 = await config.createRow()
@@ -716,6 +731,11 @@ describe.each([
   // Legacy views are not available for external
   isInternal &&
     describe("fetchView", () => {
+      beforeEach(async () => {
+        const tableConfig = generateTableConfig()
+        table = await config.createTable(tableConfig)
+      })
+
       it("should be able to fetch tables contents via 'view'", async () => {
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
@@ -770,6 +790,11 @@ describe.each([
     })
 
   describe("fetchEnrichedRows", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should allow enriching some linked rows", async () => {
       const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
@@ -836,6 +861,11 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
+      beforeEach(async () => {
+        const tableConfig = generateTableConfig()
+        table = await config.createTable(tableConfig)
+      })
+
       it("should allow enriching attachment rows", async () => {
         const table = await config.createAttachmentTable()
         const attachmentId = `${structures.uuid()}.csv`
@@ -862,6 +892,11 @@ describe.each([
     })
 
   describe("exportData", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should allow exporting all columns", async () => {
       const existing = await config.createRow()
       const res = await request

From eee8a2e5f95fd27b1b6f0082df24bf32b713d803 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:49:47 +0200
Subject: [PATCH 56/94] Run before alls

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index dfb43779a8..b4e257dc1c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -500,7 +500,7 @@ describe.each([
   })
 
   describe("patch", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -554,7 +554,7 @@ describe.each([
   })
 
   describe("destroy", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -579,7 +579,7 @@ describe.each([
   })
 
   describe("validate", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })

From 0c8a8e1b26fd1cd16585cfd84632b5359e4db414 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 21:09:25 +0200
Subject: [PATCH 57/94] Unify

---
 .../server/src/api/routes/tests/row.spec.ts   | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index b4e257dc1c..a4fe43ea1a 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -39,6 +39,7 @@ describe.each([
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
+  let tableId: string
 
   afterAll(setup.afterAll)
 
@@ -131,15 +132,13 @@ describe.each([
       }
     : undefined
 
+  beforeAll(async () => {
+    const tableConfig = generateTableConfig()
+    const table = await config.createTable(tableConfig)
+    tableId = table._id!
+  })
+
   describe("save, load, update", () => {
-    let tableId: string
-
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      const table = await config.createTable(tableConfig)
-      tableId = table._id!
-    })
-
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -626,7 +625,7 @@ describe.each([
   })
 
   describe("bulkDelete", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -790,7 +789,7 @@ describe.each([
     })
 
   describe("fetchEnrichedRows", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -861,7 +860,7 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
-      beforeEach(async () => {
+      beforeAll(async () => {
         const tableConfig = generateTableConfig()
         table = await config.createTable(tableConfig)
       })
@@ -892,7 +891,7 @@ describe.each([
     })
 
   describe("exportData", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })

From 986decb1034a0ad1589e26d394fa7614b7bab160 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:37:11 +0200
Subject: [PATCH 58/94] Reduce timings

---
 .../server/src/api/routes/tests/row.spec.ts   | 190 +++++++++---------
 1 file changed, 91 insertions(+), 99 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index a4fe43ea1a..fbf6651186 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -652,9 +652,11 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
-      const row3 = await config.createRow()
+      const [row1, row2, row3] = await Promise.all([
+        config.createRow(),
+        config.createRow(),
+        config.createRow(),
+      ])
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -1153,12 +1155,11 @@ describe.each([
 
       it("returns empty rows from view when no schema is passed", async () => {
         const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(
-            await config.api.row.save(table._id!, { tableId: table._id })
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, { tableId: table._id })
           )
-        }
+        )
 
         const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1181,22 +1182,26 @@ describe.each([
 
       it("searching respects the view filters", async () => {
         const table = await config.createTable(await userTable())
-        const expectedRows = []
-        for (let i = 0; i < 10; i++)
-          await config.api.row.save(table._id!, {
-            tableId: table._id,
-            name: generator.name(),
-            age: generator.integer({ min: 10, max: 30 }),
-          })
 
-        for (let i = 0; i < 5; i++)
-          expectedRows.push(
-            await config.api.row.save(table._id!, {
+        await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.integer({ min: 10, max: 30 }),
+            })
+          )
+        )
+
+        const expectedRows = await Promise.all(
+          Array.from({ length: 5 }, () =>
+            config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: 40,
             })
           )
+        )
 
         const createViewResponse = await config.createView({
           query: [{ operator: "equal", field: "age", value: 40 }],
@@ -1292,9 +1297,8 @@ describe.each([
         ],
       ]
 
-      it.each(sortTestOptions)(
-        "allow sorting (%s)",
-        async (sortParams, expected) => {
+      describe("sorting", () => {
+        beforeAll(async () => {
           const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
@@ -1302,82 +1306,76 @@ describe.each([
             { name: "Charly", age: 27 },
             { name: "Danny", age: 15 },
           ]
-          for (const user of users) {
-            await config.api.row.save(table._id!, {
-              tableId: table._id,
-              ...user,
+          await Promise.all(
+            users.map(u =>
+              config.api.row.save(table._id!, {
+                tableId: table._id,
+                ...u,
+              })
+            )
+          )
+        })
+
+        it.each(sortTestOptions)(
+          "allow sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: sortParams,
+              schema: viewSchema,
             })
+
+            const response = await config.api.viewV2.search(
+              createViewResponse.id
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
           }
+        )
 
-          const createViewResponse = await config.createView({
-            sort: sortParams,
-            schema: viewSchema,
-          })
-
-          const response = await config.api.viewV2.search(createViewResponse.id)
-
-          expect(response.body.rows).toHaveLength(4)
-          expect(response.body.rows).toEqual(
-            expected.map(name => expect.objectContaining({ name }))
-          )
-        }
-      )
-
-      it.each(sortTestOptions)(
-        "allow override the default view sorting (%s)",
-        async (sortParams, expected) => {
-          const table = await config.createTable(await userTable())
-          const users = [
-            { name: "Alice", age: 25 },
-            { name: "Bob", age: 30 },
-            { name: "Charly", age: 27 },
-            { name: "Danny", age: 15 },
-          ]
-          for (const user of users) {
-            await config.api.row.save(table._id!, {
-              tableId: table._id,
-              ...user,
+        it.each(sortTestOptions)(
+          "allow override the default view sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: {
+                field: "name",
+                order: SortOrder.ASCENDING,
+                type: SortType.STRING,
+              },
+              schema: viewSchema,
             })
+
+            const response = await config.api.viewV2.search(
+              createViewResponse.id,
+              {
+                sort: sortParams.field,
+                sortOrder: sortParams.order,
+                sortType: sortParams.type,
+                query: {},
+              }
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
           }
-
-          const createViewResponse = await config.createView({
-            sort: {
-              field: "name",
-              order: SortOrder.ASCENDING,
-              type: SortType.STRING,
-            },
-            schema: viewSchema,
-          })
-
-          const response = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              sort: sortParams.field,
-              sortOrder: sortParams.order,
-              sortType: sortParams.type,
-              query: {},
-            }
-          )
-
-          expect(response.body.rows).toHaveLength(4)
-          expect(response.body.rows).toEqual(
-            expected.map(name => expect.objectContaining({ name }))
-          )
-        }
-      )
+        )
+      })
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
         const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(
-            await config.api.row.save(table._id!, {
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: generator.age(),
             })
           )
-        }
+        )
 
         const view = await config.createView({
           schema: { name: { visible: true } },
@@ -1408,11 +1406,9 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow())
-        }
+        await config.createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
+
         const limit = generator.integer({ min: 1, max: 8 })
 
         const createViewResponse = await config.createView()
@@ -1425,11 +1421,8 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow())
-        }
+        await config.createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const createViewResponse = await config.createView()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
@@ -1489,11 +1482,10 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
-          const rows = []
-          for (let i = 0; i < 10; i++) {
-            rows.push(await config.createRow())
-          }
+          await config.createTable(await userTable())
+          await Promise.all(
+            Array.from({ length: 10 }, () => config.createRow())
+          )
 
           const createViewResponse = await config.createView()
 

From 14259c82f343933cf325e690f1fd5b6a9da5d1df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:57:41 +0200
Subject: [PATCH 59/94] Use test api

---
 .../server/src/api/routes/tests/row.spec.ts   | 130 ++++++------------
 .../server/src/tests/utilities/api/row.ts     |  23 +++-
 2 files changed, 60 insertions(+), 93 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fbf6651186..c9d78cd404 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -36,8 +36,8 @@ describe.each([
 ])("/rows (%s)", (_, dsProvider) => {
   const isInternal = !dsProvider
 
-  let request = setup.getRequest()
-  let config = setup.getConfig()
+  const request = setup.getRequest()
+  const config = setup.getConfig()
   let table: Table
   let tableId: string
 
@@ -90,12 +90,8 @@ describe.each([
     mocks.licenses.useCloudFree()
   })
 
-  const loadRow = async (id: string, tbl_Id: string, status = 200) =>
-    await request
-      .get(`/api/${tbl_Id}/rows/${id}`)
-      .set(config.defaultHeaders())
-      .expect("Content-Type", /json/)
-      .expect(status)
+  const loadRow = (id: string, tbl_Id: string, status = 200) =>
+    config.api.row.get(tbl_Id, id, { expectStatus: status })
 
   const getRowUsage = async () => {
     const { total } = await config.doInContext(null, () =>
@@ -193,20 +189,12 @@ describe.each([
 
       // Performing several create row requests should increment the autoID fields accordingly
       const createRow = async (id: number) => {
-        const res = await request
-          .post(`/api/${newTable._id}/rows`)
-          .send({
-            name: "row_" + id,
-          })
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect((res as any).res.statusMessage).toEqual(
-          `${newTable.name} saved successfully`
-        )
-        expect(res.body.name).toEqual("row_" + id)
-        expect(res.body._rev).toBeDefined()
-        expect(res.body["Row ID"]).toEqual(id)
+        const res = await config.api.row.save(newTable._id!, {
+          name: "row_" + id,
+        })
+        expect(res.name).toEqual("row_" + id)
+        expect(res._rev).toBeDefined()
+        expect(res["Row ID"]).toEqual(id)
       }
 
       for (let i = 0; i < ids.length; i++) {
@@ -563,14 +551,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [createdRow],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
       await assertRowUsage(rowUsage - 1)
       await assertQueryUsage(queryUsage + 1)
@@ -587,15 +568,10 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows/validate`)
-        .send({ name: "ivan" })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.validate(table._id!, { name: "ivan" })
 
-      expect(res.body.valid).toBe(true)
-      expect(Object.keys(res.body.errors)).toEqual([])
+      expect(res.valid).toBe(true)
+      expect(Object.keys(res.errors)).toEqual([])
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })
@@ -604,20 +580,15 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows/validate`)
-        .send({ name: 1 })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.validate(table._id!, { name: 1 })
 
       if (isInternal) {
-        expect(res.body.valid).toBe(false)
-        expect(Object.keys(res.body.errors)).toEqual(["name"])
+        expect(res.valid).toBe(false)
+        expect(Object.keys(res.errors)).toEqual(["name"])
       } else {
         // Validation for external is not implemented, so it will always return valid
-        expect(res.body.valid).toBe(true)
-        expect(Object.keys(res.body.errors)).toEqual([])
+        expect(res.valid).toBe(true)
+        expect(Object.keys(res.errors)).toEqual([])
       }
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
@@ -636,14 +607,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [row1, row2],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [row1, row2])
 
       expect(res.body.length).toEqual(2)
       await loadRow(row1._id!, table._id!, 404)
@@ -660,14 +624,11 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [row1, row2._id, { _id: row3._id }],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [
+        row1,
+        row2._id,
+        { _id: row3._id },
+      ])
 
       expect(res.body.length).toEqual(3)
       await loadRow(row1._id!, table._id!, 404)
@@ -680,12 +641,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send(row1)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, row1)
 
       expect(res.body.id).toEqual(row1._id)
       await loadRow(row1._id!, table._id!, 404)
@@ -697,31 +653,23 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({ not: "valid" })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res = await config.api.row.delete(
+        table._id!,
+        { not: "valid" },
+        { expectStatus: 400 }
+      )
       expect(res.body.message).toEqual("Invalid delete rows request")
 
-      const res2 = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({ rows: 123 })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res2 = await config.api.row.delete(
+        table._id!,
+        { rows: 123 },
+        { expectStatus: 400 }
+      )
       expect(res2.body.message).toEqual("Invalid delete rows request")
 
-      const res3 = await request
-        .delete(`/api/${table._id}/rows`)
-        .send("invalid")
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res3 = await config.api.row.delete(table._id!, "invalid", {
+        expectStatus: 400,
+      })
       expect(res3.body.message).toEqual("Invalid delete rows request")
 
       await assertRowUsage(rowUsage)
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 3c266f07b5..2338285ed4 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -1,4 +1,9 @@
-import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
+import {
+  PatchRowRequest,
+  SaveRowRequest,
+  Row,
+  ValidateResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -51,6 +56,20 @@ export class RowAPI extends TestAPI {
     return resp.body as Row
   }
 
+  validate = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ValidateResponse> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows/validate`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as ValidateResponse
+  }
+
   patch = async (
     sourceId: string,
     row: PatchRowRequest,
@@ -71,7 +90,7 @@ export class RowAPI extends TestAPI {
   ) => {
     return this.request
       .delete(`/api/${sourceId}/rows`)
-      .send({ rows })
+      .send(Array.isArray(rows) ? { rows } : rows)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)

From 4d12ee53dae26cf078e8f38f997b5fa384782517 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:59:17 +0200
Subject: [PATCH 60/94] LegacyView test api

---
 .../server/src/api/routes/tests/row.spec.ts     | 17 +++--------------
 .../server/src/tests/utilities/api/index.ts     |  3 +++
 .../src/tests/utilities/api/legacyView.ts       | 16 ++++++++++++++++
 3 files changed, 22 insertions(+), 14 deletions(-)
 create mode 100644 packages/server/src/tests/utilities/api/legacyView.ts

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index c9d78cd404..6a7a0210ef 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -690,11 +690,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const res = await request
-          .get(`/api/views/${table._id}`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
+        const res = await config.api.legacyView.get(table._id!)
         expect(res.body.length).toEqual(1)
         expect(res.body[0]._id).toEqual(row._id)
         await assertRowUsage(rowUsage)
@@ -705,10 +701,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        await request
-          .get(`/api/views/derp`)
-          .set(config.defaultHeaders())
-          .expect(404)
+        await config.api.legacyView.get("derp", { expectStatus: 404 })
 
         await assertRowUsage(rowUsage)
         await assertQueryUsage(queryUsage)
@@ -725,11 +718,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const res = await request
-          .get(`/api/views/${view.name}`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
+        const res = await config.api.legacyView.get(view.name)
         expect(res.body.length).toEqual(1)
         expect(res.body[0]._id).toEqual(row._id)
 
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index 0521f9b19f..31c74a0e78 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -4,9 +4,11 @@ import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
 import { DatasourceAPI } from "./datasource"
+import { LegacyViewAPI } from "./legacyView"
 
 export default class API {
   table: TableAPI
+  legacyView: LegacyViewAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
@@ -14,6 +16,7 @@ export default class API {
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
+    this.legacyView = new LegacyViewAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
diff --git a/packages/server/src/tests/utilities/api/legacyView.ts b/packages/server/src/tests/utilities/api/legacyView.ts
new file mode 100644
index 0000000000..63981cec5e
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/legacyView.ts
@@ -0,0 +1,16 @@
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class LegacyViewAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  get = async (id: string, { expectStatus } = { expectStatus: 200 }) => {
+    return await this.request
+      .get(`/api/views/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
+}

From cf74f19381d8341c6fe4658658b0ac82c3072b46 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:13:54 +0200
Subject: [PATCH 61/94] Use test api

---
 .../server/src/api/controllers/row/index.ts   |  6 ++-
 .../server/src/api/routes/tests/row.spec.ts   | 42 +++++++------------
 .../server/src/tests/utilities/api/row.ts     | 17 +++++++-
 packages/types/src/api/web/app/rows.ts        | 11 ++++-
 4 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index e86b466601..f0f2462019 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -16,6 +16,8 @@ import {
   SearchParams,
   GetRowResponse,
   ValidateResponse,
+  ExportRowsRequest,
+  ExportRowsResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -239,7 +241,9 @@ export async function fetchEnrichedRow(ctx: any) {
   )
 }
 
-export const exportRows = async (ctx: any) => {
+export const exportRows = async (
+  ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
+) => {
   const tableId = utils.getTableId(ctx)
 
   const format = ctx.query.format
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 6a7a0210ef..09cc2d1888 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -771,11 +771,10 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       // test basic enrichment
-      const resBasic = await request
-        .get(`/api/${linkedTable._id}/rows/${secondRow._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const resBasic = await config.api.row.get(
+        linkedTable._id!,
+        secondRow._id!
+      )
       expect(resBasic.body.link.length).toBe(1)
       expect(resBasic.body.link[0]).toEqual({
         _id: firstRow._id,
@@ -783,11 +782,10 @@ describe.each([
       })
 
       // test full enrichment
-      const resEnriched = await request
-        .get(`/api/${linkedTable._id}/${secondRow._id}/enrich`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const resEnriched = await config.api.row.getEnriched(
+        linkedTable._id!,
+        secondRow._id!
+      )
       expect(resEnriched.body.link.length).toBe(1)
       expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
       expect(resEnriched.body.link[0].name).toBe("Test Contact")
@@ -837,14 +835,9 @@ describe.each([
 
     it("should allow exporting all columns", async () => {
       const existing = await config.createRow()
-      const res = await request
-        .post(`/api/${table._id}/rows/exportRows?format=json`)
-        .set(config.defaultHeaders())
-        .send({
-          rows: [existing._id],
-        })
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+      })
       const results = JSON.parse(res.text)
       expect(results.length).toEqual(1)
       const row = results[0]
@@ -860,15 +853,10 @@ describe.each([
 
     it("should allow exporting only certain columns", async () => {
       const existing = await config.createRow()
-      const res = await request
-        .post(`/api/${table._id}/rows/exportRows?format=json`)
-        .set(config.defaultHeaders())
-        .send({
-          rows: [existing._id],
-          columns: ["_id"],
-        })
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+        columns: ["_id"],
+      })
       const results = JSON.parse(res.text)
       expect(results.length).toEqual(1)
       const row = results[0]
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 2338285ed4..686c8c031b 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -3,6 +3,7 @@ import {
   SaveRowRequest,
   Row,
   ValidateResponse,
+  ExportRowsRequest,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -33,7 +34,7 @@ export class RowAPI extends TestAPI {
     { expectStatus } = { expectStatus: 200 }
   ) => {
     const request = this.request
-      .get(`/api/${sourceId}/rows/${rowId}/enrich`)
+      .get(`/api/${sourceId}/${rowId}/enrich`)
       .set(this.config.defaultHeaders())
       .expect(expectStatus)
     if (expectStatus !== 404) {
@@ -107,4 +108,18 @@ export class RowAPI extends TestAPI {
 
     return (await request).body
   }
+
+  exportRows = async (
+    tableId: string,
+    body: ExportRowsRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .post(`/api/${tableId}/rows/exportRows?format=json`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return request
+  }
 }
diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts
index a99ef0e837..62ea90a6a4 100644
--- a/packages/types/src/api/web/app/rows.ts
+++ b/packages/types/src/api/web/app/rows.ts
@@ -1,5 +1,6 @@
-import { SearchParams } from "../../../sdk"
+import { SearchFilters, SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
+import { ReadStream } from "fs"
 
 export interface SaveRowRequest extends Row {}
 
@@ -28,3 +29,11 @@ export interface SearchViewRowRequest
 export interface SearchRowResponse {
   rows: any[]
 }
+
+export interface ExportRowsRequest {
+  rows: string[]
+  columns?: string[]
+  query?: SearchFilters
+}
+
+export type ExportRowsResponse = ReadStream

From 069b0d6161945288d5c937516cc9d5ca447b053e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:31:39 +0200
Subject: [PATCH 62/94] Fix types

---
 packages/server/src/migrations/tests/index.spec.ts | 5 ++++-
 packages/server/src/sdk/app/rows/search.ts         | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts
index b64cad26c1..d83e3cf31e 100644
--- a/packages/server/src/migrations/tests/index.spec.ts
+++ b/packages/server/src/migrations/tests/index.spec.ts
@@ -11,6 +11,7 @@ import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
 import tk from "timekeeper"
+import { View } from "@budibase/types"
 const timestamp = new Date().toISOString()
 tk.freeze(timestamp)
 
@@ -52,7 +53,9 @@ describe("migrations", () => {
         await config.createTable()
         await config.createLegacyView()
         await config.createTable()
-        await config.createLegacyView(structures.view(config.table!._id!))
+        await config.createLegacyView(
+          structures.view(config.table!._id!) as View
+        )
         await config.createScreen()
         await config.createScreen()
 
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index afee18b57c..b3bc220124 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -30,7 +30,7 @@ export interface ExportRowsParams {
   format: Format
   rowIds?: string[]
   columns?: string[]
-  query: SearchFilters
+  query?: SearchFilters
 }
 
 export interface ExportRowsResult {

From 47899c669dee55588353242d7cb4d56bf8f16ac8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:39:22 +0200
Subject: [PATCH 63/94] Use api

---
 packages/server/src/integration-test/postgres.spec.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index b0f6f5bb04..0e2560b7df 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -47,7 +47,7 @@ describe("postgres integrations", () => {
     )
 
     async function createAuxTable(prefix: string) {
-      return await config.createTable({
+      return await config.api.table.create({
         name: `${prefix}_${generator.word({ length: 6 })}`,
         type: "external",
         primary: ["id"],
@@ -83,7 +83,7 @@ describe("postgres integrations", () => {
       relationshipType: RelationshipType.MANY_TO_MANY,
     }
 
-    primaryPostgresTable = await config.createTable({
+    primaryPostgresTable = await config.api.table.create({
       name: `p_${generator.word({ length: 6 })}`,
       type: "external",
       primary: ["id"],
@@ -263,7 +263,7 @@ describe("postgres integrations", () => {
   }
 
   async function createDefaultPgTable() {
-    return await config.createTable({
+    return await config.api.table.create({
       name: generator.word({ length: 10 }),
       type: "external",
       primary: ["id"],
@@ -301,7 +301,7 @@ describe("postgres integrations", () => {
     )
   }
 
-  it("validate table schema", async () => {
+  it.only("validate table schema", async () => {
     const res = await makeRequest(
       "get",
       `/api/datasources/${postgresDatasource._id}`

From c7e41eacc77383918e82f2c23041935aab04aad2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 11:00:53 +0200
Subject: [PATCH 64/94] Cleanup test

---
 .../src/integration-test/postgres.spec.ts     | 74 ++++++++-----------
 1 file changed, 30 insertions(+), 44 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 0e2560b7df..2dc84a98da 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -181,32 +181,16 @@ describe("postgres integrations", () => {
     let { rowData } = opts as any
     let foreignRows: ForeignRowsInfo[] = []
 
-    async function createForeignRow(tableInfo: ForeignTableInfo) {
-      const foreignKey = `fk_${tableInfo.table.name}_${tableInfo.fieldName}`
-
-      const foreignRow = await config.createRow({
-        tableId: tableInfo.table._id,
-        title: generator.name(),
-      })
-
-      rowData = {
-        ...rowData,
-        [foreignKey]: foreignRow.id,
-      }
-      foreignRows.push({
-        row: foreignRow,
-
-        relationshipType: tableInfo.relationshipType,
-      })
-    }
-
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
-      const foreignRow = await config.createRow({
-        tableId: oneToManyRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        oneToManyRelationshipInfo.table._id!,
+        {
+          tableId: oneToManyRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -219,10 +203,13 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToOne || 0); i++) {
-      const foreignRow = await config.createRow({
-        tableId: manyToOneRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        manyToOneRelationshipInfo.table._id!,
+        {
+          tableId: manyToOneRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -237,10 +224,13 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToMany || 0); i++) {
-      const foreignRow = await config.createRow({
-        tableId: manyToManyRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        manyToManyRelationshipInfo.table._id!,
+        {
+          tableId: manyToManyRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -254,7 +244,7 @@ describe("postgres integrations", () => {
       })
     }
 
-    const row = await config.createRow({
+    const row = await config.api.row.save(primaryPostgresTable._id!, {
       tableId: primaryPostgresTable._id,
       ...rowData,
     })
@@ -277,6 +267,14 @@ describe("postgres integrations", () => {
     })
   }
 
+  const createRandomTableWithRows = async () => {
+    const tableId = (await createDefaultPgTable())._id!
+    return await config.api.row.save(tableId, {
+      tableId,
+      title: generator.name(),
+    })
+  }
+
   async function populatePrimaryRows(
     count: number,
     opts?: {
@@ -685,12 +683,6 @@ describe("postgres integrations", () => {
       describe("given than multiple tables have multiple rows", () => {
         const rowsCount = 6
         beforeEach(async () => {
-          const createRandomTableWithRows = async () =>
-            await config.createRow({
-              tableId: (await createDefaultPgTable())._id,
-              title: generator.name(),
-            })
-
           await createRandomTableWithRows()
           await createRandomTableWithRows()
 
@@ -978,12 +970,6 @@ describe("postgres integrations", () => {
       const rowsCount = 6
 
       beforeEach(async () => {
-        const createRandomTableWithRows = async () =>
-          await config.createRow({
-            tableId: (await createDefaultPgTable())._id,
-            title: generator.name(),
-          })
-
         await createRandomTableWithRows()
         await populatePrimaryRows(rowsCount)
         await createRandomTableWithRows()

From 823afa1f553116526ed1c3f150570ec4848b89ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 11:59:34 +0200
Subject: [PATCH 65/94] Run all tests

---
 packages/server/src/integration-test/postgres.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 2dc84a98da..27e9fa7206 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -299,7 +299,7 @@ describe("postgres integrations", () => {
     )
   }
 
-  it.only("validate table schema", async () => {
+  it("validate table schema", async () => {
     const res = await makeRequest(
       "get",
       `/api/datasources/${postgresDatasource._id}`

From ceb3129e221c4871c6c48dc0d14f62ff11fd8c5a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:02:13 +0200
Subject: [PATCH 66/94] Revert relationship changes

---
 .../server/src/api/controllers/table/external.ts | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 513b989a3a..327904666d 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -67,15 +67,13 @@ function cleanupRelationships(
   tables: Record<string, Table>,
   oldTable?: Table
 ) {
-  const isUpdate = !!oldTable
-  if (!isUpdate) {
-    return
-  }
-
-  // When updating a table, we want to detect if some relationships were removed.
-  // If so, we want to remove the relationship from the other end
-  for (let [key, schema] of Object.entries(oldTable.schema)) {
-    if (schema.type === FieldTypes.LINK && !table.schema[key]) {
+  const tableToIterate = oldTable ? oldTable : table
+  // clean up relationships in couch table schemas
+  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
+    if (
+      schema.type === FieldTypes.LINK &&
+      (!oldTable || table.schema[key] == null)
+    ) {
       const relatedTable = Object.values(tables).find(
         table => table._id === schema.tableId
       )

From 6bae382d81e118c12c047b19674d27d5ae5d4f09 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:23:59 +0200
Subject: [PATCH 67/94] Fix tests

---
 packages/server/src/api/controllers/table/index.ts       | 4 ++--
 packages/server/src/tests/utilities/TestConfiguration.ts | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 759974d6a7..29c41ad985 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -78,9 +78,9 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
 }
 
 export async function destroy(ctx: UserCtx) {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 44867e6b01..e521a5a6a6 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -557,6 +557,11 @@ class TestConfiguration {
     return this.updateTable(config, options)
   }
 
+  async getTable(tableId?: string) {
+    tableId = tableId || this.table!._id!
+    return this._req(null, { tableId }, controllers.table.find)
+  }
+
   async createLinkedTable(
     config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
@@ -607,6 +612,10 @@ class TestConfiguration {
     return this.api.row.save(tableId!, config)
   }
 
+  async getRow(tableId: string, rowId: string): Promise<Row> {
+    return this._req(null, { tableId, rowId }, controllers.row.find)
+  }
+
   async getRows(tableId: string) {
     if (!tableId && this.table) {
       tableId = this.table._id!

From dc5cd7cf76cf7132f742b68032f04f2119b4f7d2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:53:47 +0200
Subject: [PATCH 68/94] Fix

---
 .../src/integration-test/postgres.spec.ts     | 95 ++++++++++++-------
 1 file changed, 60 insertions(+), 35 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 27e9fa7206..8ce1530ec2 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -47,7 +47,7 @@ describe("postgres integrations", () => {
     )
 
     async function createAuxTable(prefix: string) {
-      return await config.api.table.create({
+      return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
         type: "external",
         primary: ["id"],
@@ -83,7 +83,7 @@ describe("postgres integrations", () => {
       relationshipType: RelationshipType.MANY_TO_MANY,
     }
 
-    primaryPostgresTable = await config.api.table.create({
+    primaryPostgresTable = await config.createTable({
       name: `p_${generator.word({ length: 6 })}`,
       type: "external",
       primary: ["id"],
@@ -184,13 +184,10 @@ describe("postgres integrations", () => {
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
-      const foreignRow = await config.api.row.save(
-        oneToManyRelationshipInfo.table._id!,
-        {
-          tableId: oneToManyRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: oneToManyRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -203,13 +200,10 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToOne || 0); i++) {
-      const foreignRow = await config.api.row.save(
-        manyToOneRelationshipInfo.table._id!,
-        {
-          tableId: manyToOneRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: manyToOneRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -224,13 +218,10 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToMany || 0); i++) {
-      const foreignRow = await config.api.row.save(
-        manyToManyRelationshipInfo.table._id!,
-        {
-          tableId: manyToManyRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: manyToManyRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -244,7 +235,7 @@ describe("postgres integrations", () => {
       })
     }
 
-    const row = await config.api.row.save(primaryPostgresTable._id!, {
+    const row = await config.createRow({
       tableId: primaryPostgresTable._id,
       ...rowData,
     })
@@ -253,7 +244,7 @@ describe("postgres integrations", () => {
   }
 
   async function createDefaultPgTable() {
-    return await config.api.table.create({
+    return await config.createTable({
       name: generator.word({ length: 10 }),
       type: "external",
       primary: ["id"],
@@ -395,7 +386,7 @@ describe("postgres integrations", () => {
         expect(res.status).toBe(200)
         expect(res.body).toEqual(updatedRow)
 
-        const persistedRow = await config.api.row.get(
+        const persistedRow = await config.getRow(
           primaryPostgresTable._id!,
           row.id
         )
@@ -520,7 +511,7 @@ describe("postgres integrations", () => {
           foreignRows = createdRow.foreignRows
         })
 
-        it("only one to many foreign keys are retrieved", async () => {
+        it("only one to primary keys are retrieved", async () => {
           const res = await getRow(primaryPostgresTable._id, row.id)
 
           expect(res.status).toBe(200)
@@ -528,6 +519,12 @@ describe("postgres integrations", () => {
           const one2ManyForeignRows = foreignRows.filter(
             x => x.relationshipType === RelationshipType.ONE_TO_MANY
           )
+          const many2OneForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_ONE
+          )
+          const many2ManyForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_MANY
+          )
           expect(one2ManyForeignRows).toHaveLength(1)
 
           expect(res.body).toEqual({
@@ -538,9 +535,25 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               one2ManyForeignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              one2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2OneForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -569,9 +582,13 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               foreignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -598,9 +615,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -627,9 +648,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
     })

From c530d5fa348105ead7cab139655b8444dde7424b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 13:17:51 +0200
Subject: [PATCH 69/94] Timeout issues

---
 .../src/integration-test/postgres.spec.ts     | 20 ++++++++++---------
 .../src/integrations/tests/utils/index.ts     |  2 ++
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 8ce1530ec2..86a43c8c42 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -21,8 +21,6 @@ import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
-jest.setTimeout(30000)
-
 jest.unmock("pg")
 jest.mock("../websockets")
 
@@ -39,13 +37,13 @@ describe("postgres integrations", () => {
     const apiKey = await config.generateApiKey()
 
     makeRequest = generateMakeRequest(apiKey, true)
-  })
 
-  beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
       await databaseTestProviders.postgres.getDsConfig()
     )
+  })
 
+  beforeEach(async () => {
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
@@ -345,12 +343,16 @@ describe("postgres integrations", () => {
 
       it("multiple rows can be persisted", async () => {
         const numberOfRows = 10
-        const newRows = Array(numberOfRows).fill(generateRandomPrimaryRowData())
+        const newRows: Row[] = Array(numberOfRows).fill(
+          generateRandomPrimaryRowData()
+        )
 
-        for (const newRow of newRows) {
-          const res = await createRow(primaryPostgresTable._id, newRow)
-          expect(res.status).toBe(200)
-        }
+        await Promise.all(
+          newRows.map(async newRow => {
+            const res = await createRow(primaryPostgresTable._id, newRow)
+            expect(res.status).toBe(200)
+          })
+        )
 
         const persistedRows = await config.getRows(primaryPostgresTable._id!)
         expect(persistedRows).toHaveLength(numberOfRows)
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index c145a9c809..a28141db08 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -3,6 +3,8 @@ jest.unmock("pg")
 import { Datasource } from "@budibase/types"
 import * as pg from "./postgres"
 
+jest.setTimeout(30000)
+
 export interface DatabasePlusTestProvider {
   getDsConfig(): Promise<Datasource>
 }

From 142fb18c177b1063076c98852643094299cb1137 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 14:09:48 +0200
Subject: [PATCH 70/94] Fix tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 33 +++++++++++--------
 .../src/db/tests/linkController.spec.js       |  2 +-
 .../src/tests/utilities/TestConfiguration.ts  |  4 +--
 3 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 09cc2d1888..e3bfc32320 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -10,6 +10,7 @@ import {
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
+  RelationshipType,
   Row,
   SaveTableRequest,
   SortOrder,
@@ -737,22 +738,26 @@ describe.each([
       const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
         async () => {
-          const linkedTable = await config.createLinkedTable({
-            name: generator.word(),
-            type: "table",
-            primary: ["id"],
-            primaryDisplay: "id",
-            schema: {
-              id: {
-                type: FieldType.AUTO,
-                name: "id",
-                autocolumn: true,
-                constraints: {
-                  presence: true,
+          const linkedTable = await config.createLinkedTable(
+            RelationshipType.ONE_TO_MANY,
+            ["link"],
+            {
+              name: generator.word(),
+              type: "table",
+              primary: ["id"],
+              primaryDisplay: "id",
+              schema: {
+                id: {
+                  type: FieldType.AUTO,
+                  name: "id",
+                  autocolumn: true,
+                  constraints: {
+                    presence: true,
+                  },
                 },
               },
-            },
-          })
+            }
+          )
           const firstRow = await config.createRow({
             name: "Test Contact",
             description: "original description",
diff --git a/packages/server/src/db/tests/linkController.spec.js b/packages/server/src/db/tests/linkController.spec.js
index 1c50142871..59d0f3f983 100644
--- a/packages/server/src/db/tests/linkController.spec.js
+++ b/packages/server/src/db/tests/linkController.spec.js
@@ -6,7 +6,7 @@ const { RelationshipType } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")
 
 describe("test the link controller", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
   let table1, table2, appId
 
   beforeAll(async () => {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index e521a5a6a6..94896932ad 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -563,9 +563,9 @@ class TestConfiguration {
   }
 
   async createLinkedTable(
-    config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
-    links: any = ["link"]
+    links: any = ["link"],
+    config?: Table
   ) {
     if (!this.table) {
       throw "Must have created a table first."

From 21e5219bde49817b78d8dcf217784586550e277f Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 12:10:55 +0000
Subject: [PATCH 71/94] Bump version to 2.10.7-alpha.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 63466ceedb..88bde5827a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.6",
+  "version": "2.10.7-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9c6e880479a059c099a1c17f884a9dbc4dd48a2c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:03:21 +0200
Subject: [PATCH 72/94] Fix update datasource test helper

---
 packages/server/src/tests/utilities/api/datasource.ts | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 2ce09959fc..ee698334f2 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -30,16 +30,12 @@ export class DatasourceAPI extends TestAPI {
   }
 
   update = async (
-    config: Datasource,
+    datasource: Datasource,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Datasource> => {
-    const body: CreateDatasourceRequest = {
-      datasource: config,
-      tablesFilter: [],
-    }
     const result = await this.request
-      .put(`/api/datasources/${config._id}`)
-      .send(body)
+      .put(`/api/datasources/${datasource._id}`)
+      .send(datasource)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)

From d248c7b6b8357a3bc3f1a52e7e7bce6b62c7924f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:26:29 +0200
Subject: [PATCH 73/94] Fix types

---
 packages/server/src/sdk/app/rows/search.ts          | 2 +-
 packages/server/src/sdk/app/rows/search/internal.ts | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index b3bc220124..380521a05a 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -20,7 +20,7 @@ function pickApi(tableId: any) {
 export async function search(options: SearchParams): Promise<{
   rows: any[]
   hasNextPage?: boolean
-  bookmark?: number | string | null
+  bookmark?: number | null
 }> {
   return pickApi(options.tableId).search(options)
 }
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 0a6fd69ff6..4cdeca87f6 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -59,11 +59,7 @@ export async function search(options: SearchParams) {
   if (paginate) {
     response = await paginatedSearch(query, params)
   } else {
-    response = {
-      ...(await fullSearch(query, params)),
-      hasNextPage: false,
-      bookmark: null,
-    }
+    response = await fullSearch(query, params)
   }
 
   // Enrich search results with relationships

From 891e77e2bfdc0e68fba12d4f44b4f5a48fbe4c04 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:28:38 +0200
Subject: [PATCH 74/94] Remove coverage from prettier

---
 .prettierignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.prettierignore b/.prettierignore
index a73fed4890..64607d74ab 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/sdk/sdk
\ No newline at end of file
+packages/sdk/sdk
+packages/pro/coverage
\ No newline at end of file

From 17c365d398ce0e6cb4102fb79a4d5af7840d9026 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:39:59 +0200
Subject: [PATCH 75/94] Fix tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index e3bfc32320..a74a9f7960 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1105,8 +1105,12 @@ describe.each([
               ...defaultRowFields,
             }))
           ),
-          hasNextPage: false,
-          bookmark: null,
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
         })
       })
 
@@ -1153,8 +1157,12 @@ describe.each([
               ...defaultRowFields,
             }))
           ),
-          hasNextPage: false,
-          bookmark: null,
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
         })
       })
 

From dc29ebfe273fac6cc8c195e91b705bd1b1e29fa0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:54:44 +0200
Subject: [PATCH 76/94] Fix tests

---
 .../server/src/api/controllers/query/import/tests/index.spec.js | 2 +-
 packages/server/src/db/tests/linkTests.spec.js                  | 2 +-
 .../src/migrations/functions/usageQuotas/tests/syncRows.spec.ts | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js
index 60030ae7f3..3aa9e7f916 100644
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@@ -39,7 +39,7 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
 
   beforeAll(async () => {
     await config.init()
diff --git a/packages/server/src/db/tests/linkTests.spec.js b/packages/server/src/db/tests/linkTests.spec.js
index b7764dacb7..19a0eb88d3 100644
--- a/packages/server/src/db/tests/linkTests.spec.js
+++ b/packages/server/src/db/tests/linkTests.spec.js
@@ -4,7 +4,7 @@ const linkUtils = require("../linkedRows/linkUtils")
 const { context } = require("@budibase/backend-core")
 
 describe("test link functionality", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
   let appId
 
   describe("getLinkedTable", () => {
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
index fc0f9e1aa9..5257ffc042 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
@@ -5,7 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
 
   beforeEach(async () => {
     await config.init()

From 8af3e4bf6b5d6930126cb4302e0df67a5db2f6d8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 13 Sep 2023 15:23:30 +0100
Subject: [PATCH 77/94] Found some discussion of testcontainers being
 problematic when nearly out of disk space, we have seen issues with the
 default Github runners as they have extremely limited disk space, this should
 help a bit removing android and dotnet, two pieces of functionality we will
 never need.

---
 .github/workflows/budibase_ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml
index d670e222d3..fc35575ec6 100644
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@@ -25,6 +25,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

From d1e1ad4930ae91ab595485d4f4a3696aee6567aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 16:33:04 +0200
Subject: [PATCH 78/94] Revert TestConfiguration changes

---
 .../src/tests/utilities/TestConfiguration.ts   | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 94896932ad..da7af8acd7 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -535,7 +535,7 @@ class TestConfiguration {
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    const response = await this.api.table.create(config)
+    const response = await this._req(config, null, controllers.table.save)
     if (!skipReassigning) {
       this.table = response
     }
@@ -589,7 +589,7 @@ class TestConfiguration {
       }
     }
 
-    const linkedTable = await this.api.table.create(tableConfig)
+    const linkedTable = await this.createTable(tableConfig)
     return linkedTable
   }
 
@@ -609,7 +609,7 @@ class TestConfiguration {
     }
     const tableId = (config && config.tableId) || this.table._id
     config = config || basicRow(tableId!)
-    return this.api.row.save(tableId!, config)
+    return this._req(config, { tableId }, controllers.row.save)
   }
 
   async getRow(tableId: string, rowId: string): Promise<Row> {
@@ -715,14 +715,18 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this.api.datasource.create({ ...config.datasource })
-    this.datasource = response
+    const response = await this._req(config, null, controllers.datasource.save)
+    this.datasource = response.datasource
     return this.datasource!
   }
 
   async updateDatasource(datasource: Datasource): Promise<Datasource> {
-    const response = await this.api.datasource.update(datasource)
-    this.datasource = response
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
     return this.datasource!
   }
 

From 405ea4e37b9eff0d9c0eaec1b721a9382bc2af38 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 16:39:23 +0200
Subject: [PATCH 79/94] Reverts

---
 .../server/src/api/controllers/query/import/tests/index.spec.js | 2 +-
 .../src/migrations/functions/usageQuotas/tests/syncRows.spec.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js
index 3aa9e7f916..60030ae7f3 100644
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@@ -39,7 +39,7 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
-  const config = new TestConfig()
+  const config = new TestConfig(false)
 
   beforeAll(async () => {
     await config.init()
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
index 5257ffc042..fc0f9e1aa9 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
@@ -5,7 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
-  let config = new TestConfig()
+  let config = new TestConfig(false)
 
   beforeEach(async () => {
     await config.init()

From f70531d34322ce64ed6bc2f3e20df5e489e9e809 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 15:00:27 +0000
Subject: [PATCH 80/94] Bump version to 2.10.7-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 88bde5827a..054621b521 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7-alpha.0",
+  "version": "2.10.7-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 78d0398315ec4f9e1edf47b7ec94480c0b887f61 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Wed, 13 Sep 2023 16:18:33 +0100
Subject: [PATCH 81/94] Update stale_bot.yml

---
 .github/workflows/stale_bot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml
index 8cda3a9342..f87d561db9 100644
--- a/.github/workflows/stale_bot.yml
+++ b/.github/workflows/stale_bot.yml
@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 1 * * *' # 1:30 every morning
+    - cron: '*/30 * * * *' # Every 30 mins
 
 jobs:
   stale:

From 29358cec5f070840ef6ea4a836065c9f7755d5e8 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 15:18:50 +0000
Subject: [PATCH 82/94] Bump version to 2.10.7-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 054621b521..a3a8bda618 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7-alpha.1",
+  "version": "2.10.7-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 37cb9261e8028bd62d21f0499d7b20aeba6ddd64 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 14 Sep 2023 09:28:43 +0000
Subject: [PATCH 83/94] Bump version to 2.10.9-alpha.0

---
 lerna.json | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lerna.json b/lerna.json
index d267691a6e..2394c5aa57 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,7 +1,9 @@
 {
-  "version": "2.10.8",
+  "version": "2.10.9-alpha.0",
   "npmClient": "yarn",
-  "packages": ["packages/*"],
+  "packages": [
+    "packages/*"
+  ],
   "useNx": true,
   "command": {
     "publish": {
@@ -17,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From 4daa996044837eaf344a60f1edb7ca53e1984700 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 10:38:14 +0100
Subject: [PATCH 84/94] Adding console log to trigger cache bust and run tests.

---
 packages/backend-core/src/index.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index ffffd8240a..c540b717da 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -50,5 +50,6 @@ export * from "./constants"
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {
+  console.log("test")
   db.init(opts.db)
 }

From 4ca9ea97b66c0e96b4949b01566326594714de31 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 10:55:41 +0100
Subject: [PATCH 85/94] Get server/worker tests to run.

---
 packages/server/src/index.ts | 1 +
 packages/worker/src/index.ts | 1 +
 2 files changed, 2 insertions(+)

diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index b3fbb22bde..46ec2035a7 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -11,6 +11,7 @@ function runServer() {
   process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false"
   bootstrap()
   require("./app")
+  console.log("test")
 }
 
 runServer()
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 30596d1d76..7212e8207c 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -109,6 +109,7 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.init()
+  console.log("test")
 })
 
 process.on("uncaughtException", err => {

From 26c6393f06c102379f3f4a0a2886272b3cda71a8 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 11:11:05 +0100
Subject: [PATCH 86/94] Adding retries to 409 test.

---
 packages/worker/src/api/routes/global/tests/scim.spec.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/worker/src/api/routes/global/tests/scim.spec.ts b/packages/worker/src/api/routes/global/tests/scim.spec.ts
index 5686e39fa8..fba1523cd4 100644
--- a/packages/worker/src/api/routes/global/tests/scim.spec.ts
+++ b/packages/worker/src/api/routes/global/tests/scim.spec.ts
@@ -10,6 +10,8 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
+// this test can 409 - retries reduce issues with this
+jest.retryTimes(2)
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()

From 5f82b79d40d37f66a132a7f12bd43ed281444e90 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 14 Sep 2023 11:42:04 +0100
Subject: [PATCH 87/94] Removing test logs.

---
 packages/backend-core/src/index.ts | 1 -
 packages/server/src/index.ts       | 1 -
 packages/worker/src/index.ts       | 1 -
 3 files changed, 3 deletions(-)

diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts
index c540b717da..ffffd8240a 100644
--- a/packages/backend-core/src/index.ts
+++ b/packages/backend-core/src/index.ts
@@ -50,6 +50,5 @@ export * from "./constants"
 // expose package init function
 import * as db from "./db"
 export const init = (opts: any = {}) => {
-  console.log("test")
   db.init(opts.db)
 }
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index 46ec2035a7..b3fbb22bde 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -11,7 +11,6 @@ function runServer() {
   process.env.GLOBAL_AGENT_FORCE_GLOBAL_AGENT = "false"
   bootstrap()
   require("./app")
-  console.log("test")
 }
 
 runServer()
diff --git a/packages/worker/src/index.ts b/packages/worker/src/index.ts
index 7212e8207c..30596d1d76 100644
--- a/packages/worker/src/index.ts
+++ b/packages/worker/src/index.ts
@@ -109,7 +109,6 @@ export default server.listen(parseInt(env.PORT || "4002"), async () => {
   console.log(`Worker running on ${JSON.stringify(server.address())}`)
   await initPro()
   await redis.init()
-  console.log("test")
 })
 
 process.on("uncaughtException", err => {

From f15ca351cd00ddc525055c65c2138a0c92ce3960 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Thu, 14 Sep 2023 11:03:14 +0000
Subject: [PATCH 88/94] Bump version to 2.10.9-alpha.1

---
 lerna.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lerna.json b/lerna.json
index 68cc011c5b..a428ef9af6 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.0",
+  "version": "2.10.9-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
@@ -19,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}
\ No newline at end of file

From bd5c6056d8cab9360de943716b116a3e1903fae3 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Fri, 15 Sep 2023 08:04:17 +0100
Subject: [PATCH 89/94] Update dayjs dependencies to same version and fix
 import syntax in DateCell

---
 packages/bbui/package.json                    |  2 +-
 packages/builder/package.json                 |  2 +-
 packages/client/package.json                  |  2 +-
 packages/frontend-core/package.json           |  2 +-
 .../src/components/grid/cells/DateCell.svelte |  2 +-
 packages/string-templates/package.json        |  2 +-
 yarn.lock                                     | 20 ++++++-------------
 7 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/packages/bbui/package.json b/packages/bbui/package.json
index 0b87960ab5..dc6c910be8 100644
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@@ -82,7 +82,7 @@
     "@spectrum-css/typography": "3.0.1",
     "@spectrum-css/underlay": "2.0.9",
     "@spectrum-css/vars": "3.0.1",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "easymde": "^2.16.1",
     "svelte-flatpickr": "3.2.3",
     "svelte-portal": "^1.0.0",
diff --git a/packages/builder/package.json b/packages/builder/package.json
index a6bf81201d..43f1ae3bff 100644
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@@ -69,7 +69,7 @@
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",
     "codemirror": "^5.59.0",
-    "dayjs": "^1.11.2",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "fast-json-patch": "^3.1.1",
     "lodash": "4.17.21",
diff --git a/packages/client/package.json b/packages/client/package.json
index a5ee304610..698c7bd929 100644
--- a/packages/client/package.json
+++ b/packages/client/package.json
@@ -33,7 +33,7 @@
     "@spectrum-css/typography": "^3.0.2",
     "@spectrum-css/vars": "^3.0.1",
     "apexcharts": "^3.22.1",
-    "dayjs": "^1.10.5",
+    "dayjs": "^1.10.8",
     "downloadjs": "1.4.7",
     "html5-qrcode": "^2.2.1",
     "leaflet": "^1.7.1",
diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json
index ca7e0a6d2b..1f15bb72c5 100644
--- a/packages/frontend-core/package.json
+++ b/packages/frontend-core/package.json
@@ -8,7 +8,7 @@
   "dependencies": {
     "@budibase/bbui": "0.0.0",
     "@budibase/shared-core": "0.0.0",
-    "dayjs": "^1.11.7",
+    "dayjs": "^1.10.8",
     "lodash": "^4.17.21",
     "socket.io-client": "^4.6.1",
     "svelte": "^3.46.2"
diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
index 9144f5b769..53b159ee30 100644
--- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
@@ -1,5 +1,5 @@
 <script>
-  import { dayjs } from "dayjs"
+  import dayjs from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 
diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json
index 4dc4c17ad8..aee60f4732 100644
--- a/packages/string-templates/package.json
+++ b/packages/string-templates/package.json
@@ -26,7 +26,7 @@
   },
   "dependencies": {
     "@budibase/handlebars-helpers": "^0.11.9",
-    "dayjs": "^1.10.4",
+    "dayjs": "^1.10.8",
     "handlebars": "^4.7.6",
     "handlebars-utils": "^1.0.6",
     "lodash": "^4.17.20",
diff --git a/yarn.lock b/yarn.lock
index 8c93661665..d777dca0bd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,14 +6269,6 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
-"@types/node-fetch@2.6.1":
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
-  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^3.0.0"
-
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6298,11 +6290,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
-"@types/node@14.18.20":
-  version "14.18.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
-  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
-
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"
@@ -9792,11 +9779,16 @@ dateformat@^4.5.1, dateformat@^4.6.3:
   resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-4.6.3.tgz#556fa6497e5217fedb78821424f8a1c22fa3f4b5"
   integrity sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==
 
-dayjs@^1.10.4, dayjs@^1.10.5, dayjs@^1.11.2, dayjs@^1.11.7:
+dayjs@^1.10.4, dayjs@^1.10.5:
   version "1.11.7"
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2"
   integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ==
 
+dayjs@^1.10.8:
+  version "1.11.9"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.9.tgz#9ca491933fadd0a60a2c19f6c237c03517d71d1a"
+  integrity sha512-QvzAURSbQ0pKdIye2txOzNaHmxtUBXerpY0FJsFXUMKbIZeFm5ht1LS/jFsrncjnmtv8HsG0W2g6c0zUjZWmpA==
+
 dd-trace@3.13.2:
   version "3.13.2"
   resolved "https://registry.yarnpkg.com/dd-trace/-/dd-trace-3.13.2.tgz#95b1ec480ab9ac406e1da7591a8c6f678d3799fd"

From 14d5f551c329374a12080b3c901aa39063d275f1 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Fri, 15 Sep 2023 10:27:34 +0000
Subject: [PATCH 90/94] Bump version to 2.10.9-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a428ef9af6..1673559ee9 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.1",
+  "version": "2.10.9-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 36acd69fddaa2b4dcc8ed9db697c6e0477033135 Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Mon, 18 Sep 2023 12:14:12 +0100
Subject: [PATCH 91/94] Note about MySQL SSL (#11797)

---
 packages/server/src/integrations/mysql.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts
index b88162c59a..8a688c5f3b 100644
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@@ -158,6 +158,12 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     ) {
       config.ssl.rejectUnauthorized = config.rejectUnauthorized
     }
+    // The MySQL library we use doesn't directly document the parameters that can be passed in the ssl
+    // object, it instead points to an older library that it says it is mostly API compatible with, that
+    // older library actually documents what parameters can be passed in the ssl object.
+    // https://github.com/sidorares/node-mysql2#api-and-configuration
+    // https://github.com/mysqljs/mysql#ssl-options
+
     // @ts-ignore
     delete config.rejectUnauthorized
     this.config = {

From fd518548fddb17c1e28c3f5eddf07b3f5f2a3fea Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Mon, 18 Sep 2023 11:14:32 +0000
Subject: [PATCH 92/94] Bump version to 2.10.9-alpha.3

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 1673559ee9..a57875986a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.2",
+  "version": "2.10.9-alpha.3",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 84a89ddc92bb088e9541b6afa3bf5bff6f8a18fa Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Tue, 19 Sep 2023 11:07:31 +0100
Subject: [PATCH 93/94] Support up to 3 lines of text in tooltips and fix icon
 for date fields

---
 packages/bbui/src/Tooltip/AbsTooltip.svelte              | 5 +++--
 .../backend/DataTable/modals/CreateEditColumn.svelte     | 9 +++++----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/bbui/src/Tooltip/AbsTooltip.svelte b/packages/bbui/src/Tooltip/AbsTooltip.svelte
index 9be7251445..92d5af26bb 100644
--- a/packages/bbui/src/Tooltip/AbsTooltip.svelte
+++ b/packages/bbui/src/Tooltip/AbsTooltip.svelte
@@ -126,8 +126,9 @@
     transition: top 130ms ease-out, left 130ms ease-out;
   }
   .spectrum-Tooltip-label {
-    text-overflow: ellipsis;
-    white-space: nowrap;
+    display: -webkit-box;
+    -webkit-line-clamp: 3;
+    -webkit-box-orient: vertical;
     overflow: hidden;
     font-size: 12px;
     font-weight: 600;
diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index 44c37813d6..75964af513 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -502,7 +502,7 @@
     </div>
     {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
       <div>
-        <div>
+        <div class="row">
           <Label>Time zones</Label>
           <AbsTooltip
             position="top"
@@ -690,18 +690,19 @@
     display: flex;
     align-items: center;
   }
-
   .tooltip-alignment {
     display: flex;
     align-items: center;
     gap: var(--spacing-xs);
   }
-
   .label-length {
     flex-basis: 40%;
   }
-
   .input-length {
     flex-grow: 1;
   }
+  .row {
+    gap: 8px;
+    display: flex;
+  }
 </style>

From d0097273b9d87744a9c31bcce9ccd6bdbccadd47 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 19 Sep 2023 10:18:04 +0000
Subject: [PATCH 94/94] Bump version to 2.10.9-alpha.4

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index a57875986a..219091b329 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.9-alpha.3",
+  "version": "2.10.9-alpha.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*"