From 9fb2d89a330fd99a990cd6b65ebcc7374cbabddc Mon Sep 17 00:00:00 2001
From: jvcalderon <jose@budibase.com>
Date: Thu, 31 Aug 2023 15:11:39 +0200
Subject: [PATCH 01/81] Fix daysjs import error

---
 .../frontend-core/src/components/grid/cells/DateCell.svelte     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DateCell.svelte b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
index 53b159ee30..9144f5b769 100644
--- a/packages/frontend-core/src/components/grid/cells/DateCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DateCell.svelte
@@ -1,5 +1,5 @@
 <script>
-  import dayjs from "dayjs"
+  import { dayjs } from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 

From 122cd843abc2da063eb0c5fa15f55e4f6241105b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 30 Aug 2023 17:42:26 +0200
Subject: [PATCH 02/81] Type ds in testconfiguration

---
 .../src/tests/utilities/TestConfiguration.ts    | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 81bfa0abbd..137828f188 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -78,7 +78,7 @@ class TestConfiguration {
   table?: Table
   linkedTable: any
   automation: any
-  datasource: any
+  datasource?: Datasource
   tenantId?: string
   defaultUserValues: DefaultUserValues
   api: API
@@ -528,7 +528,7 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: any,
+    config?: Table,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
@@ -543,6 +543,11 @@ class TestConfiguration {
     if (config != null && config._id) {
       delete config._id
     }
+    config = config || basicTable()
+    if (this.datasource && !config.sourceId) {
+      config.sourceId = this.datasource._id
+    }
+
     return this.updateTable(config, options)
   }
 
@@ -678,17 +683,17 @@ class TestConfiguration {
     config = config || basicDatasource()
     const response = await this._req(config, null, controllers.datasource.save)
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
-  async updateDatasource(datasource: any) {
+  async updateDatasource(datasource: Datasource): Promise<Datasource> {
     const response = await this._req(
       datasource,
       { datasourceId: datasource._id },
       controllers.datasource.update
     )
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
   async restDatasource(cfg?: any) {
@@ -772,7 +777,7 @@ class TestConfiguration {
     if (!this.datasource && !config) {
       throw "No datasource created for query."
     }
-    config = config || basicQuery(this.datasource._id)
+    config = config || basicQuery(this.datasource!._id!)
     return this._req(config, null, controllers.query.save)
   }
 

From cb6977a18b421d64db16ff22b304b16cad092fa9 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 17:01:49 +0200
Subject: [PATCH 03/81] Create non-plus datasource

---
 packages/server/src/api/routes/tests/row.spec.ts         | 7 +++++++
 packages/server/src/tests/utilities/TestConfiguration.ts | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index f7878b2182..71c4325f92 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -18,6 +18,7 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
+import { basicDatasource } from "../../../tests/utilities/structures"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
@@ -37,6 +38,12 @@ describe("/rows", () => {
   })
 
   beforeEach(async () => {
+    await config.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        plus: false,
+      },
+    })
     table = await config.createTable()
     row = basicRow(table._id!)
   })
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 137828f188..5d5d6a18e5 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -546,6 +546,9 @@ class TestConfiguration {
     config = config || basicTable()
     if (this.datasource && !config.sourceId) {
       config.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        config.type = "external"
+      }
     }
 
     return this.updateTable(config, options)

From 8931309669d9011ae69940c517e05b92f6f5afe7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 18:53:32 +0200
Subject: [PATCH 04/81] Datasource test api

---
 .../src/tests/utilities/TestConfiguration.ts  |  6 ++++-
 .../src/tests/utilities/api/datasource.ts     | 26 +++++++++++++++++++
 .../server/src/tests/utilities/api/index.ts   |  3 +++
 3 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 packages/server/src/tests/utilities/api/datasource.ts

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 5d5d6a18e5..d759431bcc 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -684,7 +684,11 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this._req(config, null, controllers.datasource.save)
+    const response = await this._req(
+      { ...config },
+      null,
+      controllers.datasource.save
+    )
     this.datasource = response.datasource
     return this.datasource!
   }
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
new file mode 100644
index 0000000000..c07fd3bc61
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -0,0 +1,26 @@
+import { CreateDatasourceRequest, Datasource } from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class DatasourceAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .post(`/api/datasources`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+}
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index 40995b62f2..0521f9b19f 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -3,17 +3,20 @@ import { PermissionAPI } from "./permission"
 import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
+import { DatasourceAPI } from "./datasource"
 
 export default class API {
   table: TableAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
+  datasource: DatasourceAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
+    this.datasource = new DatasourceAPI(config)
   }
 }

From e4479ee522b80cfce73f0575c8d90d96d3cf0ce5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:04 +0200
Subject: [PATCH 05/81] Use docker utils helpers

---
 .../src/tests/utilities/api/datasource.ts     | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index c07fd3bc61..3c85a1c332 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -1,4 +1,9 @@
-import { CreateDatasourceRequest, Datasource } from "@budibase/types"
+import {
+  CreateDatasourceRequest,
+  Datasource,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -23,4 +28,17 @@ export class DatasourceAPI extends TestAPI {
       .expect(expectStatus)
     return result.body.datasource as Datasource
   }
+
+  verify = async (
+    data: VerifyDatasourceRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const result = await this.request
+      .post(`/api/datasources/verify`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result
+  }
 }

From baab7d3fb501b96ada3859d17c92f56c0caa6466 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:11 +0200
Subject: [PATCH 06/81] Use docker utils helpers

---
 .../src/integrations/tests/utils/index.ts     |  5 +++
 .../src/integrations/tests/utils/postgres.ts  | 34 +++++++++++++++++++
 2 files changed, 39 insertions(+)
 create mode 100644 packages/server/src/integrations/tests/utils/index.ts
 create mode 100644 packages/server/src/integrations/tests/utils/postgres.ts

diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
new file mode 100644
index 0000000000..2c8c583fd4
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -0,0 +1,5 @@
+import * as postgres from "./postgres"
+
+export const testDatasourceConfig = {
+  postgres: postgres.getDatasourceConfig,
+}
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
new file mode 100644
index 0000000000..9f9e98fe17
--- /dev/null
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -0,0 +1,34 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+
+export async function getDatasourceConfig(): Promise<Datasource> {
+  const containerPostgres = await new GenericContainer("postgres")
+    .withExposedPorts(5432)
+    .withEnv("POSTGRES_PASSWORD", "password")
+    .withWaitStrategy(
+      Wait.forLogMessage(
+        "PostgreSQL init process complete; ready for start up."
+      )
+    )
+    .start()
+
+  const host = containerPostgres.getContainerIpAddress()
+  const port = containerPostgres.getMappedPort(5432)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.POSTGRES,
+    plus: true,
+    config: {
+      host,
+      port,
+      database: "postgres",
+      user: "postgres",
+      password: "password",
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
+  }
+}

From 57801b5bcc903324c023d3dea363860bfa1f2c96 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Mon, 4 Sep 2023 19:05:18 +0200
Subject: [PATCH 07/81] Use docker utils helpers

---
 .../src/integration-test/postgres.spec.ts     | 78 ++++---------------
 1 file changed, 16 insertions(+), 62 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 6b0e7b4266..7dde9f2e0c 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -12,13 +12,12 @@ import {
   FieldType,
   RelationshipType,
   Row,
-  SourceName,
   Table,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { testDatasourceConfig } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -35,61 +34,19 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let host: string
-  let port: number
-  const containers: StartedTestContainer[] = []
+  let pgDatasourceConfig: Datasource
 
   beforeAll(async () => {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
-        )
-      )
-      .start()
-
-    host = containerPostgres.getContainerIpAddress()
-    port = containerPostgres.getMappedPort(5432)
+    pgDatasourceConfig = await testDatasourceConfig.postgres()
 
     await config.init()
     const apiKey = await config.generateApiKey()
 
-    containers.push(containerPostgres)
-
     makeRequest = generateMakeRequest(apiKey, true)
   })
 
-  afterAll(async () => {
-    for (let container of containers) {
-      await container.stop()
-    }
-  })
-
-  function pgDatasourceConfig() {
-    return {
-      datasource: {
-        type: "datasource",
-        source: SourceName.POSTGRES,
-        plus: true,
-        config: {
-          host,
-          port,
-          database: "postgres",
-          user: "postgres",
-          password: "password",
-          schema: "public",
-          ssl: false,
-          rejectUnauthorized: false,
-          ca: false,
-        },
-      },
-    }
-  }
-
   beforeEach(async () => {
-    postgresDatasource = await config.createDatasource(pgDatasourceConfig())
+    postgresDatasource = await config.api.datasource.create(pgDatasourceConfig)
 
     async function createAuxTable(prefix: string) {
       return await config.createTable({
@@ -357,9 +314,9 @@ describe("postgres integrations", () => {
       config: {
         ca: false,
         database: "postgres",
-        host,
+        host: postgresDatasource.config!.host,
         password: "--secret-value--",
-        port,
+        port: postgresDatasource.config!.port,
         rejectUnauthorized: false,
         schema: "public",
         ssl: false,
@@ -1046,24 +1003,21 @@ describe("postgres integrations", () => {
 
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
-      const config = pgDatasourceConfig()
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: pgDatasourceConfig,
+      })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const config = pgDatasourceConfig()
-      config.datasource.config.password = "wrongpassword"
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: {
+          ...pgDatasourceConfig,
+          config: { ...pgDatasourceConfig.config, password: "wrongpassword" },
+        },
+      })
+
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(false)
       expect(response.body.error).toBeDefined()

From b27899b1e6469cea37ac4e417cd4291ee2b62f8d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 10:47:59 +0200
Subject: [PATCH 08/81] Redo helpers

---
 .../src/integration-test/postgres.spec.ts     | 20 +++++---
 .../src/integrations/tests/utils/index.ts     |  7 +--
 .../src/integrations/tests/utils/postgres.ts  | 46 +++++++++++++++++++
 3 files changed, 63 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 7dde9f2e0c..fd85393477 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -17,7 +17,7 @@ import {
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { testDatasourceConfig } from "../integrations/tests/utils"
+import { PostgresProvider } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -34,10 +34,10 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let pgDatasourceConfig: Datasource
+  let provider: PostgresProvider
 
   beforeAll(async () => {
-    pgDatasourceConfig = await testDatasourceConfig.postgres()
+    provider = await PostgresProvider.init()
 
     await config.init()
     const apiKey = await config.generateApiKey()
@@ -46,7 +46,9 @@ describe("postgres integrations", () => {
   })
 
   beforeEach(async () => {
-    postgresDatasource = await config.api.datasource.create(pgDatasourceConfig)
+    postgresDatasource = await config.api.datasource.create(
+      provider.getDsConfig()
+    )
 
     async function createAuxTable(prefix: string) {
       return await config.createTable({
@@ -1004,17 +1006,21 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: pgDatasourceConfig,
+        datasource: provider.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
+      const dbConfig = provider.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
-          ...pgDatasourceConfig,
-          config: { ...pgDatasourceConfig.config, password: "wrongpassword" },
+          ...dbConfig,
+          config: {
+            ...dbConfig.config,
+            password: "wrongpassword",
+          },
         },
       })
 
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 2c8c583fd4..e80dbed83a 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,5 +1,6 @@
-import * as postgres from "./postgres"
+import { Datasource } from "@budibase/types"
+export * from "./postgres"
 
-export const testDatasourceConfig = {
-  postgres: postgres.getDatasourceConfig,
+export interface DatabasePlusTestProvider {
+  getDsConfig(): Datasource
 }
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 9f9e98fe17..514605306e 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,5 +1,51 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait } from "testcontainers"
+import { DatabasePlusTestProvider } from "."
+
+export class PostgresProvider implements DatabasePlusTestProvider {
+  private host: string
+  private port: number
+
+  private constructor(host: string, port: number) {
+    this.host = host
+    this.port = port
+  }
+
+  static async init() {
+    const containerPostgres = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
+        )
+      )
+      .start()
+
+    const host = containerPostgres.getContainerIpAddress()
+    const port = containerPostgres.getMappedPort(5432)
+    return new PostgresProvider(host, port)
+  }
+
+  getDsConfig(): Datasource {
+    return {
+      type: "datasource_plus",
+      source: SourceName.POSTGRES,
+      plus: true,
+      config: {
+        host: this.host,
+        port: this.port,
+        database: "postgres",
+        user: "postgres",
+        password: "password",
+        schema: "public",
+        ssl: false,
+        rejectUnauthorized: false,
+        ca: false,
+      },
+    }
+  }
+}
 
 export async function getDatasourceConfig(): Promise<Datasource> {
   const containerPostgres = await new GenericContainer("postgres")

From 6240740a42854bd70ede3859ed676590582526ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:11:47 +0200
Subject: [PATCH 09/81] Simplify

---
 .../src/integration-test/postgres.spec.ts     | 12 +++---
 .../src/integrations/tests/utils/index.ts     |  2 +-
 .../src/integrations/tests/utils/postgres.ts  | 41 ++++++++-----------
 3 files changed, 23 insertions(+), 32 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index fd85393477..faaa22f2c8 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -26,6 +26,8 @@ jest.setTimeout(30000)
 jest.unmock("pg")
 jest.mock("../websockets")
 
+const provider = new PostgresProvider()
+
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
     postgresDatasource: Datasource,
@@ -34,11 +36,7 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let provider: PostgresProvider
-
   beforeAll(async () => {
-    provider = await PostgresProvider.init()
-
     await config.init()
     const apiKey = await config.generateApiKey()
 
@@ -47,7 +45,7 @@ describe("postgres integrations", () => {
 
   beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
-      provider.getDsConfig()
+      await provider.getDsConfig()
     )
 
     async function createAuxTable(prefix: string) {
@@ -1006,14 +1004,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: provider.getDsConfig(),
+        datasource: await provider.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = provider.getDsConfig()
+      const dbConfig = await provider.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index e80dbed83a..20b6f277dd 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -2,5 +2,5 @@ import { Datasource } from "@budibase/types"
 export * from "./postgres"
 
 export interface DatabasePlusTestProvider {
-  getDsConfig(): Datasource
+  getDsConfig(): Promise<Datasource>
 }
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 514605306e..8d79a7e762 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,40 +1,33 @@
 import { Datasource, SourceName } from "@budibase/types"
-import { GenericContainer, Wait } from "testcontainers"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
 import { DatabasePlusTestProvider } from "."
 
 export class PostgresProvider implements DatabasePlusTestProvider {
-  private host: string
-  private port: number
+  private container?: StartedTestContainer
 
-  private constructor(host: string, port: number) {
-    this.host = host
-    this.port = port
-  }
-
-  static async init() {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
+  async getDsConfig(): Promise<Datasource> {
+    if (!this.container) {
+      this.container = await new GenericContainer("postgres")
+        .withExposedPorts(5432)
+        .withEnv("POSTGRES_PASSWORD", "password")
+        .withWaitStrategy(
+          Wait.forLogMessage(
+            "PostgreSQL init process complete; ready for start up."
+          )
         )
-      )
-      .start()
+        .start()
+    }
 
-    const host = containerPostgres.getContainerIpAddress()
-    const port = containerPostgres.getMappedPort(5432)
-    return new PostgresProvider(host, port)
-  }
+    const host = this.container.getContainerIpAddress()
+    const port = this.container.getMappedPort(5432)
 
-  getDsConfig(): Datasource {
     return {
       type: "datasource_plus",
       source: SourceName.POSTGRES,
       plus: true,
       config: {
-        host: this.host,
-        port: this.port,
+        host,
+        port,
         database: "postgres",
         user: "postgres",
         password: "password",

From 887ebb2eebe5ab75a3d73190c0d8a1fc4ad29799 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:16:37 +0200
Subject: [PATCH 10/81] Simplify

---
 .../src/integration-test/postgres.spec.ts     | 10 ++-
 .../src/integrations/tests/utils/index.ts     |  6 +-
 .../src/integrations/tests/utils/postgres.ts  | 61 ++++---------------
 3 files changed, 22 insertions(+), 55 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index faaa22f2c8..0a53326cf4 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -17,7 +17,7 @@ import {
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { PostgresProvider } from "../integrations/tests/utils"
+import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
@@ -26,8 +26,6 @@ jest.setTimeout(30000)
 jest.unmock("pg")
 jest.mock("../websockets")
 
-const provider = new PostgresProvider()
-
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
     postgresDatasource: Datasource,
@@ -45,7 +43,7 @@ describe("postgres integrations", () => {
 
   beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
-      await provider.getDsConfig()
+      await databaseTestProviders.postgres.getDsConfig()
     )
 
     async function createAuxTable(prefix: string) {
@@ -1004,14 +1002,14 @@ describe("postgres integrations", () => {
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
       const response = await config.api.datasource.verify({
-        datasource: await provider.getDsConfig(),
+        datasource: await databaseTestProviders.postgres.getDsConfig(),
       })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const dbConfig = await provider.getDsConfig()
+      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
       const response = await config.api.datasource.verify({
         datasource: {
           ...dbConfig,
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 20b6f277dd..8893ce67f3 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,6 +1,10 @@
 import { Datasource } from "@budibase/types"
-export * from "./postgres"
+import * as pg from "./postgres"
 
 export interface DatabasePlusTestProvider {
   getDsConfig(): Promise<Datasource>
 }
+
+export const databaseTestProviders = {
+  postgres: pg,
+}
diff --git a/packages/server/src/integrations/tests/utils/postgres.ts b/packages/server/src/integrations/tests/utils/postgres.ts
index 8d79a7e762..036e81bbd8 100644
--- a/packages/server/src/integrations/tests/utils/postgres.ts
+++ b/packages/server/src/integrations/tests/utils/postgres.ts
@@ -1,58 +1,23 @@
 import { Datasource, SourceName } from "@budibase/types"
 import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
-import { DatabasePlusTestProvider } from "."
 
-export class PostgresProvider implements DatabasePlusTestProvider {
-  private container?: StartedTestContainer
+let container: StartedTestContainer | undefined
 
-  async getDsConfig(): Promise<Datasource> {
-    if (!this.container) {
-      this.container = await new GenericContainer("postgres")
-        .withExposedPorts(5432)
-        .withEnv("POSTGRES_PASSWORD", "password")
-        .withWaitStrategy(
-          Wait.forLogMessage(
-            "PostgreSQL init process complete; ready for start up."
-          )
+export async function getDsConfig(): Promise<Datasource> {
+  if (!container) {
+    container = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
         )
-        .start()
-    }
-
-    const host = this.container.getContainerIpAddress()
-    const port = this.container.getMappedPort(5432)
-
-    return {
-      type: "datasource_plus",
-      source: SourceName.POSTGRES,
-      plus: true,
-      config: {
-        host,
-        port,
-        database: "postgres",
-        user: "postgres",
-        password: "password",
-        schema: "public",
-        ssl: false,
-        rejectUnauthorized: false,
-        ca: false,
-      },
-    }
-  }
-}
-
-export async function getDatasourceConfig(): Promise<Datasource> {
-  const containerPostgres = await new GenericContainer("postgres")
-    .withExposedPorts(5432)
-    .withEnv("POSTGRES_PASSWORD", "password")
-    .withWaitStrategy(
-      Wait.forLogMessage(
-        "PostgreSQL init process complete; ready for start up."
       )
-    )
-    .start()
+      .start()
+  }
 
-  const host = containerPostgres.getContainerIpAddress()
-  const port = containerPostgres.getMappedPort(5432)
+  const host = container.getContainerIpAddress()
+  const port = container.getMappedPort(5432)
 
   return {
     type: "datasource_plus",

From 14303c9eb0599ddf6714fdf73331312928096c77 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 11:41:30 +0200
Subject: [PATCH 11/81] Extend test create api

---
 packages/server/src/tests/utilities/api/table.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/table.ts b/packages/server/src/tests/utilities/api/table.ts
index 70f0869650..04432a788a 100644
--- a/packages/server/src/tests/utilities/api/table.ts
+++ b/packages/server/src/tests/utilities/api/table.ts
@@ -1,4 +1,4 @@
-import { Table } from "@budibase/types"
+import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -7,6 +7,19 @@ export class TableAPI extends TestAPI {
     super(config)
   }
 
+  create = async (
+    data: SaveTableRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<SaveTableResponse> => {
+    const res = await this.request
+      .post(`/api/tables`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return res.body
+  }
+
   fetch = async (
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Table[]> => {

From f476c84bc25401dff1f1c5a78aa04968e581996a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 5 Sep 2023 12:09:11 +0200
Subject: [PATCH 12/81] Run row api for pg

---
 .../server/src/api/routes/tests/row.spec.ts   | 57 ++++++++++++++++---
 1 file changed, 49 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 71c4325f92..d9fa0938e1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -4,10 +4,12 @@ import * as setup from "./utilities"
 import { context, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
+  Datasource,
   FieldType,
   MonthlyQuotaName,
   QuotaUsageType,
   Row,
+  SaveTableRequest,
   SortOrder,
   SortType,
   StaticQuotaName,
@@ -18,18 +20,22 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import { basicDatasource } from "../../../tests/utilities/structures"
+import { databaseTestProviders } from "../../../integrations/tests/utils"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
 
 const { basicRow } = setup.structures
 
-describe("/rows", () => {
+describe.each([
+  ["internal", undefined],
+  ["postgres", databaseTestProviders.postgres],
+])("/rows (%s)", (_, dsProvider) => {
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
   let row: Row
+  let datasource: Datasource | undefined
 
   afterAll(setup.afterAll)
 
@@ -38,13 +44,48 @@ describe("/rows", () => {
   })
 
   beforeEach(async () => {
-    await config.createDatasource({
-      datasource: {
-        ...basicDatasource().datasource,
-        plus: false,
+    let tableConfig: SaveTableRequest = {
+      name: "TestTable",
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          type: FieldType.AUTO,
+          name: "id",
+          autocolumn: true,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            type: "string",
+          },
+        },
+        description: {
+          type: FieldType.STRING,
+          name: "description",
+          constraints: {
+            type: "string",
+          },
+        },
       },
-    })
-    table = await config.createTable()
+    }
+
+    if (dsProvider) {
+      datasource = await config.api.datasource.create(
+        await dsProvider.getDsConfig()
+      )
+
+      tableConfig.sourceId = datasource._id
+      if (datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
+    table = await config.api.table.create(tableConfig)
     row = basicRow(table._id!)
   })
 

From 3fe7a9f4cbd02de1eab0cf510a5001b89abc1379 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 6 Sep 2023 12:51:08 +0200
Subject: [PATCH 13/81] Unmock pg

---
 packages/server/src/api/routes/tests/row.spec.ts      | 3 ++-
 packages/server/src/integrations/tests/utils/index.ts | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d9fa0938e1..968f5ff6c3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1,3 +1,5 @@
+import { databaseTestProviders } from "../../../integrations/tests/utils"
+
 import tk from "timekeeper"
 import { outputProcessing } from "../../../utilities/rowProcessor"
 import * as setup from "./utilities"
@@ -20,7 +22,6 @@ import {
   generator,
   structures,
 } from "@budibase/backend-core/tests"
-import { databaseTestProviders } from "../../../integrations/tests/utils"
 
 const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
 tk.freeze(timestamp)
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index 8893ce67f3..c145a9c809 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -1,3 +1,5 @@
+jest.unmock("pg")
+
 import { Datasource } from "@budibase/types"
 import * as pg from "./postgres"
 

From 7a8eab863c03df98016293d0ca849bee7859bdea Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 6 Sep 2023 13:03:35 +0200
Subject: [PATCH 14/81] Fix tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 968f5ff6c3..7d23714ccd 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -46,7 +46,7 @@ describe.each([
 
   beforeEach(async () => {
     let tableConfig: SaveTableRequest = {
-      name: "TestTable",
+      name: generator.guid(),
       type: "table",
       primary: ["id"],
       schema: {
@@ -124,6 +124,8 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
+  const createRow = () => config.api.row.save(table._id!, basicRow(table._id!))
+
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
@@ -200,7 +202,7 @@ describe.each([
     })
 
     it("updates a row successfully", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 0bda4a19529f7aa460aa1a340f8b7e5f98fd706d Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 6 Sep 2023 16:56:27 +0100
Subject: [PATCH 15/81] Add option to duplicate columns

---
 .../src/components/grid/cells/DataCell.svelte |  5 ++-
 .../components/grid/cells/HeaderCell.svelte   | 36 +++++++++++++++++++
 .../grid/overlays/KeyboardManager.svelte      |  2 ++
 3 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index 19bdf1bd74..daf0a03276 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -56,7 +56,10 @@
     isActive: () => api?.isActive?.() ?? false,
     onKeyDown: (...params) => api?.onKeyDown(...params),
     isReadonly: () => readonly,
-    getType: () => column.schema.type,
+    getType: () => {
+      console.log("getType", column)
+      return column.schema.type
+    },
     getValue: () => row[column.name],
     setValue: (value, options = { save: true }) => {
       validation.actions.setError(cellId, null)
diff --git a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
index c5f1f7896a..a58b4c8fe4 100644
--- a/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/HeaderCell.svelte
@@ -20,6 +20,8 @@
     config,
     ui,
     columns,
+    definition,
+    datasource,
   } = getContext("grid")
 
   const bannedDisplayColumnTypes = [
@@ -118,6 +120,33 @@
     open = false
   }
 
+  const duplicateColumn = async () => {
+    open = false
+
+    // Generate new name
+    let newName = `${column.name} copy`
+    let attempts = 2
+    while ($definition.schema[newName]) {
+      newName = `${column.name} copy ${attempts++}`
+    }
+
+    // Save schema with new column
+    const existingColumnDefinition = $definition.schema[column.name]
+    await datasource.actions.saveDefinition({
+      ...$definition,
+      schema: {
+        ...$definition.schema,
+        [newName]: {
+          ...existingColumnDefinition,
+          name: newName,
+          schema: {
+            ...existingColumnDefinition.schema,
+          },
+        },
+      },
+    })
+  }
+
   onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
@@ -192,6 +221,13 @@
       >
         Edit column
       </MenuItem>
+      <MenuItem
+        icon="Duplicate"
+        on:click={duplicateColumn}
+        disabled={!$config.canEditColumns}
+      >
+        Duplicate column
+      </MenuItem>
       <MenuItem
         icon="Label"
         on:click={makeDisplayColumn}
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index cd23f154b5..3232f23e76 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -212,8 +212,10 @@
 
   // Focuses the cell and starts entering a new value
   const startEnteringValue = (key, keyCode) => {
+    console.log("start", $focusedCellAPI, $focusedCellAPI.isReadonly())
     if ($focusedCellAPI && !$focusedCellAPI.isReadonly()) {
       const type = $focusedCellAPI.getType()
+      console.log(type)
       if (type === "number" && keyCodeIsNumber(keyCode)) {
         // Update the value locally but don't save it yet
         $focusedCellAPI.setValue(parseInt(key), { save: false })

From 32108abbc0e6064176d7e3798a93041370f159a2 Mon Sep 17 00:00:00 2001
From: Andrew Kingston <andrew@kingston.dev>
Date: Wed, 6 Sep 2023 16:59:30 +0100
Subject: [PATCH 16/81] Remove logs

---
 .../frontend-core/src/components/grid/cells/DataCell.svelte  | 5 +----
 .../src/components/grid/overlays/KeyboardManager.svelte      | 2 --
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/packages/frontend-core/src/components/grid/cells/DataCell.svelte b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
index daf0a03276..19bdf1bd74 100644
--- a/packages/frontend-core/src/components/grid/cells/DataCell.svelte
+++ b/packages/frontend-core/src/components/grid/cells/DataCell.svelte
@@ -56,10 +56,7 @@
     isActive: () => api?.isActive?.() ?? false,
     onKeyDown: (...params) => api?.onKeyDown(...params),
     isReadonly: () => readonly,
-    getType: () => {
-      console.log("getType", column)
-      return column.schema.type
-    },
+    getType: () => column.schema.type,
     getValue: () => row[column.name],
     setValue: (value, options = { save: true }) => {
       validation.actions.setError(cellId, null)
diff --git a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
index 3232f23e76..cd23f154b5 100644
--- a/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
+++ b/packages/frontend-core/src/components/grid/overlays/KeyboardManager.svelte
@@ -212,10 +212,8 @@
 
   // Focuses the cell and starts entering a new value
   const startEnteringValue = (key, keyCode) => {
-    console.log("start", $focusedCellAPI, $focusedCellAPI.isReadonly())
     if ($focusedCellAPI && !$focusedCellAPI.isReadonly()) {
       const type = $focusedCellAPI.getType()
-      console.log(type)
       if (type === "number" && keyCodeIsNumber(keyCode)) {
         // Update the value locally but don't save it yet
         $focusedCellAPI.setValue(parseInt(key), { save: false })

From 3c46d66333713671c0d144b92ba908deb1bb3077 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 09:25:19 +0200
Subject: [PATCH 17/81] yarn.lock

---
 yarn.lock | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/yarn.lock b/yarn.lock
index ab86a87560..8c93661665 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,6 +6269,14 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
+"@types/node-fetch@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
+  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6290,6 +6298,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
+"@types/node@14.18.20":
+  version "14.18.20"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
+  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
+
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"

From c6621e8c495d518a1d3825ec302b62441c88d202 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:04:35 +0200
Subject: [PATCH 18/81] Type response

---
 packages/server/src/api/controllers/row/external.ts | 2 +-
 packages/server/src/api/controllers/row/index.ts    | 3 ++-
 packages/server/src/api/controllers/row/internal.ts | 2 +-
 packages/server/src/api/controllers/row/utils.ts    | 2 +-
 packages/types/src/api/web/app/row.ts               | 2 ++
 5 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 491cbfa8f9..5a78b55ca7 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -106,7 +106,7 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
   return sdk.rows.external.getRow(tableId, id)
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 88d3f50dbe..0bc3c4d3c7 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -14,6 +14,7 @@ import {
   SearchRowResponse,
   SearchRowRequest,
   SearchParams,
+  GetRowResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -111,7 +112,7 @@ export async function fetch(ctx: any) {
   })
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx<void, GetRowResponse>) {
   const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
     datasourceId: tableId,
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index 4a412c42eb..b80bd339e6 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -131,7 +131,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
   const table = await sdk.tables.getTable(tableId)
diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts
index e85ec4553c..c1df80600c 100644
--- a/packages/server/src/api/controllers/row/utils.ts
+++ b/packages/server/src/api/controllers/row/utils.ts
@@ -27,7 +27,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
-  let row
+  let row: Row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {
diff --git a/packages/types/src/api/web/app/row.ts b/packages/types/src/api/web/app/row.ts
index f9623a3daf..6e051facae 100644
--- a/packages/types/src/api/web/app/row.ts
+++ b/packages/types/src/api/web/app/row.ts
@@ -1,5 +1,7 @@
 import { Row } from "../../../documents/app/row"
 
+export interface GetRowResponse extends Row {}
+
 export interface DeleteRows {
   rows: (Row | string)[]
 }

From f8adbb86a0e19d228629d4e1ce231a15412e51e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:04:49 +0200
Subject: [PATCH 19/81] Fix tests

---
 .../src/api/controllers/row/external.ts       |  8 +++-
 .../server/src/api/routes/tests/row.spec.ts   | 37 ++++++++++++-------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 5a78b55ca7..5a1c7e04ff 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -109,7 +109,13 @@ export async function save(ctx: UserCtx) {
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  return sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id)
+
+  if (!row) {
+    ctx.throw(404)
+  }
+
+  return row
 }
 
 export async function destroy(ctx: UserCtx) {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9c150d8968..821842e1b4 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -12,6 +12,7 @@ import {
   PermissionLevel,
   QuotaUsageType,
   Row,
+  SaveRowRequest,
   SaveTableRequest,
   SortOrder,
   SortType,
@@ -34,6 +35,8 @@ describe.each([
   ["internal", undefined],
   ["postgres", databaseTestProviders.postgres],
 ])("/rows (%s)", (_, dsProvider) => {
+  const isInternal = !dsProvider
+
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
@@ -127,9 +130,20 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = () => config.api.row.save(table._id!, basicRow(table._id!))
+  const createRow = (row?: SaveRowRequest) =>
+    config.api.row.save(table._id!, row || basicRow(table._id!))
 
   describe("save, load, update", () => {
+    function getDefaultFields() {
+      if (isInternal) {
+        return {
+          type: "row",
+          createdAt: timestamp,
+          updatedAt: timestamp,
+        }
+      }
+    }
+
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -230,7 +244,7 @@ describe.each([
     })
 
     it("should load a row", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -243,9 +257,8 @@ describe.each([
         ...row,
         _id: existing._id,
         _rev: existing._rev,
-        type: "row",
-        createdAt: timestamp,
-        updatedAt: timestamp,
+        id: existing.id,
+        ...getDefaultFields(),
       })
       await assertQueryUsage(queryUsage + 1)
     })
@@ -256,8 +269,8 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      await config.createRow()
-      await config.createRow(newRow)
+      await createRow()
+      await createRow(newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -273,14 +286,12 @@ describe.each([
     })
 
     it("load should return 404 when row does not exist", async () => {
-      await config.createRow()
+      await createRow()
       const queryUsage = await getQueryUsage()
 
-      await request
-        .get(`/api/${table._id}/rows/not-a-valid-id`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(404)
+      await config.api.row.get(table._id!, "1234567", {
+        expectStatus: 404,
+      })
       await assertQueryUsage(queryUsage) // no change
     })
 

From 2c6df7475547a33677554dc93ab6e7de28c0a877 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:12:46 +0200
Subject: [PATCH 20/81] Fix more tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 821842e1b4..14b59e24c1 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -503,7 +503,7 @@ describe.each([
 
   describe("patch", () => {
     it("should update only the fields that are supplied", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
 
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -530,7 +530,7 @@ describe.each([
     })
 
     it("should throw an error when given improper types", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -552,7 +552,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await config.createRow(row)
+      const createdRow = await createRow(row)
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 93124d804bed15b313a6c4563dda9c11f9348e8c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:17:22 +0200
Subject: [PATCH 21/81] Fix validate test

---
 packages/server/src/api/controllers/row/index.ts | 5 +++--
 packages/types/src/api/web/app/row.ts            | 5 +++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 0bc3c4d3c7..e86b466601 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -15,6 +15,7 @@ import {
   SearchRowRequest,
   SearchParams,
   GetRowResponse,
+  ValidateResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -215,11 +216,11 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   })
 }
 
-export async function validate(ctx: Ctx) {
+export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
   if (isExternalTable(tableId)) {
-    ctx.body = { valid: true }
+    ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
diff --git a/packages/types/src/api/web/app/row.ts b/packages/types/src/api/web/app/row.ts
index 6e051facae..4ab4090461 100644
--- a/packages/types/src/api/web/app/row.ts
+++ b/packages/types/src/api/web/app/row.ts
@@ -11,3 +11,8 @@ export interface DeleteRow {
 }
 
 export type DeleteRowRequest = DeleteRows | DeleteRow
+
+export interface ValidateResponse {
+  valid: boolean
+  errors: Record<string, any>
+}

From 3912517f6733e27ba293a695b83f785c6caa411a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:22:24 +0200
Subject: [PATCH 22/81] Do not validate external

---
 packages/server/src/api/routes/tests/row.spec.ts | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 14b59e24c1..ab61a480f3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -599,8 +599,14 @@ describe.each([
         .expect("Content-Type", /json/)
         .expect(200)
 
-      expect(res.body.valid).toBe(false)
-      expect(Object.keys(res.body.errors)).toEqual(["name"])
+      if (isInternal) {
+        expect(res.body.valid).toBe(false)
+        expect(Object.keys(res.body.errors)).toEqual(["name"])
+      } else {
+        // Validation for external is not implemented, so it will always return valid
+        expect(res.body.valid).toBe(true)
+        expect(Object.keys(res.body.errors)).toEqual([])
+      }
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })

From 93e9b1b8b4d7f4e24df169b8ab33c2929dcfde52 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:34:45 +0200
Subject: [PATCH 23/81] Fix tests

---
 packages/server/src/api/controllers/row/external.ts |  2 +-
 packages/server/src/api/routes/tests/row.spec.ts    | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 5a1c7e04ff..44d50886d3 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -125,7 +125,7 @@ export async function destroy(ctx: UserCtx) {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
-  return { response: { ok: true }, row }
+  return { response: { ok: true, id: _id }, row }
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ab61a480f3..ae28c14279 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -614,8 +614,8 @@ describe.each([
 
   describe("bulkDelete", () => {
     it("should be able to delete a bulk set of rows", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
+      const row1 = await createRow()
+      const row2 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -635,9 +635,9 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
-      const row3 = await config.createRow()
+      const row1 = await createRow()
+      const row2 = await createRow()
+      const row3 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -657,7 +657,7 @@ describe.each([
     })
 
     it("should accept a valid row object and delete the row", async () => {
-      const row1 = await config.createRow()
+      const row1 = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 9e799c6b933eb6e582a53b5e021498cf8c83f6ae Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 10:43:19 +0200
Subject: [PATCH 24/81] Legacy views tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 99 ++++++++++---------
 .../src/tests/utilities/TestConfiguration.ts  |  7 +-
 2 files changed, 57 insertions(+), 49 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ae28c14279..0fab8eb426 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -710,55 +710,62 @@ describe.each([
     })
   })
 
-  describe("fetchView", () => {
-    it("should be able to fetch tables contents via 'view'", async () => {
-      const row = await config.createRow()
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
+  // Legacy views are not available for external
+  isInternal &&
+    describe("fetchView", () => {
+      it("should be able to fetch tables contents via 'view'", async () => {
+        const row = await createRow()
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/views/${table._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 1)
+        const res = await request
+          .get(`/api/views/${table._id}`)
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage + 1)
+      })
+
+      it("should throw an error if view doesn't exist", async () => {
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
+
+        await request
+          .get(`/api/views/derp`)
+          .set(config.defaultHeaders())
+          .expect(404)
+
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage)
+      })
+
+      it("should be able to run on a view", async () => {
+        const view = await config.createLegacyView({
+          tableId: table._id!,
+          name: "ViewTest",
+          filters: [],
+          schema: {},
+        })
+        const row = await createRow()
+        const rowUsage = await getRowUsage()
+        const queryUsage = await getQueryUsage()
+
+        const res = await request
+          .get(`/api/views/${view.name}`)
+          .set(config.defaultHeaders())
+          .expect("Content-Type", /json/)
+          .expect(200)
+        expect(res.body.length).toEqual(1)
+        expect(res.body[0]._id).toEqual(row._id)
+
+        await assertRowUsage(rowUsage)
+        await assertQueryUsage(queryUsage + 1)
+      })
     })
 
-    it("should throw an error if view doesn't exist", async () => {
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
-
-      await request
-        .get(`/api/views/derp`)
-        .set(config.defaultHeaders())
-        .expect(404)
-
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage)
-    })
-
-    it("should be able to run on a view", async () => {
-      const view = await config.createLegacyView()
-      const row = await config.createRow()
-      const rowUsage = await getRowUsage()
-      const queryUsage = await getQueryUsage()
-
-      const res = await request
-        .get(`/api/views/${view.name}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.length).toEqual(1)
-      expect(res.body[0]._id).toEqual(row._id)
-
-      await assertRowUsage(rowUsage)
-      await assertQueryUsage(queryUsage + 1)
-    })
-  })
-
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
       const { table, firstRow, secondRow } = await tenancy.doInTenant(
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index e835779f93..72a8b9a146 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -50,6 +50,7 @@ import {
   SearchFilters,
   UserRoles,
   Automation,
+  View,
 } from "@budibase/types"
 
 import API from "./api"
@@ -629,12 +630,12 @@ class TestConfiguration {
 
   // VIEW
 
-  async createLegacyView(config?: any) {
-    if (!this.table) {
+  async createLegacyView(config?: View) {
+    if (!this.table && !config) {
       throw "Test requires table to be configured."
     }
     const view = config || {
-      tableId: this.table._id,
+      tableId: this.table!._id,
       name: "ViewTest",
     }
     return this._req(view, null, controllers.view.v1.save)

From 5e2e43a7d7e342014d97010ee723d6b23848f6f7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 13:52:17 +0200
Subject: [PATCH 25/81] Clean configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 70 ++++++++++---------
 1 file changed, 37 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0fab8eb426..d572097469 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -49,37 +49,39 @@ describe.each([
     await config.init()
   })
 
-  beforeEach(async () => {
-    mocks.licenses.useCloudFree()
-    let tableConfig: SaveTableRequest = {
-      name: generator.guid(),
-      type: "table",
-      primary: ["id"],
-      schema: {
-        id: {
-          type: FieldType.AUTO,
-          name: "id",
-          autocolumn: true,
-          constraints: {
-            presence: true,
-          },
-        },
-        name: {
-          type: FieldType.STRING,
-          name: "name",
-          constraints: {
-            type: "string",
-          },
-        },
-        description: {
-          type: FieldType.STRING,
-          name: "description",
-          constraints: {
-            type: "string",
-          },
+  const generateTableConfig: () => SaveTableRequest = () => ({
+    name: generator.guid(),
+    type: "table",
+    primary: ["id"],
+    schema: {
+      id: {
+        type: FieldType.AUTO,
+        name: "id",
+        autocolumn: true,
+        constraints: {
+          presence: true,
         },
       },
-    }
+      name: {
+        type: FieldType.STRING,
+        name: "name",
+        constraints: {
+          type: "string",
+        },
+      },
+      description: {
+        type: FieldType.STRING,
+        name: "description",
+        constraints: {
+          type: "string",
+        },
+      },
+    },
+  })
+
+  beforeEach(async () => {
+    mocks.licenses.useCloudFree()
+    const tableConfig = generateTableConfig()
 
     if (dsProvider) {
       datasource = await config.api.datasource.create(
@@ -93,6 +95,8 @@ describe.each([
     }
 
     table = await config.api.table.create(tableConfig)
+    config.table = table
+    config.datasource = datasource
     row = basicRow(table._id!)
   })
 
@@ -130,8 +134,8 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = (row?: SaveRowRequest) =>
-    config.api.row.save(table._id!, row || basicRow(table._id!))
+  const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
+    config.api.row.save(tableId, row || basicRow(table._id!))
 
   describe("save, load, update", () => {
     function getDefaultFields() {
@@ -270,7 +274,7 @@ describe.each([
         status: "new",
       }
       await createRow()
-      await createRow(newRow)
+      await createRow(table._id, newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -552,7 +556,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await createRow(row)
+      const createdRow = await createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 

From 9b783e0804763554f949da4ed4dac85a804395a5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:08:49 +0200
Subject: [PATCH 26/81] Fix cleanup relationships

---
 .../server/src/api/controllers/table/external.ts | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 327904666d..513b989a3a 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -67,13 +67,15 @@ function cleanupRelationships(
   tables: Record<string, Table>,
   oldTable?: Table
 ) {
-  const tableToIterate = oldTable ? oldTable : table
-  // clean up relationships in couch table schemas
-  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
-    if (
-      schema.type === FieldTypes.LINK &&
-      (!oldTable || table.schema[key] == null)
-    ) {
+  const isUpdate = !!oldTable
+  if (!isUpdate) {
+    return
+  }
+
+  // When updating a table, we want to detect if some relationships were removed.
+  // If so, we want to remove the relationship from the other end
+  for (let [key, schema] of Object.entries(oldTable.schema)) {
+    if (schema.type === FieldTypes.LINK && !table.schema[key]) {
       const relatedTable = Object.values(tables).find(
         table => table._id === schema.tableId
       )

From a5142088d9f0c25b08a7e09e656a238290dd4948 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:09:50 +0200
Subject: [PATCH 27/81] Clean tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d572097469..01a6175249 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -772,22 +772,36 @@ describe.each([
 
   describe("fetchEnrichedRows", () => {
     it("should allow enriching some linked rows", async () => {
-      const { table, firstRow, secondRow } = await tenancy.doInTenant(
+      const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
         async () => {
-          const table = await config.createLinkedTable()
-          const firstRow = await config.createRow({
+          const linkedTable = await config.createLinkedTable({
+            name: generator.guid(),
+            type: "table",
+            primary: ["id"],
+            schema: {
+              id: {
+                type: FieldType.AUTO,
+                name: "id",
+                autocolumn: true,
+                constraints: {
+                  presence: true,
+                },
+              },
+            },
+          })
+          const firstRow = await createRow(table._id, {
             name: "Test Contact",
             description: "original description",
             tableId: table._id,
           })
-          const secondRow = await config.createRow({
+          const secondRow = await createRow(linkedTable._id, {
             name: "Test 2",
             description: "og desc",
             link: [{ _id: firstRow._id }],
-            tableId: table._id,
+            tableId: linkedTable._id,
           })
-          return { table, firstRow, secondRow }
+          return { linkedTable, firstRow, secondRow }
         }
       )
       const rowUsage = await getRowUsage()
@@ -795,16 +809,16 @@ describe.each([
 
       // test basic enrichment
       const resBasic = await request
-        .get(`/api/${table._id}/rows/${secondRow._id}`)
+        .get(`/api/${linkedTable._id}/rows/${secondRow._id}`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(resBasic.body.link[0]._id).toBe(firstRow._id)
-      expect(resBasic.body.link[0].primaryDisplay).toBe("Test Contact")
+      expect(resBasic.body.link.length).toBe(1)
+      expect(resBasic.body.link[0]).toEqual({ _id: firstRow._id })
 
       // test full enrichment
       const resEnriched = await request
-        .get(`/api/${table._id}/${secondRow._id}/enrich`)
+        .get(`/api/${linkedTable._id}/${secondRow._id}/enrich`)
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)

From 45ddd46f4cb79ff71a76a1ce1c45e887115a8073 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:29:05 +0200
Subject: [PATCH 28/81] Fix tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 49 ++++++++++---------
 1 file changed, 27 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 01a6175249..9d9bcb302c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -137,17 +137,15 @@ describe.each([
   const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
     config.api.row.save(tableId, row || basicRow(table._id!))
 
-  describe("save, load, update", () => {
-    function getDefaultFields() {
-      if (isInternal) {
-        return {
-          type: "row",
-          createdAt: timestamp,
-          updatedAt: timestamp,
-        }
+  const defaultRowFields = isInternal
+    ? {
+        type: "row",
+        createdAt: timestamp,
+        updatedAt: timestamp,
       }
-    }
+    : undefined
 
+  describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -262,7 +260,7 @@ describe.each([
         _id: existing._id,
         _rev: existing._rev,
         id: existing.id,
-        ...getDefaultFields(),
+        ...defaultRowFields,
       })
       await assertQueryUsage(queryUsage + 1)
     })
@@ -835,7 +833,7 @@ describe.each([
     it("should allow enriching attachment rows", async () => {
       const table = await config.createAttachmentTable()
       const attachmentId = `${structures.uuid()}.csv`
-      const row = await config.createRow({
+      const row = await createRow(table._id, {
         name: "test",
         description: "test",
         attachment: [
@@ -906,15 +904,24 @@ describe.each([
     function userTable(): Table {
       return {
         name: "user",
-        type: "user",
+        type: "table",
+        primary: ["id"],
         schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+            autocolumn: true,
+            constraints: {
+              presence: true,
+            },
+          },
           name: {
             type: FieldType.STRING,
             name: "name",
           },
           surname: {
             type: FieldType.STRING,
-            name: "name",
+            name: "surname",
           },
           age: {
             type: FieldType.NUMBER,
@@ -965,11 +972,10 @@ describe.each([
           surname: data.surname,
           address: data.address,
           tableId: config.table!._id,
-          type: "row",
-          _id: expect.any(String),
-          _rev: expect.any(String),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
+          _id: newRow._id,
+          _rev: newRow._rev,
+          id: newRow.id,
+          ...defaultRowFields,
         })
         expect(row.body._viewId).toBeUndefined()
         expect(row.body.age).toBeUndefined()
@@ -1006,13 +1012,12 @@ describe.each([
         const row = await config.api.row.get(tableId, newRow._id!)
         expect(row.body).toEqual({
           ...newRow,
-          type: "row",
           name: newData.name,
           address: newData.address,
-          _id: expect.any(String),
+          _id: newRow._id,
           _rev: expect.any(String),
-          createdAt: expect.any(String),
-          updatedAt: expect.any(String),
+          id: newRow.id,
+          ...defaultRowFields,
         })
         expect(row.body._viewId).toBeUndefined()
         expect(row.body.age).toBeUndefined()

From c26a4c3a1135e42d321f88edb596db1a84293630 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:46:10 +0200
Subject: [PATCH 29/81] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 99 ++++++++++---------
 1 file changed, 53 insertions(+), 46 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9d9bcb302c..fa72776f30 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -49,51 +49,57 @@ describe.each([
     await config.init()
   })
 
-  const generateTableConfig: () => SaveTableRequest = () => ({
-    name: generator.guid(),
-    type: "table",
-    primary: ["id"],
-    schema: {
-      id: {
-        type: FieldType.AUTO,
-        name: "id",
-        autocolumn: true,
-        constraints: {
-          presence: true,
-        },
-      },
-      name: {
-        type: FieldType.STRING,
-        name: "name",
-        constraints: {
-          type: "string",
-        },
-      },
-      description: {
-        type: FieldType.STRING,
-        name: "description",
-        constraints: {
-          type: "string",
-        },
-      },
-    },
-  })
-
-  beforeEach(async () => {
-    mocks.licenses.useCloudFree()
-    const tableConfig = generateTableConfig()
-
+  const tableDatasourceConfig = async () => {
+    const result: Partial<SaveTableRequest> = {}
     if (dsProvider) {
       datasource = await config.api.datasource.create(
         await dsProvider.getDsConfig()
       )
 
-      tableConfig.sourceId = datasource._id
+      result.sourceId = datasource._id
       if (datasource.plus) {
-        tableConfig.type = "external"
+        result.type = "external"
       }
     }
+    return result
+  }
 
+  const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
+    return {
+      name: generator.guid(),
+      type: "table",
+      primary: ["id"],
+      schema: {
+        id: {
+          type: FieldType.AUTO,
+          name: "id",
+          autocolumn: true,
+          constraints: {
+            presence: true,
+          },
+        },
+        name: {
+          type: FieldType.STRING,
+          name: "name",
+          constraints: {
+            type: "string",
+          },
+        },
+        description: {
+          type: FieldType.STRING,
+          name: "description",
+          constraints: {
+            type: "string",
+          },
+        },
+      },
+      ...(await tableDatasourceConfig()),
+    }
+  }
+
+  beforeEach(async () => {
+    mocks.licenses.useCloudFree()
+    const tableConfig = await generateTableConfig()
     table = await config.api.table.create(tableConfig)
     config.table = table
     config.datasource = datasource
@@ -901,9 +907,9 @@ describe.each([
   })
 
   describe("view 2.0", () => {
-    function userTable(): Table {
+    async function userTable(): Promise<Table> {
       return {
-        name: "user",
+        name: `users_${generator.guid()}`,
         type: "table",
         primary: ["id"],
         schema: {
@@ -936,6 +942,7 @@ describe.each([
             name: "jobTitle",
           },
         },
+        ...(await tableDatasourceConfig()),
       }
     }
 
@@ -949,7 +956,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -985,7 +992,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1027,7 +1034,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1037,7 +1044,7 @@ describe.each([
           },
         })
 
-        const createdRow = await config.createRow()
+        const createdRow = await createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1052,7 +1059,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1063,9 +1070,9 @@ describe.each([
         })
 
         const rows = [
-          await config.createRow(),
-          await config.createRow(),
-          await config.createRow(),
+          await createRow(tableId),
+          await createRow(tableId),
+          await createRow(tableId),
         ]
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()

From b63b61655b3b4e25e4466f0516f6dbfa9a1ab411 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 15:54:27 +0200
Subject: [PATCH 30/81] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 36 ++++++++++++-------
 .../src/tests/utilities/TestConfiguration.ts  | 28 ++++++++++-----
 2 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fa72776f30..0193ff095d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1094,11 +1094,20 @@ describe.each([
 
     describe("view search", () => {
       const viewSchema = { age: { visible: true }, name: { visible: true } }
-      function userTable(): Table {
+      async function userTable(): Promise<Table> {
         return {
-          name: "user",
-          type: "user",
+          name: `users_${generator.guid()}`,
+          type: "table",
+          primary: ["id"],
           schema: {
+            id: {
+              type: FieldType.AUTO,
+              name: "id",
+              autocolumn: true,
+              constraints: {
+                presence: true,
+              },
+            },
             name: {
               type: FieldType.STRING,
               name: "name",
@@ -1110,11 +1119,12 @@ describe.each([
               constraints: {},
             },
           },
+          ...(await tableDatasourceConfig()),
         }
       }
 
-      it("returns table rows from view", async () => {
-        const table = await config.createTable(userTable())
+      it.only("returns table rows from view", async () => {
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1130,7 +1140,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
           await config.createRow({
@@ -1235,7 +1245,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(userTable())
+          await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1266,7 +1276,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(userTable())
+          await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1307,7 +1317,7 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1337,7 +1347,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        await config.createTable(userTable())
+        await config.createTable(await userTable())
 
         const createViewResponse = await config.api.viewV2.create()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1346,7 +1356,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1363,7 +1373,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
@@ -1428,7 +1438,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(userTable())
+          const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
             rows.push(await config.createRow({ tableId: table._id }))
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 72a8b9a146..030a08cee7 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -51,6 +51,8 @@ import {
   UserRoles,
   Automation,
   View,
+  FieldType,
+  RelationshipType,
 } from "@budibase/types"
 
 import API from "./api"
@@ -76,7 +78,6 @@ class TestConfiguration {
   globalUserId: any
   userMetadataId: any
   table?: Table
-  linkedTable: any
   automation: any
   datasource?: Datasource
   tenantId?: string
@@ -559,25 +560,34 @@ class TestConfiguration {
     return this._req(null, { tableId }, controllers.table.find)
   }
 
-  async createLinkedTable(relationshipType?: string, links: any = ["link"]) {
+  async createLinkedTable(
+    config?: Table,
+    relationshipType = RelationshipType.ONE_TO_MANY,
+    links: any = ["link"]
+  ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
-    const tableConfig: any = basicTable()
+    const tableConfig = config || basicTable()
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
-        type: "link",
+        type: FieldType.LINK,
         fieldName: link,
         tableId: this.table._id,
         name: link,
-      }
-      if (relationshipType) {
-        tableConfig.schema[link].relationshipType = relationshipType
+        relationshipType,
       }
     }
-    const linkedTable = await this.createTable(tableConfig)
-    this.linkedTable = linkedTable
+
+    if (this.datasource && !tableConfig.sourceId) {
+      tableConfig.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
+    const linkedTable = await this.api.table.create(tableConfig)
     return linkedTable
   }
 

From 1a7a1cdd1bbcbe473987e17d14a0e53d9287b2e7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:23:34 +0200
Subject: [PATCH 31/81] Fix view test

---
 packages/server/src/api/routes/tests/row.spec.ts   | 14 ++++++++++++--
 packages/server/src/sdk/app/rows/search.ts         |  2 +-
 .../server/src/sdk/app/rows/search/internal.ts     |  6 +++++-
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0193ff095d..905304356c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1123,7 +1123,7 @@ describe.each([
         }
       }
 
-      it.only("returns table rows from view", async () => {
+      it("returns empty rows from view when no schema is passed", async () => {
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
@@ -1135,7 +1135,17 @@ describe.each([
 
         expect(response.body.rows).toHaveLength(10)
         expect(response.body).toEqual({
-          rows: expect.arrayContaining(rows.map(expect.objectContaining)),
+          rows: expect.arrayContaining(
+            rows.map(r => ({
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              _id: r._id,
+              _rev: r._rev,
+              ...defaultRowFields,
+            }))
+          ),
+          hasNextPage: false,
+          bookmark: null,
         })
       })
 
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index 4861f473ea..afee18b57c 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -20,7 +20,7 @@ function pickApi(tableId: any) {
 export async function search(options: SearchParams): Promise<{
   rows: any[]
   hasNextPage?: boolean
-  bookmark?: number | null
+  bookmark?: number | string | null
 }> {
   return pickApi(options.tableId).search(options)
 }
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 4cdeca87f6..0a6fd69ff6 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -59,7 +59,11 @@ export async function search(options: SearchParams) {
   if (paginate) {
     response = await paginatedSearch(query, params)
   } else {
-    response = await fullSearch(query, params)
+    response = {
+      ...(await fullSearch(query, params)),
+      hasNextPage: false,
+      bookmark: null,
+    }
   }
 
   // Enrich search results with relationships

From 8858fe38873d1b9d4c128cfa4d8f8b2b537d4d5f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:31:47 +0200
Subject: [PATCH 32/81] Fixes

---
 .../server/src/api/routes/tests/row.spec.ts   | 29 ++++++++++++++-----
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 905304356c..94014d458d 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -20,6 +20,7 @@ import {
   Table,
 } from "@budibase/types"
 import {
+  expectAnyExternalColsAttributes,
   expectAnyInternalColsAttributes,
   generator,
   mocks,
@@ -1178,8 +1179,18 @@ describe.each([
         expect(response.body.rows).toHaveLength(5)
         expect(response.body).toEqual({
           rows: expect.arrayContaining(
-            expectedRows.map(expect.objectContaining)
+            expectedRows.map(r => ({
+              _viewId: createViewResponse.id,
+              tableId: table._id,
+              name: r.name,
+              age: r.age,
+              _id: r._id,
+              _rev: r._rev,
+              ...defaultRowFields,
+            }))
           ),
+          hasNextPage: false,
+          bookmark: null,
         })
       })
 
@@ -1277,9 +1288,9 @@ describe.each([
           const response = await config.api.viewV2.search(createViewResponse.id)
 
           expect(response.body.rows).toHaveLength(4)
-          expect(response.body).toEqual({
-            rows: expected.map(name => expect.objectContaining({ name })),
-          })
+          expect(response.body.rows).toEqual(
+            expected.map(name => expect.objectContaining({ name }))
+          )
         }
       )
 
@@ -1320,9 +1331,9 @@ describe.each([
           )
 
           expect(response.body.rows).toHaveLength(4)
-          expect(response.body).toEqual({
-            rows: expected.map(name => expect.objectContaining({ name })),
-          })
+          expect(response.body.rows).toEqual(
+            expected.map(name => expect.objectContaining({ name }))
+          )
         }
       )
 
@@ -1348,7 +1359,9 @@ describe.each([
         expect(response.body.rows).toEqual(
           expect.arrayContaining(
             rows.map(r => ({
-              ...expectAnyInternalColsAttributes,
+              ...(isInternal
+                ? expectAnyInternalColsAttributes
+                : expectAnyExternalColsAttributes),
               _viewId: view.id,
               name: r.name,
             }))

From 2f5aadec4bdc09b71dc16f34e4c9b2339d27dad8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 16:39:13 +0200
Subject: [PATCH 33/81] More fixes

---
 packages/server/src/api/routes/tests/row.spec.ts | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 94014d458d..2373b8f9cc 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1401,7 +1401,6 @@ describe.each([
         for (let i = 0; i < 10; i++) {
           rows.push(await config.createRow({ tableId: table._id }))
         }
-        // rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
 
         const createViewResponse = await config.api.viewV2.create()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
@@ -1417,9 +1416,9 @@ describe.each([
         )
         expect(firstPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(0, 4)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
 
         const secondPageResponse = await config.api.viewV2.search(
@@ -1434,9 +1433,9 @@ describe.each([
         )
         expect(secondPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(4, 8)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: true,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
 
         const lastPageResponse = await config.api.viewV2.search(
@@ -1450,9 +1449,9 @@ describe.each([
         )
         expect(lastPageResponse.body).toEqual({
           rows: expect.arrayContaining(allRows.slice(8)),
-          totalRows: 10,
+          totalRows: isInternal ? 10 : undefined,
           hasNextPage: false,
-          bookmark: expect.any(String),
+          bookmark: expect.anything(),
         })
       })
 

From 4e69e51ccad85c9663f13d411ea6fd6f265cd1d8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 8 Sep 2023 17:42:54 +0200
Subject: [PATCH 34/81] Fix autocolumns

---
 .../src/api/controllers/row/ExternalRequest.ts | 18 ++++++++++++------
 .../server/src/api/controllers/row/external.ts | 18 +++++++++++++++++-
 .../server/src/api/routes/tests/row.spec.ts    |  4 ++--
 packages/server/src/sdk/app/tables/index.ts    | 15 +++++++++++++--
 .../server/src/sdk/app/tables/validation.ts    |  7 -------
 5 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts
index b24b642035..2c4428599b 100644
--- a/packages/server/src/api/controllers/row/ExternalRequest.ts
+++ b/packages/server/src/api/controllers/row/ExternalRequest.ts
@@ -1,4 +1,5 @@
 import {
+  AutoReason,
   Datasource,
   FieldSchema,
   FieldType,
@@ -24,7 +25,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
-import { FieldTypes } from "../../../constants"
+import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
@@ -259,6 +260,15 @@ function isOneSide(field: FieldSchema) {
   )
 }
 
+function isEditableColumn(column: FieldSchema) {
+  const isExternalAutoColumn =
+    column.autocolumn &&
+    column.autoReason !== AutoReason.FOREIGN_KEY &&
+    column.subtype !== AutoFieldSubTypes.AUTO_ID
+  const isFormula = column.type === FieldTypes.FORMULA
+  return !(isExternalAutoColumn || isFormula)
+}
+
 export class ExternalRequest {
   private operation: Operation
   private tableId: string
@@ -295,11 +305,7 @@ export class ExternalRequest {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (
-        row[key] == null ||
-        newRow[key] ||
-        !sdk.tables.isEditableColumn(field)
-      ) {
+      if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
         continue
       }
       // if its an empty string then it means return the column to null (if possible)
diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 44d50886d3..acae075165 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -18,6 +18,8 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
+import { inputProcessing } from "../../../utilities/rowProcessor"
+import { cloneDeep, isEqual } from "lodash"
 
 export async function handleRequest(
   operation: Operation,
@@ -88,10 +90,24 @@ export async function save(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = inputProcessing(
+    ctx.user,
+    cloneDeep(table),
+    inputs
+  )
+
   const response = await handleRequest(Operation.CREATE, tableId, {
-    row: inputs,
+    row,
   })
+
   const responseRow = response as { row: Row }
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
   const rowId = responseRow.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 2373b8f9cc..63850602ba 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -177,8 +177,8 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const newTable = await config.createTable({
+        ...table,
         name: "TestTableAuto",
-        type: "table",
         schema: {
           ...table.schema,
           "Row ID": {
@@ -189,7 +189,7 @@ describe.each([
             autocolumn: true,
             constraints: {
               type: "number",
-              presence: false,
+              presence: true,
               numericality: {
                 greaterThanOrEqualTo: "",
                 lessThanOrEqualTo: "",
diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts
index 1bf9117837..64fcde4bff 100644
--- a/packages/server/src/sdk/app/tables/index.ts
+++ b/packages/server/src/sdk/app/tables/index.ts
@@ -12,7 +12,7 @@ import {
   TableViewsResponse,
 } from "@budibase/types"
 import datasources from "../datasources"
-import { isEditableColumn, populateExternalTableSchemas } from "./validation"
+import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -73,12 +73,23 @@ function enrichViewSchemas(table: Table): TableResponse {
   }
 }
 
+async function saveTable(table: Table) {
+  const db = context.getAppDB()
+  if (isExternalTable(table._id!)) {
+    const datasource = await sdk.datasources.get(table.sourceId!)
+    datasource.entities![table.name] = table
+    await db.put(datasource)
+  } else {
+    await db.put(table)
+  }
+}
+
 export default {
   getAllInternalTables,
   getAllExternalTables,
   getExternalTable,
   getTable,
   populateExternalTableSchemas,
-  isEditableColumn,
   enrichViewSchemas,
+  saveTable,
 }
diff --git a/packages/server/src/sdk/app/tables/validation.ts b/packages/server/src/sdk/app/tables/validation.ts
index 8dc41107d3..56f3e84c7a 100644
--- a/packages/server/src/sdk/app/tables/validation.ts
+++ b/packages/server/src/sdk/app/tables/validation.ts
@@ -55,13 +55,6 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   return datasource
 }
 
-export function isEditableColumn(column: FieldSchema) {
-  const isAutoColumn =
-    column.autocolumn && column.autoReason !== AutoReason.FOREIGN_KEY
-  const isFormula = column.type === FieldTypes.FORMULA
-  return !(isAutoColumn || isFormula)
-}
-
 export function populateExternalTableSchemas(datasource: Datasource) {
   return checkForeignKeysAreAutoColumns(datasource)
 }

From 33be2d6c969c52be48f833c4f8eadb9b6c1b7e2a Mon Sep 17 00:00:00 2001
From: FlaminWrap <97764630+FlaminWrap@users.noreply.github.com>
Date: Fri, 8 Sep 2023 21:28:25 +0100
Subject: [PATCH 35/81] Fix #11353

Removes the slice as the v prefix has been removed

Addresses:
#11353
---
 .../builder/src/pages/builder/portal/settings/version.svelte   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/packages/builder/src/pages/builder/portal/settings/version.svelte b/packages/builder/src/pages/builder/portal/settings/version.svelte
index fb9144c730..c3898b7861 100644
--- a/packages/builder/src/pages/builder/portal/settings/version.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/version.svelte
@@ -63,8 +63,7 @@
       )
       const githubResponse = await githubCheck.json()
 
-      //Get tag and remove the v infront of the tage name e.g. v1.0.0 is 1.0.0
-      githubVersion = githubResponse.tag_name.slice(1)
+      githubVersion = githubResponse.tag_name
 
       //Get the release date and output it in the local time format
       githubPublishedDate = new Date(githubResponse.published_at)

From ca743e77fbfc00a8c21003ad4cb8db0e3e3978f4 Mon Sep 17 00:00:00 2001
From: Conor Webb <conorwebb59@gmail.com>
Date: Mon, 11 Sep 2023 07:52:43 +0100
Subject: [PATCH 36/81] Fixed typo of label for number type column.

---
 .../components/backend/DataTable/modals/CreateEditColumn.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
index fd13767a63..44c37813d6 100644
--- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
+++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
@@ -523,7 +523,7 @@
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
-        <Label size="M">Max Value</Label>
+        <Label size="M">Min Value</Label>
       </div>
       <div class="input-length">
         <Input

From 55514653c34d3cada6009ccf2d88b92fb72e8991 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:25:40 +0200
Subject: [PATCH 37/81] yarn.lock

---
 yarn.lock | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/yarn.lock b/yarn.lock
index 8c93661665..ab86a87560 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6269,14 +6269,6 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
-"@types/node-fetch@2.6.1":
-  version "2.6.1"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
-  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^3.0.0"
-
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6298,11 +6290,6 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
-"@types/node@14.18.20":
-  version "14.18.20"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
-  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
-
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"

From c70c627fc9f0ae12ca8f047e24cc08e44126e391 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:52:46 +0200
Subject: [PATCH 38/81] Fix view test

---
 .../server/src/api/routes/tests/row.spec.ts   | 37 +++++++++++++++----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 63850602ba..d66a30684e 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -471,7 +471,12 @@ describe.each([
     function orderTable(): Table {
       return {
         name: "orders",
+        primary: ["id"],
         schema: {
+          id: {
+            type: FieldType.AUTO,
+            name: "id",
+          },
           Country: {
             type: FieldType.STRING,
             name: "Country",
@@ -494,19 +499,35 @@ describe.each([
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
         schema: {
-          Country: {},
-          OrderID: {},
+          Country: {
+            visible: true,
+          },
+          OrderID: {
+            visible: true,
+          },
         },
       })
 
-      const response = await config.api.row.save(createViewResponse.id, {
-        Country: "Aussy",
-        OrderID: "1111",
-        Story: "aaaaa",
-      })
+      const createRowResponse = await config.api.row.save(
+        createViewResponse.id,
+        {
+          OrderID: "1111",
+          Country: "Aussy",
+          Story: "aaaaa",
+        }
+      )
 
-      const row = await config.api.row.get(table._id!, response._id!)
+      const row = await config.api.row.get(table._id!, createRowResponse._id!)
       expect(row.body.Story).toBeUndefined()
+      expect(row.body).toEqual({
+        ...defaultRowFields,
+        OrderID: "1111",
+        Country: "Aussy",
+        id: 1,
+        _id: "%5B1%5D",
+        _rev: createRowResponse._rev,
+        tableId: table._id,
+      })
     })
   })
 

From 8ee23e6168f16456bafc06f464a8790dc3c4c212 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 09:57:47 +0200
Subject: [PATCH 39/81] Fix

---
 packages/server/src/api/routes/tests/row.spec.ts | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index d66a30684e..bbf5a13574 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -471,18 +471,14 @@ describe.each([
     function orderTable(): Table {
       return {
         name: "orders",
-        primary: ["id"],
+        primary: ["OrderID"],
         schema: {
-          id: {
-            type: FieldType.AUTO,
-            name: "id",
-          },
           Country: {
             type: FieldType.STRING,
             name: "Country",
           },
           OrderID: {
-            type: FieldType.STRING,
+            type: FieldType.NUMBER,
             name: "OrderID",
           },
           Story: {
@@ -521,10 +517,9 @@ describe.each([
       expect(row.body.Story).toBeUndefined()
       expect(row.body).toEqual({
         ...defaultRowFields,
-        OrderID: "1111",
+        OrderID: 1111,
         Country: "Aussy",
-        id: 1,
-        _id: "%5B1%5D",
+        _id: createRowResponse._id,
         _rev: createRowResponse._rev,
         tableId: table._id,
       })

From 66524d998b746b1ad92e265bb5eb9436aee8446c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 11:02:44 +0200
Subject: [PATCH 40/81] Run coerced only for internal

---
 .../server/src/api/routes/tests/row.spec.ts   | 305 +++++++++---------
 1 file changed, 153 insertions(+), 152 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index bbf5a13574..5fa3745987 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -304,167 +304,168 @@ describe.each([
       await assertQueryUsage(queryUsage) // no change
     })
 
-    it("row values are coerced", async () => {
-      const str = {
-        type: FieldType.STRING,
-        name: "str",
-        constraints: { type: "string", presence: false },
-      }
-      const attachment = {
-        type: FieldType.ATTACHMENT,
-        name: "attachment",
-        constraints: { type: "array", presence: false },
-      }
-      const bool = {
-        type: FieldType.BOOLEAN,
-        name: "boolean",
-        constraints: { type: "boolean", presence: false },
-      }
-      const number = {
-        type: FieldType.NUMBER,
-        name: "str",
-        constraints: { type: "number", presence: false },
-      }
-      const datetime = {
-        type: FieldType.DATETIME,
-        name: "datetime",
-        constraints: {
-          type: "string",
-          presence: false,
-          datetime: { earliest: "", latest: "" },
-        },
-      }
-      const arrayField = {
-        type: FieldType.ARRAY,
-        constraints: {
-          type: "array",
-          presence: false,
-          inclusion: ["One", "Two", "Three"],
-        },
-        name: "Sample Tags",
-        sortable: false,
-      }
-      const optsField = {
-          fieldName: "Sample Opts",
-          name: "Sample Opts",
-          type: FieldType.OPTIONS,
+    isInternal &&
+      it("row values are coerced", async () => {
+        const str = {
+          type: FieldType.STRING,
+          name: "str",
+          constraints: { type: "string", presence: false },
+        }
+        const attachment = {
+          type: FieldType.ATTACHMENT,
+          name: "attachment",
+          constraints: { type: "array", presence: false },
+        }
+        const bool = {
+          type: FieldType.BOOLEAN,
+          name: "boolean",
+          constraints: { type: "boolean", presence: false },
+        }
+        const number = {
+          type: FieldType.NUMBER,
+          name: "str",
+          constraints: { type: "number", presence: false },
+        }
+        const datetime = {
+          type: FieldType.DATETIME,
+          name: "datetime",
           constraints: {
             type: "string",
             presence: false,
-            inclusion: ["Alpha", "Beta", "Gamma"],
+            datetime: { earliest: "", latest: "" },
           },
-        },
-        table = await config.createTable({
-          name: "TestTable2",
-          type: "table",
-          schema: {
-            name: str,
-            stringUndefined: str,
-            stringNull: str,
-            stringString: str,
-            numberEmptyString: number,
-            numberNull: number,
-            numberUndefined: number,
-            numberString: number,
-            numberNumber: number,
-            datetimeEmptyString: datetime,
-            datetimeNull: datetime,
-            datetimeUndefined: datetime,
-            datetimeString: datetime,
-            datetimeDate: datetime,
-            boolNull: bool,
-            boolEmpty: bool,
-            boolUndefined: bool,
-            boolString: bool,
-            boolBool: bool,
-            attachmentNull: attachment,
-            attachmentUndefined: attachment,
-            attachmentEmpty: attachment,
-            attachmentEmptyArrayStr: attachment,
-            arrayFieldEmptyArrayStr: arrayField,
-            arrayFieldArrayStrKnown: arrayField,
-            arrayFieldNull: arrayField,
-            arrayFieldUndefined: arrayField,
-            optsFieldEmptyStr: optsField,
-            optsFieldUndefined: optsField,
-            optsFieldNull: optsField,
-            optsFieldStrKnown: optsField,
+        }
+        const arrayField = {
+          type: FieldType.ARRAY,
+          constraints: {
+            type: "array",
+            presence: false,
+            inclusion: ["One", "Two", "Three"],
           },
-        })
+          name: "Sample Tags",
+          sortable: false,
+        }
+        const optsField = {
+            fieldName: "Sample Opts",
+            name: "Sample Opts",
+            type: FieldType.OPTIONS,
+            constraints: {
+              type: "string",
+              presence: false,
+              inclusion: ["Alpha", "Beta", "Gamma"],
+            },
+          },
+          table = await config.createTable({
+            name: "TestTable2",
+            type: "table",
+            schema: {
+              name: str,
+              stringUndefined: str,
+              stringNull: str,
+              stringString: str,
+              numberEmptyString: number,
+              numberNull: number,
+              numberUndefined: number,
+              numberString: number,
+              numberNumber: number,
+              datetimeEmptyString: datetime,
+              datetimeNull: datetime,
+              datetimeUndefined: datetime,
+              datetimeString: datetime,
+              datetimeDate: datetime,
+              boolNull: bool,
+              boolEmpty: bool,
+              boolUndefined: bool,
+              boolString: bool,
+              boolBool: bool,
+              attachmentNull: attachment,
+              attachmentUndefined: attachment,
+              attachmentEmpty: attachment,
+              attachmentEmptyArrayStr: attachment,
+              arrayFieldEmptyArrayStr: arrayField,
+              arrayFieldArrayStrKnown: arrayField,
+              arrayFieldNull: arrayField,
+              arrayFieldUndefined: arrayField,
+              optsFieldEmptyStr: optsField,
+              optsFieldUndefined: optsField,
+              optsFieldNull: optsField,
+              optsFieldStrKnown: optsField,
+            },
+          })
 
-      row = {
-        name: "Test Row",
-        stringUndefined: undefined,
-        stringNull: null,
-        stringString: "i am a string",
-        numberEmptyString: "",
-        numberNull: null,
-        numberUndefined: undefined,
-        numberString: "123",
-        numberNumber: 123,
-        datetimeEmptyString: "",
-        datetimeNull: null,
-        datetimeUndefined: undefined,
-        datetimeString: "1984-04-20T00:00:00.000Z",
-        datetimeDate: new Date("1984-04-20"),
-        boolNull: null,
-        boolEmpty: "",
-        boolUndefined: undefined,
-        boolString: "true",
-        boolBool: true,
-        tableId: table._id,
-        attachmentNull: null,
-        attachmentUndefined: undefined,
-        attachmentEmpty: "",
-        attachmentEmptyArrayStr: "[]",
-        arrayFieldEmptyArrayStr: "[]",
-        arrayFieldUndefined: undefined,
-        arrayFieldNull: null,
-        arrayFieldArrayStrKnown: "['One']",
-        optsFieldEmptyStr: "",
-        optsFieldUndefined: undefined,
-        optsFieldNull: null,
-        optsFieldStrKnown: "Alpha",
-      }
+        row = {
+          name: "Test Row",
+          stringUndefined: undefined,
+          stringNull: null,
+          stringString: "i am a string",
+          numberEmptyString: "",
+          numberNull: null,
+          numberUndefined: undefined,
+          numberString: "123",
+          numberNumber: 123,
+          datetimeEmptyString: "",
+          datetimeNull: null,
+          datetimeUndefined: undefined,
+          datetimeString: "1984-04-20T00:00:00.000Z",
+          datetimeDate: new Date("1984-04-20"),
+          boolNull: null,
+          boolEmpty: "",
+          boolUndefined: undefined,
+          boolString: "true",
+          boolBool: true,
+          tableId: table._id,
+          attachmentNull: null,
+          attachmentUndefined: undefined,
+          attachmentEmpty: "",
+          attachmentEmptyArrayStr: "[]",
+          arrayFieldEmptyArrayStr: "[]",
+          arrayFieldUndefined: undefined,
+          arrayFieldNull: null,
+          arrayFieldArrayStrKnown: "['One']",
+          optsFieldEmptyStr: "",
+          optsFieldUndefined: undefined,
+          optsFieldNull: null,
+          optsFieldStrKnown: "Alpha",
+        }
 
-      const createdRow = await config.createRow(row)
-      const id = createdRow._id!
+        const createdRow = await config.createRow(row)
+        const id = createdRow._id!
 
-      const saved = (await loadRow(id, table._id!)).body
+        const saved = (await loadRow(id, table._id!)).body
 
-      expect(saved.stringUndefined).toBe(undefined)
-      expect(saved.stringNull).toBe("")
-      expect(saved.stringString).toBe("i am a string")
-      expect(saved.numberEmptyString).toBe(null)
-      expect(saved.numberNull).toBe(null)
-      expect(saved.numberUndefined).toBe(undefined)
-      expect(saved.numberString).toBe(123)
-      expect(saved.numberNumber).toBe(123)
-      expect(saved.datetimeEmptyString).toBe(null)
-      expect(saved.datetimeNull).toBe(null)
-      expect(saved.datetimeUndefined).toBe(undefined)
-      expect(saved.datetimeString).toBe(
-        new Date(row.datetimeString).toISOString()
-      )
-      expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
-      expect(saved.boolNull).toBe(null)
-      expect(saved.boolEmpty).toBe(null)
-      expect(saved.boolUndefined).toBe(undefined)
-      expect(saved.boolString).toBe(true)
-      expect(saved.boolBool).toBe(true)
-      expect(saved.attachmentNull).toEqual([])
-      expect(saved.attachmentUndefined).toBe(undefined)
-      expect(saved.attachmentEmpty).toEqual([])
-      expect(saved.attachmentEmptyArrayStr).toEqual([])
-      expect(saved.arrayFieldEmptyArrayStr).toEqual([])
-      expect(saved.arrayFieldNull).toEqual([])
-      expect(saved.arrayFieldUndefined).toEqual(undefined)
-      expect(saved.optsFieldEmptyStr).toEqual(null)
-      expect(saved.optsFieldUndefined).toEqual(undefined)
-      expect(saved.optsFieldNull).toEqual(null)
-      expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
-      expect(saved.optsFieldStrKnown).toEqual("Alpha")
-    })
+        expect(saved.stringUndefined).toBe(undefined)
+        expect(saved.stringNull).toBe("")
+        expect(saved.stringString).toBe("i am a string")
+        expect(saved.numberEmptyString).toBe(null)
+        expect(saved.numberNull).toBe(null)
+        expect(saved.numberUndefined).toBe(undefined)
+        expect(saved.numberString).toBe(123)
+        expect(saved.numberNumber).toBe(123)
+        expect(saved.datetimeEmptyString).toBe(null)
+        expect(saved.datetimeNull).toBe(null)
+        expect(saved.datetimeUndefined).toBe(undefined)
+        expect(saved.datetimeString).toBe(
+          new Date(row.datetimeString).toISOString()
+        )
+        expect(saved.datetimeDate).toBe(row.datetimeDate.toISOString())
+        expect(saved.boolNull).toBe(null)
+        expect(saved.boolEmpty).toBe(null)
+        expect(saved.boolUndefined).toBe(undefined)
+        expect(saved.boolString).toBe(true)
+        expect(saved.boolBool).toBe(true)
+        expect(saved.attachmentNull).toEqual([])
+        expect(saved.attachmentUndefined).toBe(undefined)
+        expect(saved.attachmentEmpty).toEqual([])
+        expect(saved.attachmentEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldEmptyArrayStr).toEqual([])
+        expect(saved.arrayFieldNull).toEqual([])
+        expect(saved.arrayFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldEmptyStr).toEqual(null)
+        expect(saved.optsFieldUndefined).toEqual(undefined)
+        expect(saved.optsFieldNull).toEqual(null)
+        expect(saved.arrayFieldArrayStrKnown).toEqual(["One"])
+        expect(saved.optsFieldStrKnown).toEqual("Alpha")
+      })
   })
 
   describe("view save", () => {

From 45605100697e6f0b9a3df63a357661261e546c9d Mon Sep 17 00:00:00 2001
From: Gerard Burns <Ghrehh@users.noreply.github.com>
Date: Tue, 12 Sep 2023 12:16:13 +0100
Subject: [PATCH 41/81] Customize signup flow BB changes (#11706)

* wip

* wip

* wip

* wip
---
 packages/bbui/src/Banner/Banner.svelte        |  4 +++
 .../bbui/src/Notification/Notification.svelte |  6 ++++-
 .../Notification/NotificationDisplay.svelte   |  3 ++-
 packages/bbui/src/Stores/banner.js            |  1 +
 packages/bbui/src/Stores/notifications.js     |  6 ++++-
 .../builder/src/pages/builder/_layout.svelte  | 27 +++++++++++++++++++
 6 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/packages/bbui/src/Banner/Banner.svelte b/packages/bbui/src/Banner/Banner.svelte
index 3810021a61..a04d469cc7 100644
--- a/packages/bbui/src/Banner/Banner.svelte
+++ b/packages/bbui/src/Banner/Banner.svelte
@@ -66,6 +66,10 @@
     pointer-events: all;
     width: 100%;
   }
+
+  .spectrum-Toast--neutral {
+    background-color: var(--grey-2);
+  }
   .spectrum-Button {
     border: 1px solid rgba(255, 255, 255, 0.2);
   }
diff --git a/packages/bbui/src/Notification/Notification.svelte b/packages/bbui/src/Notification/Notification.svelte
index eb2922a5de..26e60ed366 100644
--- a/packages/bbui/src/Notification/Notification.svelte
+++ b/packages/bbui/src/Notification/Notification.svelte
@@ -27,7 +27,11 @@
   <div class="spectrum-Toast-body" class:actionBody={!!action}>
     <div class="wrap spectrum-Toast-content">{message || ""}</div>
     {#if action}
-      <ActionButton quiet emphasized on:click={action}>
+      <ActionButton
+        quiet
+        emphasized
+        on:click={() => action(() => dispatch("dismiss"))}
+      >
         <div style="color: white; font-weight: 600;">{actionMessage}</div>
       </ActionButton>
     {/if}
diff --git a/packages/bbui/src/Notification/NotificationDisplay.svelte b/packages/bbui/src/Notification/NotificationDisplay.svelte
index 0f7e93eb23..6b7e68cece 100644
--- a/packages/bbui/src/Notification/NotificationDisplay.svelte
+++ b/packages/bbui/src/Notification/NotificationDisplay.svelte
@@ -8,7 +8,7 @@
 
 <Portal target=".modal-container">
   <div class="notifications">
-    {#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)}
+    {#each $notifications as { type, icon, message, id, dismissable, action, actionMessage, wide } (id)}
       <div transition:fly={{ y: 30 }}>
         <Notification
           {type}
@@ -16,6 +16,7 @@
           {message}
           {dismissable}
           {action}
+          {actionMessage}
           {wide}
           on:dismiss={() => notifications.dismiss(id)}
         />
diff --git a/packages/bbui/src/Stores/banner.js b/packages/bbui/src/Stores/banner.js
index 1a0b2d9ecc..fc93e7be99 100644
--- a/packages/bbui/src/Stores/banner.js
+++ b/packages/bbui/src/Stores/banner.js
@@ -1,6 +1,7 @@
 import { writable } from "svelte/store"
 
 export const BANNER_TYPES = {
+  NEUTRAL: "neutral",
   INFO: "info",
   NEGATIVE: "negative",
   WARNING: "warning",
diff --git a/packages/bbui/src/Stores/notifications.js b/packages/bbui/src/Stores/notifications.js
index 449d282f24..28331fffd8 100644
--- a/packages/bbui/src/Stores/notifications.js
+++ b/packages/bbui/src/Stores/notifications.js
@@ -27,7 +27,9 @@ export const createNotificationStore = () => {
       icon = "",
       autoDismiss = true,
       action = null,
+      actionMessage = null,
       wide = false,
+      dismissTimeout = NOTIFICATION_TIMEOUT,
     }
   ) => {
     if (block) {
@@ -44,14 +46,16 @@ export const createNotificationStore = () => {
           icon,
           dismissable: !autoDismiss,
           action,
+          actionMessage,
           wide,
+          dismissTimeout,
         },
       ]
     })
     if (autoDismiss) {
       const timeoutId = setTimeout(() => {
         dismissNotification(_id)
-      }, NOTIFICATION_TIMEOUT)
+      }, dismissTimeout)
       timeoutIds.add(timeoutId)
     }
   }
diff --git a/packages/builder/src/pages/builder/_layout.svelte b/packages/builder/src/pages/builder/_layout.svelte
index b216958045..960822a39f 100644
--- a/packages/builder/src/pages/builder/_layout.svelte
+++ b/packages/builder/src/pages/builder/_layout.svelte
@@ -3,6 +3,7 @@
   import { admin, auth, licensing } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
+  import { banner, BANNER_TYPES } from "@budibase/bbui"
   import { API } from "api"
   import Branding from "./Branding.svelte"
 
@@ -16,6 +17,32 @@
   $: user = $auth.user
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
+  let showVerificationPrompt = false
+
+  const checkVerification = user => {
+    if (!showVerificationPrompt && user?.account?.verified === false) {
+      showVerificationPrompt = true
+      banner.queue([
+        {
+          message: `Please verify your account. We've sent the verification link to ${user.email}`,
+          type: BANNER_TYPES.NEUTRAL,
+          showCloseButton: false,
+          extraButtonAction: () => {
+            fetch(`${$admin.accountPortalUrl}/api/auth/reset`, {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({ email: user.email }),
+            })
+          },
+          extraButtonText: "Resend email",
+        },
+      ])
+    }
+  }
+
+  $: checkVerification(user)
 
   const validateTenantId = async () => {
     const host = window.location.host

From 6e23d746117ce6b760a0c338ab79de85fd5a5da7 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 12 Sep 2023 11:16:28 +0000
Subject: [PATCH 42/81] Bump version to 2.10.3-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 1669e15246..ebdf2a4466 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.3-alpha.0",
+  "version": "2.10.3-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9a8b43b218ce7274c0d5b96d45e79da8dc7aab2b Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Tue, 12 Sep 2023 15:39:25 +0000
Subject: [PATCH 43/81] Bump version to 2.10.3-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index ebdf2a4466..e070f6579c 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.3-alpha.1",
+  "version": "2.10.3-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 88c68cfe5858e758d80fbeac2df3eb3dd14901bf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:07:20 +0200
Subject: [PATCH 44/81] Fetch relationships on external row find

---
 packages/server/src/api/controllers/row/external.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index acae075165..3881d1dd08 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -125,7 +125,9 @@ export async function save(ctx: UserCtx) {
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  const row = await sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id, {
+    relationships: true,
+  })
 
   if (!row) {
     ctx.throw(404)

From a537b17b239d8c8b166ba0e1e0cbb898a044f870 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:07:50 +0200
Subject: [PATCH 45/81] Run attachments only on internal

---
 .../server/src/api/routes/tests/row.spec.ts   | 45 ++++++++++---------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 5fa3745987..90eb4820b8 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -853,31 +853,32 @@ describe.each([
     })
   })
 
-  describe("attachments", () => {
-    it("should allow enriching attachment rows", async () => {
-      const table = await config.createAttachmentTable()
-      const attachmentId = `${structures.uuid()}.csv`
-      const row = await createRow(table._id, {
-        name: "test",
-        description: "test",
-        attachment: [
-          {
-            key: `${config.getAppId()}/attachments/${attachmentId}`,
-          },
-        ],
-        tableId: table._id,
-      })
-      // the environment needs configured for this
-      await setup.switchToSelfHosted(async () => {
-        return context.doInAppContext(config.getAppId(), async () => {
-          const enriched = await outputProcessing(table, [row])
-          expect((enriched as Row[])[0].attachment[0].url).toBe(
-            `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
-          )
+  isInternal &&
+    describe("attachments", () => {
+      it("should allow enriching attachment rows", async () => {
+        const table = await config.createAttachmentTable()
+        const attachmentId = `${structures.uuid()}.csv`
+        const row = await createRow(table._id, {
+          name: "test",
+          description: "test",
+          attachment: [
+            {
+              key: `${config.getAppId()}/attachments/${attachmentId}`,
+            },
+          ],
+          tableId: table._id,
+        })
+        // the environment needs configured for this
+        await setup.switchToSelfHosted(async () => {
+          return context.doInAppContext(config.getAppId(), async () => {
+            const enriched = await outputProcessing(table, [row])
+            expect((enriched as Row[])[0].attachment[0].url).toBe(
+              `/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
+            )
+          })
         })
       })
     })
-  })
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {

From 8b644555e3e108bebdfa7f5bbcac835c0f1705a5 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:09:09 +0200
Subject: [PATCH 46/81] Fetch primaryDisplay

---
 packages/server/src/api/routes/tests/row.spec.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 90eb4820b8..0bf678ef93 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -70,6 +70,7 @@ describe.each([
       name: generator.guid(),
       type: "table",
       primary: ["id"],
+      primaryDisplay: "name",
       schema: {
         id: {
           type: FieldType.AUTO,
@@ -801,6 +802,7 @@ describe.each([
             name: generator.guid(),
             type: "table",
             primary: ["id"],
+            primaryDisplay: "id",
             schema: {
               id: {
                 type: FieldType.AUTO,
@@ -836,7 +838,10 @@ describe.each([
         .expect("Content-Type", /json/)
         .expect(200)
       expect(resBasic.body.link.length).toBe(1)
-      expect(resBasic.body.link[0]).toEqual({ _id: firstRow._id })
+      expect(resBasic.body.link[0]).toEqual({
+        _id: firstRow._id,
+        primaryDisplay: firstRow.name,
+      })
 
       // test full enrichment
       const resEnriched = await request

From dd47c79ef92921c560dcd8949f0cdc26be73402d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 18:09:42 +0200
Subject: [PATCH 47/81] Replace guids by words

---
 packages/server/src/api/routes/tests/row.spec.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 0bf678ef93..fdf1a861db 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -67,7 +67,7 @@ describe.each([
 
   const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
     return {
-      name: generator.guid(),
+      name: generator.word(),
       type: "table",
       primary: ["id"],
       primaryDisplay: "name",
@@ -799,7 +799,7 @@ describe.each([
         config.getTenantId(),
         async () => {
           const linkedTable = await config.createLinkedTable({
-            name: generator.guid(),
+            name: generator.word(),
             type: "table",
             primary: ["id"],
             primaryDisplay: "id",
@@ -933,7 +933,7 @@ describe.each([
   describe("view 2.0", () => {
     async function userTable(): Promise<Table> {
       return {
-        name: `users_${generator.guid()}`,
+        name: `users_${generator.word()}`,
         type: "table",
         primary: ["id"],
         schema: {
@@ -1120,7 +1120,7 @@ describe.each([
       const viewSchema = { age: { visible: true }, name: { visible: true } }
       async function userTable(): Promise<Table> {
         return {
-          name: `users_${generator.guid()}`,
+          name: `users_${generator.word()}`,
           type: "table",
           primary: ["id"],
           schema: {

From cb762cc336237e87a695700c4cecba8c83655ebe Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:03:24 +0200
Subject: [PATCH 48/81] Use api for testing

---
 .../server/src/api/routes/tests/row.spec.ts   | 95 +++++++++++--------
 .../server/src/tests/utilities/api/viewV2.ts  |  4 +-
 2 files changed, 59 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fdf1a861db..2f1647b994 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -177,11 +177,12 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
+      const tableConfig = await generateTableConfig()
       const newTable = await config.createTable({
-        ...table,
+        ...tableConfig,
         name: "TestTableAuto",
         schema: {
-          ...table.schema,
+          ...tableConfig.schema,
           "Row ID": {
             name: "Row ID",
             type: FieldType.NUMBER,
@@ -356,7 +357,7 @@ describe.each([
               inclusion: ["Alpha", "Beta", "Gamma"],
             },
           },
-          table = await config.createTable({
+          table = await config.api.table.create({
             name: "TestTable2",
             type: "table",
             schema: {
@@ -392,6 +393,7 @@ describe.each([
               optsFieldNull: optsField,
               optsFieldStrKnown: optsField,
             },
+            ...(await tableDatasourceConfig()),
           })
 
         row = {
@@ -470,7 +472,7 @@ describe.each([
   })
 
   describe("view save", () => {
-    function orderTable(): Table {
+    async function orderTable(): Promise<Table> {
       return {
         name: "orders",
         primary: ["OrderID"],
@@ -488,11 +490,12 @@ describe.each([
             name: "Story",
           },
         },
+        ...(await tableDatasourceConfig()),
       }
     }
 
     it("views have extra data trimmed", async () => {
-      const table = await config.createTable(orderTable())
+      const table = await config.api.table.create(await orderTable())
 
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
@@ -887,7 +890,7 @@ describe.each([
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -910,7 +913,7 @@ describe.each([
     })
 
     it("should allow exporting only certain columns", async () => {
-      const existing = await config.createRow()
+      const existing = await createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -980,7 +983,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -992,7 +995,7 @@ describe.each([
 
         const data = randomRowData()
         const newRow = await config.api.row.save(view.id, {
-          tableId: config.table!._id,
+          tableId: table!._id,
           _viewId: view.id,
           ...data,
         })
@@ -1002,7 +1005,7 @@ describe.each([
           name: data.name,
           surname: data.surname,
           address: data.address,
-          tableId: config.table!._id,
+          tableId: table!._id,
           _id: newRow._id,
           _rev: newRow._rev,
           id: newRow.id,
@@ -1016,7 +1019,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1058,7 +1061,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1083,7 +1086,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1148,13 +1151,17 @@ describe.each([
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(
+            await config.api.row.save(table._id!, { tableId: table._id })
+          )
         }
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(10)
@@ -1174,10 +1181,10 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
-          await config.createRow({
+          await config.api.row.save(table._id!, {
             tableId: table._id,
             name: generator.name(),
             age: generator.integer({ min: 10, max: 30 }),
@@ -1185,7 +1192,7 @@ describe.each([
 
         for (let i = 0; i < 5; i++)
           expectedRows.push(
-            await config.createRow({
+            await config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: 40,
@@ -1193,6 +1200,7 @@ describe.each([
           )
 
         const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id!,
           query: [{ operator: "equal", field: "age", value: 40 }],
           schema: viewSchema,
         })
@@ -1289,7 +1297,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1297,13 +1305,14 @@ describe.each([
             { name: "Danny", age: 15 },
           ]
           for (const user of users) {
-            await config.createRow({
-              tableId: config.table!._id,
+            await config.api.row.save(table._id!, {
+              tableId: table._id,
               ...user,
             })
           }
 
           const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
             sort: sortParams,
             schema: viewSchema,
           })
@@ -1320,7 +1329,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1328,13 +1337,14 @@ describe.each([
             { name: "Danny", age: 15 },
           ]
           for (const user of users) {
-            await config.createRow({
-              tableId: config.table!._id,
+            await config.api.row.save(table._id!, {
+              tableId: table._id,
               ...user,
             })
           }
 
           const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
             sort: {
               field: "name",
               order: SortOrder.ASCENDING,
@@ -1361,11 +1371,11 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
-            await config.createRow({
+            await config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: generator.age(),
@@ -1374,6 +1384,7 @@ describe.each([
         }
 
         const view = await config.api.viewV2.create({
+          tableId: table._id,
           schema: { name: { visible: true } },
         })
         const response = await config.api.viewV2.search(view.id)
@@ -1393,23 +1404,27 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(0)
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(await createRow(table._id, { tableId: table._id }))
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const response = await config.api.viewV2.search(createViewResponse.id, {
           limit,
           query: {},
@@ -1419,13 +1434,15 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(await userTable())
+        const table = await config.api.table.create(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow({ tableId: table._id }))
+          rows.push(await createRow(table._id, { tableId: table._id }))
         }
 
-        const createViewResponse = await config.api.viewV2.create()
+        const createViewResponse = await config.api.viewV2.create({
+          tableId: table._id,
+        })
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
           .body.rows
 
@@ -1483,13 +1500,15 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
+          const table = await config.api.table.create(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
-            rows.push(await config.createRow({ tableId: table._id }))
+            rows.push(await createRow(table._id, { tableId: table._id }))
           }
 
-          const createViewResponse = await config.api.viewV2.create()
+          const createViewResponse = await config.api.viewV2.create({
+            tableId: table._id,
+          })
 
           tableId = table._id!
           viewId = createViewResponse.id
diff --git a/packages/server/src/tests/utilities/api/viewV2.ts b/packages/server/src/tests/utilities/api/viewV2.ts
index 0682361e16..92a6d394bf 100644
--- a/packages/server/src/tests/utilities/api/viewV2.ts
+++ b/packages/server/src/tests/utilities/api/viewV2.ts
@@ -23,8 +23,8 @@ export class ViewV2API extends TestAPI {
     if (!tableId && !this.config.table) {
       throw "Test requires table to be configured."
     }
-    const table = this.config.table
-    tableId = table!._id!
+
+    tableId = tableId || this.config.table!._id!
     const view = {
       tableId,
       name: generator.guid(),

From 9c37f2f05690e7d145ed6f475c0ce9a3b2f0f45a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:18:19 +0200
Subject: [PATCH 49/81] Use configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 84 +++++++------------
 1 file changed, 31 insertions(+), 53 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 2f1647b994..9282d5c83b 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -41,31 +41,20 @@ describe.each([
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
-  let row: Row
-  let datasource: Datasource | undefined
 
   afterAll(setup.afterAll)
 
   beforeAll(async () => {
     await config.init()
+
+    if (dsProvider) {
+      await config.createDatasource({
+        datasource: await dsProvider.getDsConfig(),
+      })
+    }
   })
 
-  const tableDatasourceConfig = async () => {
-    const result: Partial<SaveTableRequest> = {}
-    if (dsProvider) {
-      datasource = await config.api.datasource.create(
-        await dsProvider.getDsConfig()
-      )
-
-      result.sourceId = datasource._id
-      if (datasource.plus) {
-        result.type = "external"
-      }
-    }
-    return result
-  }
-
-  const generateTableConfig: () => Promise<SaveTableRequest> = async () => {
+  const generateTableConfig: () => SaveTableRequest = () => {
     return {
       name: generator.word(),
       type: "table",
@@ -95,17 +84,13 @@ describe.each([
           },
         },
       },
-      ...(await tableDatasourceConfig()),
     }
   }
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    const tableConfig = await generateTableConfig()
-    table = await config.api.table.create(tableConfig)
-    config.table = table
-    config.datasource = datasource
-    row = basicRow(table._id!)
+    const tableConfig = generateTableConfig()
+    table = await config.createTable(tableConfig)
   })
 
   const loadRow = async (id: string, tbl_Id: string, status = 200) =>
@@ -159,8 +144,8 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const res = await request
-        .post(`/api/${row.tableId}/rows`)
-        .send(row)
+        .post(`/api/${config.table!._id}/rows`)
+        .send(basicRow(config.table!._id!))
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
@@ -177,7 +162,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const tableConfig = await generateTableConfig()
+      const tableConfig = generateTableConfig()
       const newTable = await config.createTable({
         ...tableConfig,
         name: "TestTableAuto",
@@ -265,10 +250,7 @@ describe.each([
         .expect(200)
 
       expect(res.body).toEqual({
-        ...row,
-        _id: existing._id,
-        _rev: existing._rev,
-        id: existing.id,
+        ...existing,
         ...defaultRowFields,
       })
       await assertQueryUsage(queryUsage + 1)
@@ -280,7 +262,7 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      await createRow()
+      const firstRow = await createRow()
       await createRow(table._id, newRow)
       const queryUsage = await getQueryUsage()
 
@@ -292,7 +274,7 @@ describe.each([
 
       expect(res.body.length).toBe(2)
       expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find((r: Row) => r.name === row.name)).toBeDefined()
+      expect(res.body.find((r: Row) => r.name === firstRow.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -357,7 +339,7 @@ describe.each([
               inclusion: ["Alpha", "Beta", "Gamma"],
             },
           },
-          table = await config.api.table.create({
+          table = await config.createTable({
             name: "TestTable2",
             type: "table",
             schema: {
@@ -393,10 +375,9 @@ describe.each([
               optsFieldNull: optsField,
               optsFieldStrKnown: optsField,
             },
-            ...(await tableDatasourceConfig()),
           })
 
-        row = {
+        const row = {
           name: "Test Row",
           stringUndefined: undefined,
           stringNull: null,
@@ -490,12 +471,11 @@ describe.each([
             name: "Story",
           },
         },
-        ...(await tableDatasourceConfig()),
       }
     }
 
     it("views have extra data trimmed", async () => {
-      const table = await config.api.table.create(await orderTable())
+      const table = await config.createTable(await orderTable())
 
       const createViewResponse = await config.api.viewV2.create({
         tableId: table._id,
@@ -969,7 +949,6 @@ describe.each([
             name: "jobTitle",
           },
         },
-        ...(await tableDatasourceConfig()),
       }
     }
 
@@ -983,7 +962,7 @@ describe.each([
 
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const view = await config.api.viewV2.create({
           tableId: table._id!,
           schema: {
@@ -1019,7 +998,7 @@ describe.each([
 
     describe("patch", () => {
       it("should update only the view fields for a row", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1061,7 +1040,7 @@ describe.each([
 
     describe("destroy", () => {
       it("should be able to delete a row", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1086,7 +1065,7 @@ describe.each([
       })
 
       it("should be able to delete multiple rows", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const tableId = table._id!
         const view = await config.api.viewV2.create({
           tableId,
@@ -1146,12 +1125,11 @@ describe.each([
               constraints: {},
             },
           },
-          ...(await tableDatasourceConfig()),
         }
       }
 
       it("returns empty rows from view when no schema is passed", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1181,7 +1159,7 @@ describe.each([
       })
 
       it("searching respects the view filters", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const expectedRows = []
         for (let i = 0; i < 10; i++)
           await config.api.row.save(table._id!, {
@@ -1297,7 +1275,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow sorting (%s)",
         async (sortParams, expected) => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1329,7 +1307,7 @@ describe.each([
       it.each(sortTestOptions)(
         "allow override the default view sorting (%s)",
         async (sortParams, expected) => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
             { name: "Bob", age: 30 },
@@ -1371,7 +1349,7 @@ describe.each([
       )
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(
@@ -1404,7 +1382,7 @@ describe.each([
       })
 
       it("views without data can be returned", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
 
         const createViewResponse = await config.api.viewV2.create({
           tableId: table._id,
@@ -1415,7 +1393,7 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await createRow(table._id, { tableId: table._id }))
@@ -1434,7 +1412,7 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.api.table.create(await userTable())
+        const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
           rows.push(await createRow(table._id, { tableId: table._id }))
@@ -1500,7 +1478,7 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.api.table.create(await userTable())
+          const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
             rows.push(await createRow(table._id, { tableId: table._id }))

From 3843581e56d7e71eeafb80c6ebf2e5b68ffcea76 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:31:58 +0200
Subject: [PATCH 50/81] Unify

---
 .../server/src/api/routes/tests/row.spec.ts   | 63 +++++++++----------
 .../src/tests/utilities/TestConfiguration.ts  | 10 +--
 2 files changed, 33 insertions(+), 40 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 9282d5c83b..7488056bc3 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -127,9 +127,6 @@ describe.each([
     expect(usage).toBe(expected)
   }
 
-  const createRow = (tableId = table._id!, row?: SaveRowRequest) =>
-    config.api.row.save(tableId, row || basicRow(table._id!))
-
   const defaultRowFields = isInternal
     ? {
         type: "row",
@@ -215,7 +212,7 @@ describe.each([
     })
 
     it("updates a row successfully", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -240,7 +237,7 @@ describe.each([
     })
 
     it("should load a row", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -262,8 +259,8 @@ describe.each([
         name: "Second Contact",
         status: "new",
       }
-      const firstRow = await createRow()
-      await createRow(table._id, newRow)
+      const firstRow = await config.createRow()
+      await config.createRow(newRow)
       const queryUsage = await getQueryUsage()
 
       const res = await request
@@ -279,7 +276,7 @@ describe.each([
     })
 
     it("load should return 404 when row does not exist", async () => {
-      await createRow()
+      await config.createRow()
       const queryUsage = await getQueryUsage()
 
       await config.api.row.get(table._id!, "1234567", {
@@ -513,7 +510,7 @@ describe.each([
 
   describe("patch", () => {
     it("should update only the fields that are supplied", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
 
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -540,7 +537,7 @@ describe.each([
     })
 
     it("should throw an error when given improper types", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -562,7 +559,7 @@ describe.each([
 
   describe("destroy", () => {
     it("should be able to delete a row", async () => {
-      const createdRow = await createRow()
+      const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -624,8 +621,8 @@ describe.each([
 
   describe("bulkDelete", () => {
     it("should be able to delete a bulk set of rows", async () => {
-      const row1 = await createRow()
-      const row2 = await createRow()
+      const row1 = await config.createRow()
+      const row2 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -645,9 +642,9 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await createRow()
-      const row2 = await createRow()
-      const row3 = await createRow()
+      const row1 = await config.createRow()
+      const row2 = await config.createRow()
+      const row3 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -667,7 +664,7 @@ describe.each([
     })
 
     it("should accept a valid row object and delete the row", async () => {
-      const row1 = await createRow()
+      const row1 = await config.createRow()
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -724,7 +721,7 @@ describe.each([
   isInternal &&
     describe("fetchView", () => {
       it("should be able to fetch tables contents via 'view'", async () => {
-        const row = await createRow()
+        const row = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -759,7 +756,7 @@ describe.each([
           filters: [],
           schema: {},
         })
-        const row = await createRow()
+        const row = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -797,12 +794,12 @@ describe.each([
               },
             },
           })
-          const firstRow = await createRow(table._id, {
+          const firstRow = await config.createRow({
             name: "Test Contact",
             description: "original description",
             tableId: table._id,
           })
-          const secondRow = await createRow(linkedTable._id, {
+          const secondRow = await config.createRow({
             name: "Test 2",
             description: "og desc",
             link: [{ _id: firstRow._id }],
@@ -846,7 +843,7 @@ describe.each([
       it("should allow enriching attachment rows", async () => {
         const table = await config.createAttachmentTable()
         const attachmentId = `${structures.uuid()}.csv`
-        const row = await createRow(table._id, {
+        const row = await config.createRow({
           name: "test",
           description: "test",
           attachment: [
@@ -870,7 +867,7 @@ describe.each([
 
   describe("exportData", () => {
     it("should allow exporting all columns", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -893,7 +890,7 @@ describe.each([
     })
 
     it("should allow exporting only certain columns", async () => {
-      const existing = await createRow()
+      const existing = await config.createRow()
       const res = await request
         .post(`/api/${table._id}/rows/exportRows?format=json`)
         .set(config.defaultHeaders())
@@ -1050,7 +1047,7 @@ describe.each([
           },
         })
 
-        const createdRow = await createRow()
+        const createdRow = await config.createRow()
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1075,11 +1072,11 @@ describe.each([
           },
         })
 
-        const rows = [
-          await createRow(tableId),
-          await createRow(tableId),
-          await createRow(tableId),
-        ]
+        const rows = await Promise.all([
+          config.createRow(),
+          config.createRow(),
+          config.createRow(),
+        ])
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
@@ -1396,7 +1393,7 @@ describe.each([
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await createRow(table._id, { tableId: table._id }))
+          rows.push(await config.createRow())
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
@@ -1415,7 +1412,7 @@ describe.each([
         const table = await config.createTable(await userTable())
         const rows = []
         for (let i = 0; i < 10; i++) {
-          rows.push(await createRow(table._id, { tableId: table._id }))
+          rows.push(await config.createRow())
         }
 
         const createViewResponse = await config.api.viewV2.create({
@@ -1481,7 +1478,7 @@ describe.each([
           const table = await config.createTable(await userTable())
           const rows = []
           for (let i = 0; i < 10; i++) {
-            rows.push(await createRow(table._id, { tableId: table._id }))
+            rows.push(await config.createRow())
           }
 
           const createViewResponse = await config.api.viewV2.create({
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 030a08cee7..119a926085 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -533,7 +533,7 @@ class TestConfiguration {
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    const response = await this._req(config, null, controllers.table.save)
+    const response = await this.api.table.create(config)
     if (!skipReassigning) {
       this.table = response
     }
@@ -694,12 +694,8 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this._req(
-      { ...config },
-      null,
-      controllers.datasource.save
-    )
-    this.datasource = response.datasource
+    const response = await this.api.datasource.create({ ...config.datasource })
+    this.datasource = response
     return this.datasource!
   }
 

From fccb2f625c902b5699d1f815a15c1fc517b988ad Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:39:40 +0200
Subject: [PATCH 51/81] Use configs

---
 .../server/src/api/routes/tests/row.spec.ts   | 48 ++++++-------------
 .../src/tests/utilities/TestConfiguration.ts  | 21 ++++++++
 2 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 7488056bc3..ee3f12b449 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -474,8 +474,8 @@ describe.each([
     it("views have extra data trimmed", async () => {
       const table = await config.createTable(await orderTable())
 
-      const createViewResponse = await config.api.viewV2.create({
-        tableId: table._id,
+      const createViewResponse = await config.createView({
+        name: generator.word(),
         schema: {
           Country: {
             visible: true,
@@ -960,8 +960,7 @@ describe.each([
     describe("create", () => {
       it("should persist a new row with only the provided view fields", async () => {
         const table = await config.createTable(await userTable())
-        const view = await config.api.viewV2.create({
-          tableId: table._id!,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             surname: { visible: true },
@@ -997,8 +996,7 @@ describe.each([
       it("should update only the view fields for a row", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1039,8 +1037,7 @@ describe.each([
       it("should be able to delete a row", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1064,8 +1061,7 @@ describe.each([
       it("should be able to delete multiple rows", async () => {
         const table = await config.createTable(await userTable())
         const tableId = table._id!
-        const view = await config.api.viewV2.create({
-          tableId,
+        const view = await config.createView({
           schema: {
             name: { visible: true },
             address: { visible: true },
@@ -1134,9 +1130,7 @@ describe.each([
           )
         }
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(10)
@@ -1174,8 +1168,7 @@ describe.each([
             })
           )
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id!,
+        const createViewResponse = await config.createView({
           query: [{ operator: "equal", field: "age", value: 40 }],
           schema: viewSchema,
         })
@@ -1286,8 +1279,7 @@ describe.each([
             })
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
+          const createViewResponse = await config.createView({
             sort: sortParams,
             schema: viewSchema,
           })
@@ -1318,8 +1310,7 @@ describe.each([
             })
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
+          const createViewResponse = await config.createView({
             sort: {
               field: "name",
               order: SortOrder.ASCENDING,
@@ -1358,8 +1349,7 @@ describe.each([
           )
         }
 
-        const view = await config.api.viewV2.create({
-          tableId: table._id,
+        const view = await config.createView({
           schema: { name: { visible: true } },
         })
         const response = await config.api.viewV2.search(view.id)
@@ -1381,9 +1371,7 @@ describe.each([
       it("views without data can be returned", async () => {
         const table = await config.createTable(await userTable())
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
 
         expect(response.body.rows).toHaveLength(0)
@@ -1397,9 +1385,7 @@ describe.each([
         }
         const limit = generator.integer({ min: 1, max: 8 })
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id, {
           limit,
           query: {},
@@ -1415,9 +1401,7 @@ describe.each([
           rows.push(await config.createRow())
         }
 
-        const createViewResponse = await config.api.viewV2.create({
-          tableId: table._id,
-        })
+        const createViewResponse = await config.createView()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
           .body.rows
 
@@ -1481,9 +1465,7 @@ describe.each([
             rows.push(await config.createRow())
           }
 
-          const createViewResponse = await config.api.viewV2.create({
-            tableId: table._id,
-          })
+          const createViewResponse = await config.createView()
 
           tableId = table._id!
           viewId = createViewResponse.id
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 119a926085..fbb6e05c57 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -53,6 +53,8 @@ import {
   View,
   FieldType,
   RelationshipType,
+  ViewV2,
+  CreateViewRequest,
 } from "@budibase/types"
 
 import API from "./api"
@@ -651,6 +653,25 @@ class TestConfiguration {
     return this._req(view, null, controllers.view.v1.save)
   }
 
+  async createView(
+    config?: Omit<CreateViewRequest, "tableId" | "name"> & {
+      name?: string
+      tableId?: string
+    }
+  ) {
+    if (!this.table && !config?.tableId) {
+      throw "Test requires table to be configured."
+    }
+
+    const view: CreateViewRequest = {
+      ...config,
+      tableId: config?.tableId || this.table!._id!,
+      name: config?.name || generator.word(),
+    }
+
+    return await this.api.viewV2.create(view)
+  }
+
   // AUTOMATION
 
   async createAutomation(config?: any) {

From f0872c1fa3135bd6b1cd6e976d8c0c9a1ec75790 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 19:59:37 +0200
Subject: [PATCH 52/81] Improve api

---
 packages/server/src/tests/utilities/api/row.ts | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index c6ef4606d2..2f6760ebeb 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -22,6 +22,21 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  getEnriched = async (
+    sourceId: string,
+    rowId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows/${rowId}/enrich`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+    if (expectStatus !== 404) {
+      request.expect("Content-Type", /json/)
+    }
+    return request
+  }
+
   save = async (
     sourceId: string,
     row: SaveRowRequest,
@@ -51,7 +66,7 @@ export class RowAPI extends TestAPI {
 
   delete = async (
     sourceId: string,
-    rows: Row[],
+    rows: Row | string | (Row | string)[],
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request

From 458de1282ec1ab6cf4b93f5269464013529fd2df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:17:21 +0200
Subject: [PATCH 53/81] Use api for testing

---
 .../server/src/api/routes/tests/row.spec.ts   |  2 --
 .../server/src/api/routes/tests/table.spec.ts | 12 ++++++-----
 .../src/api/routes/tests/viewV2.spec.ts       | 10 ++++++---
 .../src/integration-test/postgres.spec.ts     |  2 +-
 .../src/tests/utilities/TestConfiguration.ts  | 21 ++++---------------
 .../src/tests/utilities/api/datasource.ts     | 17 +++++++++++++++
 6 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index ee3f12b449..7ec2ef8ef9 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -6,13 +6,11 @@ import * as setup from "./utilities"
 import { context, roles, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
-  Datasource,
   FieldType,
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
   Row,
-  SaveRowRequest,
   SaveTableRequest,
   SortOrder,
   SortType,
diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts
index 9914e6d66f..f56c6e4e44 100644
--- a/packages/server/src/api/routes/tests/table.spec.ts
+++ b/packages/server/src/api/routes/tests/table.spec.ts
@@ -1,6 +1,6 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table } from "@budibase/types"
+import { FieldType, Table, ViewCalculation } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -90,8 +90,10 @@ describe("/tables", () => {
       await config.createLegacyView({
         name: "TestView",
         field: "Price",
-        calculation: "stats",
-        tableId: testTable._id,
+        calculation: ViewCalculation.STATISTICS,
+        tableId: testTable._id!,
+        schema: {},
+        filters: [],
       })
 
       const testRow = await request
@@ -254,7 +256,7 @@ describe("/tables", () => {
       }))
 
       await config.api.viewV2.create({ tableId })
-      await config.createLegacyView({ tableId, name: generator.guid() })
+      await config.createLegacyView()
 
       const res = await config.api.table.fetch()
 
@@ -366,7 +368,7 @@ describe("/tables", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
-      const dependentTable = await config.getTable(linkedTable._id)
+      const dependentTable = await config.api.table.get(linkedTable._id!)
       expect(dependentTable.schema.TestTable).not.toBeDefined()
     })
 
diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts
index 5e8ae09e55..6d893c1c7f 100644
--- a/packages/server/src/api/routes/tests/viewV2.spec.ts
+++ b/packages/server/src/api/routes/tests/viewV2.spec.ts
@@ -6,6 +6,7 @@ import {
   SortOrder,
   SortType,
   Table,
+  UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
 } from "@budibase/types"
@@ -418,9 +419,12 @@ describe.each([
       const res = await config.api.viewV2.create(newView)
       const view = await config.api.viewV2.get(res.id)
       expect(view!.schema?.Price).toBeUndefined()
-      const updatedTable = await config.getTable(table._id!)
-      const viewSchema = updatedTable.views[view!.name!].schema
-      expect(viewSchema.Price.visible).toEqual(false)
+      const updatedTable = await config.api.table.get(table._id!)
+      const viewSchema = updatedTable.views![view!.name!].schema as Record<
+        string,
+        UIFieldMetadata
+      >
+      expect(viewSchema.Price?.visible).toEqual(false)
     })
   })
 })
diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 0a53326cf4..b0f6f5bb04 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -397,7 +397,7 @@ describe("postgres integrations", () => {
         expect(res.status).toBe(200)
         expect(res.body).toEqual(updatedRow)
 
-        const persistedRow = await config.getRow(
+        const persistedRow = await config.api.row.get(
           primaryPostgresTable._id!,
           row.id
         )
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index fbb6e05c57..44867e6b01 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -557,11 +557,6 @@ class TestConfiguration {
     return this.updateTable(config, options)
   }
 
-  async getTable(tableId?: string) {
-    tableId = tableId || this.table?._id
-    return this._req(null, { tableId }, controllers.table.find)
-  }
-
   async createLinkedTable(
     config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
@@ -609,11 +604,7 @@ class TestConfiguration {
     }
     const tableId = (config && config.tableId) || this.table._id
     config = config || basicRow(tableId!)
-    return this._req(config, { tableId }, controllers.row.save)
-  }
-
-  async getRow(tableId: string, rowId: string): Promise<Row> {
-    return this._req(null, { tableId, rowId }, controllers.row.find)
+    return this.api.row.save(tableId!, config)
   }
 
   async getRows(tableId: string) {
@@ -648,7 +639,7 @@ class TestConfiguration {
     }
     const view = config || {
       tableId: this.table!._id,
-      name: "ViewTest",
+      name: generator.guid(),
     }
     return this._req(view, null, controllers.view.v1.save)
   }
@@ -721,12 +712,8 @@ class TestConfiguration {
   }
 
   async updateDatasource(datasource: Datasource): Promise<Datasource> {
-    const response = await this._req(
-      datasource,
-      { datasourceId: datasource._id },
-      controllers.datasource.update
-    )
-    this.datasource = response.datasource
+    const response = await this.api.datasource.update(datasource)
+    this.datasource = response
     return this.datasource!
   }
 
diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 3c85a1c332..2ce09959fc 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -29,6 +29,23 @@ export class DatasourceAPI extends TestAPI {
     return result.body.datasource as Datasource
   }
 
+  update = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .put(`/api/datasources/${config._id}`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
   verify = async (
     data: VerifyDatasourceRequest,
     { expectStatus } = { expectStatus: 200 }

From b5e6b42db20dca220fc46ebd534377ec5852dab3 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:40:00 +0200
Subject: [PATCH 54/81] Reuse tables

---
 .../server/src/api/routes/tests/row.spec.ts   | 196 +++++++++---------
 .../server/src/tests/utilities/api/row.ts     |  12 ++
 2 files changed, 109 insertions(+), 99 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 7ec2ef8ef9..b610e9689e 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -133,19 +133,26 @@ describe.each([
       }
     : undefined
 
-  describe("save, load, update", () => {
+    let tableId: string
+
+    beforeAll(async () => {
+      const tableConfig = generateTableConfig()
+      const table = await config.createTable(tableConfig)
+      tableId = table._id!
+    })
+
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
       const res = await request
-        .post(`/api/${config.table!._id}/rows`)
-        .send(basicRow(config.table!._id!))
+        .post(`/api/${tableId}/rows`)
+        .send(basicRow(tableId))
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
       expect((res as any).res.statusMessage).toEqual(
-        `${table.name} saved successfully`
+        `${config.table!.name} saved successfully`
       )
       expect(res.body.name).toEqual("Test Contact")
       expect(res.body._rev).toBeDefined()
@@ -158,28 +165,31 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       const tableConfig = generateTableConfig()
-      const newTable = await config.createTable({
-        ...tableConfig,
-        name: "TestTableAuto",
-        schema: {
-          ...tableConfig.schema,
-          "Row ID": {
-            name: "Row ID",
-            type: FieldType.NUMBER,
-            subtype: "autoID",
-            icon: "ri-magic-line",
-            autocolumn: true,
-            constraints: {
-              type: "number",
-              presence: true,
-              numericality: {
-                greaterThanOrEqualTo: "",
-                lessThanOrEqualTo: "",
+      const newTable = await config.createTable(
+        {
+          ...tableConfig,
+          name: "TestTableAuto",
+          schema: {
+            ...tableConfig.schema,
+            "Row ID": {
+              name: "Row ID",
+              type: FieldType.NUMBER,
+              subtype: "autoID",
+              icon: "ri-magic-line",
+              autocolumn: true,
+              constraints: {
+                type: "number",
+                presence: true,
+                numericality: {
+                  greaterThanOrEqualTo: "",
+                  lessThanOrEqualTo: "",
+                },
               },
             },
           },
         },
-      })
+        { skipReassigning: true }
+      )
 
       const ids = [1, 2, 3]
 
@@ -214,22 +224,14 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows`)
-        .send({
-          _id: existing._id,
-          _rev: existing._rev,
-          tableId: table._id,
-          name: "Updated Name",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.save(tableId, {
+        _id: existing._id,
+        _rev: existing._rev,
+        tableId,
+        name: "Updated Name",
+      })
 
-      expect((res as any).res.statusMessage).toEqual(
-        `${table.name} updated successfully.`
-      )
-      expect(res.body.name).toEqual("Updated Name")
+      expect(res.name).toEqual("Updated Name")
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage + 1)
     })
@@ -238,11 +240,7 @@ describe.each([
       const existing = await config.createRow()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/${table._id}/rows/${existing._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.get(tableId, existing._id!)
 
       expect(res.body).toEqual({
         ...existing,
@@ -252,24 +250,24 @@ describe.each([
     })
 
     it("should list all rows for given tableId", async () => {
+      const table = await config.createTable(generateTableConfig(), {
+        skipReassigning: true,
+      })
+      const tableId = table._id!
       const newRow = {
-        tableId: table._id,
+        tableId,
         name: "Second Contact",
-        status: "new",
+        description: "new",
       }
-      const firstRow = await config.createRow()
+      const firstRow = await config.createRow({ tableId })
       await config.createRow(newRow)
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .get(`/api/${table._id}/rows`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.fetch(tableId)
 
-      expect(res.body.length).toBe(2)
-      expect(res.body.find((r: Row) => r.name === newRow.name)).toBeDefined()
-      expect(res.body.find((r: Row) => r.name === firstRow.name)).toBeDefined()
+      expect(res.length).toBe(2)
+      expect(res.find((r: Row) => r.name === newRow.name)).toBeDefined()
+      expect(res.find((r: Row) => r.name === firstRow.name)).toBeDefined()
       await assertQueryUsage(queryUsage + 1)
     })
 
@@ -277,7 +275,7 @@ describe.each([
       await config.createRow()
       const queryUsage = await getQueryUsage()
 
-      await config.api.row.get(table._id!, "1234567", {
+      await config.api.row.get(tableId, "1234567", {
         expectStatus: 404,
       })
       await assertQueryUsage(queryUsage) // no change
@@ -325,52 +323,52 @@ describe.each([
           sortable: false,
         }
         const optsField = {
-            fieldName: "Sample Opts",
-            name: "Sample Opts",
-            type: FieldType.OPTIONS,
-            constraints: {
-              type: "string",
-              presence: false,
-              inclusion: ["Alpha", "Beta", "Gamma"],
-            },
+          fieldName: "Sample Opts",
+          name: "Sample Opts",
+          type: FieldType.OPTIONS,
+          constraints: {
+            type: "string",
+            presence: false,
+            inclusion: ["Alpha", "Beta", "Gamma"],
           },
-          table = await config.createTable({
-            name: "TestTable2",
-            type: "table",
-            schema: {
-              name: str,
-              stringUndefined: str,
-              stringNull: str,
-              stringString: str,
-              numberEmptyString: number,
-              numberNull: number,
-              numberUndefined: number,
-              numberString: number,
-              numberNumber: number,
-              datetimeEmptyString: datetime,
-              datetimeNull: datetime,
-              datetimeUndefined: datetime,
-              datetimeString: datetime,
-              datetimeDate: datetime,
-              boolNull: bool,
-              boolEmpty: bool,
-              boolUndefined: bool,
-              boolString: bool,
-              boolBool: bool,
-              attachmentNull: attachment,
-              attachmentUndefined: attachment,
-              attachmentEmpty: attachment,
-              attachmentEmptyArrayStr: attachment,
-              arrayFieldEmptyArrayStr: arrayField,
-              arrayFieldArrayStrKnown: arrayField,
-              arrayFieldNull: arrayField,
-              arrayFieldUndefined: arrayField,
-              optsFieldEmptyStr: optsField,
-              optsFieldUndefined: optsField,
-              optsFieldNull: optsField,
-              optsFieldStrKnown: optsField,
-            },
-          })
+        }
+        const table = await config.createTable({
+          name: "TestTable2",
+          type: "table",
+          schema: {
+            name: str,
+            stringUndefined: str,
+            stringNull: str,
+            stringString: str,
+            numberEmptyString: number,
+            numberNull: number,
+            numberUndefined: number,
+            numberString: number,
+            numberNumber: number,
+            datetimeEmptyString: datetime,
+            datetimeNull: datetime,
+            datetimeUndefined: datetime,
+            datetimeString: datetime,
+            datetimeDate: datetime,
+            boolNull: bool,
+            boolEmpty: bool,
+            boolUndefined: bool,
+            boolString: bool,
+            boolBool: bool,
+            attachmentNull: attachment,
+            attachmentUndefined: attachment,
+            attachmentEmpty: attachment,
+            attachmentEmptyArrayStr: attachment,
+            arrayFieldEmptyArrayStr: arrayField,
+            arrayFieldArrayStrKnown: arrayField,
+            arrayFieldNull: arrayField,
+            arrayFieldUndefined: arrayField,
+            optsFieldEmptyStr: optsField,
+            optsFieldUndefined: optsField,
+            optsFieldNull: optsField,
+            optsFieldStrKnown: optsField,
+          },
+        })
 
         const row = {
           name: "Test Row",
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 2f6760ebeb..3c266f07b5 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -76,4 +76,16 @@ export class RowAPI extends TestAPI {
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  fetch = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
 }

From d522ff70b4eb6ec77eb30a842945c303e48c22c7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:47:06 +0200
Subject: [PATCH 55/81] Don't create tables for each test

---
 .../server/src/api/routes/tests/row.spec.ts   | 53 +++++++++++++++----
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index b610e9689e..dfb43779a8 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -87,8 +87,6 @@ describe.each([
 
   beforeEach(async () => {
     mocks.licenses.useCloudFree()
-    const tableConfig = generateTableConfig()
-    table = await config.createTable(tableConfig)
   })
 
   const loadRow = async (id: string, tbl_Id: string, status = 200) =>
@@ -133,6 +131,7 @@ describe.each([
       }
     : undefined
 
+  describe("save, load, update", () => {
     let tableId: string
 
     beforeAll(async () => {
@@ -446,8 +445,8 @@ describe.each([
   })
 
   describe("view save", () => {
-    async function orderTable(): Promise<Table> {
-      return {
+    it("views have extra data trimmed", async () => {
+      const table = await config.createTable({
         name: "orders",
         primary: ["OrderID"],
         schema: {
@@ -464,11 +463,7 @@ describe.each([
             name: "Story",
           },
         },
-      }
-    }
-
-    it("views have extra data trimmed", async () => {
-      const table = await config.createTable(await orderTable())
+      })
 
       const createViewResponse = await config.createView({
         name: generator.word(),
@@ -505,6 +500,11 @@ describe.each([
   })
 
   describe("patch", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should update only the fields that are supplied", async () => {
       const existing = await config.createRow()
 
@@ -554,6 +554,11 @@ describe.each([
   })
 
   describe("destroy", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should be able to delete a row", async () => {
       const createdRow = await config.createRow()
       const rowUsage = await getRowUsage()
@@ -574,6 +579,11 @@ describe.each([
   })
 
   describe("validate", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should return no errors on valid row", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -616,6 +626,11 @@ describe.each([
   })
 
   describe("bulkDelete", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should be able to delete a bulk set of rows", async () => {
       const row1 = await config.createRow()
       const row2 = await config.createRow()
@@ -716,6 +731,11 @@ describe.each([
   // Legacy views are not available for external
   isInternal &&
     describe("fetchView", () => {
+      beforeEach(async () => {
+        const tableConfig = generateTableConfig()
+        table = await config.createTable(tableConfig)
+      })
+
       it("should be able to fetch tables contents via 'view'", async () => {
         const row = await config.createRow()
         const rowUsage = await getRowUsage()
@@ -770,6 +790,11 @@ describe.each([
     })
 
   describe("fetchEnrichedRows", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should allow enriching some linked rows", async () => {
       const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
@@ -836,6 +861,11 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
+      beforeEach(async () => {
+        const tableConfig = generateTableConfig()
+        table = await config.createTable(tableConfig)
+      })
+
       it("should allow enriching attachment rows", async () => {
         const table = await config.createAttachmentTable()
         const attachmentId = `${structures.uuid()}.csv`
@@ -862,6 +892,11 @@ describe.each([
     })
 
   describe("exportData", () => {
+    beforeEach(async () => {
+      const tableConfig = generateTableConfig()
+      table = await config.createTable(tableConfig)
+    })
+
     it("should allow exporting all columns", async () => {
       const existing = await config.createRow()
       const res = await request

From eee8a2e5f95fd27b1b6f0082df24bf32b713d803 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 20:49:47 +0200
Subject: [PATCH 56/81] Run before alls

---
 packages/server/src/api/routes/tests/row.spec.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index dfb43779a8..b4e257dc1c 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -500,7 +500,7 @@ describe.each([
   })
 
   describe("patch", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -554,7 +554,7 @@ describe.each([
   })
 
   describe("destroy", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -579,7 +579,7 @@ describe.each([
   })
 
   describe("validate", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })

From 0c8a8e1b26fd1cd16585cfd84632b5359e4db414 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 12 Sep 2023 21:09:25 +0200
Subject: [PATCH 57/81] Unify

---
 .../server/src/api/routes/tests/row.spec.ts   | 23 +++++++++----------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index b4e257dc1c..a4fe43ea1a 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -39,6 +39,7 @@ describe.each([
   let request = setup.getRequest()
   let config = setup.getConfig()
   let table: Table
+  let tableId: string
 
   afterAll(setup.afterAll)
 
@@ -131,15 +132,13 @@ describe.each([
       }
     : undefined
 
+  beforeAll(async () => {
+    const tableConfig = generateTableConfig()
+    const table = await config.createTable(tableConfig)
+    tableId = table._id!
+  })
+
   describe("save, load, update", () => {
-    let tableId: string
-
-    beforeAll(async () => {
-      const tableConfig = generateTableConfig()
-      const table = await config.createTable(tableConfig)
-      tableId = table._id!
-    })
-
     it("returns a success message when the row is created", async () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
@@ -626,7 +625,7 @@ describe.each([
   })
 
   describe("bulkDelete", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -790,7 +789,7 @@ describe.each([
     })
 
   describe("fetchEnrichedRows", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })
@@ -861,7 +860,7 @@ describe.each([
 
   isInternal &&
     describe("attachments", () => {
-      beforeEach(async () => {
+      beforeAll(async () => {
         const tableConfig = generateTableConfig()
         table = await config.createTable(tableConfig)
       })
@@ -892,7 +891,7 @@ describe.each([
     })
 
   describe("exportData", () => {
-    beforeEach(async () => {
+    beforeAll(async () => {
       const tableConfig = generateTableConfig()
       table = await config.createTable(tableConfig)
     })

From 986decb1034a0ad1589e26d394fa7614b7bab160 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:37:11 +0200
Subject: [PATCH 58/81] Reduce timings

---
 .../server/src/api/routes/tests/row.spec.ts   | 190 +++++++++---------
 1 file changed, 91 insertions(+), 99 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index a4fe43ea1a..fbf6651186 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -652,9 +652,11 @@ describe.each([
     })
 
     it("should be able to delete a variety of row set types", async () => {
-      const row1 = await config.createRow()
-      const row2 = await config.createRow()
-      const row3 = await config.createRow()
+      const [row1, row2, row3] = await Promise.all([
+        config.createRow(),
+        config.createRow(),
+        config.createRow(),
+      ])
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
@@ -1153,12 +1155,11 @@ describe.each([
 
       it("returns empty rows from view when no schema is passed", async () => {
         const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(
-            await config.api.row.save(table._id!, { tableId: table._id })
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, { tableId: table._id })
           )
-        }
+        )
 
         const createViewResponse = await config.createView()
         const response = await config.api.viewV2.search(createViewResponse.id)
@@ -1181,22 +1182,26 @@ describe.each([
 
       it("searching respects the view filters", async () => {
         const table = await config.createTable(await userTable())
-        const expectedRows = []
-        for (let i = 0; i < 10; i++)
-          await config.api.row.save(table._id!, {
-            tableId: table._id,
-            name: generator.name(),
-            age: generator.integer({ min: 10, max: 30 }),
-          })
 
-        for (let i = 0; i < 5; i++)
-          expectedRows.push(
-            await config.api.row.save(table._id!, {
+        await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
+              tableId: table._id,
+              name: generator.name(),
+              age: generator.integer({ min: 10, max: 30 }),
+            })
+          )
+        )
+
+        const expectedRows = await Promise.all(
+          Array.from({ length: 5 }, () =>
+            config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: 40,
             })
           )
+        )
 
         const createViewResponse = await config.createView({
           query: [{ operator: "equal", field: "age", value: 40 }],
@@ -1292,9 +1297,8 @@ describe.each([
         ],
       ]
 
-      it.each(sortTestOptions)(
-        "allow sorting (%s)",
-        async (sortParams, expected) => {
+      describe("sorting", () => {
+        beforeAll(async () => {
           const table = await config.createTable(await userTable())
           const users = [
             { name: "Alice", age: 25 },
@@ -1302,82 +1306,76 @@ describe.each([
             { name: "Charly", age: 27 },
             { name: "Danny", age: 15 },
           ]
-          for (const user of users) {
-            await config.api.row.save(table._id!, {
-              tableId: table._id,
-              ...user,
+          await Promise.all(
+            users.map(u =>
+              config.api.row.save(table._id!, {
+                tableId: table._id,
+                ...u,
+              })
+            )
+          )
+        })
+
+        it.each(sortTestOptions)(
+          "allow sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: sortParams,
+              schema: viewSchema,
             })
+
+            const response = await config.api.viewV2.search(
+              createViewResponse.id
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
           }
+        )
 
-          const createViewResponse = await config.createView({
-            sort: sortParams,
-            schema: viewSchema,
-          })
-
-          const response = await config.api.viewV2.search(createViewResponse.id)
-
-          expect(response.body.rows).toHaveLength(4)
-          expect(response.body.rows).toEqual(
-            expected.map(name => expect.objectContaining({ name }))
-          )
-        }
-      )
-
-      it.each(sortTestOptions)(
-        "allow override the default view sorting (%s)",
-        async (sortParams, expected) => {
-          const table = await config.createTable(await userTable())
-          const users = [
-            { name: "Alice", age: 25 },
-            { name: "Bob", age: 30 },
-            { name: "Charly", age: 27 },
-            { name: "Danny", age: 15 },
-          ]
-          for (const user of users) {
-            await config.api.row.save(table._id!, {
-              tableId: table._id,
-              ...user,
+        it.each(sortTestOptions)(
+          "allow override the default view sorting (%s)",
+          async (sortParams, expected) => {
+            const createViewResponse = await config.createView({
+              sort: {
+                field: "name",
+                order: SortOrder.ASCENDING,
+                type: SortType.STRING,
+              },
+              schema: viewSchema,
             })
+
+            const response = await config.api.viewV2.search(
+              createViewResponse.id,
+              {
+                sort: sortParams.field,
+                sortOrder: sortParams.order,
+                sortType: sortParams.type,
+                query: {},
+              }
+            )
+
+            expect(response.body.rows).toHaveLength(4)
+            expect(response.body.rows).toEqual(
+              expected.map(name => expect.objectContaining({ name }))
+            )
           }
-
-          const createViewResponse = await config.createView({
-            sort: {
-              field: "name",
-              order: SortOrder.ASCENDING,
-              type: SortType.STRING,
-            },
-            schema: viewSchema,
-          })
-
-          const response = await config.api.viewV2.search(
-            createViewResponse.id,
-            {
-              sort: sortParams.field,
-              sortOrder: sortParams.order,
-              sortType: sortParams.type,
-              query: {},
-            }
-          )
-
-          expect(response.body.rows).toHaveLength(4)
-          expect(response.body.rows).toEqual(
-            expected.map(name => expect.objectContaining({ name }))
-          )
-        }
-      )
+        )
+      })
 
       it("when schema is defined, defined columns and row attributes are returned", async () => {
         const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(
-            await config.api.row.save(table._id!, {
+        const rows = await Promise.all(
+          Array.from({ length: 10 }, () =>
+            config.api.row.save(table._id!, {
               tableId: table._id,
               name: generator.name(),
               age: generator.age(),
             })
           )
-        }
+        )
 
         const view = await config.createView({
           schema: { name: { visible: true } },
@@ -1408,11 +1406,9 @@ describe.each([
       })
 
       it("respects the limit parameter", async () => {
-        const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow())
-        }
+        await config.createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
+
         const limit = generator.integer({ min: 1, max: 8 })
 
         const createViewResponse = await config.createView()
@@ -1425,11 +1421,8 @@ describe.each([
       })
 
       it("can handle pagination", async () => {
-        const table = await config.createTable(await userTable())
-        const rows = []
-        for (let i = 0; i < 10; i++) {
-          rows.push(await config.createRow())
-        }
+        await config.createTable(await userTable())
+        await Promise.all(Array.from({ length: 10 }, () => config.createRow()))
 
         const createViewResponse = await config.createView()
         const allRows = (await config.api.viewV2.search(createViewResponse.id))
@@ -1489,11 +1482,10 @@ describe.each([
         let tableId: string
 
         beforeAll(async () => {
-          const table = await config.createTable(await userTable())
-          const rows = []
-          for (let i = 0; i < 10; i++) {
-            rows.push(await config.createRow())
-          }
+          await config.createTable(await userTable())
+          await Promise.all(
+            Array.from({ length: 10 }, () => config.createRow())
+          )
 
           const createViewResponse = await config.createView()
 

From 14259c82f343933cf325e690f1fd5b6a9da5d1df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:57:41 +0200
Subject: [PATCH 59/81] Use test api

---
 .../server/src/api/routes/tests/row.spec.ts   | 130 ++++++------------
 .../server/src/tests/utilities/api/row.ts     |  23 +++-
 2 files changed, 60 insertions(+), 93 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index fbf6651186..c9d78cd404 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -36,8 +36,8 @@ describe.each([
 ])("/rows (%s)", (_, dsProvider) => {
   const isInternal = !dsProvider
 
-  let request = setup.getRequest()
-  let config = setup.getConfig()
+  const request = setup.getRequest()
+  const config = setup.getConfig()
   let table: Table
   let tableId: string
 
@@ -90,12 +90,8 @@ describe.each([
     mocks.licenses.useCloudFree()
   })
 
-  const loadRow = async (id: string, tbl_Id: string, status = 200) =>
-    await request
-      .get(`/api/${tbl_Id}/rows/${id}`)
-      .set(config.defaultHeaders())
-      .expect("Content-Type", /json/)
-      .expect(status)
+  const loadRow = (id: string, tbl_Id: string, status = 200) =>
+    config.api.row.get(tbl_Id, id, { expectStatus: status })
 
   const getRowUsage = async () => {
     const { total } = await config.doInContext(null, () =>
@@ -193,20 +189,12 @@ describe.each([
 
       // Performing several create row requests should increment the autoID fields accordingly
       const createRow = async (id: number) => {
-        const res = await request
-          .post(`/api/${newTable._id}/rows`)
-          .send({
-            name: "row_" + id,
-          })
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
-        expect((res as any).res.statusMessage).toEqual(
-          `${newTable.name} saved successfully`
-        )
-        expect(res.body.name).toEqual("row_" + id)
-        expect(res.body._rev).toBeDefined()
-        expect(res.body["Row ID"]).toEqual(id)
+        const res = await config.api.row.save(newTable._id!, {
+          name: "row_" + id,
+        })
+        expect(res.name).toEqual("row_" + id)
+        expect(res._rev).toBeDefined()
+        expect(res["Row ID"]).toEqual(id)
       }
 
       for (let i = 0; i < ids.length; i++) {
@@ -563,14 +551,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [createdRow],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [createdRow])
       expect(res.body[0]._id).toEqual(createdRow._id)
       await assertRowUsage(rowUsage - 1)
       await assertQueryUsage(queryUsage + 1)
@@ -587,15 +568,10 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows/validate`)
-        .send({ name: "ivan" })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.validate(table._id!, { name: "ivan" })
 
-      expect(res.body.valid).toBe(true)
-      expect(Object.keys(res.body.errors)).toEqual([])
+      expect(res.valid).toBe(true)
+      expect(Object.keys(res.errors)).toEqual([])
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
     })
@@ -604,20 +580,15 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .post(`/api/${table._id}/rows/validate`)
-        .send({ name: 1 })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.validate(table._id!, { name: 1 })
 
       if (isInternal) {
-        expect(res.body.valid).toBe(false)
-        expect(Object.keys(res.body.errors)).toEqual(["name"])
+        expect(res.valid).toBe(false)
+        expect(Object.keys(res.errors)).toEqual(["name"])
       } else {
         // Validation for external is not implemented, so it will always return valid
-        expect(res.body.valid).toBe(true)
-        expect(Object.keys(res.body.errors)).toEqual([])
+        expect(res.valid).toBe(true)
+        expect(Object.keys(res.errors)).toEqual([])
       }
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
@@ -636,14 +607,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [row1, row2],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [row1, row2])
 
       expect(res.body.length).toEqual(2)
       await loadRow(row1._id!, table._id!, 404)
@@ -660,14 +624,11 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({
-          rows: [row1, row2._id, { _id: row3._id }],
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, [
+        row1,
+        row2._id,
+        { _id: row3._id },
+      ])
 
       expect(res.body.length).toEqual(3)
       await loadRow(row1._id!, table._id!, 404)
@@ -680,12 +641,7 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send(row1)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.delete(table._id!, row1)
 
       expect(res.body.id).toEqual(row1._id)
       await loadRow(row1._id!, table._id!, 404)
@@ -697,31 +653,23 @@ describe.each([
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({ not: "valid" })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res = await config.api.row.delete(
+        table._id!,
+        { not: "valid" },
+        { expectStatus: 400 }
+      )
       expect(res.body.message).toEqual("Invalid delete rows request")
 
-      const res2 = await request
-        .delete(`/api/${table._id}/rows`)
-        .send({ rows: 123 })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res2 = await config.api.row.delete(
+        table._id!,
+        { rows: 123 },
+        { expectStatus: 400 }
+      )
       expect(res2.body.message).toEqual("Invalid delete rows request")
 
-      const res3 = await request
-        .delete(`/api/${table._id}/rows`)
-        .send("invalid")
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(400)
-
+      const res3 = await config.api.row.delete(table._id!, "invalid", {
+        expectStatus: 400,
+      })
       expect(res3.body.message).toEqual("Invalid delete rows request")
 
       await assertRowUsage(rowUsage)
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 3c266f07b5..2338285ed4 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -1,4 +1,9 @@
-import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
+import {
+  PatchRowRequest,
+  SaveRowRequest,
+  Row,
+  ValidateResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -51,6 +56,20 @@ export class RowAPI extends TestAPI {
     return resp.body as Row
   }
 
+  validate = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ValidateResponse> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows/validate`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as ValidateResponse
+  }
+
   patch = async (
     sourceId: string,
     row: PatchRowRequest,
@@ -71,7 +90,7 @@ export class RowAPI extends TestAPI {
   ) => {
     return this.request
       .delete(`/api/${sourceId}/rows`)
-      .send({ rows })
+      .send(Array.isArray(rows) ? { rows } : rows)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)

From 4d12ee53dae26cf078e8f38f997b5fa384782517 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 09:59:17 +0200
Subject: [PATCH 60/81] LegacyView test api

---
 .../server/src/api/routes/tests/row.spec.ts     | 17 +++--------------
 .../server/src/tests/utilities/api/index.ts     |  3 +++
 .../src/tests/utilities/api/legacyView.ts       | 16 ++++++++++++++++
 3 files changed, 22 insertions(+), 14 deletions(-)
 create mode 100644 packages/server/src/tests/utilities/api/legacyView.ts

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index c9d78cd404..6a7a0210ef 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -690,11 +690,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const res = await request
-          .get(`/api/views/${table._id}`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
+        const res = await config.api.legacyView.get(table._id!)
         expect(res.body.length).toEqual(1)
         expect(res.body[0]._id).toEqual(row._id)
         await assertRowUsage(rowUsage)
@@ -705,10 +701,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        await request
-          .get(`/api/views/derp`)
-          .set(config.defaultHeaders())
-          .expect(404)
+        await config.api.legacyView.get("derp", { expectStatus: 404 })
 
         await assertRowUsage(rowUsage)
         await assertQueryUsage(queryUsage)
@@ -725,11 +718,7 @@ describe.each([
         const rowUsage = await getRowUsage()
         const queryUsage = await getQueryUsage()
 
-        const res = await request
-          .get(`/api/views/${view.name}`)
-          .set(config.defaultHeaders())
-          .expect("Content-Type", /json/)
-          .expect(200)
+        const res = await config.api.legacyView.get(view.name)
         expect(res.body.length).toEqual(1)
         expect(res.body[0]._id).toEqual(row._id)
 
diff --git a/packages/server/src/tests/utilities/api/index.ts b/packages/server/src/tests/utilities/api/index.ts
index 0521f9b19f..31c74a0e78 100644
--- a/packages/server/src/tests/utilities/api/index.ts
+++ b/packages/server/src/tests/utilities/api/index.ts
@@ -4,9 +4,11 @@ import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
 import { DatasourceAPI } from "./datasource"
+import { LegacyViewAPI } from "./legacyView"
 
 export default class API {
   table: TableAPI
+  legacyView: LegacyViewAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
@@ -14,6 +16,7 @@ export default class API {
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
+    this.legacyView = new LegacyViewAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
diff --git a/packages/server/src/tests/utilities/api/legacyView.ts b/packages/server/src/tests/utilities/api/legacyView.ts
new file mode 100644
index 0000000000..63981cec5e
--- /dev/null
+++ b/packages/server/src/tests/utilities/api/legacyView.ts
@@ -0,0 +1,16 @@
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class LegacyViewAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  get = async (id: string, { expectStatus } = { expectStatus: 200 }) => {
+    return await this.request
+      .get(`/api/views/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
+}

From cf74f19381d8341c6fe4658658b0ac82c3072b46 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:13:54 +0200
Subject: [PATCH 61/81] Use test api

---
 .../server/src/api/controllers/row/index.ts   |  6 ++-
 .../server/src/api/routes/tests/row.spec.ts   | 42 +++++++------------
 .../server/src/tests/utilities/api/row.ts     | 17 +++++++-
 packages/types/src/api/web/app/rows.ts        | 11 ++++-
 4 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index e86b466601..f0f2462019 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -16,6 +16,8 @@ import {
   SearchParams,
   GetRowResponse,
   ValidateResponse,
+  ExportRowsRequest,
+  ExportRowsResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -239,7 +241,9 @@ export async function fetchEnrichedRow(ctx: any) {
   )
 }
 
-export const exportRows = async (ctx: any) => {
+export const exportRows = async (
+  ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
+) => {
   const tableId = utils.getTableId(ctx)
 
   const format = ctx.query.format
diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 6a7a0210ef..09cc2d1888 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -771,11 +771,10 @@ describe.each([
       const queryUsage = await getQueryUsage()
 
       // test basic enrichment
-      const resBasic = await request
-        .get(`/api/${linkedTable._id}/rows/${secondRow._id}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const resBasic = await config.api.row.get(
+        linkedTable._id!,
+        secondRow._id!
+      )
       expect(resBasic.body.link.length).toBe(1)
       expect(resBasic.body.link[0]).toEqual({
         _id: firstRow._id,
@@ -783,11 +782,10 @@ describe.each([
       })
 
       // test full enrichment
-      const resEnriched = await request
-        .get(`/api/${linkedTable._id}/${secondRow._id}/enrich`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const resEnriched = await config.api.row.getEnriched(
+        linkedTable._id!,
+        secondRow._id!
+      )
       expect(resEnriched.body.link.length).toBe(1)
       expect(resEnriched.body.link[0]._id).toBe(firstRow._id)
       expect(resEnriched.body.link[0].name).toBe("Test Contact")
@@ -837,14 +835,9 @@ describe.each([
 
     it("should allow exporting all columns", async () => {
       const existing = await config.createRow()
-      const res = await request
-        .post(`/api/${table._id}/rows/exportRows?format=json`)
-        .set(config.defaultHeaders())
-        .send({
-          rows: [existing._id],
-        })
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+      })
       const results = JSON.parse(res.text)
       expect(results.length).toEqual(1)
       const row = results[0]
@@ -860,15 +853,10 @@ describe.each([
 
     it("should allow exporting only certain columns", async () => {
       const existing = await config.createRow()
-      const res = await request
-        .post(`/api/${table._id}/rows/exportRows?format=json`)
-        .set(config.defaultHeaders())
-        .send({
-          rows: [existing._id],
-          columns: ["_id"],
-        })
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.exportRows(table._id!, {
+        rows: [existing._id!],
+        columns: ["_id"],
+      })
       const results = JSON.parse(res.text)
       expect(results.length).toEqual(1)
       const row = results[0]
diff --git a/packages/server/src/tests/utilities/api/row.ts b/packages/server/src/tests/utilities/api/row.ts
index 2338285ed4..686c8c031b 100644
--- a/packages/server/src/tests/utilities/api/row.ts
+++ b/packages/server/src/tests/utilities/api/row.ts
@@ -3,6 +3,7 @@ import {
   SaveRowRequest,
   Row,
   ValidateResponse,
+  ExportRowsRequest,
 } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
@@ -33,7 +34,7 @@ export class RowAPI extends TestAPI {
     { expectStatus } = { expectStatus: 200 }
   ) => {
     const request = this.request
-      .get(`/api/${sourceId}/rows/${rowId}/enrich`)
+      .get(`/api/${sourceId}/${rowId}/enrich`)
       .set(this.config.defaultHeaders())
       .expect(expectStatus)
     if (expectStatus !== 404) {
@@ -107,4 +108,18 @@ export class RowAPI extends TestAPI {
 
     return (await request).body
   }
+
+  exportRows = async (
+    tableId: string,
+    body: ExportRowsRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .post(`/api/${tableId}/rows/exportRows?format=json`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return request
+  }
 }
diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts
index a99ef0e837..62ea90a6a4 100644
--- a/packages/types/src/api/web/app/rows.ts
+++ b/packages/types/src/api/web/app/rows.ts
@@ -1,5 +1,6 @@
-import { SearchParams } from "../../../sdk"
+import { SearchFilters, SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
+import { ReadStream } from "fs"
 
 export interface SaveRowRequest extends Row {}
 
@@ -28,3 +29,11 @@ export interface SearchViewRowRequest
 export interface SearchRowResponse {
   rows: any[]
 }
+
+export interface ExportRowsRequest {
+  rows: string[]
+  columns?: string[]
+  query?: SearchFilters
+}
+
+export type ExportRowsResponse = ReadStream

From 069b0d6161945288d5c937516cc9d5ca447b053e Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:31:39 +0200
Subject: [PATCH 62/81] Fix types

---
 packages/server/src/migrations/tests/index.spec.ts | 5 ++++-
 packages/server/src/sdk/app/rows/search.ts         | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts
index b64cad26c1..d83e3cf31e 100644
--- a/packages/server/src/migrations/tests/index.spec.ts
+++ b/packages/server/src/migrations/tests/index.spec.ts
@@ -11,6 +11,7 @@ import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
 import tk from "timekeeper"
+import { View } from "@budibase/types"
 const timestamp = new Date().toISOString()
 tk.freeze(timestamp)
 
@@ -52,7 +53,9 @@ describe("migrations", () => {
         await config.createTable()
         await config.createLegacyView()
         await config.createTable()
-        await config.createLegacyView(structures.view(config.table!._id!))
+        await config.createLegacyView(
+          structures.view(config.table!._id!) as View
+        )
         await config.createScreen()
         await config.createScreen()
 
diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index afee18b57c..b3bc220124 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -30,7 +30,7 @@ export interface ExportRowsParams {
   format: Format
   rowIds?: string[]
   columns?: string[]
-  query: SearchFilters
+  query?: SearchFilters
 }
 
 export interface ExportRowsResult {

From 47899c669dee55588353242d7cb4d56bf8f16ac8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 10:39:22 +0200
Subject: [PATCH 63/81] Use api

---
 packages/server/src/integration-test/postgres.spec.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index b0f6f5bb04..0e2560b7df 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -47,7 +47,7 @@ describe("postgres integrations", () => {
     )
 
     async function createAuxTable(prefix: string) {
-      return await config.createTable({
+      return await config.api.table.create({
         name: `${prefix}_${generator.word({ length: 6 })}`,
         type: "external",
         primary: ["id"],
@@ -83,7 +83,7 @@ describe("postgres integrations", () => {
       relationshipType: RelationshipType.MANY_TO_MANY,
     }
 
-    primaryPostgresTable = await config.createTable({
+    primaryPostgresTable = await config.api.table.create({
       name: `p_${generator.word({ length: 6 })}`,
       type: "external",
       primary: ["id"],
@@ -263,7 +263,7 @@ describe("postgres integrations", () => {
   }
 
   async function createDefaultPgTable() {
-    return await config.createTable({
+    return await config.api.table.create({
       name: generator.word({ length: 10 }),
       type: "external",
       primary: ["id"],
@@ -301,7 +301,7 @@ describe("postgres integrations", () => {
     )
   }
 
-  it("validate table schema", async () => {
+  it.only("validate table schema", async () => {
     const res = await makeRequest(
       "get",
       `/api/datasources/${postgresDatasource._id}`

From c7e41eacc77383918e82f2c23041935aab04aad2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 11:00:53 +0200
Subject: [PATCH 64/81] Cleanup test

---
 .../src/integration-test/postgres.spec.ts     | 74 ++++++++-----------
 1 file changed, 30 insertions(+), 44 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 0e2560b7df..2dc84a98da 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -181,32 +181,16 @@ describe("postgres integrations", () => {
     let { rowData } = opts as any
     let foreignRows: ForeignRowsInfo[] = []
 
-    async function createForeignRow(tableInfo: ForeignTableInfo) {
-      const foreignKey = `fk_${tableInfo.table.name}_${tableInfo.fieldName}`
-
-      const foreignRow = await config.createRow({
-        tableId: tableInfo.table._id,
-        title: generator.name(),
-      })
-
-      rowData = {
-        ...rowData,
-        [foreignKey]: foreignRow.id,
-      }
-      foreignRows.push({
-        row: foreignRow,
-
-        relationshipType: tableInfo.relationshipType,
-      })
-    }
-
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
-      const foreignRow = await config.createRow({
-        tableId: oneToManyRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        oneToManyRelationshipInfo.table._id!,
+        {
+          tableId: oneToManyRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -219,10 +203,13 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToOne || 0); i++) {
-      const foreignRow = await config.createRow({
-        tableId: manyToOneRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        manyToOneRelationshipInfo.table._id!,
+        {
+          tableId: manyToOneRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -237,10 +224,13 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToMany || 0); i++) {
-      const foreignRow = await config.createRow({
-        tableId: manyToManyRelationshipInfo.table._id,
-        title: generator.name(),
-      })
+      const foreignRow = await config.api.row.save(
+        manyToManyRelationshipInfo.table._id!,
+        {
+          tableId: manyToManyRelationshipInfo.table._id,
+          title: generator.name(),
+        }
+      )
 
       rowData = {
         ...rowData,
@@ -254,7 +244,7 @@ describe("postgres integrations", () => {
       })
     }
 
-    const row = await config.createRow({
+    const row = await config.api.row.save(primaryPostgresTable._id!, {
       tableId: primaryPostgresTable._id,
       ...rowData,
     })
@@ -277,6 +267,14 @@ describe("postgres integrations", () => {
     })
   }
 
+  const createRandomTableWithRows = async () => {
+    const tableId = (await createDefaultPgTable())._id!
+    return await config.api.row.save(tableId, {
+      tableId,
+      title: generator.name(),
+    })
+  }
+
   async function populatePrimaryRows(
     count: number,
     opts?: {
@@ -685,12 +683,6 @@ describe("postgres integrations", () => {
       describe("given than multiple tables have multiple rows", () => {
         const rowsCount = 6
         beforeEach(async () => {
-          const createRandomTableWithRows = async () =>
-            await config.createRow({
-              tableId: (await createDefaultPgTable())._id,
-              title: generator.name(),
-            })
-
           await createRandomTableWithRows()
           await createRandomTableWithRows()
 
@@ -978,12 +970,6 @@ describe("postgres integrations", () => {
       const rowsCount = 6
 
       beforeEach(async () => {
-        const createRandomTableWithRows = async () =>
-          await config.createRow({
-            tableId: (await createDefaultPgTable())._id,
-            title: generator.name(),
-          })
-
         await createRandomTableWithRows()
         await populatePrimaryRows(rowsCount)
         await createRandomTableWithRows()

From 823afa1f553116526ed1c3f150570ec4848b89ee Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 11:59:34 +0200
Subject: [PATCH 65/81] Run all tests

---
 packages/server/src/integration-test/postgres.spec.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 2dc84a98da..27e9fa7206 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -299,7 +299,7 @@ describe("postgres integrations", () => {
     )
   }
 
-  it.only("validate table schema", async () => {
+  it("validate table schema", async () => {
     const res = await makeRequest(
       "get",
       `/api/datasources/${postgresDatasource._id}`

From ceb3129e221c4871c6c48dc0d14f62ff11fd8c5a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:02:13 +0200
Subject: [PATCH 66/81] Revert relationship changes

---
 .../server/src/api/controllers/table/external.ts | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts
index 513b989a3a..327904666d 100644
--- a/packages/server/src/api/controllers/table/external.ts
+++ b/packages/server/src/api/controllers/table/external.ts
@@ -67,15 +67,13 @@ function cleanupRelationships(
   tables: Record<string, Table>,
   oldTable?: Table
 ) {
-  const isUpdate = !!oldTable
-  if (!isUpdate) {
-    return
-  }
-
-  // When updating a table, we want to detect if some relationships were removed.
-  // If so, we want to remove the relationship from the other end
-  for (let [key, schema] of Object.entries(oldTable.schema)) {
-    if (schema.type === FieldTypes.LINK && !table.schema[key]) {
+  const tableToIterate = oldTable ? oldTable : table
+  // clean up relationships in couch table schemas
+  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
+    if (
+      schema.type === FieldTypes.LINK &&
+      (!oldTable || table.schema[key] == null)
+    ) {
       const relatedTable = Object.values(tables).find(
         table => table._id === schema.tableId
       )

From 6bae382d81e118c12c047b19674d27d5ae5d4f09 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:23:59 +0200
Subject: [PATCH 67/81] Fix tests

---
 packages/server/src/api/controllers/table/index.ts       | 4 ++--
 packages/server/src/tests/utilities/TestConfiguration.ts | 9 +++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts
index 759974d6a7..29c41ad985 100644
--- a/packages/server/src/api/controllers/table/index.ts
+++ b/packages/server/src/api/controllers/table/index.ts
@@ -78,9 +78,9 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
 }
 
 export async function destroy(ctx: UserCtx) {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 44867e6b01..e521a5a6a6 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -557,6 +557,11 @@ class TestConfiguration {
     return this.updateTable(config, options)
   }
 
+  async getTable(tableId?: string) {
+    tableId = tableId || this.table!._id!
+    return this._req(null, { tableId }, controllers.table.find)
+  }
+
   async createLinkedTable(
     config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
@@ -607,6 +612,10 @@ class TestConfiguration {
     return this.api.row.save(tableId!, config)
   }
 
+  async getRow(tableId: string, rowId: string): Promise<Row> {
+    return this._req(null, { tableId, rowId }, controllers.row.find)
+  }
+
   async getRows(tableId: string) {
     if (!tableId && this.table) {
       tableId = this.table._id!

From dc5cd7cf76cf7132f742b68032f04f2119b4f7d2 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 12:53:47 +0200
Subject: [PATCH 68/81] Fix

---
 .../src/integration-test/postgres.spec.ts     | 95 ++++++++++++-------
 1 file changed, 60 insertions(+), 35 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 27e9fa7206..8ce1530ec2 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -47,7 +47,7 @@ describe("postgres integrations", () => {
     )
 
     async function createAuxTable(prefix: string) {
-      return await config.api.table.create({
+      return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
         type: "external",
         primary: ["id"],
@@ -83,7 +83,7 @@ describe("postgres integrations", () => {
       relationshipType: RelationshipType.MANY_TO_MANY,
     }
 
-    primaryPostgresTable = await config.api.table.create({
+    primaryPostgresTable = await config.createTable({
       name: `p_${generator.word({ length: 6 })}`,
       type: "external",
       primary: ["id"],
@@ -184,13 +184,10 @@ describe("postgres integrations", () => {
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
-      const foreignRow = await config.api.row.save(
-        oneToManyRelationshipInfo.table._id!,
-        {
-          tableId: oneToManyRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: oneToManyRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -203,13 +200,10 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToOne || 0); i++) {
-      const foreignRow = await config.api.row.save(
-        manyToOneRelationshipInfo.table._id!,
-        {
-          tableId: manyToOneRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: manyToOneRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -224,13 +218,10 @@ describe("postgres integrations", () => {
     }
 
     for (let i = 0; i < (opts?.createForeignRows?.createManyToMany || 0); i++) {
-      const foreignRow = await config.api.row.save(
-        manyToManyRelationshipInfo.table._id!,
-        {
-          tableId: manyToManyRelationshipInfo.table._id,
-          title: generator.name(),
-        }
-      )
+      const foreignRow = await config.createRow({
+        tableId: manyToManyRelationshipInfo.table._id,
+        title: generator.name(),
+      })
 
       rowData = {
         ...rowData,
@@ -244,7 +235,7 @@ describe("postgres integrations", () => {
       })
     }
 
-    const row = await config.api.row.save(primaryPostgresTable._id!, {
+    const row = await config.createRow({
       tableId: primaryPostgresTable._id,
       ...rowData,
     })
@@ -253,7 +244,7 @@ describe("postgres integrations", () => {
   }
 
   async function createDefaultPgTable() {
-    return await config.api.table.create({
+    return await config.createTable({
       name: generator.word({ length: 10 }),
       type: "external",
       primary: ["id"],
@@ -395,7 +386,7 @@ describe("postgres integrations", () => {
         expect(res.status).toBe(200)
         expect(res.body).toEqual(updatedRow)
 
-        const persistedRow = await config.api.row.get(
+        const persistedRow = await config.getRow(
           primaryPostgresTable._id!,
           row.id
         )
@@ -520,7 +511,7 @@ describe("postgres integrations", () => {
           foreignRows = createdRow.foreignRows
         })
 
-        it("only one to many foreign keys are retrieved", async () => {
+        it("only one to primary keys are retrieved", async () => {
           const res = await getRow(primaryPostgresTable._id, row.id)
 
           expect(res.status).toBe(200)
@@ -528,6 +519,12 @@ describe("postgres integrations", () => {
           const one2ManyForeignRows = foreignRows.filter(
             x => x.relationshipType === RelationshipType.ONE_TO_MANY
           )
+          const many2OneForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_ONE
+          )
+          const many2ManyForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_MANY
+          )
           expect(one2ManyForeignRows).toHaveLength(1)
 
           expect(res.body).toEqual({
@@ -538,9 +535,25 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               one2ManyForeignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              one2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2OneForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -569,9 +582,13 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               foreignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -598,9 +615,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -627,9 +648,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
     })

From c530d5fa348105ead7cab139655b8444dde7424b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 13:17:51 +0200
Subject: [PATCH 69/81] Timeout issues

---
 .../src/integration-test/postgres.spec.ts     | 20 ++++++++++---------
 .../src/integrations/tests/utils/index.ts     |  2 ++
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/integration-test/postgres.spec.ts b/packages/server/src/integration-test/postgres.spec.ts
index 8ce1530ec2..86a43c8c42 100644
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@@ -21,8 +21,6 @@ import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
-jest.setTimeout(30000)
-
 jest.unmock("pg")
 jest.mock("../websockets")
 
@@ -39,13 +37,13 @@ describe("postgres integrations", () => {
     const apiKey = await config.generateApiKey()
 
     makeRequest = generateMakeRequest(apiKey, true)
-  })
 
-  beforeEach(async () => {
     postgresDatasource = await config.api.datasource.create(
       await databaseTestProviders.postgres.getDsConfig()
     )
+  })
 
+  beforeEach(async () => {
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
@@ -345,12 +343,16 @@ describe("postgres integrations", () => {
 
       it("multiple rows can be persisted", async () => {
         const numberOfRows = 10
-        const newRows = Array(numberOfRows).fill(generateRandomPrimaryRowData())
+        const newRows: Row[] = Array(numberOfRows).fill(
+          generateRandomPrimaryRowData()
+        )
 
-        for (const newRow of newRows) {
-          const res = await createRow(primaryPostgresTable._id, newRow)
-          expect(res.status).toBe(200)
-        }
+        await Promise.all(
+          newRows.map(async newRow => {
+            const res = await createRow(primaryPostgresTable._id, newRow)
+            expect(res.status).toBe(200)
+          })
+        )
 
         const persistedRows = await config.getRows(primaryPostgresTable._id!)
         expect(persistedRows).toHaveLength(numberOfRows)
diff --git a/packages/server/src/integrations/tests/utils/index.ts b/packages/server/src/integrations/tests/utils/index.ts
index c145a9c809..a28141db08 100644
--- a/packages/server/src/integrations/tests/utils/index.ts
+++ b/packages/server/src/integrations/tests/utils/index.ts
@@ -3,6 +3,8 @@ jest.unmock("pg")
 import { Datasource } from "@budibase/types"
 import * as pg from "./postgres"
 
+jest.setTimeout(30000)
+
 export interface DatabasePlusTestProvider {
   getDsConfig(): Promise<Datasource>
 }

From 142fb18c177b1063076c98852643094299cb1137 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 14:09:48 +0200
Subject: [PATCH 70/81] Fix tests

---
 .../server/src/api/routes/tests/row.spec.ts   | 33 +++++++++++--------
 .../src/db/tests/linkController.spec.js       |  2 +-
 .../src/tests/utilities/TestConfiguration.ts  |  4 +--
 3 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index 09cc2d1888..e3bfc32320 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -10,6 +10,7 @@ import {
   MonthlyQuotaName,
   PermissionLevel,
   QuotaUsageType,
+  RelationshipType,
   Row,
   SaveTableRequest,
   SortOrder,
@@ -737,22 +738,26 @@ describe.each([
       const { linkedTable, firstRow, secondRow } = await tenancy.doInTenant(
         config.getTenantId(),
         async () => {
-          const linkedTable = await config.createLinkedTable({
-            name: generator.word(),
-            type: "table",
-            primary: ["id"],
-            primaryDisplay: "id",
-            schema: {
-              id: {
-                type: FieldType.AUTO,
-                name: "id",
-                autocolumn: true,
-                constraints: {
-                  presence: true,
+          const linkedTable = await config.createLinkedTable(
+            RelationshipType.ONE_TO_MANY,
+            ["link"],
+            {
+              name: generator.word(),
+              type: "table",
+              primary: ["id"],
+              primaryDisplay: "id",
+              schema: {
+                id: {
+                  type: FieldType.AUTO,
+                  name: "id",
+                  autocolumn: true,
+                  constraints: {
+                    presence: true,
+                  },
                 },
               },
-            },
-          })
+            }
+          )
           const firstRow = await config.createRow({
             name: "Test Contact",
             description: "original description",
diff --git a/packages/server/src/db/tests/linkController.spec.js b/packages/server/src/db/tests/linkController.spec.js
index 1c50142871..59d0f3f983 100644
--- a/packages/server/src/db/tests/linkController.spec.js
+++ b/packages/server/src/db/tests/linkController.spec.js
@@ -6,7 +6,7 @@ const { RelationshipType } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")
 
 describe("test the link controller", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
   let table1, table2, appId
 
   beforeAll(async () => {
diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index e521a5a6a6..94896932ad 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -563,9 +563,9 @@ class TestConfiguration {
   }
 
   async createLinkedTable(
-    config?: Table,
     relationshipType = RelationshipType.ONE_TO_MANY,
-    links: any = ["link"]
+    links: any = ["link"],
+    config?: Table
   ) {
     if (!this.table) {
       throw "Must have created a table first."

From 21e5219bde49817b78d8dcf217784586550e277f Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 12:10:55 +0000
Subject: [PATCH 71/81] Bump version to 2.10.7-alpha.0

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 63466ceedb..88bde5827a 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.6",
+  "version": "2.10.7-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 9c6e880479a059c099a1c17f884a9dbc4dd48a2c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:03:21 +0200
Subject: [PATCH 72/81] Fix update datasource test helper

---
 packages/server/src/tests/utilities/api/datasource.ts | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts
index 2ce09959fc..ee698334f2 100644
--- a/packages/server/src/tests/utilities/api/datasource.ts
+++ b/packages/server/src/tests/utilities/api/datasource.ts
@@ -30,16 +30,12 @@ export class DatasourceAPI extends TestAPI {
   }
 
   update = async (
-    config: Datasource,
+    datasource: Datasource,
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Datasource> => {
-    const body: CreateDatasourceRequest = {
-      datasource: config,
-      tablesFilter: [],
-    }
     const result = await this.request
-      .put(`/api/datasources/${config._id}`)
-      .send(body)
+      .put(`/api/datasources/${datasource._id}`)
+      .send(datasource)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)

From d248c7b6b8357a3bc3f1a52e7e7bce6b62c7924f Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:26:29 +0200
Subject: [PATCH 73/81] Fix types

---
 packages/server/src/sdk/app/rows/search.ts          | 2 +-
 packages/server/src/sdk/app/rows/search/internal.ts | 6 +-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts
index b3bc220124..380521a05a 100644
--- a/packages/server/src/sdk/app/rows/search.ts
+++ b/packages/server/src/sdk/app/rows/search.ts
@@ -20,7 +20,7 @@ function pickApi(tableId: any) {
 export async function search(options: SearchParams): Promise<{
   rows: any[]
   hasNextPage?: boolean
-  bookmark?: number | string | null
+  bookmark?: number | null
 }> {
   return pickApi(options.tableId).search(options)
 }
diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts
index 0a6fd69ff6..4cdeca87f6 100644
--- a/packages/server/src/sdk/app/rows/search/internal.ts
+++ b/packages/server/src/sdk/app/rows/search/internal.ts
@@ -59,11 +59,7 @@ export async function search(options: SearchParams) {
   if (paginate) {
     response = await paginatedSearch(query, params)
   } else {
-    response = {
-      ...(await fullSearch(query, params)),
-      hasNextPage: false,
-      bookmark: null,
-    }
+    response = await fullSearch(query, params)
   }
 
   // Enrich search results with relationships

From 891e77e2bfdc0e68fba12d4f44b4f5a48fbe4c04 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:28:38 +0200
Subject: [PATCH 74/81] Remove coverage from prettier

---
 .prettierignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.prettierignore b/.prettierignore
index a73fed4890..64607d74ab 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/sdk/sdk
\ No newline at end of file
+packages/sdk/sdk
+packages/pro/coverage
\ No newline at end of file

From 17c365d398ce0e6cb4102fb79a4d5af7840d9026 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:39:59 +0200
Subject: [PATCH 75/81] Fix tests

---
 packages/server/src/api/routes/tests/row.spec.ts | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts
index e3bfc32320..a74a9f7960 100644
--- a/packages/server/src/api/routes/tests/row.spec.ts
+++ b/packages/server/src/api/routes/tests/row.spec.ts
@@ -1105,8 +1105,12 @@ describe.each([
               ...defaultRowFields,
             }))
           ),
-          hasNextPage: false,
-          bookmark: null,
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
         })
       })
 
@@ -1153,8 +1157,12 @@ describe.each([
               ...defaultRowFields,
             }))
           ),
-          hasNextPage: false,
-          bookmark: null,
+          ...(isInternal
+            ? {}
+            : {
+                hasNextPage: false,
+                bookmark: null,
+              }),
         })
       })
 

From dc29ebfe273fac6cc8c195e91b705bd1b1e29fa0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 15:54:44 +0200
Subject: [PATCH 76/81] Fix tests

---
 .../server/src/api/controllers/query/import/tests/index.spec.js | 2 +-
 packages/server/src/db/tests/linkTests.spec.js                  | 2 +-
 .../src/migrations/functions/usageQuotas/tests/syncRows.spec.ts | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js
index 60030ae7f3..3aa9e7f916 100644
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@@ -39,7 +39,7 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
 
   beforeAll(async () => {
     await config.init()
diff --git a/packages/server/src/db/tests/linkTests.spec.js b/packages/server/src/db/tests/linkTests.spec.js
index b7764dacb7..19a0eb88d3 100644
--- a/packages/server/src/db/tests/linkTests.spec.js
+++ b/packages/server/src/db/tests/linkTests.spec.js
@@ -4,7 +4,7 @@ const linkUtils = require("../linkedRows/linkUtils")
 const { context } = require("@budibase/backend-core")
 
 describe("test link functionality", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
   let appId
 
   describe("getLinkedTable", () => {
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
index fc0f9e1aa9..5257ffc042 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
@@ -5,7 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
 
   beforeEach(async () => {
     await config.init()

From d1e1ad4930ae91ab595485d4f4a3696aee6567aa Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 16:33:04 +0200
Subject: [PATCH 77/81] Revert TestConfiguration changes

---
 .../src/tests/utilities/TestConfiguration.ts   | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts
index 94896932ad..da7af8acd7 100644
--- a/packages/server/src/tests/utilities/TestConfiguration.ts
+++ b/packages/server/src/tests/utilities/TestConfiguration.ts
@@ -535,7 +535,7 @@ class TestConfiguration {
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
-    const response = await this.api.table.create(config)
+    const response = await this._req(config, null, controllers.table.save)
     if (!skipReassigning) {
       this.table = response
     }
@@ -589,7 +589,7 @@ class TestConfiguration {
       }
     }
 
-    const linkedTable = await this.api.table.create(tableConfig)
+    const linkedTable = await this.createTable(tableConfig)
     return linkedTable
   }
 
@@ -609,7 +609,7 @@ class TestConfiguration {
     }
     const tableId = (config && config.tableId) || this.table._id
     config = config || basicRow(tableId!)
-    return this.api.row.save(tableId!, config)
+    return this._req(config, { tableId }, controllers.row.save)
   }
 
   async getRow(tableId: string, rowId: string): Promise<Row> {
@@ -715,14 +715,18 @@ class TestConfiguration {
     datasource: Datasource
   }): Promise<Datasource> {
     config = config || basicDatasource()
-    const response = await this.api.datasource.create({ ...config.datasource })
-    this.datasource = response
+    const response = await this._req(config, null, controllers.datasource.save)
+    this.datasource = response.datasource
     return this.datasource!
   }
 
   async updateDatasource(datasource: Datasource): Promise<Datasource> {
-    const response = await this.api.datasource.update(datasource)
-    this.datasource = response
+    const response = await this._req(
+      datasource,
+      { datasourceId: datasource._id },
+      controllers.datasource.update
+    )
+    this.datasource = response.datasource
     return this.datasource!
   }
 

From 405ea4e37b9eff0d9c0eaec1b721a9382bc2af38 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 13 Sep 2023 16:39:23 +0200
Subject: [PATCH 78/81] Reverts

---
 .../server/src/api/controllers/query/import/tests/index.spec.js | 2 +-
 .../src/migrations/functions/usageQuotas/tests/syncRows.spec.ts | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/api/controllers/query/import/tests/index.spec.js b/packages/server/src/api/controllers/query/import/tests/index.spec.js
index 3aa9e7f916..60030ae7f3 100644
--- a/packages/server/src/api/controllers/query/import/tests/index.spec.js
+++ b/packages/server/src/api/controllers/query/import/tests/index.spec.js
@@ -39,7 +39,7 @@ const datasets = {
 }
 
 describe("Rest Importer", () => {
-  const config = new TestConfig()
+  const config = new TestConfig(false)
 
   beforeAll(async () => {
     await config.init()
diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
index 5257ffc042..fc0f9e1aa9 100644
--- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
+++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts
@@ -5,7 +5,7 @@ import { QuotaUsageType, StaticQuotaName } from "@budibase/types"
 import { db as dbCore, context } from "@budibase/backend-core"
 
 describe("syncRows", () => {
-  let config = new TestConfig()
+  let config = new TestConfig(false)
 
   beforeEach(async () => {
     await config.init()

From f70531d34322ce64ed6bc2f3e20df5e489e9e809 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 15:00:27 +0000
Subject: [PATCH 79/81] Bump version to 2.10.7-alpha.1

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 88bde5827a..054621b521 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7-alpha.0",
+  "version": "2.10.7-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

From 78d0398315ec4f9e1edf47b7ec94480c0b887f61 Mon Sep 17 00:00:00 2001
From: Martin McKeaveney <martin@budibase.com>
Date: Wed, 13 Sep 2023 16:18:33 +0100
Subject: [PATCH 80/81] Update stale_bot.yml

---
 .github/workflows/stale_bot.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml
index 8cda3a9342..f87d561db9 100644
--- a/.github/workflows/stale_bot.yml
+++ b/.github/workflows/stale_bot.yml
@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 1 * * *' # 1:30 every morning
+    - cron: '*/30 * * * *' # Every 30 mins
 
 jobs:
   stale:

From 29358cec5f070840ef6ea4a836065c9f7755d5e8 Mon Sep 17 00:00:00 2001
From: Budibase Staging Release Bot <>
Date: Wed, 13 Sep 2023 15:18:50 +0000
Subject: [PATCH 81/81] Bump version to 2.10.7-alpha.2

---
 lerna.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lerna.json b/lerna.json
index 054621b521..a3a8bda618 100644
--- a/lerna.json
+++ b/lerna.json
@@ -1,5 +1,5 @@
 {
-  "version": "2.10.7-alpha.1",
+  "version": "2.10.7-alpha.2",
   "npmClient": "yarn",
   "packages": [
     "packages/*"