From 1b0c551602cb1841489e354a1ca0bf211e3476d6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 19 Dec 2023 16:59:23 +0100
Subject: [PATCH 01/31] Create create-many script

---
 .../scripts/load/create-many-relationships.js | 47 +++++++++++++++++++
 packages/server/scripts/load/utils.js         | 43 ++++++++++-------
 2 files changed, 73 insertions(+), 17 deletions(-)
 create mode 100755 packages/server/scripts/load/create-many-relationships.js

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
new file mode 100755
index 0000000000..78353c029c
--- /dev/null
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -0,0 +1,47 @@
+#!/bin/node
+const { createApp, getTable, createRow, createTable } = require("./utils")
+
+const ROW_COUNT = 10
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const app = await createApp(apiKey)
+  console.log(`App created: ${app._id}`)
+  const table = await getTable(apiKey, app._id)
+  console.log(`Table found: ${table.name}`)
+
+  const subjectTable = await createTable(apiKey, app._id, {
+    schema: {
+      title: {
+        name: "Name",
+        type: "string",
+      },
+    },
+    name: "Subjects",
+  })
+  for (let i = 0; i < 10; i++) {
+    await createRow(apiKey, app._id, subjectTable)
+  }
+
+  const promises = []
+  for (let i = 0; i < ROW_COUNT; i++) {
+    promises.push(await createRow(apiKey, app._id, table))
+    console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
+  }
+  await Promise.all(promises)
+
+  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+}
+
+run()
+  .then(() => {
+    console.log(`Finished creating ${ROW_COUNT} rows.`)
+  })
+  .catch(err => {
+    console.error(err)
+  })
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 97ff8f1e13..7af867ab1d 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -2,7 +2,8 @@ const fetch = require("node-fetch")
 const uuid = require("uuid/v4")
 
 const URL_APP = "http://localhost:10000/api/public/v1/applications"
-const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search"
+const URL_TABLE = "http://localhost:10000/api/public/v1/tables"
+const URL_SEARCH_TABLE = "http://localhost:10000/api/public/v1/tables/search"
 
 async function request(apiKey, url, method, body, appId = undefined) {
   const headers = {
@@ -38,29 +39,37 @@ exports.createApp = async apiKey => {
 }
 
 exports.getTable = async (apiKey, appId) => {
-  const res = await request(apiKey, URL_TABLE, "POST", {}, appId)
+  const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()
   return json.data[0]
 }
 
-exports.createRow = async (apiKey, appId, table) => {
-  const body = {}
-  for (let [key, schema] of Object.entries(table.schema)) {
-    let fake
-    switch (schema.type) {
-      default:
-      case "string":
-        fake = schema.constraints.inclusion
-          ? schema.constraints.inclusion[0]
-          : "a"
-        break
-      case "number":
-        fake = 1
-        break
+exports.createRow = async (apiKey, appId, table, body) => {
+  if (!body) {
+    body = {}
+    for (let [key, schema] of Object.entries(table.schema)) {
+      let fake
+      switch (schema.type) {
+        default:
+        case "string":
+          fake = schema.constraints?.inclusion
+            ? schema.constraints.inclusion[0]
+            : "a"
+          break
+        case "number":
+          fake = 1
+          break
+      }
+      body[key] = fake
     }
-    body[key] = fake
   }
   const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows`
   const res = await request(apiKey, url, "POST", body, appId)
   return (await res.json()).data
 }
+
+exports.createTable = async (apiKey, appId, config) => {
+  const res = await request(apiKey, URL_TABLE, "POST", config, appId)
+  const json = await res.json()
+  return json.data
+}

From 1b6c7f27297f72b28f67966b5d10fde117c27506 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 11:51:35 +0100
Subject: [PATCH 02/31] Add content to subjects

---
 .../scripts/load/create-many-relationships.js  | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 78353c029c..1589c1111e 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,6 +1,9 @@
 #!/bin/node
 const { createApp, getTable, createRow, createTable } = require("./utils")
 
+const Chance = require("chance")
+const generator = new Chance()
+
 const ROW_COUNT = 10
 
 if (!process.argv[2]) {
@@ -17,23 +20,26 @@ async function run() {
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
-      title: {
+      Name: {
         name: "Name",
         type: "string",
       },
     },
     name: "Subjects",
   })
-  for (let i = 0; i < 10; i++) {
-    await createRow(apiKey, app._id, subjectTable)
+
+  const SUBJECT_COUNT = 10
+  for (let i = 0; i < SUBJECT_COUNT; i++) {
+    await createRow(apiKey, app._id, subjectTable, {
+      Name: generator.profession(),
+    })
+    console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
   }
 
-  const promises = []
   for (let i = 0; i < ROW_COUNT; i++) {
-    promises.push(await createRow(apiKey, app._id, table))
+    await createRow(apiKey, app._id, table)
     console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
   }
-  await Promise.all(promises)
 
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 }

From e986374c23618703e6fc3dce7b00490ac65808b6 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:18:29 +0100
Subject: [PATCH 03/31] Check fetched table

---
 .../scripts/load/create-many-relationships.js | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 1589c1111e..926d0ac342 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,19 +4,27 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const ROW_COUNT = 10
+const STUDENT_COUNT = 10
+const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
 }
 
+async function sleep(ms) {
+  return new Promise(r => setTimeout(() => r(), ms))
+}
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
   console.log(`App created: ${app._id}`)
-  const table = await getTable(apiKey, app._id)
-  console.log(`Table found: ${table.name}`)
+  const studentsTable = await getTable(apiKey, app._id)
+  if (studentsTable.name !== "Students") {
+    throw 'Fetched table should be "Students"'
+  }
+  console.log(`Table found: ${studentsTable.name}`)
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -28,17 +36,17 @@ async function run() {
     name: "Subjects",
   })
 
-  const SUBJECT_COUNT = 10
   for (let i = 0; i < SUBJECT_COUNT; i++) {
     await createRow(apiKey, app._id, subjectTable, {
       Name: generator.profession(),
     })
     console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
+    await sleep(50)
   }
 
-  for (let i = 0; i < ROW_COUNT; i++) {
-    await createRow(apiKey, app._id, table)
-    console.log(`Row ${i + 1} of ${ROW_COUNT} created`)
+  for (let i = 0; i < STUDENT_COUNT; i++) {
+    await createRow(apiKey, app._id, studentsTable)
+    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
   }
 
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)

From e4acd91237a0cb56b849d85cdeab162b2701df4c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:30:35 +0100
Subject: [PATCH 04/31] Populate rows

---
 .../scripts/load/create-many-relationships.js | 29 +++++++++++--------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 926d0ac342..374dda58c2 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,7 +4,7 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const STUDENT_COUNT = 10
+const STUDENT_COUNT = 100
 const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
@@ -19,7 +19,8 @@ async function sleep(ms) {
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
-  console.log(`App created: ${app._id}`)
+  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+
   const studentsTable = await getTable(apiKey, app._id)
   if (studentsTable.name !== "Students") {
     throw 'Fetched table should be "Students"'
@@ -44,18 +45,22 @@ async function run() {
     await sleep(50)
   }
 
+  let studentNumber = studentsTable.schema["Auto ID"].lastID
   for (let i = 0; i < STUDENT_COUNT; i++) {
-    await createRow(apiKey, app._id, studentsTable)
+    await createRow(apiKey, app._id, studentsTable, {
+      "Student Number": (++studentNumber).toString(),
+      "First Name": generator.first(),
+      "Last Name": generator.last(),
+      Gender: generator.pickone(["M", "F"]),
+      Grade: generator.pickone(["8", "9", "10", "11"]),
+      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+      "Home Number": generator.phone(),
+      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+    })
     console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
   }
-
-  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 }
 
-run()
-  .then(() => {
-    console.log(`Finished creating ${ROW_COUNT} rows.`)
-  })
-  .catch(err => {
-    console.error(err)
-  })
+run().catch(err => {
+  console.error(err)
+})

From 6d6100eaf40f552b46f1bf94f4229dda2526024c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 12:52:25 +0100
Subject: [PATCH 05/31] Add grades

---
 .../scripts/load/create-many-relationships.js | 79 ++++++++++++++++---
 1 file changed, 67 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 374dda58c2..5d0cf3046a 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -27,6 +27,26 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
+  const students = [],
+    subjects = []
+
+  let studentNumber = studentsTable.schema["Auto ID"].lastID
+  for (let i = 0; i < STUDENT_COUNT; i++) {
+    students.push(
+      await createRow(apiKey, app._id, studentsTable, {
+        "Student Number": (++studentNumber).toString(),
+        "First Name": generator.first(),
+        "Last Name": generator.last(),
+        Gender: generator.pickone(["M", "F"]),
+        Grade: generator.pickone(["8", "9", "10", "11"]),
+        "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+        "Home Number": generator.phone(),
+        "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+      })
+    )
+    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+  }
+
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
       Name: {
@@ -35,29 +55,64 @@ async function run() {
       },
     },
     name: "Subjects",
+    primaryDisplay: "Name",
   })
 
   for (let i = 0; i < SUBJECT_COUNT; i++) {
+    subjects.push(
     await createRow(apiKey, app._id, subjectTable, {
       Name: generator.profession(),
     })
+    )
     console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
     await sleep(50)
   }
 
-  let studentNumber = studentsTable.schema["Auto ID"].lastID
-  for (let i = 0; i < STUDENT_COUNT; i++) {
-    await createRow(apiKey, app._id, studentsTable, {
-      "Student Number": (++studentNumber).toString(),
-      "First Name": generator.first(),
-      "Last Name": generator.last(),
-      Gender: generator.pickone(["M", "F"]),
-      Grade: generator.pickone(["8", "9", "10", "11"]),
-      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
-      "Home Number": generator.phone(),
-      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
+  const gradesTable = await createTable(apiKey, app._id, {
+    schema: {
+      Score: {
+        name: "Score",
+        type: "number",
+      },
+      Student: {
+        name: "Student",
+        tableId: studentsTable._id,
+        constraints: {
+          presence: true,
+          type: "array",
+        },
+        fieldName: "Grades",
+        relationshipType: "one-to-many",
+        type: "link",
+      },
+      Subject: {
+        name: "Subject",
+        tableId: subjectTable._id,
+        constraints: {
+          presence: true,
+          type: "array",
+        },
+        fieldName: "Grades",
+        relationshipType: "one-to-many",
+        type: "link",
+      },
+    },
+    name: "Grades",
+  })
+
+  let i = 0
+  for (const student of students) {
+    for (const subject of subjects) {
+      await createRow(apiKey, app._id, gradesTable, {
+        Score: generator.integer({ min: 0, max: 100 }),
+        Student: [student],
+        Subject: [subject],
     })
-    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+      console.log(
+        `Grade ${++i} of ${students.length * subjects.length} created`
+      )
+      await sleep(20)
+    }
   }
 }
 

From 192d980e53d7e6e9c2c215de6e8857436661976a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 13:04:16 +0100
Subject: [PATCH 06/31] Log timings

---
 .../scripts/load/create-many-relationships.js | 38 +++++++++++++------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 5d0cf3046a..1c146ccaa0 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -4,7 +4,7 @@ const { createApp, getTable, createRow, createTable } = require("./utils")
 const Chance = require("chance")
 const generator = new Chance()
 
-const STUDENT_COUNT = 100
+const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
 if (!process.argv[2]) {
@@ -16,6 +16,8 @@ async function sleep(ms) {
   return new Promise(r => setTimeout(() => r(), ms))
 }
 
+const start = Date.now()
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
@@ -44,7 +46,11 @@ async function run() {
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
     )
-    console.log(`Row ${i + 1} of ${STUDENT_COUNT} created`)
+    console.log(
+      `Student ${i + 1} of ${STUDENT_COUNT} created (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
   }
 
   const subjectTable = await createTable(apiKey, app._id, {
@@ -60,11 +66,15 @@ async function run() {
 
   for (let i = 0; i < SUBJECT_COUNT; i++) {
     subjects.push(
-    await createRow(apiKey, app._id, subjectTable, {
-      Name: generator.profession(),
-    })
+      await createRow(apiKey, app._id, subjectTable, {
+        Name: generator.profession(),
+      })
+    )
+    console.log(
+      `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
+        (Date.now() - start) / 1000
+      }s)`
     )
-    console.log(`Subject ${i + 1} of ${SUBJECT_COUNT} created`)
     await sleep(50)
   }
 
@@ -107,15 +117,21 @@ async function run() {
         Score: generator.integer({ min: 0, max: 100 }),
         Student: [student],
         Subject: [subject],
-    })
+      })
       console.log(
-        `Grade ${++i} of ${students.length * subjects.length} created`
+        `Grade ${++i} of ${students.length * subjects.length} created (${
+          (Date.now() - start) / 1000
+        }s)`
       )
       await sleep(20)
     }
   }
 }
 
-run().catch(err => {
-  console.error(err)
-})
+run()
+  .then(() => {
+    console.log(`Done in(${(Date.now() - start) / 1000} seconds`)
+  })
+  .catch(err => {
+    console.error(err)
+  })

From 5240b04a3a73997842d904c13dd9add656722da4 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 13:09:02 +0100
Subject: [PATCH 07/31] Remove 429 in dev

---
 packages/server/scripts/load/create-many-relationships.js | 6 ------
 packages/server/src/api/routes/public/index.ts            | 2 +-
 2 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 1c146ccaa0..ee05cc8fba 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -12,10 +12,6 @@ if (!process.argv[2]) {
   process.exit(-1)
 }
 
-async function sleep(ms) {
-  return new Promise(r => setTimeout(() => r(), ms))
-}
-
 const start = Date.now()
 
 async function run() {
@@ -75,7 +71,6 @@ async function run() {
         (Date.now() - start) / 1000
       }s)`
     )
-    await sleep(50)
   }
 
   const gradesTable = await createTable(apiKey, app._id, {
@@ -123,7 +118,6 @@ async function run() {
           (Date.now() - start) / 1000
         }s)`
       )
-      await sleep(20)
     }
   }
 }
diff --git a/packages/server/src/api/routes/public/index.ts b/packages/server/src/api/routes/public/index.ts
index f27f3f8857..36e0f74bee 100644
--- a/packages/server/src/api/routes/public/index.ts
+++ b/packages/server/src/api/routes/public/index.ts
@@ -77,7 +77,7 @@ const publicRouter = new Router({
   prefix: PREFIX,
 })
 
-if (limiter) {
+if (limiter && !env.isDev()) {
   publicRouter.use(limiter)
 }
 

From 57a7be7a266d7d49379166766805ab8a05523051 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 15:02:21 +0100
Subject: [PATCH 08/31] Run creation in parallel

---
 .../scripts/load/create-many-relationships.js | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index ee05cc8fba..49d4efb28d 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -25,12 +25,11 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
-  const students = [],
-    subjects = []
-
+  const studentsPromise = []
   let studentNumber = studentsTable.schema["Auto ID"].lastID
   for (let i = 0; i < STUDENT_COUNT; i++) {
-    students.push(
+    studentsPromise.push(
+      (async () => {
       await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
@@ -41,14 +40,18 @@ async function run() {
         "Home Number": generator.phone(),
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
-    )
+
     console.log(
       `Student ${i + 1} of ${STUDENT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
+        )
+      })()
     )
   }
 
+  const students = await Promise.all(studentsPromise)
+
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
       Name: {
@@ -60,19 +63,24 @@ async function run() {
     primaryDisplay: "Name",
   })
 
+  const subjectPromises = []
   for (let i = 0; i < SUBJECT_COUNT; i++) {
-    subjects.push(
+    subjectPromises.push(
+      (async () => {
       await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
-    )
     console.log(
       `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
+        )
+      })()
     )
   }
 
+  const subjects = await Promise.all(subjectPromises)
+
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {
       Score: {
@@ -124,7 +132,7 @@ async function run() {
 
 run()
   .then(() => {
-    console.log(`Done in(${(Date.now() - start) / 1000} seconds`)
+    console.log(`Done in ${(Date.now() - start) / 1000} seconds`)
   })
   .catch(err => {
     console.error(err)

From ed32b701ecd82bd0d1971839a830ec0b46b3c4df Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 15:41:25 +0100
Subject: [PATCH 09/31] Move row save to sdk

---
 .../src/api/controllers/row/external.ts       | 46 +---------------
 .../server/src/api/controllers/row/index.ts   | 11 ++--
 .../src/api/controllers/row/internal.ts       | 41 +-------------
 packages/server/src/sdk/app/rows/external.ts  | 54 ++++++++++++++++++-
 packages/server/src/sdk/app/rows/internal.ts  | 49 +++++++++++++++++
 packages/server/src/sdk/app/rows/rows.ts      | 18 +++++++
 6 files changed, 129 insertions(+), 90 deletions(-)
 create mode 100644 packages/server/src/sdk/app/rows/internal.ts

diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts
index 287b2ae6aa..d741247687 100644
--- a/packages/server/src/api/controllers/row/external.ts
+++ b/packages/server/src/api/controllers/row/external.ts
@@ -26,7 +26,7 @@ import {
   inputProcessing,
   outputProcessing,
 } from "../../../utilities/rowProcessor"
-import { cloneDeep, isEqual } from "lodash"
+import { cloneDeep } from "lodash"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
@@ -86,50 +86,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   }
 }
 
-export async function save(ctx: UserCtx) {
-  const inputs = ctx.request.body
-  const tableId = utils.getTableId(ctx)
-
-  const table = await sdk.tables.getTable(tableId)
-  const { table: updatedTable, row } = await inputProcessing(
-    ctx.user?._id,
-    cloneDeep(table),
-    inputs
-  )
-
-  const validateResult = await sdk.rows.utils.validate({
-    row,
-    tableId,
-  })
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
-
-  const response = await handleRequest(Operation.CREATE, tableId, {
-    row,
-  })
-
-  if (!isEqual(table, updatedTable)) {
-    await sdk.tables.saveTable(updatedTable)
-  }
-
-  const rowId = response.row._id
-  if (rowId) {
-    const row = await sdk.rows.external.getRow(tableId, rowId, {
-      relationships: true,
-    })
-    return {
-      ...response,
-      row: await outputProcessing(table, row, {
-        preserveLinks: true,
-        squash: true,
-      }),
-    }
-  } else {
-    return response
-  }
-}
-
 export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts
index 38731a87e1..7ff8d83e71 100644
--- a/packages/server/src/api/controllers/row/index.ts
+++ b/packages/server/src/api/controllers/row/index.ts
@@ -30,7 +30,7 @@ import { Format } from "../view/exporters"
 
 export * as views from "./views"
 
-function pickApi(tableId: any) {
+function pickApi(tableId: string) {
   if (isExternalTableID(tableId)) {
     return external
   }
@@ -84,9 +84,12 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
     return patch(ctx as UserCtx<PatchRowRequest, PatchRowResponse>)
   }
   const { row, table, squashed } = await quotas.addRow(() =>
-    quotas.addQuery(() => pickApi(tableId).save(ctx), {
-      datasourceId: tableId,
-    })
+    quotas.addQuery(
+      () => sdk.rows.save(tableId, ctx.request.body, ctx.user?._id),
+      {
+        datasourceId: tableId,
+      }
+    )
   )
   ctx.status = 200
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts
index a251724b0a..5c714c098a 100644
--- a/packages/server/src/api/controllers/row/internal.ts
+++ b/packages/server/src/api/controllers/row/internal.ts
@@ -1,5 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import { generateRowID, InternalTables } from "../../../db/utils"
+import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
   AttachmentCleanup,
@@ -94,45 +94,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   })
 }
 
-export async function save(ctx: UserCtx) {
-  let inputs = ctx.request.body
-  inputs.tableId = utils.getTableId(ctx)
-
-  if (!inputs._rev && !inputs._id) {
-    inputs._id = generateRowID(inputs.tableId)
-  }
-
-  // this returns the table and row incase they have been updated
-  const dbTable = await sdk.tables.getTable(inputs.tableId)
-
-  // need to copy the table so it can be differenced on way out
-  const tableClone = cloneDeep(dbTable)
-
-  let { table, row } = await inputProcessing(ctx.user?._id, tableClone, inputs)
-
-  const validateResult = await sdk.rows.utils.validate({
-    row,
-    table,
-  })
-
-  if (!validateResult.valid) {
-    throw { validation: validateResult.errors }
-  }
-
-  // make sure link rows are up-to-date
-  row = (await linkRows.updateLinks({
-    eventType: linkRows.EventType.ROW_SAVE,
-    row,
-    tableId: row.tableId,
-    table,
-  })) as Row
-
-  return finaliseRow(table, row, {
-    oldTable: dbTable,
-    updateFormula: true,
-  })
-}
-
 export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
diff --git a/packages/server/src/sdk/app/rows/external.ts b/packages/server/src/sdk/app/rows/external.ts
index beae02e134..7ad5ea37ff 100644
--- a/packages/server/src/sdk/app/rows/external.ts
+++ b/packages/server/src/sdk/app/rows/external.ts
@@ -1,6 +1,13 @@
-import { IncludeRelationship, Operation } from "@budibase/types"
+import { IncludeRelationship, Operation, Row } from "@budibase/types"
 import { handleRequest } from "../../../api/controllers/row/external"
 import { breakRowIdField } from "../../../integrations/utils"
+import sdk from "../../../sdk"
+import {
+  inputProcessing,
+  outputProcessing,
+} from "../../../utilities/rowProcessor"
+import cloneDeep from "lodash/fp/cloneDeep"
+import isEqual from "lodash/fp/isEqual"
 
 export async function getRow(
   tableId: string,
@@ -15,3 +22,48 @@ export async function getRow(
   })
   return response ? response[0] : response
 }
+
+export async function save(
+  tableId: string,
+  inputs: Row,
+  userId: string | undefined
+) {
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = await inputProcessing(
+    userId,
+    cloneDeep(table),
+    inputs
+  )
+
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
+  const response = await handleRequest(Operation.CREATE, tableId, {
+    row,
+  })
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
+  const rowId = response.row._id
+  if (rowId) {
+    const row = await sdk.rows.external.getRow(tableId, rowId, {
+      relationships: true,
+    })
+    return {
+      ...response,
+      row: await outputProcessing(table, row, {
+        preserveLinks: true,
+        squash: true,
+      }),
+    }
+  } else {
+    return response
+  }
+}
diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts
new file mode 100644
index 0000000000..c735d3e430
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/internal.ts
@@ -0,0 +1,49 @@
+import { db } from "@budibase/backend-core"
+import { Row } from "@budibase/types"
+import sdk from "../../../sdk"
+import cloneDeep from "lodash/fp/cloneDeep"
+import { finaliseRow } from "src/api/controllers/row/staticFormula"
+import { inputProcessing } from "src/utilities/rowProcessor"
+import * as linkRows from "../../../db/linkedRows"
+
+export async function save(
+  tableId: string,
+  inputs: Row,
+  userId: string | undefined
+) {
+  inputs.tableId = tableId
+
+  if (!inputs._rev && !inputs._id) {
+    inputs._id = db.generateRowID(inputs.tableId)
+  }
+
+  // this returns the table and row incase they have been updated
+  const dbTable = await sdk.tables.getTable(inputs.tableId)
+
+  // need to copy the table so it can be differenced on way out
+  const tableClone = cloneDeep(dbTable)
+
+  let { table, row } = await inputProcessing(userId, tableClone, inputs)
+
+  const validateResult = await sdk.rows.utils.validate({
+    row,
+    table,
+  })
+
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
+
+  // make sure link rows are up-to-date
+  row = (await linkRows.updateLinks({
+    eventType: linkRows.EventType.ROW_SAVE,
+    row,
+    tableId: row.tableId,
+    table,
+  })) as Row
+
+  return finaliseRow(table, row, {
+    oldTable: dbTable,
+    updateFormula: true,
+  })
+}
diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts
index 8709180f0b..bfd84a715c 100644
--- a/packages/server/src/sdk/app/rows/rows.ts
+++ b/packages/server/src/sdk/app/rows/rows.ts
@@ -1,6 +1,9 @@
 import { db as dbCore, context } from "@budibase/backend-core"
 import { Database, Row } from "@budibase/types"
 import { getRowParams } from "../../../db/utils"
+import { isExternalTableID } from "../../../integrations/utils"
+import * as internal from "./internal"
+import * as external from "./external"
 
 export async function getAllInternalRows(appId?: string) {
   let db: Database
@@ -16,3 +19,18 @@ export async function getAllInternalRows(appId?: string) {
   )
   return response.rows.map(row => row.doc) as Row[]
 }
+
+function pickApi(tableId: any) {
+  if (isExternalTableID(tableId)) {
+    return external
+  }
+  return internal
+}
+
+export async function save(
+  tableId: string,
+  row: Row,
+  userId: string | undefined
+) {
+  return pickApi(tableId).save(tableId, row, userId)
+}

From 0452b1a307967184f02e8074ae1321d4fd3725a8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 16:25:08 +0100
Subject: [PATCH 10/31] Fix imports

---
 packages/server/src/sdk/app/rows/internal.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/internal.ts b/packages/server/src/sdk/app/rows/internal.ts
index c735d3e430..14e771b36e 100644
--- a/packages/server/src/sdk/app/rows/internal.ts
+++ b/packages/server/src/sdk/app/rows/internal.ts
@@ -2,8 +2,8 @@ import { db } from "@budibase/backend-core"
 import { Row } from "@budibase/types"
 import sdk from "../../../sdk"
 import cloneDeep from "lodash/fp/cloneDeep"
-import { finaliseRow } from "src/api/controllers/row/staticFormula"
-import { inputProcessing } from "src/utilities/rowProcessor"
+import { finaliseRow } from "../../../api/controllers/row/staticFormula"
+import { inputProcessing } from "../../../utilities/rowProcessor"
 import * as linkRows from "../../../db/linkedRows"
 
 export async function save(

From becb7bd46d3b6c5f829359da3ee0d72d5d31fab7 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 17:26:51 +0100
Subject: [PATCH 11/31] Add tests

---
 .../src/sdk/app/rows/tests/internal.spec.ts   | 127 ++++++++++++++++++
 1 file changed, 127 insertions(+)
 create mode 100644 packages/server/src/sdk/app/rows/tests/internal.spec.ts

diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
new file mode 100644
index 0000000000..0d19543889
--- /dev/null
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -0,0 +1,127 @@
+import * as internalSdk from "../internal"
+
+import { generator } from "@budibase/backend-core/tests"
+import {
+  INTERNAL_TABLE_SOURCE_ID,
+  TableSourceType,
+  FieldType,
+  Table,
+} from "@budibase/types"
+
+import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+
+describe("sdk >> rows >> internal", () => {
+  const config = new TestConfiguration()
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  function makeRow() {
+    return {
+      name: generator.first(),
+      surname: generator.last(),
+      age: generator.age(),
+      address: generator.address(),
+    }
+  }
+
+  describe("save", () => {
+    const tableData: Table = {
+      name: generator.word(),
+      type: "table",
+      sourceId: INTERNAL_TABLE_SOURCE_ID,
+      sourceType: TableSourceType.INTERNAL,
+      schema: {
+        id: {
+          name: "id",
+          type: FieldType.AUTO,
+          autocolumn: true,
+          lastID: 0,
+        },
+        name: {
+          name: "name",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+        surname: {
+          name: "surname",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+        age: {
+          name: "age",
+          type: FieldType.NUMBER,
+          constraints: {
+            type: FieldType.NUMBER,
+          },
+        },
+        address: {
+          name: "address",
+          type: FieldType.STRING,
+          constraints: {
+            type: FieldType.STRING,
+          },
+        },
+      },
+    }
+
+    beforeEach(() => {
+      jest.clearAllMocks()
+    })
+
+    it("save will persist the row properly", async () => {
+      const table = await config.createTable(tableData)
+      const row = makeRow()
+
+      await config.doInContext(config.appId, async () => {
+        const response = await internalSdk.save(
+          table._id!,
+          row,
+          config.user._id
+        )
+
+        expect(response).toEqual({
+          table,
+          row: {
+            ...row,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+          squashed: {
+            ...row,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+        })
+
+        const persistedRow = await config.getRow(table._id!, response.row._id!)
+        expect(persistedRow).toEqual({
+          ...row,
+          type: "row",
+          _rev: expect.stringMatching("1-.*"),
+          createdAt: expect.any(String),
+          updatedAt: expect.any(String),
+        })
+      })
+    })
+
+    it.only("can persist in parallel", async () => {
+      const table = await config.createTable(tableData)
+      const rows = Array.from({ length: 100 }, () => makeRow())
+
+      await config.doInContext(config.appId, async () => {
+        await Promise.all(
+          rows.map(row => internalSdk.save(table._id!, row, config.user._id))
+        )
+
+        const persistedRows = await config.getRows(table._id!)
+        expect(persistedRows).toHaveLength(100)
+      })
+    })
+  })
+})

From 3824156ca5b90908d5a078ffb5d375194dd31ba8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Wed, 20 Dec 2023 17:36:23 +0100
Subject: [PATCH 12/31] Add delete all apps script

---
 .../server/scripts/load/delete-all-apps.js    | 29 +++++++++++++++++++
 packages/server/scripts/load/utils.js         | 11 +++++++
 2 files changed, 40 insertions(+)
 create mode 100755 packages/server/scripts/load/delete-all-apps.js

diff --git a/packages/server/scripts/load/delete-all-apps.js b/packages/server/scripts/load/delete-all-apps.js
new file mode 100755
index 0000000000..5c9e974c7d
--- /dev/null
+++ b/packages/server/scripts/load/delete-all-apps.js
@@ -0,0 +1,29 @@
+#!/bin/node
+const { searchApps, deleteApp } = require("./utils")
+
+if (!process.argv[2]) {
+  console.error("Please specify an API key as script argument.")
+  process.exit(-1)
+}
+
+async function run() {
+  const apiKey = process.argv[2]
+  const apps = await searchApps(apiKey)
+  console.log(`Deleting ${apps.length} apps`)
+
+  let deletedApps = 0
+  await Promise.all(
+    apps.map(async app => {
+      await deleteApp(apiKey, app._id)
+      console.log(`App ${++deletedApps} of ${apps.length} deleted`)
+    })
+  )
+}
+
+run()
+  .then(() => {
+    console.log("Done!")
+  })
+  .catch(err => {
+    console.error(err)
+  })
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 7af867ab1d..04ebb6f3d6 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -38,6 +38,17 @@ exports.createApp = async apiKey => {
   return json.data
 }
 
+exports.searchApps = async apiKey => {
+  const res = await request(apiKey, `${URL_APP}/search`, "POST", {})
+  const json = await res.json()
+  return json.data
+}
+
+exports.deleteApp = async (apiKey, appId) => {
+  const res = await request(apiKey, `${URL_APP}/${appId}`, "DELETE")
+  return res
+}
+
 exports.getTable = async (apiKey, appId) => {
   const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()

From f722ae970cdd79f7cbdd7bf7c4d1c6182932b91b Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 10:30:27 +0100
Subject: [PATCH 13/31] Add autoid tests (failing)

---
 .../src/sdk/app/rows/tests/internal.spec.ts   | 112 ++++++++++++++++--
 1 file changed, 99 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
index 0d19543889..62a056bef3 100644
--- a/packages/server/src/sdk/app/rows/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -1,3 +1,4 @@
+import tk from "timekeeper"
 import * as internalSdk from "../internal"
 
 import { generator } from "@budibase/backend-core/tests"
@@ -6,9 +7,13 @@ import {
   TableSourceType,
   FieldType,
   Table,
+  AutoFieldSubTypes,
 } from "@budibase/types"
 
 import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
+import { cache } from "@budibase/backend-core"
+
+tk.freeze(Date.now())
 
 describe("sdk >> rows >> internal", () => {
   const config = new TestConfiguration()
@@ -33,12 +38,6 @@ describe("sdk >> rows >> internal", () => {
       sourceId: INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
       schema: {
-        id: {
-          name: "id",
-          type: FieldType.AUTO,
-          autocolumn: true,
-          lastID: 0,
-        },
         name: {
           name: "name",
           type: FieldType.STRING,
@@ -110,18 +109,105 @@ describe("sdk >> rows >> internal", () => {
       })
     })
 
-    it.only("can persist in parallel", async () => {
-      const table = await config.createTable(tableData)
-      const rows = Array.from({ length: 100 }, () => makeRow())
+    it("auto ids will update when creating new rows", async () => {
+      const table = await config.createTable({
+        ...tableData,
+        schema: {
+          ...tableData.schema,
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubTypes.AUTO_ID,
+            autocolumn: true,
+            lastID: 0,
+          },
+        },
+      })
+      const row = makeRow()
 
       await config.doInContext(config.appId, async () => {
-        await Promise.all(
-          rows.map(row => internalSdk.save(table._id!, row, config.user._id))
+        const response = await internalSdk.save(
+          table._id!,
+          row,
+          config.user._id
         )
 
-        const persistedRows = await config.getRows(table._id!)
-        expect(persistedRows).toHaveLength(100)
+        expect(response).toEqual({
+          table: {
+            ...table,
+            schema: {
+              ...table.schema,
+              id: {
+                ...table.schema.id,
+                lastID: 1,
+              },
+            },
+          },
+          row: {
+            ...row,
+            id: 1,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+          squashed: {
+            ...row,
+            id: 1,
+            type: "row",
+            _rev: expect.stringMatching("1-.*"),
+          },
+        })
+
+        const persistedRow = await config.getRow(table._id!, response.row._id!)
+        expect(persistedRow).toEqual({
+          ...row,
+          type: "row",
+          id: 1,
+          _rev: expect.stringMatching("1-.*"),
+          createdAt: expect.any(String),
+          updatedAt: expect.any(String),
+        })
       })
     })
+
+    it("auto ids will update when creating new rows in parallel", async () => {
+      function makeRows(count: number) {
+        return Array.from({ length: count }, () => makeRow())
+      }
+
+      const table = await config.createTable({
+        ...tableData,
+        schema: {
+          ...tableData.schema,
+          id: {
+            name: "id",
+            type: FieldType.AUTO,
+            subtype: AutoFieldSubTypes.AUTO_ID,
+            autocolumn: true,
+            lastID: 0,
+          },
+        },
+      })
+
+      await config.doInContext(config.appId, async () => {
+        for (const row of makeRows(30)) {
+          await internalSdk.save(table._id!, row, config.user._id)
+        }
+        await Promise.all(
+          makeRows(200).map(row =>
+            internalSdk.save(table._id!, row, config.user._id)
+          )
+        )
+        for (const row of makeRows(20)) {
+          await internalSdk.save(table._id!, row, config.user._id)
+        }
+      })
+
+      const persistedRows = await config.getRows(table._id!)
+      expect(persistedRows).toHaveLength(250)
+
+      const persistedTable = await config.getTable(table._id)
+      expect((table as any).schema.id.lastID).toBe(0)
+      expect(persistedTable.schema.id.lastID).toBe(250)
+    })
   })
 })

From c0a19c2a7ef5a7319997ec019caa3be7c5895dda Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 15:45:30 +0100
Subject: [PATCH 14/31] Type cache arguments

---
 packages/backend-core/src/cache/generic.ts | 23 +++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/packages/backend-core/src/cache/generic.ts b/packages/backend-core/src/cache/generic.ts
index 7a2be5a0f0..3ac323a8d4 100644
--- a/packages/backend-core/src/cache/generic.ts
+++ b/packages/backend-core/src/cache/generic.ts
@@ -18,14 +18,15 @@ export enum TTL {
   ONE_DAY = 86400,
 }
 
-function performExport(funcName: string) {
-  // @ts-ignore
-  return (...args: any) => GENERIC[funcName](...args)
-}
-
-export const keys = performExport("keys")
-export const get = performExport("get")
-export const store = performExport("store")
-export const destroy = performExport("delete")
-export const withCache = performExport("withCache")
-export const bustCache = performExport("bustCache")
+export const keys = (...args: Parameters<typeof GENERIC.keys>) =>
+  GENERIC.keys(...args)
+export const get = (...args: Parameters<typeof GENERIC.get>) =>
+  GENERIC.get(...args)
+export const store = (...args: Parameters<typeof GENERIC.store>) =>
+  GENERIC.store(...args)
+export const destroy = (...args: Parameters<typeof GENERIC.delete>) =>
+  GENERIC.delete(...args)
+export const withCache = (...args: Parameters<typeof GENERIC.withCache>) =>
+  GENERIC.withCache(...args)
+export const bustCache = (...args: Parameters<typeof GENERIC.bustCache>) =>
+  GENERIC.bustCache(...args)

From 0c77cf2b40d6fcec40de872934e7a90e79278ea8 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:14:34 +0100
Subject: [PATCH 15/31] Fix concurrent saves

---
 .../src/api/controllers/row/staticFormula.ts  | 26 +++++++++++++------
 .../src/sdk/app/rows/tests/internal.spec.ts   | 17 ++++++++----
 packages/types/src/sdk/locks.ts               |  1 +
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts
index 8d52b6a05c..3cb7348ee3 100644
--- a/packages/server/src/api/controllers/row/staticFormula.ts
+++ b/packages/server/src/api/controllers/row/staticFormula.ts
@@ -5,8 +5,8 @@ import {
   processFormulas,
 } from "../../../utilities/rowProcessor"
 import { FieldTypes, FormulaTypes } from "../../../constants"
-import { context } from "@budibase/backend-core"
-import { Table, Row } from "@budibase/types"
+import { context, locks } from "@budibase/backend-core"
+import { Table, Row, LockType, LockName } from "@budibase/types"
 import * as linkRows from "../../../db/linkedRows"
 import sdk from "../../../sdk"
 import isEqual from "lodash/isEqual"
@@ -149,12 +149,22 @@ export async function finaliseRow(
       await db.put(table)
     } catch (err: any) {
       if (err.status === 409) {
-        const updatedTable = await sdk.tables.getTable(table._id!)
-        let response = processAutoColumn(null, updatedTable, row, {
-          reprocessing: true,
-        })
-        await db.put(response.table)
-        row = response.row
+        // Some conflicts with the autocolumns occurred, we need to refetch the table and recalculate
+        await locks.doWithLock(
+          {
+            type: LockType.AUTO_EXTEND,
+            name: LockName.PROCESS_AUTO_COLUMNS,
+            resource: table._id,
+          },
+          async () => {
+            const latestTable = await sdk.tables.getTable(table._id!)
+            let response = processAutoColumn(null, latestTable, row, {
+              reprocessing: true,
+            })
+            await db.put(response.table)
+            row = response.row
+          }
+        )
       } else {
         throw err
       }
diff --git a/packages/server/src/sdk/app/rows/tests/internal.spec.ts b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
index 62a056bef3..b60d31b226 100644
--- a/packages/server/src/sdk/app/rows/tests/internal.spec.ts
+++ b/packages/server/src/sdk/app/rows/tests/internal.spec.ts
@@ -189,25 +189,32 @@ describe("sdk >> rows >> internal", () => {
       })
 
       await config.doInContext(config.appId, async () => {
-        for (const row of makeRows(30)) {
+        for (const row of makeRows(5)) {
           await internalSdk.save(table._id!, row, config.user._id)
         }
         await Promise.all(
-          makeRows(200).map(row =>
+          makeRows(10).map(row =>
             internalSdk.save(table._id!, row, config.user._id)
           )
         )
-        for (const row of makeRows(20)) {
+        for (const row of makeRows(5)) {
           await internalSdk.save(table._id!, row, config.user._id)
         }
       })
 
       const persistedRows = await config.getRows(table._id!)
-      expect(persistedRows).toHaveLength(250)
+      expect(persistedRows).toHaveLength(20)
+      expect(persistedRows).toEqual(
+        expect.arrayContaining(
+          Array.from({ length: 20 }).map((_, i) =>
+            expect.objectContaining({ id: i + 1 })
+          )
+        )
+      )
 
       const persistedTable = await config.getTable(table._id)
       expect((table as any).schema.id.lastID).toBe(0)
-      expect(persistedTable.schema.id.lastID).toBe(250)
+      expect(persistedTable.schema.id.lastID).toBe(20)
     })
   })
 })
diff --git a/packages/types/src/sdk/locks.ts b/packages/types/src/sdk/locks.ts
index 82a7089b3f..0e6053a4db 100644
--- a/packages/types/src/sdk/locks.ts
+++ b/packages/types/src/sdk/locks.ts
@@ -21,6 +21,7 @@ export enum LockName {
   PERSIST_WRITETHROUGH = "persist_writethrough",
   QUOTA_USAGE_EVENT = "quota_usage_event",
   APP_MIGRATION = "app_migrations",
+  PROCESS_AUTO_COLUMNS = "process_auto_columns",
 }
 
 export type LockOptions = {

From 69527cd4b964ab6fffdf7009161ecf9ec3c493be Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:22:41 +0100
Subject: [PATCH 16/31] Parallel creations

---
 .../scripts/load/create-many-relationships.js | 30 +++++++------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 49d4efb28d..dba953e4c7 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -25,11 +25,10 @@ async function run() {
   }
   console.log(`Table found: ${studentsTable.name}`)
 
-  const studentsPromise = []
   let studentNumber = studentsTable.schema["Auto ID"].lastID
-  for (let i = 0; i < STUDENT_COUNT; i++) {
-    studentsPromise.push(
-      (async () => {
+  let i = 0
+  const students = await Promise.all(
+    Array.from({ length: STUDENT_COUNT }).map(async () => {
       await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
@@ -42,15 +41,12 @@ async function run() {
       })
 
     console.log(
-      `Student ${i + 1} of ${STUDENT_COUNT} created (${
+        `Student ${++i} of ${STUDENT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
         )
-      })()
+    })
     )
-  }
-
-  const students = await Promise.all(studentsPromise)
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -63,23 +59,19 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  const subjectPromises = []
-  for (let i = 0; i < SUBJECT_COUNT; i++) {
-    subjectPromises.push(
-      (async () => {
+  i = 0
+  const subjects = await Promise.all(
+    Array.from({ length: SUBJECT_COUNT }).map(async () => {
       await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
     console.log(
-      `Subject ${i + 1} of ${SUBJECT_COUNT} created (${
+        `Subject ${++i} of ${SUBJECT_COUNT} created (${
         (Date.now() - start) / 1000
       }s)`
         )
-      })()
+    })
     )
-  }
-
-  const subjects = await Promise.all(subjectPromises)
 
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {
@@ -113,7 +105,7 @@ async function run() {
     name: "Grades",
   })
 
-  let i = 0
+  i = 0
   for (const student of students) {
     for (const subject of subjects) {
       await createRow(apiKey, app._id, gradesTable, {

From b034542536b9a2b922bed014697447ed50c9b85c Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:24:48 +0100
Subject: [PATCH 17/31] Fix

---
 .../scripts/load/create-many-relationships.js | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index dba953e4c7..c1e59bd586 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -29,7 +29,7 @@ async function run() {
   let i = 0
   const students = await Promise.all(
     Array.from({ length: STUDENT_COUNT }).map(async () => {
-      await createRow(apiKey, app._id, studentsTable, {
+      const row = await createRow(apiKey, app._id, studentsTable, {
         "Student Number": (++studentNumber).toString(),
         "First Name": generator.first(),
         "Last Name": generator.last(),
@@ -40,13 +40,14 @@ async function run() {
         "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
       })
 
-    console.log(
+      console.log(
         `Student ${++i} of ${STUDENT_COUNT} created (${
-        (Date.now() - start) / 1000
-      }s)`
-        )
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
     })
-    )
+  )
 
   const subjectTable = await createTable(apiKey, app._id, {
     schema: {
@@ -62,16 +63,17 @@ async function run() {
   i = 0
   const subjects = await Promise.all(
     Array.from({ length: SUBJECT_COUNT }).map(async () => {
-      await createRow(apiKey, app._id, subjectTable, {
+      const row = await createRow(apiKey, app._id, subjectTable, {
         Name: generator.profession(),
       })
-    console.log(
+      console.log(
         `Subject ${++i} of ${SUBJECT_COUNT} created (${
-        (Date.now() - start) / 1000
-      }s)`
-        )
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
     })
-    )
+  )
 
   const gradesTable = await createTable(apiKey, app._id, {
     schema: {

From c46509d6cb2673cb8738a63c67ea8c65e85eeeeb Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:39:14 +0100
Subject: [PATCH 18/31] Use batches

---
 .../scripts/load/create-many-relationships.js | 80 +++++++++++--------
 1 file changed, 48 insertions(+), 32 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index c1e59bd586..f034cb0cc2 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -7,6 +7,8 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
+const batchSize = 100
+
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
@@ -14,6 +16,31 @@ if (!process.argv[2]) {
 
 const start = Date.now()
 
+async function batchCreate(apiKey, appId, count, table, generator) {
+  let i = 0
+
+  async function createSingleRow() {
+    const row = await createRow(apiKey, appId, table, generator())
+    console.log(
+      `${table.name} - ${++i} of ${count} created (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
+    return row
+  }
+
+  const rows = []
+  for (let j = 0; j < count; j += batchSize) {
+    const batchPromises = Array.from(
+      { length: Math.min(batchSize, count - j) },
+      createSingleRow
+    )
+    const batchRows = await Promise.all(batchPromises)
+    rows.push(...batchRows)
+  }
+  return rows
+}
+
 async function run() {
   const apiKey = process.argv[2]
   const app = await createApp(apiKey)
@@ -26,26 +53,20 @@ async function run() {
   console.log(`Table found: ${studentsTable.name}`)
 
   let studentNumber = studentsTable.schema["Auto ID"].lastID
-  let i = 0
-  const students = await Promise.all(
-    Array.from({ length: STUDENT_COUNT }).map(async () => {
-      const row = await createRow(apiKey, app._id, studentsTable, {
-        "Student Number": (++studentNumber).toString(),
-        "First Name": generator.first(),
-        "Last Name": generator.last(),
-        Gender: generator.pickone(["M", "F"]),
-        Grade: generator.pickone(["8", "9", "10", "11"]),
-        "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
-        "Home Number": generator.phone(),
-        "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
-      })
-
-      console.log(
-        `Student ${++i} of ${STUDENT_COUNT} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-      return row
+  const students = await batchCreate(
+    apiKey,
+    app._id,
+    STUDENT_COUNT,
+    studentsTable,
+    () => ({
+      "Student Number": (++studentNumber).toString(),
+      "First Name": generator.first(),
+      "Last Name": generator.last(),
+      Gender: generator.pickone(["M", "F"]),
+      Grade: generator.pickone(["8", "9", "10", "11"]),
+      "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
+      "Home Number": generator.phone(),
+      "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
     })
   )
 
@@ -60,18 +81,13 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  i = 0
-  const subjects = await Promise.all(
-    Array.from({ length: SUBJECT_COUNT }).map(async () => {
-      const row = await createRow(apiKey, app._id, subjectTable, {
-        Name: generator.profession(),
-      })
-      console.log(
-        `Subject ${++i} of ${SUBJECT_COUNT} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-      return row
+  const subjects = await batchCreate(
+    apiKey,
+    app._id,
+    SUBJECT_COUNT,
+    subjectTable,
+    () => ({
+      Name: generator.profession(),
     })
   )
 

From dbf4d6dbdd5d76fb52b43ae4434f2e719ae9dddf Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:48:39 +0100
Subject: [PATCH 19/31] Create grades in batch

---
 .../scripts/load/create-many-relationships.js | 48 ++++++++-----------
 1 file changed, 19 insertions(+), 29 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index f034cb0cc2..a90c21d250 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -7,8 +7,6 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-const batchSize = 100
-
 if (!process.argv[2]) {
   console.error("Please specify an API key as script argument.")
   process.exit(-1)
@@ -16,13 +14,13 @@ if (!process.argv[2]) {
 
 const start = Date.now()
 
-async function batchCreate(apiKey, appId, count, table, generator) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
-  async function createSingleRow() {
-    const row = await createRow(apiKey, appId, table, generator())
+  async function createSingleRow(item) {
+    const row = await createRow(apiKey, appId, table, item)
     console.log(
-      `${table.name} - ${++i} of ${count} created (${
+      `${table.name} - ${++i} of ${items.length} created (${
         (Date.now() - start) / 1000
       }s)`
     )
@@ -30,11 +28,8 @@ async function batchCreate(apiKey, appId, count, table, generator) {
   }
 
   const rows = []
-  for (let j = 0; j < count; j += batchSize) {
-    const batchPromises = Array.from(
-      { length: Math.min(batchSize, count - j) },
-      createSingleRow
-    )
+  for (let j = 0; j < items.length; j += batchSize) {
+    const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
     const batchRows = await Promise.all(batchPromises)
     rows.push(...batchRows)
   }
@@ -56,9 +51,8 @@ async function run() {
   const students = await batchCreate(
     apiKey,
     app._id,
-    STUDENT_COUNT,
     studentsTable,
-    () => ({
+    Array.from({ length: SUBJECT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
       "First Name": generator.first(),
       "Last Name": generator.last(),
@@ -67,7 +61,7 @@ async function run() {
       "Tardiness (Days)": generator.integer({ min: 1, max: 100 }),
       "Home Number": generator.phone(),
       "Attendance_(%)": generator.integer({ min: 0, max: 100 }),
-    })
+    }))
   )
 
   const subjectTable = await createTable(apiKey, app._id, {
@@ -84,11 +78,10 @@ async function run() {
   const subjects = await batchCreate(
     apiKey,
     app._id,
-    SUBJECT_COUNT,
     subjectTable,
-    () => ({
+    Array.from({ length: SUBJECT_COUNT }).map(() => ({
       Name: generator.profession(),
-    })
+    }))
   )
 
   const gradesTable = await createTable(apiKey, app._id, {
@@ -123,21 +116,18 @@ async function run() {
     name: "Grades",
   })
 
-  i = 0
-  for (const student of students) {
-    for (const subject of subjects) {
-      await createRow(apiKey, app._id, gradesTable, {
+  await batchCreate(
+    apiKey,
+    app._id,
+    gradesTable,
+    students.flatMap(student =>
+      subjects.map(subject => ({
         Score: generator.integer({ min: 0, max: 100 }),
         Student: [student],
         Subject: [subject],
-      })
-      console.log(
-        `Grade ${++i} of ${students.length * subjects.length} created (${
-          (Date.now() - start) / 1000
-        }s)`
-      )
-    }
-  }
+      }))
+    )
+  )
 }
 
 run()

From 3eee77c5498328547c5b7c5ff9368eda5757718d Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 17:52:04 +0100
Subject: [PATCH 20/31] Create 500 students

---
 packages/server/scripts/load/create-many-relationships.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index a90c21d250..4626b8fe24 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -52,7 +52,7 @@ async function run() {
     apiKey,
     app._id,
     studentsTable,
-    Array.from({ length: SUBJECT_COUNT }).map(() => ({
+    Array.from({ length: STUDENT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
       "First Name": generator.first(),
       "Last Name": generator.last(),

From c518d8a7fad730d9c7798080840dcf0abfb3bbf1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:02:44 +0100
Subject: [PATCH 21/31] Add logs

---
 packages/server/scripts/load/create-many-relationships.js | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 4626b8fe24..92db2f83a1 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -128,6 +128,10 @@ async function run() {
       }))
     )
   )
+
+  console.log(
+    `Access the app here: http://localhost:10000/builder/app/${app._id}`
+  )
 }
 
 run()

From 92f1107e6caf686bc24820f0a11527299de3fbb1 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:28:10 +0100
Subject: [PATCH 22/31] Prevent one single request to stop the process

---
 .../scripts/load/create-many-relationships.js  | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 92db2f83a1..d7526a70ff 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -18,13 +18,17 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
   async function createSingleRow(item) {
-    const row = await createRow(apiKey, appId, table, item)
-    console.log(
-      `${table.name} - ${++i} of ${items.length} created (${
-        (Date.now() - start) / 1000
-      }s)`
-    )
-    return row
+    try {
+      const row = await createRow(apiKey, appId, table, item)
+      console.log(
+        `${table.name} - ${++i} of ${items.length} created (${
+          (Date.now() - start) / 1000
+        }s)`
+      )
+      return row
+    } catch {
+      console.error("Error creating row", item)
+    }
   }
 
   const rows = []

From 494a2ff91f5b87e85658f1ffcd06c1da9887de89 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 18:29:57 +0100
Subject: [PATCH 23/31] Errors

---
 packages/server/scripts/load/create-many-relationships.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index d7526a70ff..15ef9bcb1e 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -17,6 +17,7 @@ const start = Date.now()
 async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
 
+  let errors = 0
   async function createSingleRow(item) {
     try {
       const row = await createRow(apiKey, appId, table, item)
@@ -27,7 +28,7 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
       )
       return row
     } catch {
-      console.error("Error creating row", item)
+      errors++
     }
   }
 
@@ -37,6 +38,9 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
     const batchRows = await Promise.all(batchPromises)
     rows.push(...batchRows)
   }
+  if (errors) {
+    console.error(`Error creating ${errors} row`)
+  }
   return rows
 }
 

From abf025b3f7e5fbe2458b53995f362268323cb376 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 19:06:26 +0100
Subject: [PATCH 24/31] Use inflight max instead of batch wait

---
 .../scripts/load/create-many-relationships.js | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 15ef9bcb1e..0f4fe3470c 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -13,11 +13,10 @@ if (!process.argv[2]) {
 }
 
 const start = Date.now()
-
-async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
   let i = 0
-
   let errors = 0
+
   async function createSingleRow(item) {
     try {
       const row = await createRow(apiKey, appId, table, item)
@@ -33,14 +32,25 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   }
 
   const rows = []
-  for (let j = 0; j < items.length; j += batchSize) {
-    const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
-    const batchRows = await Promise.all(batchPromises)
-    rows.push(...batchRows)
-  }
-  if (errors) {
-    console.error(`Error creating ${errors} row`)
+  const maxConcurrency = Math.min(batchSize, items.length)
+  const inFlight = {}
+
+  for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+    const item = items[itemIndex]
+    const promise = createSingleRow(item).then(result => {
+      rows.push(result)
+      delete inFlight[itemIndex]
+    })
+
+    inFlight[itemIndex] = promise
+
+    if (Object.keys(inFlight).length >= maxConcurrency) {
+      await Promise.race(Object.values(inFlight))
+    }
   }
+
+  await Promise.all(Object.values(inFlight))
+
   return rows
 }
 

From 31c01e0a6b8338aa6c58772ff572e08c16bf6620 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 10:05:49 +0100
Subject: [PATCH 25/31] Display errors

---
 .../server/scripts/load/create-many-relationships.js  | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 0f4fe3470c..14eaccd99c 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -2,6 +2,7 @@
 const { createApp, getTable, createRow, createTable } = require("./utils")
 
 const Chance = require("chance")
+
 const generator = new Chance()
 
 const STUDENT_COUNT = 500
@@ -13,7 +14,7 @@ if (!process.argv[2]) {
 }
 
 const start = Date.now()
-async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
   let i = 0
   let errors = 0
 
@@ -51,6 +52,14 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
 
   await Promise.all(Object.values(inFlight))
 
+  if (errors) {
+    console.error(
+      `${table.name} - ${errors} creation errored (${
+        (Date.now() - start) / 1000
+      }s)`
+    )
+  }
+
   return rows
 }
 

From 3e991cc2f13ba80aca8fc85e63b1bf087a1da412 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:36:29 +0100
Subject: [PATCH 26/31] Use yargs

---
 .../server/scripts/load/create-many-relationships.js   | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 14eaccd99c..84d1f94dda 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,5 +1,5 @@
 #!/bin/node
-const { createApp, getTable, createRow, createTable } = require("./utils")
+const yargs = require("yargs")
 
 const Chance = require("chance")
 
@@ -8,10 +8,9 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-if (!process.argv[2]) {
-  console.error("Please specify an API key as script argument.")
-  process.exit(-1)
-}
+const { apiKey, appId } = require("yargs")
+  .demandOption(["apiKey"])
+  .option("appId").argv
 
 const start = Date.now()
 async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
@@ -64,7 +63,6 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
 }
 
 async function run() {
-  const apiKey = process.argv[2]
   const app = await createApp(apiKey)
   console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
 

From c5a50a911f64b7094c8941b12b5eb6b34e70b9a0 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:46:49 +0100
Subject: [PATCH 27/31] Allow reusing app

---
 .../scripts/load/create-many-relationships.js | 54 ++++++++++++-------
 packages/server/scripts/load/utils.js         | 13 ++++-
 2 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 84d1f94dda..f683d3aed4 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -1,5 +1,11 @@
 #!/bin/node
-const yargs = require("yargs")
+const {
+  createApp,
+  getTable,
+  createRow,
+  createTable,
+  getApp,
+} = require("./utils")
 
 const Chance = require("chance")
 
@@ -8,12 +14,12 @@ const generator = new Chance()
 const STUDENT_COUNT = 500
 const SUBJECT_COUNT = 10
 
-const { apiKey, appId } = require("yargs")
+let { apiKey, appId } = require("yargs")
   .demandOption(["apiKey"])
   .option("appId").argv
 
 const start = Date.now()
-async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   let i = 0
   let errors = 0
 
@@ -62,20 +68,34 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 1000) {
   return rows
 }
 
-async function run() {
-  const app = await createApp(apiKey)
-  console.log(`App created: http://localhost:10000/builder/app/${app._id}`)
+const useExistingApp = !!appId
 
-  const studentsTable = await getTable(apiKey, app._id)
-  if (studentsTable.name !== "Students") {
-    throw 'Fetched table should be "Students"'
+async function upsertTable(appId, tableName, tableData) {
+  if (useExistingApp) {
+    return await getTable(apiKey, appId, tableName)
   }
-  console.log(`Table found: ${studentsTable.name}`)
+
+  const table = await createTable(apiKey, appId, {
+    ...tableData,
+    name: tableName,
+  })
+  return table
+}
+
+async function run() {
+  if (!appId) {
+    const app = appId ? await getApp(apiKey, appId) : await createApp(apiKey)
+    appId = app._id
+  }
+
+  console.log(`App created: http://localhost:10000/builder/app/${appId}`)
+
+  const studentsTable = await getTable(apiKey, appId, "Students")
 
   let studentNumber = studentsTable.schema["Auto ID"].lastID
   const students = await batchCreate(
     apiKey,
-    app._id,
+    appId,
     studentsTable,
     Array.from({ length: STUDENT_COUNT }).map(() => ({
       "Student Number": (++studentNumber).toString(),
@@ -89,27 +109,26 @@ async function run() {
     }))
   )
 
-  const subjectTable = await createTable(apiKey, app._id, {
+  const subjectTable = await upsertTable(appId, "Subjects", {
     schema: {
       Name: {
         name: "Name",
         type: "string",
       },
     },
-    name: "Subjects",
     primaryDisplay: "Name",
   })
 
   const subjects = await batchCreate(
     apiKey,
-    app._id,
+    appId,
     subjectTable,
     Array.from({ length: SUBJECT_COUNT }).map(() => ({
       Name: generator.profession(),
     }))
   )
 
-  const gradesTable = await createTable(apiKey, app._id, {
+  const gradesTable = await upsertTable(appId, "Grades", {
     schema: {
       Score: {
         name: "Score",
@@ -138,12 +157,11 @@ async function run() {
         type: "link",
       },
     },
-    name: "Grades",
   })
 
   await batchCreate(
     apiKey,
-    app._id,
+    appId,
     gradesTable,
     students.flatMap(student =>
       subjects.map(subject => ({
@@ -155,7 +173,7 @@ async function run() {
   )
 
   console.log(
-    `Access the app here: http://localhost:10000/builder/app/${app._id}`
+    `Access the app here: http://localhost:10000/builder/app/${appId}`
   )
 }
 
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 04ebb6f3d6..2b789cde52 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -38,6 +38,11 @@ exports.createApp = async apiKey => {
   return json.data
 }
 
+exports.getApp = async (apiKey, appId) => {
+  const res = await request(apiKey, `${URL_APP}/${appId}`, "GET")
+  const json = await res.json()
+  return json.data
+}
 exports.searchApps = async apiKey => {
   const res = await request(apiKey, `${URL_APP}/search`, "POST", {})
   const json = await res.json()
@@ -49,10 +54,14 @@ exports.deleteApp = async (apiKey, appId) => {
   return res
 }
 
-exports.getTable = async (apiKey, appId) => {
+exports.getTable = async (apiKey, appId, tableName) => {
   const res = await request(apiKey, URL_SEARCH_TABLE, "POST", {}, appId)
   const json = await res.json()
-  return json.data[0]
+  const table = json.data.find(t => t.name === tableName)
+  if (!table) {
+    throw `Table '${tableName} not found`
+  }
+  return table
 }
 
 exports.createRow = async (apiKey, appId, table, body) => {

From bd221f27270a56bb6b9d363866dfc3c07a5dfa9a Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 11:52:48 +0100
Subject: [PATCH 28/31] Prevent locks

---
 .../server/scripts/load/create-many-relationships.js  | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index f683d3aed4..07861cc632 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -43,10 +43,13 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
 
   for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
     const item = items[itemIndex]
-    const promise = createSingleRow(item).then(result => {
-      rows.push(result)
-      delete inFlight[itemIndex]
-    })
+    const promise = createSingleRow(item)
+      .then(result => {
+        rows.push(result)
+      })
+      .finally(() => {
+        delete inFlight[itemIndex]
+      })
 
     inFlight[itemIndex] = promise
 

From 5a45249ad4f802d5b6a0d724d99207218ae7a025 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Fri, 22 Dec 2023 12:18:24 +0100
Subject: [PATCH 29/31] Reuse subjects

---
 .../scripts/load/create-many-relationships.js | 19 +++++++++++--------
 packages/server/scripts/load/utils.js         |  6 ++++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 07861cc632..13f861ec0b 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -5,6 +5,7 @@ const {
   createRow,
   createTable,
   getApp,
+  getRows,
 } = require("./utils")
 
 const Chance = require("chance")
@@ -122,14 +123,16 @@ async function run() {
     primaryDisplay: "Name",
   })
 
-  const subjects = await batchCreate(
-    apiKey,
-    appId,
-    subjectTable,
-    Array.from({ length: SUBJECT_COUNT }).map(() => ({
-      Name: generator.profession(),
-    }))
-  )
+  const subjects = useExistingApp
+    ? await getRows(apiKey, appId, subjectTable._id)
+    : await batchCreate(
+        apiKey,
+        appId,
+        subjectTable,
+        Array.from({ length: SUBJECT_COUNT }).map(() => ({
+          Name: generator.profession(),
+        }))
+      )
 
   const gradesTable = await upsertTable(appId, "Grades", {
     schema: {
diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js
index 2b789cde52..1dabdcec9a 100644
--- a/packages/server/scripts/load/utils.js
+++ b/packages/server/scripts/load/utils.js
@@ -88,6 +88,12 @@ exports.createRow = async (apiKey, appId, table, body) => {
   return (await res.json()).data
 }
 
+exports.getRows = async (apiKey, appId, tableId) => {
+  const url = `${URL_TABLE}/${tableId}/rows/search`
+  const res = await request(apiKey, url, "POST", {}, appId)
+  return (await res.json()).data
+}
+
 exports.createTable = async (apiKey, appId, config) => {
   const res = await request(apiKey, URL_TABLE, "POST", config, appId)
   const json = await res.json()

From e26c39677dde005f5133d0401322d8474393f541 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jan 2024 12:31:08 +0100
Subject: [PATCH 30/31] Update messages

---
 packages/server/scripts/load/create-many-relationships.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 13f861ec0b..b81aed3d5d 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -90,9 +90,13 @@ async function run() {
   if (!appId) {
     const app = appId ? await getApp(apiKey, appId) : await createApp(apiKey)
     appId = app._id
-  }
 
-  console.log(`App created: http://localhost:10000/builder/app/${appId}`)
+    console.log(`App created. Url: http://localhost:10000/builder/app/${appId}`)
+  } else {
+    console.log(
+      `App retrieved. Url: http://localhost:10000/builder/app/${appId}`
+    )
+  }
 
   const studentsTable = await getTable(apiKey, appId, "Students")
 

From 5392d89fde6e1cc920d79c251e443d8ea61dde82 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Tue, 2 Jan 2024 12:38:50 +0100
Subject: [PATCH 31/31] Update account-portal ref

---
 packages/account-portal | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/account-portal b/packages/account-portal
index d6a1f89aa5..c1a53bb2f4 160000
--- a/packages/account-portal
+++ b/packages/account-portal
@@ -1 +1 @@
-Subproject commit d6a1f89aa543bdce7acde5fbe4ce650a1344e2fe
+Subproject commit c1a53bb2f4cafcb4c55ad7181146617b449907f2