From abf025b3f7e5fbe2458b53995f362268323cb376 Mon Sep 17 00:00:00 2001
From: Adria Navarro <adria@budibase.com>
Date: Thu, 21 Dec 2023 19:06:26 +0100
Subject: [PATCH] Use inflight max instead of batch wait

---
 .../scripts/load/create-many-relationships.js | 30 ++++++++++++-------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/packages/server/scripts/load/create-many-relationships.js b/packages/server/scripts/load/create-many-relationships.js
index 15ef9bcb1e..0f4fe3470c 100755
--- a/packages/server/scripts/load/create-many-relationships.js
+++ b/packages/server/scripts/load/create-many-relationships.js
@@ -13,11 +13,10 @@ if (!process.argv[2]) {
 }
 
 const start = Date.now()
-
-async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
+async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
   let i = 0
-
   let errors = 0
+
   async function createSingleRow(item) {
     try {
       const row = await createRow(apiKey, appId, table, item)
@@ -33,14 +32,25 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
   }
 
   const rows = []
-  for (let j = 0; j < items.length; j += batchSize) {
-    const batchPromises = items.slice(j, j + batchSize).map(createSingleRow)
-    const batchRows = await Promise.all(batchPromises)
-    rows.push(...batchRows)
-  }
-  if (errors) {
-    console.error(`Error creating ${errors} row`)
+  const maxConcurrency = Math.min(batchSize, items.length)
+  const inFlight = {}
+
+  for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
+    const item = items[itemIndex]
+    const promise = createSingleRow(item).then(result => {
+      rows.push(result)
+      delete inFlight[itemIndex]
+    })
+
+    inFlight[itemIndex] = promise
+
+    if (Object.keys(inFlight).length >= maxConcurrency) {
+      await Promise.race(Object.values(inFlight))
+    }
   }
+
+  await Promise.all(Object.values(inFlight))
+
   return rows
 }