MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use inflight max instead of batch wait

This commit is contained in:
Adria Navarro 2023-12-21 19:06:26 +01:00
parent 494a2ff91f
commit abf025b3f7
1 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
packages/server/scripts/load/create-many-relationships.js
View File

@ -13,11 +13,10 @@ if (!process.argv[2]) {
} }
const start = Date.now() const start = Date.now()
async function batchCreate(apiKey, appId, table, items, batchSize = 10) {
async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
let i = 0 let i = 0
let errors = 0 let errors = 0
async function createSingleRow(item) { async function createSingleRow(item) {
try { try {
const row = await createRow(apiKey, appId, table, item) const row = await createRow(apiKey, appId, table, item)
@ -33,14 +32,25 @@ async function batchCreate(apiKey, appId, table, items, batchSize = 100) {
} }
const rows = [] const rows = []
for (let j = 0; j < items.length; j += batchSize) { const maxConcurrency = Math.min(batchSize, items.length)
const batchPromises = items.slice(j, j + batchSize).map(createSingleRow) const inFlight = {}
const batchRows = await Promise.all(batchPromises)
rows.push(...batchRows) for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
} const item = items[itemIndex]
if (errors) { const promise = createSingleRow(item).then(result => {
console.error(`Error creating ${errors} row`) rows.push(result)
delete inFlight[itemIndex]
})
inFlight[itemIndex] = promise
if (Object.keys(inFlight).length >= maxConcurrency) {
await Promise.race(Object.values(inFlight))
}
} }
await Promise.all(Object.values(inFlight))
return rows return rows
} }