JavaScript
Process URLs with bounded workers
The worker count is explicit and the queue can later be replaced by the application's durable job system.
async function runBatch(urls, workerCount) {
const queue = [...new Set(urls)].map((url) => ({ url }));
const results = [];
async function worker() {
while (queue.length) {
const job = queue.shift();
try {
results.push({
url: job.url,
status: "ready",
result: await captureUrl(job.url),
});
} catch (error) {
results.push({
url: job.url,
status: "failed",
error: classify(error),
});
}
}
}
await Promise.all(
Array.from({ length: workerCount }, () => worker()),
);
return results;
}