feat(server): Enqueue jobs in bulk (#5974)
* feat(server): Enqueue jobs in bulk The Job Repository now has a `queueAll` method, that enqueues messages in bulk (using BullMQ's [`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)), improving performance when many jobs must be enqueued within the same operation. Primary change is in `src/domain/job/job.service.ts`, and other services have been refactored to use `queueAll` when useful. As a simple local benchmark, triggering a full thumbnail generation process over a library of ~1,200 assets and ~350 faces went from **~600ms** to **~250ms**. * fix: Review feedback
This commit is contained in:
committed by
GitHub
parent
7dd88c4114
commit
4a5b8c3770
@@ -116,12 +116,31 @@ export class JobRepository implements IJobRepository {
|
||||
) as unknown as Promise<JobCounts>;
|
||||
}
|
||||
|
||||
async queue(item: JobItem): Promise<void> {
|
||||
const jobName = item.name;
|
||||
const jobData = (item as { data?: any })?.data || {};
|
||||
const jobOptions = this.getJobOptions(item) || undefined;
|
||||
async queueAll(items: JobItem[]): Promise<void> {
|
||||
if (!items.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions);
|
||||
const itemsByQueue = items.reduce<Record<string, JobItem[]>>((acc, item) => {
|
||||
const queueName = JOBS_TO_QUEUE[item.name];
|
||||
acc[queueName] = acc[queueName] || [];
|
||||
acc[queueName].push(item);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
for (const [queueName, items] of Object.entries(itemsByQueue)) {
|
||||
const queue = this.getQueue(queueName as QueueName);
|
||||
const jobs = items.map((item) => ({
|
||||
name: item.name,
|
||||
data: (item as { data?: any })?.data || {},
|
||||
options: this.getJobOptions(item) || undefined,
|
||||
}));
|
||||
await queue.addBulk(jobs);
|
||||
}
|
||||
}
|
||||
|
||||
async queue(item: JobItem): Promise<void> {
|
||||
await this.queueAll([item]);
|
||||
}
|
||||
|
||||
private getJobOptions(item: JobItem): JobsOptions | null {
|
||||
|
||||
Reference in New Issue
Block a user