feat(server): Enqueue jobs in bulk (#5974)

* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello
2024-01-01 15:45:42 -05:00
committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
20 changed files with 323 additions and 227 deletions
@@ -116,12 +116,31 @@ export class JobRepository implements IJobRepository {
) as unknown as Promise<JobCounts>;
}
async queue(item: JobItem): Promise<void> {
const jobName = item.name;
const jobData = (item as { data?: any })?.data || {};
const jobOptions = this.getJobOptions(item) || undefined;
async queueAll(items: JobItem[]): Promise<void> {
if (!items.length) {
return;
}
await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions);
const itemsByQueue = items.reduce<Record<string, JobItem[]>>((acc, item) => {
const queueName = JOBS_TO_QUEUE[item.name];
acc[queueName] = acc[queueName] || [];
acc[queueName].push(item);
return acc;
}, {});
for (const [queueName, items] of Object.entries(itemsByQueue)) {
const queue = this.getQueue(queueName as QueueName);
const jobs = items.map((item) => ({
name: item.name,
data: (item as { data?: any })?.data || {},
options: this.getJobOptions(item) || undefined,
}));
await queue.addBulk(jobs);
}
}
async queue(item: JobItem): Promise<void> {
await this.queueAll([item]);
}
private getJobOptions(item: JobItem): JobsOptions | null {