feat(server): Enqueue jobs in bulk (#5974)

* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback

server/src/domain/library/library.service.spec.ts

@@ -135,18 +135,16 @@ describe(LibraryService.name, () => {
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
 
-      expect(jobMock.queue.mock.calls).toEqual([
-        [
-          {
-            name: JobName.LIBRARY_SCAN_ASSET,
-            data: {
-              id: libraryStub.externalLibrary1.id,
-              ownerId: libraryStub.externalLibrary1.owner.id,
-              assetPath: '/data/user1/photo.jpg',
-              force: false,
-            },
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.LIBRARY_SCAN_ASSET,
+          data: {
+            id: libraryStub.externalLibrary1.id,
+            ownerId: libraryStub.externalLibrary1.owner.id,
+            assetPath: '/data/user1/photo.jpg',
+            force: false,
           },
-        ],
+        },
       ]);
     });
 
@@ -420,6 +418,7 @@ describe(LibraryService.name, () => {
       await expect(sut.handleAssetRefresh(mockLibraryJob)).resolves.toBe(true);
 
       expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
     });
 
     it('should import an asset when mtime differs from db asset', async () => {
@@ -468,6 +467,7 @@ describe(LibraryService.name, () => {
 
       expect(assetMock.save).toHaveBeenCalledWith({ id: assetStub.image.id, isOffline: true });
       expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
     });
 
     it('should online a previously-offline asset', async () => {
@@ -607,6 +607,7 @@ describe(LibraryService.name, () => {
       );
 
       expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
       expect(libraryMock.softDelete).not.toHaveBeenCalled();
     });
 
@@ -953,9 +954,9 @@ describe(LibraryService.name, () => {
       libraryMock.getAllDeleted.mockResolvedValue([libraryStub.uploadLibrary1, libraryStub.externalLibrary1]);
       await expect(sut.handleQueueCleanup()).resolves.toBe(true);
 
-      expect(jobMock.queue.mock.calls).toEqual([
-        [{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } }],
-        [{ name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } }],
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        { name: JobName.LIBRARY_DELETE, data: { id: libraryStub.uploadLibrary1.id } },
+        { name: JobName.LIBRARY_DELETE, data: { id: libraryStub.externalLibrary1.id } },
       ]);
     });
   });
@@ -1101,16 +1102,16 @@ describe(LibraryService.name, () => {
             data: {},
           },
         ],
-        [
-          {
-            name: JobName.LIBRARY_SCAN,
-            data: {
-              id: libraryStub.externalLibrary1.id,
-              refreshModifiedFiles: true,
-              refreshAllFiles: false,
-            },
+      ]);
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.LIBRARY_SCAN,
+          data: {
+            id: libraryStub.externalLibrary1.id,
+            refreshModifiedFiles: true,
+            refreshAllFiles: false,
           },
-        ],
+        },
       ]);
     });
 
@@ -1126,16 +1127,16 @@ describe(LibraryService.name, () => {
             data: {},
           },
         ],
-        [
-          {
-            name: JobName.LIBRARY_SCAN,
-            data: {
-              id: libraryStub.externalLibrary1.id,
-              refreshModifiedFiles: false,
-              refreshAllFiles: true,
-            },
+      ]);
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.LIBRARY_SCAN,
+          data: {
+            id: libraryStub.externalLibrary1.id,
+            refreshModifiedFiles: false,
+            refreshAllFiles: true,
           },
-        ],
+        },
       ]);
     });
   });
@@ -1147,13 +1148,11 @@ describe(LibraryService.name, () => {
 
       await expect(sut.handleOfflineRemoval({ id: libraryStub.externalLibrary1.id })).resolves.toBe(true);
 
-      expect(jobMock.queue.mock.calls).toEqual([
-        [
-          {
-            name: JobName.ASSET_DELETION,
-            data: { id: assetStub.image1.id, fromExternal: true },
-          },
-        ],
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.ASSET_DELETION,
+          data: { id: assetStub.image1.id, fromExternal: true },
+        },
       ]);
     });
   });

server/src/domain/library/library.service.ts

@@ -94,9 +94,9 @@ export class LibraryService {
   async handleQueueCleanup(): Promise<boolean> {
     this.logger.debug('Cleaning up any pending library deletions');
     const pendingDeletion = await this.repository.getAllDeleted();
-    for (const libraryToDelete of pendingDeletion) {
-      await this.jobRepository.queue({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } });
-    }
+    await this.jobRepository.queueAll(
+      pendingDeletion.map((libraryToDelete) => ({ name: JobName.LIBRARY_DELETE, data: { id: libraryToDelete.id } })),
+    );
     return true;
   }
 
@@ -160,9 +160,9 @@ export class LibraryService {
     // TODO use pagination
     const assetIds = await this.repository.getAssetIds(job.id, true);
     this.logger.debug(`Will delete ${assetIds.length} asset(s) in library ${job.id}`);
-    for (const assetId of assetIds) {
-      await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: assetId, fromExternal: true } });
-    }
+    await this.jobRepository.queueAll(
+      assetIds.map((assetId) => ({ name: JobName.ASSET_DELETION, data: { id: assetId, fromExternal: true } })),
+    );
 
     if (assetIds.length === 0) {
       this.logger.log(`Deleting library ${job.id}`);
@@ -333,16 +333,16 @@ export class LibraryService {
 
     // Queue all library refresh
     const libraries = await this.repository.getAll(true, LibraryType.EXTERNAL);
-    for (const library of libraries) {
-      await this.jobRepository.queue({
+    await this.jobRepository.queueAll(
+      libraries.map((library) => ({
         name: JobName.LIBRARY_SCAN,
         data: {
           id: library.id,
           refreshModifiedFiles: !job.force,
           refreshAllFiles: job.force ?? false,
         },
-      });
-    }
+      })),
+    );
     return true;
   }
 
@@ -353,9 +353,9 @@ export class LibraryService {
 
     for await (const assets of assetPagination) {
       this.logger.debug(`Removing ${assets.length} offline assets`);
-      for (const asset of assets) {
-        await this.jobRepository.queue({ name: JobName.ASSET_DELETION, data: { id: asset.id, fromExternal: true } });
-      }
+      await this.jobRepository.queueAll(
+        assets.map((asset) => ({ name: JobName.ASSET_DELETION, data: { id: asset.id, fromExternal: true } })),
+      );
     }
 
     return true;
@@ -411,16 +411,17 @@ export class LibraryService {
         this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
       }
 
-      for (const assetPath of filteredPaths) {
-        const libraryJobData: ILibraryFileJob = {
-          id: job.id,
-          assetPath: path.normalize(assetPath),
-          ownerId: library.ownerId,
-          force: job.refreshAllFiles ?? false,
-        };
-
-        await this.jobRepository.queue({ name: JobName.LIBRARY_SCAN_ASSET, data: libraryJobData });
-      }
+      await this.jobRepository.queueAll(
+        filteredPaths.map((assetPath) => ({
+          name: JobName.LIBRARY_SCAN_ASSET,
+          data: {
+            id: job.id,
+            assetPath: path.normalize(assetPath),
+            ownerId: library.ownerId,
+            force: job.refreshAllFiles ?? false,
+          },
+        })),
+      );
     }
 
     await this.repository.update({ id: job.id, refreshedAt: new Date() });