cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): Enqueue jobs in bulk (#5974)

Browse Source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello
2024-01-01 15:45:42 -05:00
committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
20 changed files with 323 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/asset/asset.service.spec.ts
+9 -6
View File
@@ -784,9 +784,9 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: true });
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.ASSET_DELETION, data: { id: 'asset1' } }],
[{ name: JobName.ASSET_DELETION, data: { id: 'asset2' } }],
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.ASSET_DELETION, data: { id: 'asset1' } },
{ name: JobName.ASSET_DELETION, data: { id: 'asset2' } },
]);
});
@@ -895,6 +895,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.external.id });
expect(jobMock.queue).not.toBeCalled();
expect(jobMock.queueAll).not.toBeCalled();
expect(assetMock.remove).not.toBeCalled();
});
@@ -952,19 +953,21 @@ describe(AssetService.name, () => {
it('should run the refresh metadata job', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REFRESH_METADATA }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } });
expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } }]);
});
it('should run the refresh thumbnails job', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REGENERATE_THUMBNAIL }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } });
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
]);
});
it('should run the transcode video', async () => {
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.TRANSCODE_VIDEO }),
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.VIDEO_CONVERSION, data: { id: 'asset-1' } });
expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.VIDEO_CONVERSION, data: { id: 'asset-1' } }]);
});
});