cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): Enqueue jobs in bulk (#5974)

Browse Source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello
2024-01-01 15:45:42 -05:00
committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
20 changed files with 323 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/media/media.service.spec.ts
+49 -33
View File
@@ -77,17 +77,21 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).toHaveBeenCalled();
expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_JPEG_THUMBNAIL,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_JPEG_THUMBNAIL,
data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: { id: personStub.newThumbnail.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: { id: personStub.newThumbnail.id },
},
]);
});
it('should queue all people with missing thumbnail path', async () => {
@@ -106,12 +110,14 @@ describe(MediaService.name, () => {
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getRandomFace).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: {
id: personStub.newThumbnail.id,
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: {
id: personStub.newThumbnail.id,
},
},
});
]);
});
it('should queue all assets with missing resize path', async () => {
@@ -125,10 +131,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_JPEG_THUMBNAIL,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_JPEG_THUMBNAIL,
data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -145,10 +153,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_WEBP_THUMBNAIL,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_WEBP_THUMBNAIL,
data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -165,10 +175,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_THUMBHASH_THUMBNAIL,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBHASH_THUMBNAIL,
data: { id: assetStub.image.id },
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
@@ -388,10 +400,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { type: AssetType.VIDEO });
expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.VIDEO_CONVERSION,
data: { id: assetStub.video.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.VIDEO_CONVERSION,
data: { id: assetStub.video.id },
},
]);
});
it('should queue all video assets without encoded videos', async () => {
@@ -404,10 +418,12 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.ENCODED_VIDEO);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.VIDEO_CONVERSION,
data: { id: assetStub.video.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.VIDEO_CONVERSION,
data: { id: assetStub.video.id },
},
]);
});
});