cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): Enqueue jobs in bulk (#5974)

Browse Source 
* feat(server): Enqueue jobs in bulk

The Job Repository now has a `queueAll` method, that enqueues messages
in bulk (using BullMQ's
[`addBulk`](https://docs.bullmq.io/guide/queues/adding-bulks)),
improving performance when many jobs must be enqueued within the same
operation.

Primary change is in `src/domain/job/job.service.ts`, and other services
have been refactored to use `queueAll` when useful.

As a simple local benchmark, triggering a full thumbnail generation
process over a library of ~1,200 assets and ~350 faces went from
**~600ms** to **~250ms**.

* fix: Review feedback
This commit is contained in:
Michael Manganiello
2024-01-01 15:45:42 -05:00
committed by GitHub
parent 7dd88c4114
commit 4a5b8c3770
20 changed files with 323 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/metadata/metadata.service.spec.ts
+25 -16
View File
@@ -208,10 +208,12 @@ describe(MetadataService.name, () => {
await expect(sut.handleQueueMetadataExtraction({ force: false })).resolves.toBe(true);
expect(assetMock.getWithout).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.METADATA_EXTRACTION,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.METADATA_EXTRACTION,
data: { id: assetStub.image.id },
},
]);
});
it('should queue metadata extraction for all assets', async () => {
@@ -219,10 +221,12 @@ describe(MetadataService.name, () => {
await expect(sut.handleQueueMetadataExtraction({ force: true })).resolves.toBe(true);
expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.METADATA_EXTRACTION,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.METADATA_EXTRACTION,
data: { id: assetStub.image.id },
},
]);
});
});
@@ -320,6 +324,7 @@ describe(MetadataService.name, () => {
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
expect(storageMock.writeFile).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(assetMock.save).not.toHaveBeenCalledWith(
expect.objectContaining({ assetType: AssetType.VIDEO, isVisible: false }),
);
@@ -512,10 +517,12 @@ describe(MetadataService.name, () => {
expect(assetMock.getWith).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithProperty.SIDECAR);
expect(assetMock.getWithout).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SIDECAR_SYNC,
data: { id: assetStub.sidecar.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.SIDECAR_SYNC,
data: { id: assetStub.sidecar.id },
},
]);
});
it('should queue assets without sidecar files', async () => {
@@ -525,10 +532,12 @@ describe(MetadataService.name, () => {
expect(assetMock.getWithout).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithoutProperty.SIDECAR);
expect(assetMock.getWith).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SIDECAR_DISCOVERY,
data: { id: assetStub.image.id },
});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.SIDECAR_DISCOVERY,
data: { id: assetStub.image.id },
},
]);
});
});
server/src/domain/metadata/metadata.service.ts
+9 -7
View File
@@ -196,9 +196,9 @@ export class MetadataService {
});
for await (const assets of assetPagination) {
for (const asset of assets) {
await this.jobRepository.queue({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.METADATA_EXTRACTION, data: { id: asset.id } })),
);
}
return true;
@@ -264,10 +264,12 @@ export class MetadataService {
});
for await (const assets of assetPagination) {
for (const asset of assets) {
const name = force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY;
await this.jobRepository.queue({ name, data: { id: asset.id } });
}
await this.jobRepository.queueAll(
assets.map((asset) => ({
name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY,
data: { id: asset.id },
})),
);
}
return true;