cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Use postgres as a queue

Browse Source 
We've been keen to try this for a while as it means we can remove redis as a
dependency, which makes Immich easier to setup and run.

This replaces bullmq with a bespoke postgres queue. Jobs in the queue are
processed either immediately via triggers and notifications, or eventually if a
notification is missed.
This commit is contained in:
Thomas Way
2025-04-30 20:43:51 +01:00
parent b845184c80
commit 8c0c8a8d0e
46 changed files with 731 additions and 915 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
server/src/services/job.service.spec.ts
View File

@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
import { defaults, SystemConfig } from 'src/config';
import { ImmichWorker, JobCommand, JobName, JobStatus, QueueName } from 'src/enum';
import { JobService } from 'src/services/job.service';
import { JobItem } from 'src/types';
import { JobCounts, JobItem } from 'src/types';
import { assetStub } from 'test/fixtures/asset.stub';
import { newTestService, ServiceMocks } from 'test/utils';
@@ -21,14 +21,14 @@ describe(JobService.name, () => {
});
describe('onConfigUpdate', () => {
it('should update concurrency', () => {
sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
it('should update concurrency', async () => {
await sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(15);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FACIAL_RECOGNITION, 1);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DUPLICATE_DETECTION, 1);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BACKGROUND_TASK, 5);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(9, QueueName.STORAGE_TEMPLATE_MIGRATION, 1);
expect(mocks.job.start).toHaveBeenCalledTimes(15);
expect(mocks.job.start).toHaveBeenNthCalledWith(5, QueueName.FACIAL_RECOGNITION, 1);
expect(mocks.job.start).toHaveBeenNthCalledWith(7, QueueName.DUPLICATE_DETECTION, 1);
expect(mocks.job.start).toHaveBeenNthCalledWith(8, QueueName.BACKGROUND_TASK, 5);
expect(mocks.job.start).toHaveBeenNthCalledWith(9, QueueName.STORAGE_TEMPLATE_MIGRATION, 1);
});
});
@@ -55,29 +55,20 @@ describe(JobService.name, () => {
it('should get all job statuses', async () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
completed: 1,
failed: 1,
delayed: 1,
waiting: 1,
paused: 1,
});
mocks.job.getQueueStatus.mockResolvedValue({
isActive: true,
isPaused: true,
delayed: 1,
failed: 1,
});
const expectedJobStatus = {
jobCounts: {
active: 1,
completed: 1,
waiting: 1,
delayed: 1,
failed: 1,
waiting: 1,
paused: 1,
},
queueStatus: {
isActive: true,
isPaused: true,
paused: true,
},
};
@@ -114,14 +105,20 @@ describe(JobService.name, () => {
expect(mocks.job.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should handle an empty command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.EMPTY, force: false });
it('should handle a clear command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.CLEAR, force: false });
expect(mocks.job.empty).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
expect(mocks.job.clear).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should handle a clear-failed command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.CLEAR_FAILED, force: false });
expect(mocks.job.clearFailed).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
it('should not start a job that is already running', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: true, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 1 } as JobCounts);
await expect(
sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false }),
@@ -132,7 +129,7 @@ describe(JobService.name, () => {
});
it('should handle a start video conversion command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false });
@@ -140,7 +137,7 @@ describe(JobService.name, () => {
});
it('should handle a start storage template migration command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.STORAGE_TEMPLATE_MIGRATION, { command: JobCommand.START, force: false });
@@ -148,7 +145,7 @@ describe(JobService.name, () => {
});
it('should handle a start smart search command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.SMART_SEARCH, { command: JobCommand.START, force: false });
@@ -156,7 +153,7 @@ describe(JobService.name, () => {
});
it('should handle a start metadata extraction command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.START, force: false });
@@ -164,7 +161,7 @@ describe(JobService.name, () => {
});
it('should handle a start sidecar command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false });
@@ -172,7 +169,7 @@ describe(JobService.name, () => {
});
it('should handle a start thumbnail generation command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.THUMBNAIL_GENERATION, { command: JobCommand.START, force: false });
@@ -180,7 +177,7 @@ describe(JobService.name, () => {
});
it('should handle a start face detection command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.FACE_DETECTION, { command: JobCommand.START, force: false });
@@ -188,7 +185,7 @@ describe(JobService.name, () => {
});
it('should handle a start facial recognition command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.FACIAL_RECOGNITION, { command: JobCommand.START, force: false });
@@ -196,7 +193,7 @@ describe(JobService.name, () => {
});
it('should handle a start backup database command', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await sut.handleCommand(QueueName.BACKUP_DATABASE, { command: JobCommand.START, force: false });
@@ -204,7 +201,7 @@ describe(JobService.name, () => {
});
it('should throw a bad request when an invalid queue is used', async () => {
mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
await expect(
sut.handleCommand(QueueName.BACKGROUND_TASK, { command: JobCommand.START, force: false }),

47
server/src/services/job.service.ts
View File

@@ -12,7 +12,6 @@ import {
JobName,
JobStatus,
ManualJobName,
QueueCleanType,
QueueName,
} from 'src/enum';
import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
@@ -56,7 +55,7 @@ export class JobService extends BaseService {
private services: ClassConstructor<unknown>[] = [];
@OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })
onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
this.logger.debug(`Updating queue concurrency settings`);
for (const queueName of Object.values(QueueName)) {
let concurrency = 1;
@@ -64,21 +63,18 @@ export class JobService extends BaseService {
concurrency = config.job[queueName].concurrency;
}
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
this.jobRepository.setConcurrency(queueName, concurrency);
await this.jobRepository.start(queueName, concurrency);
}
}
@OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })
onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
this.onConfigInit({ newConfig: config });
async onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
await this.onConfigInit({ newConfig: config });
}
@OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
onBootstrap() {
this.jobRepository.setup(this.services);
if (this.worker === ImmichWorker.MICROSERVICES) {
this.jobRepository.startWorkers();
}
async onBootstrap() {
await this.jobRepository.setup(this.services);
}
setServices(services: ClassConstructor<unknown>[]) {
@@ -97,25 +93,20 @@ export class JobService extends BaseService {
await this.start(queueName, dto);
break;
}
case JobCommand.PAUSE: {
await this.jobRepository.pause(queueName);
this.eventRepository.serverSend('queue.pause', queueName);
break;
}
case JobCommand.RESUME: {
await this.jobRepository.resume(queueName);
this.eventRepository.serverSend('queue.resume', queueName);
break;
}
case JobCommand.EMPTY: {
await this.jobRepository.empty(queueName);
case JobCommand.CLEAR: {
await this.jobRepository.clear(queueName);
break;
}
case JobCommand.CLEAR_FAILED: {
const failedJobs = await this.jobRepository.clear(queueName, QueueCleanType.FAILED);
this.logger.debug(`Cleared failed jobs: ${failedJobs}`);
await this.jobRepository.clearFailed(queueName);
break;
}
}
@@ -141,9 +132,9 @@ export class JobService extends BaseService {
}
private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
const { isActive } = await this.jobRepository.getQueueStatus(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
const { active } = await this.jobRepository.getJobCounts(name);
if (active > 0) {
throw new BadRequestException(`Jobs are already running`);
}
this.telemetryRepository.jobs.addToCounter(`immich.queues.${snakeCase(name)}.started`, 1);
@@ -203,6 +194,16 @@ export class JobService extends BaseService {
}
}
@OnEvent({ name: 'queue.pause', server: true, workers: [ImmichWorker.MICROSERVICES] })
async pause(...[queueName]: ArgsOf<'queue.pause'>): Promise<void> {
await this.jobRepository.pause(queueName);
}
@OnEvent({ name: 'queue.resume', server: true, workers: [ImmichWorker.MICROSERVICES] })
async resume(...[queueName]: ArgsOf<'queue.resume'>): Promise<void> {
await this.jobRepository.resume(queueName);
}
@OnEvent({ name: 'job.start' })
async onJobStart(...[queueName, job]: ArgsOf<'job.start'>) {
const queueMetric = `immich.queues.${snakeCase(queueName)}.active`;

6
server/src/services/metadata.service.spec.ts
View File

@@ -67,16 +67,12 @@ describe(MetadataService.name, () => {
});
describe('onBootstrapEvent', () => {
it('should pause and resume queue during init', async () => {
mocks.job.pause.mockResolvedValue();
it('should init', async () => {
mocks.map.init.mockResolvedValue();
mocks.job.resume.mockResolvedValue();
await sut.onBootstrap();
expect(mocks.job.pause).toHaveBeenCalledTimes(1);
expect(mocks.map.init).toHaveBeenCalledTimes(1);
expect(mocks.job.resume).toHaveBeenCalledTimes(1);
});
});

2
server/src/services/metadata.service.ts
View File

@@ -121,9 +121,7 @@ export class MetadataService extends BaseService {
this.logger.log('Initializing metadata service');
try {
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
await this.databaseRepository.withLock(DatabaseLock.GeodataImport, () => this.mapRepository.init());
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
this.logger.log(`Initialized local reverse geocoder`);
} catch (error: Error | any) {

3
server/src/services/notification.service.spec.ts
View File

@@ -499,14 +499,13 @@ describe(NotificationService.name, () => {
});
it('should add new recipients for new images if job is already queued', async () => {
mocks.job.removeJob.mockResolvedValue({ id: '1', recipientIds: ['2', '3', '4'] } as INotifyAlbumUpdateJob);
await sut.onAlbumUpdate({ id: '1', recipientIds: ['1', '2', '3'] } as INotifyAlbumUpdateJob);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.NOTIFY_ALBUM_UPDATE,
data: {
id: '1',
delay: 300_000,
recipientIds: ['1', '2', '3', '4'],
recipientIds: ['1', '2', '3'],
},
});
});

17
server/src/services/notification.service.ts
View File

@@ -196,14 +196,15 @@ export class NotificationService extends BaseService {
data: { id, recipientIds, delay: NotificationService.albumUpdateEmailDelayMs },
};
const previousJobData = await this.jobRepository.removeJob(id, JobName.NOTIFY_ALBUM_UPDATE);
if (previousJobData && this.isAlbumUpdateJob(previousJobData)) {
for (const id of previousJobData.recipientIds) {
if (!recipientIds.includes(id)) {
recipientIds.push(id);
}
}
}
// todo: https://github.com/immich-app/immich/pull/17879
// const previousJobData = await this.jobRepository.removeJob(id, JobName.NOTIFY_ALBUM_UPDATE);
// if (previousJobData && this.isAlbumUpdateJob(previousJobData)) {
// for (const id of previousJobData.recipientIds) {
// if (!recipientIds.includes(id)) {
// recipientIds.push(id);
// }
// }
// }
await this.jobRepository.queue(job);
}

24
server/src/services/person.service.spec.ts
View File

@@ -529,10 +529,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 0,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
@@ -546,10 +544,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 1,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED);
@@ -561,10 +557,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 0,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
mocks.person.getAllWithoutFaces.mockResolvedValue([]);
@@ -590,10 +584,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 0,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
mocks.person.getAll.mockReturnValue(makeStream());
mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
@@ -619,10 +611,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 0,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
mocks.person.getAll.mockReturnValue(makeStream());
mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
@@ -666,10 +656,8 @@ describe(PersonService.name, () => {
mocks.job.getJobCounts.mockResolvedValue({
active: 1,
waiting: 0,
paused: 0,
completed: 0,
failed: 0,
delayed: 0,
failed: 0,
});
mocks.person.getAll.mockReturnValue(makeStream([faceStub.face1.person, personStub.randomPerson]));
mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));

3
server/src/services/person.service.ts
View File

@@ -392,7 +392,8 @@ export class PersonService extends BaseService {
return JobStatus.SKIPPED;
}
await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
// todo
// await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
if (nightly) {
const [state, latestFaceDate] = await Promise.all([