feat(server): separate face clustering job (#5598)

* separate facial clustering job

* update api

* fixed some tests

* invert clustering

* hdbscan

* update api

* remove commented code

* wip dbscan

* cleanup

removed cluster endpoint

remove commented code

* fixes

updated tests

minor fixes and formatting

fixed queuing

refinements

* scale search range based on library size

* defer non-core faces

* optimizations

removed unused query option

* assign faces individually for correctness

fixed unit tests

remove unused method

* don't select face embedding

update sql

linting

fixed ml typing

* updated job mock

* paginate people query

* select face embeddings because typeorm

* fix setting face detection concurrency

* update sql

formatting

linting

* simplify logic

remove unused imports

* more specific delete signature

* more accurate typing for face stubs

* add migration

formatting

* chore: better typing

* don't select embedding by default

remove unused import

* updated sql

* use normal try/catch

* stricter concurrency typing and enforcement

* update api

* update job concurrency panel to show disabled queues

formatting

* check jobId in queueAll

fix tests

* remove outdated comment

* better facial recognition icon

* wording

wording

formatting

* fixed tests

* fix

* formatting & sql

* try to fix sql check

* more detailed description

* update sql

* formatting

* wording

* update `minFaces` description

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/domain/job/job.constants.ts

@@ -2,7 +2,8 @@ export enum QueueName {
   THUMBNAIL_GENERATION = 'thumbnailGeneration',
   METADATA_EXTRACTION = 'metadataExtraction',
   VIDEO_CONVERSION = 'videoConversion',
-  RECOGNIZE_FACES = 'recognizeFaces',
+  FACE_DETECTION = 'faceDetection',
+  FACIAL_RECOGNITION = 'facialRecognition',
   SMART_SEARCH = 'smartSearch',
   BACKGROUND_TASK = 'backgroundTask',
   STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
@@ -12,6 +13,11 @@ export enum QueueName {
   LIBRARY = 'library',
 }
 
+export type ConcurrentQueueName = Exclude<
+  QueueName,
+  QueueName.STORAGE_TEMPLATE_MIGRATION | QueueName.FACIAL_RECOGNITION
+>;
+
 export enum JobCommand {
   START = 'start',
   PAUSE = 'pause',
@@ -57,9 +63,10 @@ export enum JobName {
 
   // facial recognition
   PERSON_CLEANUP = 'person-cleanup',
-  PERSON_DELETE = 'person-delete',
-  QUEUE_RECOGNIZE_FACES = 'queue-recognize-faces',
-  RECOGNIZE_FACES = 'recognize-faces',
+  QUEUE_FACE_DETECTION = 'queue-face-detection',
+  FACE_DETECTION = 'face-detection',
+  QUEUE_FACIAL_RECOGNITION = 'queue-facial-recognition',
+  FACIAL_RECOGNITION = 'facial-recognition',
 
   // library managment
   LIBRARY_SCAN = 'library-refresh',
@@ -95,7 +102,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK,
   [JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK,
   [JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
-  [JobName.PERSON_DELETE]: QueueName.BACKGROUND_TASK,
   [JobName.USER_SYNC_USAGE]: QueueName.BACKGROUND_TASK,
 
   // conversion
@@ -124,8 +130,10 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.MIGRATE_PERSON]: QueueName.MIGRATION,
 
   // facial recognition
-  [JobName.QUEUE_RECOGNIZE_FACES]: QueueName.RECOGNIZE_FACES,
-  [JobName.RECOGNIZE_FACES]: QueueName.RECOGNIZE_FACES,
+  [JobName.QUEUE_FACE_DETECTION]: QueueName.FACE_DETECTION,
+  [JobName.FACE_DETECTION]: QueueName.FACE_DETECTION,
+  [JobName.QUEUE_FACIAL_RECOGNITION]: QueueName.FACIAL_RECOGNITION,
+  [JobName.FACIAL_RECOGNITION]: QueueName.FACIAL_RECOGNITION,
 
   // clip
   [JobName.QUEUE_ENCODE_CLIP]: QueueName.SMART_SEARCH,

server/src/domain/job/job.dto.ts

@@ -75,7 +75,10 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
   [QueueName.SEARCH]!: JobStatusDto;
 
   @ApiProperty({ type: JobStatusDto })
-  [QueueName.RECOGNIZE_FACES]!: JobStatusDto;
+  [QueueName.FACE_DETECTION]!: JobStatusDto;
+
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.FACIAL_RECOGNITION]!: JobStatusDto;
 
   @ApiProperty({ type: JobStatusDto })
   [QueueName.SIDECAR]!: JobStatusDto;

server/src/domain/job/job.interface.ts

@@ -35,3 +35,7 @@ export interface ISidecarWriteJob extends IEntityJob {
   latitude?: number;
   longitude?: number;
 }
+
+export interface IDeferrableJob extends IEntityJob {
+  deferred?: boolean;
+}

server/src/domain/job/job.service.spec.ts

@@ -104,7 +104,8 @@ describe(JobService.name, () => {
         [QueueName.MIGRATION]: expectedJobStatus,
         [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
         [QueueName.VIDEO_CONVERSION]: expectedJobStatus,
-        [QueueName.RECOGNIZE_FACES]: expectedJobStatus,
+        [QueueName.FACE_DETECTION]: expectedJobStatus,
+        [QueueName.FACIAL_RECOGNITION]: expectedJobStatus,
         [QueueName.SIDECAR]: expectedJobStatus,
         [QueueName.LIBRARY]: expectedJobStatus,
       });
@@ -189,12 +190,20 @@ describe(JobService.name, () => {
       expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
     });
 
-    it('should handle a start recognize faces command', async () => {
+    it('should handle a start face detection command', async () => {
       jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
 
-      await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false });
+      await sut.handleCommand(QueueName.FACE_DETECTION, { command: JobCommand.START, force: false });
 
-      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } });
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_FACE_DETECTION, data: { force: false } });
+    });
+
+    it('should handle a start facial recognition command', async () => {
+      jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+
+      await sut.handleCommand(QueueName.FACIAL_RECOGNITION, { command: JobCommand.START, force: false });
+
+      expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } });
     });
 
     it('should throw a bad request when an invalid queue is used', async () => {
@@ -224,7 +233,7 @@ describe(JobService.name, () => {
           [QueueName.BACKGROUND_TASK]: { concurrency: 10 },
           [QueueName.SMART_SEARCH]: { concurrency: 10 },
           [QueueName.METADATA_EXTRACTION]: { concurrency: 10 },
-          [QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
+          [QueueName.FACE_DETECTION]: { concurrency: 10 },
           [QueueName.SEARCH]: { concurrency: 10 },
           [QueueName.SIDECAR]: { concurrency: 10 },
           [QueueName.LIBRARY]: { concurrency: 10 },
@@ -237,7 +246,7 @@ describe(JobService.name, () => {
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.BACKGROUND_TASK, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SMART_SEARCH, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION, 10);
-      expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
+      expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.FACE_DETECTION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
@@ -280,7 +289,7 @@ describe(JobService.name, () => {
           JobName.GENERATE_WEBP_THUMBNAIL,
           JobName.GENERATE_THUMBHASH_THUMBNAIL,
           JobName.ENCODE_CLIP,
-          JobName.RECOGNIZE_FACES,
+          JobName.FACE_DETECTION,
         ],
       },
       {
@@ -289,7 +298,7 @@ describe(JobService.name, () => {
           JobName.GENERATE_WEBP_THUMBNAIL,
           JobName.GENERATE_THUMBHASH_THUMBNAIL,
           JobName.ENCODE_CLIP,
-          JobName.RECOGNIZE_FACES,
+          JobName.FACE_DETECTION,
           JobName.VIDEO_CONVERSION,
         ],
       },
@@ -299,7 +308,7 @@ describe(JobService.name, () => {
           JobName.GENERATE_WEBP_THUMBNAIL,
           JobName.GENERATE_THUMBHASH_THUMBNAIL,
           JobName.ENCODE_CLIP,
-          JobName.RECOGNIZE_FACES,
+          JobName.FACE_DETECTION,
           JobName.VIDEO_CONVERSION,
         ],
       },
@@ -308,7 +317,11 @@ describe(JobService.name, () => {
         jobs: [],
       },
       {
-        item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
+        item: { name: JobName.FACE_DETECTION, data: { id: 'asset-1' } },
+        jobs: [JobName.QUEUE_FACIAL_RECOGNITION],
+      },
+      {
+        item: { name: JobName.FACIAL_RECOGNITION, data: { id: 'asset-1' } },
         jobs: [],
       },
     ];
@@ -355,7 +368,12 @@ describe(JobService.name, () => {
         configKey: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED,
       },
       {
-        queue: QueueName.RECOGNIZE_FACES,
+        queue: QueueName.FACE_DETECTION,
+        feature: FeatureFlag.FACIAL_RECOGNITION,
+        configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
+      },
+      {
+        queue: QueueName.FACIAL_RECOGNITION,
         feature: FeatureFlag.FACIAL_RECOGNITION,
         configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
       },

server/src/domain/job/job.service.ts

@@ -14,7 +14,7 @@ import {
   QueueCleanType,
 } from '../repositories';
 import { FeatureFlag, SystemConfigCore } from '../system-config/system-config.core';
-import { JobCommand, JobName, QueueName } from './job.constants';
+import { ConcurrentQueueName, JobCommand, JobName, QueueName } from './job.constants';
 import { AllJobStatusResponseDto, JobCommandDto, JobStatusDto } from './job.dto';
 
 @Injectable()
@@ -108,9 +108,13 @@ export class JobService {
       case QueueName.THUMBNAIL_GENERATION:
         return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } });
 
-      case QueueName.RECOGNIZE_FACES:
+      case QueueName.FACE_DETECTION:
         await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
-        return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
+        return this.jobRepository.queue({ name: JobName.QUEUE_FACE_DETECTION, data: { force } });
+
+      case QueueName.FACIAL_RECOGNITION:
+        await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
+        return this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force } });
 
       case QueueName.LIBRARY:
         return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force } });
@@ -124,7 +128,8 @@ export class JobService {
     const config = await this.configCore.getConfig();
     for (const queueName of Object.values(QueueName)) {
       let concurrency = 1;
-      if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
+
+      if (this.isConcurrentQueue(queueName)) {
         concurrency = config.job[queueName].concurrency;
       }
 
@@ -145,10 +150,10 @@ export class JobService {
     }
 
     this.configCore.config$.subscribe((config) => {
-      this.logger.log(`Updating queue concurrency settings`);
+      this.logger.debug(`Updating queue concurrency settings`);
       for (const queueName of Object.values(QueueName)) {
         let concurrency = 1;
-        if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
+        if (this.isConcurrentQueue(queueName)) {
           concurrency = config.job[queueName].concurrency;
         }
         this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
@@ -157,6 +162,10 @@ export class JobService {
     });
   }
 
+  private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
+    return ![QueueName.FACIAL_RECOGNITION, QueueName.STORAGE_TEMPLATE_MIGRATION].includes(name);
+  }
+
   async handleNightlyJobs() {
     await this.jobRepository.queueAll([
       { name: JobName.ASSET_DELETION_CHECK },
@@ -217,7 +226,7 @@ export class JobService {
           { name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data },
           { name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data },
           { name: JobName.ENCODE_CLIP, data: item.data },
-          { name: JobName.RECOGNIZE_FACES, data: item.data },
+          { name: JobName.FACE_DETECTION, data: item.data },
         ];
 
         const [asset] = await this.assetRepository.getByIds([item.data.id]);
@@ -244,6 +253,12 @@ export class JobService {
         if (asset && asset.isVisible) {
           this.communicationRepository.send(ClientEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
         }
+        break;
+      }
+
+      case JobName.FACE_DETECTION: {
+        await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: item.data });
+        break;
       }
     }
   }