feat(server): separate face clustering job (#5598)

* separate facial clustering job * update api * fixed some tests * invert clustering * hdbscan * update api * remove commented code * wip dbscan * cleanup removed cluster endpoint remove commented code * fixes updated tests minor fixes and formatting fixed queuing refinements * scale search range based on library size * defer non-core faces * optimizations removed unused query option * assign faces individually for correctness fixed unit tests remove unused method * don't select face embedding update sql linting fixed ml typing * updated job mock * paginate people query * select face embeddings because typeorm * fix setting face detection concurrency * update sql formatting linting * simplify logic remove unused imports * more specific delete signature * more accurate typing for face stubs * add migration formatting * chore: better typing * don't select embedding by default remove unused import * updated sql * use normal try/catch * stricter concurrency typing and enforcement * update api * update job concurrency panel to show disabled queues formatting * check jobId in queueAll fix tests * remove outdated comment * better facial recognition icon * wording wording formatting * fixed tests * fix * formatting & sql * try to fix sql check * more detailed description * update sql * formatting * wording * update `minFaces` description --------- Co-authored-by: Jason Rasmussen <jrasm91@gmail.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-18 00:08:48 -05:00
parent 44873b4224
commit 68f52818ae
57 changed files with 1081 additions and 631 deletions
@@ -18,10 +18,12 @@ import {
  newSystemConfigRepositoryMock,
  personStub,
 } from '@test';
+import { IsNull } from 'typeorm';
 import { BulkIdErrorReason } from '../asset';
 import { CacheControl, ImmichFileResponse } from '../domain.util';
 import { JobName } from '../job';
 import {
+  FaceSearchResult,
  IAssetRepository,
  ICryptoRepository,
  IJobRepository,
@@ -120,7 +122,7 @@ describe(PersonService.name, () => {
        people: [responseDto],
      });
      expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
-        minimumFaceCount: 1,
+        minimumFaceCount: 3,
        withHidden: false,
      });
    });
@@ -132,7 +134,7 @@ describe(PersonService.name, () => {
        people: [responseDto],
      });
      expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
-        minimumFaceCount: 1,
+        minimumFaceCount: 3,
        withHidden: false,
      });
    });
@@ -153,7 +155,7 @@ describe(PersonService.name, () => {
        ],
      });
      expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
-        minimumFaceCount: 1,
+        minimumFaceCount: 3,
        withHidden: true,
      });
    });
@@ -516,51 +518,22 @@ describe(PersonService.name, () => {
    });
  });

-  describe('handlePersonDelete', () => {
-    it('should stop if a person has not be found', async () => {
-      personMock.getById.mockResolvedValue(null);
-
-      await expect(sut.handlePersonDelete({ id: 'person-1' })).resolves.toBe(false);
-      expect(personMock.update).not.toHaveBeenCalled();
-      expect(storageMock.unlink).not.toHaveBeenCalled();
-    });
-    it('should delete a person', async () => {
-      personMock.getById.mockResolvedValue(personStub.primaryPerson);
-
-      await expect(sut.handlePersonDelete({ id: 'person-1' })).resolves.toBe(true);
-      expect(personMock.delete).toHaveBeenCalledWith(personStub.primaryPerson);
-      expect(storageMock.unlink).toHaveBeenCalledWith(personStub.primaryPerson.thumbnailPath);
-    });
-  });
-
-  describe('handlePersonDelete', () => {
-    it('should delete person', async () => {
-      personMock.getById.mockResolvedValue(personStub.withName);
-
-      await sut.handlePersonDelete({ id: personStub.withName.id });
-
-      expect(personMock.delete).toHaveBeenCalledWith(personStub.withName);
-      expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
-    });
-  });
-
  describe('handlePersonCleanup', () => {
    it('should delete people without faces', async () => {
      personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);

      await sut.handlePersonCleanup();

-      expect(jobMock.queueAll).toHaveBeenCalledWith([
-        { name: JobName.PERSON_DELETE, data: { id: personStub.noName.id } },
-      ]);
+      expect(personMock.delete).toHaveBeenCalledWith([personStub.noName]);
+      expect(storageMock.unlink).toHaveBeenCalledWith(personStub.noName.thumbnailPath);
    });
  });

-  describe('handleQueueRecognizeFaces', () => {
+  describe('handleQueueDetectFaces', () => {
    it('should return if machine learning is disabled', async () => {
      configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);

-      await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
+      await expect(sut.handleQueueDetectFaces({})).resolves.toBe(true);
      expect(jobMock.queue).not.toHaveBeenCalled();
      expect(jobMock.queueAll).not.toHaveBeenCalled();
      expect(configMock.load).toHaveBeenCalled();
@@ -571,12 +544,13 @@ describe(PersonService.name, () => {
        items: [assetStub.image],
        hasNextPage: false,
      });
-      await sut.handleQueueRecognizeFaces({});
+
+      await sut.handleQueueDetectFaces({});

      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
      expect(jobMock.queueAll).toHaveBeenCalledWith([
        {
-          name: JobName.RECOGNIZE_FACES,
+          name: JobName.FACE_DETECTION,
          data: { id: assetStub.image.id },
        },
      ]);
@@ -587,39 +561,133 @@ describe(PersonService.name, () => {
        items: [assetStub.image],
        hasNextPage: false,
      });
-      personMock.getAll.mockResolvedValue([personStub.withName]);
-      personMock.deleteAll.mockResolvedValue(5);
+      personMock.getAll.mockResolvedValue({
+        items: [personStub.withName],
+        hasNextPage: false,
+      });

-      await sut.handleQueueRecognizeFaces({ force: true });
+      await sut.handleQueueDetectFaces({ force: true });

      expect(assetMock.getAll).toHaveBeenCalled();
      expect(jobMock.queueAll).toHaveBeenCalledWith([
        {
-          name: JobName.RECOGNIZE_FACES,
+          name: JobName.FACE_DETECTION,
          data: { id: assetStub.image.id },
        },
      ]);
+    });
+
+    it('should delete existing people and faces if forced', async () => {
+      personMock.getAll.mockResolvedValue({
+        items: [faceStub.face1.person],
+        hasNextPage: false,
+      });
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+      assetMock.getAll.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueDetectFaces({ force: true });
+
+      expect(assetMock.getAll).toHaveBeenCalled();
      expect(jobMock.queueAll).toHaveBeenCalledWith([
        {
-          name: JobName.PERSON_DELETE,
-          data: { id: personStub.withName.id },
+          name: JobName.FACE_DETECTION,
+          data: { id: assetStub.image.id },
        },
      ]);
+      expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
+      expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
    });
  });

-  describe('handleRecognizeFaces', () => {
+  describe('handleQueueRecognizeFaces', () => {
    it('should return if machine learning is disabled', async () => {
      configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);

-      await expect(sut.handleRecognizeFaces({ id: 'foo' })).resolves.toBe(true);
+      await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(configMock.load).toHaveBeenCalled();
+    });
+
+    it('should queue missing assets', async () => {
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueRecognizeFaces({});
+
+      expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { personId: IsNull() } });
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.FACIAL_RECOGNITION,
+          data: { id: faceStub.face1.id, deferred: false },
+        },
+      ]);
+    });
+
+    it('should queue all assets', async () => {
+      personMock.getAll.mockResolvedValue({
+        items: [],
+        hasNextPage: false,
+      });
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueRecognizeFaces({ force: true });
+
+      expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.FACIAL_RECOGNITION,
+          data: { id: faceStub.face1.id, deferred: false },
+        },
+      ]);
+    });
+
+    it('should delete existing people and faces if forced', async () => {
+      personMock.getAll.mockResolvedValue({
+        items: [faceStub.face1.person],
+        hasNextPage: false,
+      });
+      personMock.getAllFaces.mockResolvedValue({
+        items: [faceStub.face1],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueRecognizeFaces({ force: true });
+
+      expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
+      expect(jobMock.queueAll).toHaveBeenCalledWith([
+        {
+          name: JobName.FACIAL_RECOGNITION,
+          data: { id: faceStub.face1.id, deferred: false },
+        },
+      ]);
+      expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
+      expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
+    });
+  });
+
+  describe('handleDetectFaces', () => {
+    it('should return if machine learning is disabled', async () => {
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
+
+      await expect(sut.handleDetectFaces({ id: 'foo' })).resolves.toBe(true);
      expect(assetMock.getByIds).not.toHaveBeenCalled();
      expect(configMock.load).toHaveBeenCalled();
    });

    it('should skip when no resize path', async () => {
      assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
-      await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
+      await sut.handleDetectFaces({ id: assetStub.noResizePath.id });
      expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
    });

@@ -636,7 +704,7 @@ describe(PersonService.name, () => {
          ],
        },
      ]);
-      await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
+      await sut.handleDetectFaces({ id: assetStub.noResizePath.id });
      expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
    });

@@ -645,7 +713,7 @@ describe(PersonService.name, () => {

      machineLearningMock.detectFaces.mockResolvedValue([]);
      assetMock.getByIds.mockResolvedValue([assetStub.image]);
-      await sut.handleRecognizeFaces({ id: assetStub.image.id });
+      await sut.handleDetectFaces({ id: assetStub.image.id });
      expect(machineLearningMock.detectFaces).toHaveBeenCalledWith(
        'http://immich-machine-learning:3003',
        {
@@ -655,7 +723,7 @@ describe(PersonService.name, () => {
          enabled: true,
          maxDistance: 0.6,
          minScore: 0.7,
-          minFaces: 1,
+          minFaces: 3,
          modelName: 'buffalo_l',
        },
      );
@@ -670,37 +738,13 @@ describe(PersonService.name, () => {
      expect(assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt?.getTime()).toBeGreaterThan(start);
    });

-    it('should match existing people', async () => {
+    it('should create a face with no person', async () => {
      machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
-      smartInfoMock.searchFaces.mockResolvedValue([faceStub.face1]);
+      smartInfoMock.searchFaces.mockResolvedValue([{ face: faceStub.face1, distance: 0.7 }]);
      assetMock.getByIds.mockResolvedValue([assetStub.image]);
-      await sut.handleRecognizeFaces({ id: assetStub.image.id });
+      await sut.handleDetectFaces({ id: assetStub.image.id });

      expect(personMock.createFace).toHaveBeenCalledWith({
-        personId: 'person-1',
-        assetId: 'asset-id',
-        embedding: [1, 2, 3, 4],
-        boundingBoxX1: 100,
-        boundingBoxY1: 100,
-        boundingBoxX2: 200,
-        boundingBoxY2: 200,
-        imageHeight: 500,
-        imageWidth: 400,
-      });
-    });
-
-    it('should create a new person', async () => {
-      machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
-      smartInfoMock.searchFaces.mockResolvedValue([]);
-      personMock.create.mockResolvedValue(personStub.noName);
-      assetMock.getByIds.mockResolvedValue([assetStub.image]);
-      personMock.createFace.mockResolvedValue(faceStub.primaryFace1);
-
-      await sut.handleRecognizeFaces({ id: assetStub.image.id });
-
-      expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetStub.image.ownerId });
-      expect(personMock.createFace).toHaveBeenCalledWith({
-        personId: 'person-1',
        assetId: 'asset-id',
        embedding: [1, 2, 3, 4],
        boundingBoxX1: 100,
@@ -710,8 +754,130 @@ describe(PersonService.name, () => {
        imageHeight: 500,
        imageWidth: 400,
      });
+      expect(personMock.reassignFace).not.toHaveBeenCalled();
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
    });
  });
+
+  describe('handleRecognizeFaces', () => {
+    it('should return false if face does not exist', async () => {
+      personMock.getFaceByIdWithAssets.mockResolvedValue(null);
+
+      expect(await sut.handleRecognizeFaces({ id: faceStub.face1.id })).toBe(false);
+
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.createFace).not.toHaveBeenCalled();
+    });
+
+    it('should return true if face already has an assigned person', async () => {
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.face1);
+
+      expect(await sut.handleRecognizeFaces({ id: faceStub.face1.id })).toBe(true);
+
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.createFace).not.toHaveBeenCalled();
+    });
+
+    it('should match existing person', async () => {
+      if (!faceStub.primaryFace1.person) {
+        throw new Error('faceStub.primaryFace1.person is null');
+      }
+
+      const faces = [
+        { face: faceStub.noPerson1, distance: 0.0 },
+        { face: faceStub.primaryFace1, distance: 0.2 },
+        { face: faceStub.noPerson2, distance: 0.3 },
+        { face: faceStub.face1, distance: 0.4 },
+      ] as FaceSearchResult[];
+
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 1 },
+      ]);
+      smartInfoMock.searchFaces.mockResolvedValue(faces);
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      personMock.create.mockResolvedValue(faceStub.primaryFace1.person);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.reassignFaces).toHaveBeenCalledTimes(1);
+      expect(personMock.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.arrayContaining([faceStub.noPerson1.id]),
+        newPersonId: faceStub.primaryFace1.person.id,
+      });
+      expect(personMock.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.not.arrayContaining([faceStub.face1.id]),
+        newPersonId: faceStub.primaryFace1.person.id,
+      });
+    });
+
+    it('should create a new person if the face is a core point with no person', async () => {
+      const faces = [
+        { face: faceStub.noPerson1, distance: 0.0 },
+        { face: faceStub.noPerson2, distance: 0.3 },
+      ] as FaceSearchResult[];
+
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 1 },
+      ]);
+      smartInfoMock.searchFaces.mockResolvedValue(faces);
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      personMock.create.mockResolvedValue(personStub.withName);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(personMock.create).toHaveBeenCalledWith({
+        ownerId: faceStub.noPerson1.asset.ownerId,
+        faceAssetId: faceStub.noPerson1.id,
+      });
+      expect(personMock.reassignFaces).toHaveBeenCalledWith({
+        faceIds: [faceStub.noPerson1.id],
+        newPersonId: personStub.withName.id,
+      });
+    });
+
+    it('should defer non-core faces to end of queue', async () => {
+      const faces = [{ face: faceStub.noPerson1, distance: 0.0 }] as FaceSearchResult[];
+
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
+      ]);
+      smartInfoMock.searchFaces.mockResolvedValue(faces);
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      personMock.create.mockResolvedValue(personStub.withName);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.FACIAL_RECOGNITION,
+        data: { id: faceStub.noPerson1.id, deferred: true },
+      });
+      expect(smartInfoMock.searchFaces).toHaveBeenCalledTimes(1);
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
+    });
+
+    it('should not assign person to non-core face with no matching person', async () => {
+      const faces = [{ face: faceStub.noPerson1, distance: 0.0 }] as FaceSearchResult[];
+
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
+      ]);
+      smartInfoMock.searchFaces.mockResolvedValueOnce(faces).mockResolvedValueOnce([]);
+      personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      personMock.create.mockResolvedValue(personStub.withName);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id, deferred: true });
+
+      expect(jobMock.queue).not.toHaveBeenCalled();
+      expect(smartInfoMock.searchFaces).toHaveBeenCalledTimes(2);
+      expect(personMock.create).not.toHaveBeenCalled();
+      expect(personMock.reassignFaces).not.toHaveBeenCalled();
+    });
+  });
+
  describe('handleGeneratePersonThumbnail', () => {
    it('should return if machine learning is disabled', async () => {
      configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
@@ -822,7 +988,6 @@ describe(PersonService.name, () => {
    it('should require person.write and person.merge permission', async () => {
      personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
      personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
-      personMock.delete.mockResolvedValue(personStub.mergePerson);

      await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).rejects.toBeInstanceOf(
        BadRequestException,
@@ -837,7 +1002,6 @@ describe(PersonService.name, () => {
    it('should merge two people without smart merge', async () => {
      personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
      personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
-      personMock.delete.mockResolvedValue(personStub.mergePerson);
      accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1']));
      accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-2']));

@@ -852,17 +1016,12 @@ describe(PersonService.name, () => {

      expect(personMock.update).not.toHaveBeenCalled();

-      expect(jobMock.queue).toHaveBeenCalledWith({
-        name: JobName.PERSON_DELETE,
-        data: { id: personStub.mergePerson.id },
-      });
      expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
    });

    it('should merge two people with smart merge', async () => {
      personMock.getById.mockResolvedValueOnce(personStub.randomPerson);
      personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
-      personMock.delete.mockResolvedValue(personStub.primaryPerson);
      personMock.update.mockResolvedValue({ ...personStub.randomPerson, name: personStub.primaryPerson.name });
      accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-3']));
      accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1']));
@@ -881,10 +1040,7 @@ describe(PersonService.name, () => {
        name: personStub.primaryPerson.name,
      });

-      expect(jobMock.queue).toHaveBeenCalledWith({
-        name: JobName.PERSON_DELETE,
-        data: { id: personStub.primaryPerson.id },
-      });
+      expect(personMock.delete).toHaveBeenCalledWith([personStub.primaryPerson]);
      expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
    });

@@ -954,7 +1110,7 @@ describe(PersonService.name, () => {
        boundingBoxX2: 1,
        boundingBoxY1: 0,
        boundingBoxY2: 1,
-        id: 'assetFaceId',
+        id: faceStub.face1.id,
        imageHeight: 1024,
        imageWidth: 1024,
        person: mapPerson(personStub.withName),
@@ -2,12 +2,13 @@ import { PersonEntity } from '@app/infra/entities';
 import { PersonPathType } from '@app/infra/entities/move.entity';
 import { ImmichLogger } from '@app/infra/logger';
 import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
+import { IsNull } from 'typeorm';
 import { AccessCore, Permission } from '../access';
 import { AssetResponseDto, BulkIdErrorReason, BulkIdResponseDto, mapAsset } from '../asset';
 import { AuthDto } from '../auth';
 import { mimeTypes } from '../domain.constant';
 import { CacheControl, ImmichFileResponse, usePagination } from '../domain.util';
-import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
+import { IBaseJob, IDeferrableJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
 import { FACE_THUMBNAIL_SIZE } from '../media';
 import {
  CropOptions,
@@ -249,64 +250,63 @@ export class PersonService {
    return results;
  }

-  async handlePersonDelete({ id }: IEntityJob) {
-    const person = await this.repository.getById(id);
-    if (!person) {
-      return false;
-    }
+  private async delete(people: PersonEntity[]) {
+    await Promise.all(people.map((person) => this.storageRepository.unlink(person.thumbnailPath)));
+    await this.repository.delete(people);
+    this.logger.debug(`Deleted ${people.length} people`);
+  }

-    try {
-      await this.repository.delete(person);
-      await this.storageRepository.unlink(person.thumbnailPath);
-    } catch (error: Error | any) {
-      this.logger.error(`Unable to delete person: ${error}`, error?.stack);
-    }
+  private async deleteAllPeople() {
+    const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.repository.getAll(pagination),
+    );

-    return true;
+    for await (const people of personPagination) {
+      await this.delete(people); // deletes thumbnails too
+    }
  }

  async handlePersonCleanup() {
    const people = await this.repository.getAllWithoutFaces();
-    for (const person of people) {
-      this.logger.debug(`Person ${person.name || person.id} no longer has any faces, deleting.`);
-    }
-    await this.jobRepository.queueAll(
-      people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
-    );
-
+    await this.delete(people);
    return true;
  }

-  async handleQueueRecognizeFaces({ force }: IBaseJob) {
+  async handleQueueDetectFaces({ force }: IBaseJob) {
    const { machineLearning } = await this.configCore.getConfig();
    if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
      return true;
    }

+    if (force) {
+      await this.deleteAllPeople();
+      await this.repository.deleteAllFaces();
+    }
+
    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
      return force
-        ? this.assetRepository.getAll(pagination, { order: 'DESC' })
+        ? this.assetRepository.getAll(pagination, {
+            order: 'DESC',
+            withFaces: true,
+            withPeople: false,
+            withSmartInfo: false,
+            withSmartSearch: false,
+            withExif: false,
+            withStacked: false,
+          })
        : this.assetRepository.getWithout(pagination, WithoutProperty.FACES);
    });

-    if (force) {
-      const people = await this.repository.getAll();
-      await this.jobRepository.queueAll(
-        people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
-      );
-      this.logger.debug(`Deleted ${people.length} people`);
-    }
-
    for await (const assets of assetPagination) {
      await this.jobRepository.queueAll(
-        assets.map((asset) => ({ name: JobName.RECOGNIZE_FACES, data: { id: asset.id } })),
+        assets.map((asset) => ({ name: JobName.FACE_DETECTION, data: { id: asset.id } })),
      );
    }

    return true;
  }

-  async handleRecognizeFaces({ id }: IEntityJob) {
+  async handleDetectFaces({ id }: IEntityJob) {
    const { machineLearning } = await this.configCore.getConfig();
    if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
      return true;
@@ -315,7 +315,7 @@ export class PersonService {
    const relations = {
      exifInfo: true,
      faces: {
-        person: true,
+        person: false,
      },
    };
    const [asset] = await this.assetRepository.getByIds([id], relations);
@@ -332,38 +332,19 @@ export class PersonService {
    this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
    this.logger.verbose(faces.map((face) => ({ ...face, embedding: `vector(${face.embedding.length})` })));

-    for (const { embedding, ...rest } of faces) {
-      const matches = await this.smartInfoRepository.searchFaces({
-        userIds: [asset.ownerId],
-        embedding,
-        numResults: 1,
-        maxDistance: machineLearning.facialRecognition.maxDistance,
-      });
-
-      let personId = matches[0]?.personId || null;
-      let newPerson: PersonEntity | null = null;
-      if (!personId) {
-        this.logger.debug('No matches, creating a new person.');
-        newPerson = await this.repository.create({ ownerId: asset.ownerId });
-        personId = newPerson.id;
-      }
-
-      const face = await this.repository.createFace({
+    for (const face of faces) {
+      const mappedFace = {
        assetId: asset.id,
-        personId,
-        embedding,
-        imageHeight: rest.imageHeight,
-        imageWidth: rest.imageWidth,
-        boundingBoxX1: rest.boundingBox.x1,
-        boundingBoxX2: rest.boundingBox.x2,
-        boundingBoxY1: rest.boundingBox.y1,
-        boundingBoxY2: rest.boundingBox.y2,
-      });
+        embedding: face.embedding,
+        imageHeight: face.imageHeight,
+        imageWidth: face.imageWidth,
+        boundingBoxX1: face.boundingBox.x1,
+        boundingBoxX2: face.boundingBox.x2,
+        boundingBoxY1: face.boundingBox.y1,
+        boundingBoxY2: face.boundingBox.y2,
+      };

-      if (newPerson) {
-        await this.repository.update({ id: personId, faceAssetId: face.id });
-        await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
-      }
+      await this.repository.createFace(mappedFace);
    }

    await this.assetRepository.upsertJobStatus({
@@ -374,6 +355,98 @@ export class PersonService {
    return true;
  }

+  async handleQueueRecognizeFaces({ force }: IBaseJob) {
+    const { machineLearning } = await this.configCore.getConfig();
+    if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
+      return true;
+    }
+
+    await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
+
+    if (force) {
+      await this.deleteAllPeople();
+    }
+
+    const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }),
+    );
+
+    for await (const page of facePagination) {
+      await this.jobRepository.queueAll(
+        page.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id, deferred: false } })),
+      );
+    }
+
+    return true;
+  }
+
+  async handleRecognizeFaces({ id, deferred }: IDeferrableJob) {
+    const { machineLearning } = await this.configCore.getConfig();
+    if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
+      return true;
+    }
+
+    const face = await this.repository.getFaceByIdWithAssets(
+      id,
+      { person: true, asset: true },
+      { id: true, personId: true, embedding: true },
+    );
+    if (!face) {
+      this.logger.warn(`Face ${id} not found`);
+      return false;
+    }
+
+    if (face.personId) {
+      this.logger.debug(`Face ${id} already has a person assigned`);
+      return true;
+    }
+
+    const matches = await this.smartInfoRepository.searchFaces({
+      userIds: [face.asset.ownerId],
+      embedding: face.embedding,
+      maxDistance: machineLearning.facialRecognition.maxDistance,
+      numResults: machineLearning.facialRecognition.minFaces,
+    });
+
+    this.logger.debug(`Face ${id} has ${matches.length} match${matches.length != 1 ? 'es' : ''}`);
+
+    const isCore = matches.length >= machineLearning.facialRecognition.minFaces;
+    if (!isCore && !deferred) {
+      this.logger.debug(`Deferring non-core face ${id} for later processing`);
+      await this.jobRepository.queue({ name: JobName.FACIAL_RECOGNITION, data: { id, deferred: true } });
+      return true;
+    }
+
+    let personId = matches.find((match) => match.face.personId)?.face.personId; // `matches` also includes the face itself
+    if (!personId) {
+      const matchWithPerson = await this.smartInfoRepository.searchFaces({
+        userIds: [face.asset.ownerId],
+        embedding: face.embedding,
+        maxDistance: machineLearning.facialRecognition.maxDistance,
+        numResults: 1,
+        hasPerson: true,
+      });
+
+      if (matchWithPerson.length > 0) {
+        personId = matchWithPerson[0].face.personId;
+      }
+    }
+
+    if (isCore && !personId) {
+      this.logger.log(`Creating new person for face ${id}`);
+      const newPerson = await this.repository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id });
+      await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
+      personId = newPerson.id;
+    }
+
+    if (personId) {
+      this.logger.debug(`Assigning face ${id} to person ${personId}`);
+      await this.repository.reassignFaces({ faceIds: [id], newPersonId: personId });
+    }
+
+    return true;
+  }
+
  async handlePersonMigration({ id }: IEntityJob) {
    const person = await this.repository.getById(id);
    if (!person) {
@@ -499,7 +572,7 @@ export class PersonService {
        this.logger.log(`Merging ${mergeName} into ${primaryName}`);

        await this.repository.reassignFaces(mergeData);
-        await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: mergePerson.id } });
+        await this.delete([mergePerson]);

        this.logger.log(`Merged ${mergeName} into ${primaryName}`);
        results.push({ id: mergeId, success: true });