feat(server): Avoid face match with people born after file creation #4743 (#16918)

* feat(server): Avoid face matching with people born after file creation date (#4743)

* lint

* add medium tests for facial recognition

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>

server/src/queries/search.repository.sql

@@ -136,6 +136,7 @@ with
       "asset_faces"
       inner join "assets" on "assets"."id" = "asset_faces"."assetId"
       inner join "face_search" on "face_search"."faceId" = "asset_faces"."id"
+      left join "person" on "person"."id" = "asset_faces"."personId"
     where
       "assets"."ownerId" = any ($2::uuid[])
       and "assets"."deletedAt" is null

server/src/repositories/search.repository.ts

@@ -163,6 +163,7 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
   hasPerson?: boolean;
   numResults: number;
   maxDistance: number;
+  minBirthDate?: Date;
 }
 
 export interface AssetDuplicateSearch {
@@ -338,7 +339,7 @@ export class SearchRepository {
       },
     ],
   })
-  searchFaces({ userIds, embedding, numResults, maxDistance, hasPerson }: FaceEmbeddingSearch) {
+  searchFaces({ userIds, embedding, numResults, maxDistance, hasPerson, minBirthDate }: FaceEmbeddingSearch) {
     if (!isValidInteger(numResults, { min: 1, max: 1000 })) {
       throw new Error(`Invalid value for 'numResults': ${numResults}`);
     }
@@ -354,9 +355,13 @@ export class SearchRepository {
           ])
           .innerJoin('assets', 'assets.id', 'asset_faces.assetId')
           .innerJoin('face_search', 'face_search.faceId', 'asset_faces.id')
+          .leftJoin('person', 'person.id', 'asset_faces.personId')
           .where('assets.ownerId', '=', anyUuid(userIds))
           .where('assets.deletedAt', 'is', null)
           .$if(!!hasPerson, (qb) => qb.where('asset_faces.personId', 'is not', null))
+          .$if(!!minBirthDate, (qb) =>
+            qb.where((eb) => eb.or([eb('person.birthDate', 'is', null), eb('person.birthDate', '<=', minBirthDate!)])),
+          )
           .orderBy(sql`face_search.embedding <=> ${embedding}`)
           .limit(numResults),
       )

server/src/services/person.service.spec.ts

@@ -896,6 +896,66 @@ describe(PersonService.name, () => {
       });
     });
 
+    it('should match existing person if their birth date is unknown', async () => {
+      if (!faceStub.primaryFace1.person) {
+        throw new Error('faceStub.primaryFace1.person is null');
+      }
+
+      const faces = [
+        { ...faceStub.noPerson1, distance: 0 },
+        { ...faceStub.primaryFace1, distance: 0.2 },
+        { ...faceStub.withBirthDate, distance: 0.3 },
+      ] as FaceSearchResult[];
+
+      mocks.systemMetadata.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
+      mocks.search.searchFaces.mockResolvedValue(faces);
+      mocks.person.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      mocks.person.create.mockResolvedValue(faceStub.primaryFace1.person);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(mocks.person.create).not.toHaveBeenCalled();
+      expect(mocks.person.reassignFaces).toHaveBeenCalledTimes(1);
+      expect(mocks.person.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.arrayContaining([faceStub.noPerson1.id]),
+        newPersonId: faceStub.primaryFace1.person.id,
+      });
+      expect(mocks.person.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.not.arrayContaining([faceStub.face1.id]),
+        newPersonId: faceStub.primaryFace1.person.id,
+      });
+    });
+
+    it('should match existing person if their birth date is before file creation', async () => {
+      if (!faceStub.primaryFace1.person) {
+        throw new Error('faceStub.primaryFace1.person is null');
+      }
+
+      const faces = [
+        { ...faceStub.noPerson1, distance: 0 },
+        { ...faceStub.withBirthDate, distance: 0.2 },
+        { ...faceStub.primaryFace1, distance: 0.3 },
+      ] as FaceSearchResult[];
+
+      mocks.systemMetadata.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
+      mocks.search.searchFaces.mockResolvedValue(faces);
+      mocks.person.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
+      mocks.person.create.mockResolvedValue(faceStub.primaryFace1.person);
+
+      await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
+
+      expect(mocks.person.create).not.toHaveBeenCalled();
+      expect(mocks.person.reassignFaces).toHaveBeenCalledTimes(1);
+      expect(mocks.person.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.arrayContaining([faceStub.noPerson1.id]),
+        newPersonId: faceStub.withBirthDate.person?.id,
+      });
+      expect(mocks.person.reassignFaces).toHaveBeenCalledWith({
+        faceIds: expect.not.arrayContaining([faceStub.face1.id]),
+        newPersonId: faceStub.withBirthDate.person?.id,
+      });
+    });
+
     it('should create a new person if the face is a core point with no person', async () => {
       const faces = [
         { ...faceStub.noPerson1, distance: 0 },

server/src/services/person.service.ts

@@ -483,6 +483,7 @@ export class PersonService extends BaseService {
       embedding: face.faceSearch.embedding,
       maxDistance: machineLearning.facialRecognition.maxDistance,
       numResults: machineLearning.facialRecognition.minFaces,
+      minBirthDate: face.asset.fileCreatedAt,
     });
 
     // `matches` also includes the face itself
@@ -508,6 +509,7 @@ export class PersonService extends BaseService {
         maxDistance: machineLearning.facialRecognition.maxDistance,
         numResults: 1,
         hasPerson: true,
+        minBirthDate: face.asset.fileCreatedAt,
       });
 
       if (matchWithPerson.length > 0) {