feat(server): Import face regions from metadata (#6455)
* feat: faces-from-metadata - Import face regions from metadata Implements immich-app#1692. - OpenAPI spec changes to accomodate metadata face import configs. New settings to enable the feature. - Updates admin UI compoments - ML faces detection/recognition & Exif/Metadata faces compatibility Signed-off-by: BugFest <bugfest.dev@pm.me> * chore(web): remove unused file confirm-enable-import-faces * chore(web): format metadata-settings * fix(server): faces-from-metadata tests and format * fix(server): code refinements, nullable face asset sourceType * fix(server): Add RegionInfo to ImmichTags interface * fix(server): deleteAllFaces sourceType param can be undefined * fix(server): exiftool-vendored 27.0.0 moves readArgs into ExifToolOptions * fix(server): rename isImportFacesFromMetadataEnabled to isFaceImportEnabled * fix(server): simplify sourceType conditional * fix(server): small fixes * fix(server): handling sourceType * fix(server): sourceType enum * fix(server): refactor metadata applyTaggedFaces * fix(server): create/update signature changes * fix(server): reduce computational cost of Person.getManyByName * fix(server): use faceList instead of faceSet * fix(server): Skip regions without Name defined * fix(mobile): Update open-api (face assets feature changes) * fix(server): Face-Person reconciliation with map/index * fix(server): tags.RegionInfo.AppliedToDimensions must be defined to process face-region * fix(server): fix shared-link.service.ts format * fix(mobile): Update open-api after branch update * simplify * fix(server): minor fixes * fix(server): person create/update methods type enforcement * fix(server): style fixes * fix(server): remove unused metadata code * fix(server): metadata faces unit tests * fix(server): top level config metadata category * fix(server): rename upsertFaces to replaceFaces * fix(server): remove sourceType when unnecessary * fix(server): sourceType as ENUM * fix(server): format fixes * fix(server): fix tests after sourceType ENUM change * fix(server): remove unnecessary JobItem cast * fix(server): fix asset enum imports * fix(open-api): add metadata config * fix(mobile): update open-api after metadata open-api spec changes * fix(web): update web/api metadata config * fix(server): remove duplicated sourceType def * fix(server): update generated sql queries * fix(e2e): tests for metadata face import feature * fix(web): Fix check:typescript * fix(e2e): update subproject ref * fix(server): revert format changes to pass format checks after ci * fix(mobile): update open-api * fix(server,movile,open-api,mobile): sourceType as DB data type * fix(e2e): upload face asset after enabling metadata face import * fix(web): simplify metadata admin settings and i18n keys * Update person.repository.ts Co-authored-by: Jason Rasmussen <jason@rasm.me> * fix(server): asset_faces.sourceType column not nullable * fix(server): simplified syntax * fix(e2e): use SDK for everything except the endpoint being tested * fix(e2e): fix test format * chore: clean up * chore: clean up * chore: update e2e/test-assets --------- Signed-off-by: BugFest <bugfest.dev@pm.me> Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
BugFest
@@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto';
|
||||
import { Stats } from 'node:fs';
|
||||
import { constants } from 'node:fs/promises';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import { AssetType } from 'src/enum';
|
||||
import { AssetType, SourceType } from 'src/enum';
|
||||
import { IAlbumRepository } from 'src/interfaces/album.interface';
|
||||
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
|
||||
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
|
||||
@@ -24,6 +24,8 @@ import { MetadataService, Orientation } from 'src/services/metadata.service';
|
||||
import { assetStub } from 'test/fixtures/asset.stub';
|
||||
import { fileStub } from 'test/fixtures/file.stub';
|
||||
import { probeStub } from 'test/fixtures/media.stub';
|
||||
import { metadataStub } from 'test/fixtures/metadata.stub';
|
||||
import { personStub } from 'test/fixtures/person.stub';
|
||||
import { tagStub } from 'test/fixtures/tag.stub';
|
||||
import { newAlbumRepositoryMock } from 'test/repositories/album.repository.mock';
|
||||
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
|
||||
@@ -956,6 +958,123 @@ describe(MetadataService.name, () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should skip importing metadata when the feature is disabled', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: false } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.getDistinctNames).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should skip importing metadata face for assets without tags.RegionInfo', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.empty);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.getDistinctNames).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('should skip importing faces without name', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.create.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue([]);
|
||||
personMock.update.mockResolvedValue([]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.create).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
|
||||
expect(personMock.update).toHaveBeenCalledWith([]);
|
||||
});
|
||||
|
||||
it('should skip importing faces with empty name', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.create.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue([]);
|
||||
personMock.update.mockResolvedValue([]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.image.id });
|
||||
expect(personMock.create).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
|
||||
expect(personMock.update).toHaveBeenCalledWith([]);
|
||||
});
|
||||
|
||||
it('should apply metadata face tags creating new persons', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
|
||||
personMock.getDistinctNames.mockResolvedValue([]);
|
||||
personMock.create.mockResolvedValue([personStub.withName]);
|
||||
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
|
||||
personMock.update.mockResolvedValue([personStub.withName]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
|
||||
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
|
||||
expect(personMock.create).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(
|
||||
assetStub.primaryImage.id,
|
||||
[
|
||||
{
|
||||
id: 'random-uuid',
|
||||
assetId: assetStub.primaryImage.id,
|
||||
personId: 'random-uuid',
|
||||
imageHeight: 100,
|
||||
imageWidth: 100,
|
||||
boundingBoxX1: 0,
|
||||
boundingBoxX2: 10,
|
||||
boundingBoxY1: 0,
|
||||
boundingBoxY2: 10,
|
||||
sourceType: SourceType.EXIF,
|
||||
},
|
||||
],
|
||||
SourceType.EXIF,
|
||||
);
|
||||
expect(personMock.update).toHaveBeenCalledWith([{ id: 'random-uuid', faceAssetId: 'random-uuid' }]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([
|
||||
{
|
||||
name: JobName.GENERATE_PERSON_THUMBNAIL,
|
||||
data: { id: personStub.withName.id },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('should assign metadata face tags to existing persons', async () => {
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
|
||||
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
|
||||
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
|
||||
personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]);
|
||||
personMock.create.mockResolvedValue([]);
|
||||
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
|
||||
personMock.update.mockResolvedValue([personStub.withName]);
|
||||
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
|
||||
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
|
||||
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
|
||||
expect(personMock.create).toHaveBeenCalledWith([]);
|
||||
expect(personMock.replaceFaces).toHaveBeenCalledWith(
|
||||
assetStub.primaryImage.id,
|
||||
[
|
||||
{
|
||||
id: 'random-uuid',
|
||||
assetId: assetStub.primaryImage.id,
|
||||
personId: personStub.withName.id,
|
||||
imageHeight: 100,
|
||||
imageWidth: 100,
|
||||
boundingBoxX1: 0,
|
||||
boundingBoxX2: 10,
|
||||
boundingBoxY1: 0,
|
||||
boundingBoxY2: 10,
|
||||
sourceType: SourceType.EXIF,
|
||||
},
|
||||
],
|
||||
SourceType.EXIF,
|
||||
);
|
||||
expect(personMock.update).toHaveBeenCalledWith([]);
|
||||
expect(jobMock.queueAll).toHaveBeenCalledWith([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleQueueSidecar', () => {
|
||||
|
||||
Reference in New Issue
Block a user