feat(server): Import face regions from metadata (#6455)
* feat: faces-from-metadata - Import face regions from metadata Implements immich-app#1692. - OpenAPI spec changes to accomodate metadata face import configs. New settings to enable the feature. - Updates admin UI compoments - ML faces detection/recognition & Exif/Metadata faces compatibility Signed-off-by: BugFest <bugfest.dev@pm.me> * chore(web): remove unused file confirm-enable-import-faces * chore(web): format metadata-settings * fix(server): faces-from-metadata tests and format * fix(server): code refinements, nullable face asset sourceType * fix(server): Add RegionInfo to ImmichTags interface * fix(server): deleteAllFaces sourceType param can be undefined * fix(server): exiftool-vendored 27.0.0 moves readArgs into ExifToolOptions * fix(server): rename isImportFacesFromMetadataEnabled to isFaceImportEnabled * fix(server): simplify sourceType conditional * fix(server): small fixes * fix(server): handling sourceType * fix(server): sourceType enum * fix(server): refactor metadata applyTaggedFaces * fix(server): create/update signature changes * fix(server): reduce computational cost of Person.getManyByName * fix(server): use faceList instead of faceSet * fix(server): Skip regions without Name defined * fix(mobile): Update open-api (face assets feature changes) * fix(server): Face-Person reconciliation with map/index * fix(server): tags.RegionInfo.AppliedToDimensions must be defined to process face-region * fix(server): fix shared-link.service.ts format * fix(mobile): Update open-api after branch update * simplify * fix(server): minor fixes * fix(server): person create/update methods type enforcement * fix(server): style fixes * fix(server): remove unused metadata code * fix(server): metadata faces unit tests * fix(server): top level config metadata category * fix(server): rename upsertFaces to replaceFaces * fix(server): remove sourceType when unnecessary * fix(server): sourceType as ENUM * fix(server): format fixes * fix(server): fix tests after sourceType ENUM change * fix(server): remove unnecessary JobItem cast * fix(server): fix asset enum imports * fix(open-api): add metadata config * fix(mobile): update open-api after metadata open-api spec changes * fix(web): update web/api metadata config * fix(server): remove duplicated sourceType def * fix(server): update generated sql queries * fix(e2e): tests for metadata face import feature * fix(web): Fix check:typescript * fix(e2e): update subproject ref * fix(server): revert format changes to pass format checks after ci * fix(mobile): update open-api * fix(server,movile,open-api,mobile): sourceType as DB data type * fix(e2e): upload face asset after enabling metadata face import * fix(web): simplify metadata admin settings and i18n keys * Update person.repository.ts Co-authored-by: Jason Rasmussen <jason@rasm.me> * fix(server): asset_faces.sourceType column not nullable * fix(server): simplified syntax * fix(e2e): use SDK for everything except the endpoint being tested * fix(e2e): fix test format * chore: clean up * chore: clean up * chore: update e2e/test-assets --------- Signed-off-by: BugFest <bugfest.dev@pm.me> Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
BugFest
@@ -9,9 +9,11 @@ import { SystemConfig } from 'src/config';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { SystemConfigCore } from 'src/cores/system-config.core';
|
||||
import { OnEmit } from 'src/decorators';
|
||||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import { AssetType } from 'src/enum';
|
||||
import { PersonEntity } from 'src/entities/person.entity';
|
||||
import { AssetType, SourceType } from 'src/enum';
|
||||
import { IAlbumRepository } from 'src/interfaces/album.interface';
|
||||
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
|
||||
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
|
||||
@@ -37,6 +39,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
|
||||
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
|
||||
import { ITagRepository } from 'src/interfaces/tag.interface';
|
||||
import { IUserRepository } from 'src/interfaces/user.interface';
|
||||
import { isFaceImportEnabled } from 'src/utils/misc';
|
||||
import { usePagination } from 'src/utils/pagination';
|
||||
import { upsertTags } from 'src/utils/tag';
|
||||
|
||||
@@ -104,7 +107,7 @@ export class MetadataService {
|
||||
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
|
||||
@Inject(IMetadataRepository) private repository: IMetadataRepository,
|
||||
@Inject(IMoveRepository) moveRepository: IMoveRepository,
|
||||
@Inject(IPersonRepository) personRepository: IPersonRepository,
|
||||
@Inject(IPersonRepository) private personRepository: IPersonRepository,
|
||||
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
|
||||
@Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository,
|
||||
@Inject(ITagRepository) private tagRepository: ITagRepository,
|
||||
@@ -215,6 +218,7 @@ export class MetadataService {
|
||||
}
|
||||
|
||||
async handleMetadataExtraction({ id }: IEntityJob): Promise<JobStatus> {
|
||||
const { metadata } = await this.configCore.getConfig({ withCache: true });
|
||||
const [asset] = await this.assetRepository.getByIds([id]);
|
||||
if (!asset) {
|
||||
return JobStatus.FAILED;
|
||||
@@ -253,6 +257,10 @@ export class MetadataService {
|
||||
metadataExtractedAt: new Date(),
|
||||
});
|
||||
|
||||
if (isFaceImportEnabled(metadata)) {
|
||||
await this.applyTaggedFaces(asset, exifTags);
|
||||
}
|
||||
|
||||
return JobStatus.SUCCESS;
|
||||
}
|
||||
|
||||
@@ -512,6 +520,65 @@ export class MetadataService {
|
||||
}
|
||||
}
|
||||
|
||||
private async applyTaggedFaces(asset: AssetEntity, tags: ImmichTags) {
|
||||
if (!tags.RegionInfo?.AppliedToDimensions || tags.RegionInfo.RegionList.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const discoveredFaces: Partial<AssetFaceEntity>[] = [];
|
||||
const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true });
|
||||
const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id]));
|
||||
const missing: Partial<PersonEntity>[] = [];
|
||||
const missingWithFaceAsset: Partial<PersonEntity>[] = [];
|
||||
for (const region of tags.RegionInfo.RegionList) {
|
||||
if (!region.Name) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const imageWidth = tags.RegionInfo.AppliedToDimensions.W;
|
||||
const imageHeight = tags.RegionInfo.AppliedToDimensions.H;
|
||||
const loweredName = region.Name.toLowerCase();
|
||||
const personId = existingNameMap.get(loweredName) || this.cryptoRepository.randomUUID();
|
||||
|
||||
const face = {
|
||||
id: this.cryptoRepository.randomUUID(),
|
||||
personId,
|
||||
assetId: asset.id,
|
||||
imageWidth,
|
||||
imageHeight,
|
||||
boundingBoxX1: Math.floor((region.Area.X - region.Area.W / 2) * imageWidth),
|
||||
boundingBoxY1: Math.floor((region.Area.Y - region.Area.H / 2) * imageHeight),
|
||||
boundingBoxX2: Math.floor((region.Area.X + region.Area.W / 2) * imageWidth),
|
||||
boundingBoxY2: Math.floor((region.Area.Y + region.Area.H / 2) * imageHeight),
|
||||
sourceType: SourceType.EXIF,
|
||||
};
|
||||
|
||||
discoveredFaces.push(face);
|
||||
if (!existingNameMap.has(loweredName)) {
|
||||
missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name });
|
||||
missingWithFaceAsset.push({ id: personId, faceAssetId: face.id });
|
||||
}
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
this.logger.debug(`Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`);
|
||||
}
|
||||
|
||||
const newPersons = await this.personRepository.create(missing);
|
||||
|
||||
const faceIds = await this.personRepository.replaceFaces(asset.id, discoveredFaces, SourceType.EXIF);
|
||||
this.logger.debug(`Created ${faceIds.length} faces for asset ${asset.id}`);
|
||||
|
||||
await this.personRepository.update(missingWithFaceAsset);
|
||||
|
||||
await this.jobRepository.queueAll(
|
||||
newPersons.map((person) => ({
|
||||
name: JobName.GENERATE_PERSON_THUMBNAIL,
|
||||
data: { id: person.id },
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
private async exifData(
|
||||
asset: AssetEntity,
|
||||
): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> {
|
||||
|
||||
Reference in New Issue
Block a user