cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): separate face clustering job (#5598)

Browse Source 
* separate facial clustering job

* update api

* fixed some tests

* invert clustering

* hdbscan

* update api

* remove commented code

* wip dbscan

* cleanup

removed cluster endpoint

remove commented code

* fixes

updated tests

minor fixes and formatting

fixed queuing

refinements

* scale search range based on library size

* defer non-core faces

* optimizations

removed unused query option

* assign faces individually for correctness

fixed unit tests

remove unused method

* don't select face embedding

update sql

linting

fixed ml typing

* updated job mock

* paginate people query

* select face embeddings because typeorm

* fix setting face detection concurrency

* update sql

formatting

linting

* simplify logic

remove unused imports

* more specific delete signature

* more accurate typing for face stubs

* add migration

formatting

* chore: better typing

* don't select embedding by default

remove unused import

* updated sql

* use normal try/catch

* stricter concurrency typing and enforcement

* update api

* update job concurrency panel to show disabled queues

formatting

* check jobId in queueAll

fix tests

* remove outdated comment

* better facial recognition icon

* wording

wording

formatting

* fixed tests

* fix

* formatting & sql

* try to fix sql check

* more detailed description

* update sql

* formatting

* wording

* update `minFaces` description

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Mert
2024-01-18 00:08:48 -05:00
committed by GitHub
parent 44873b4224
commit 68f52818ae
57 changed files with 1081 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/audit/audit.service.ts
+12 -8
View File
@@ -204,16 +204,20 @@ export class AuditService {
}
}
const people = await this.personRepository.getAll();
for (const { id, thumbnailPath } of people) {
track(thumbnailPath);
const entity = { entityId: id, entityType: PathEntityType.PERSON };
if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination),
);
for await (const people of personPagination) {
for (const { id, thumbnailPath } of people) {
track(thumbnailPath);
const entity = { entityId: id, entityType: PathEntityType.PERSON };
if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
}
}
}
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
}
const extras: string[] = [];
for (const file of allFiles) {
server/src/domain/job/job.constants.ts
+15 -7
View File
@@ -2,7 +2,8 @@ export enum QueueName {
THUMBNAIL_GENERATION = 'thumbnailGeneration',
METADATA_EXTRACTION = 'metadataExtraction',
VIDEO_CONVERSION = 'videoConversion',
RECOGNIZE_FACES = 'recognizeFaces',
FACE_DETECTION = 'faceDetection',
FACIAL_RECOGNITION = 'facialRecognition',
SMART_SEARCH = 'smartSearch',
BACKGROUND_TASK = 'backgroundTask',
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
@@ -12,6 +13,11 @@ export enum QueueName {
LIBRARY = 'library',
}
export type ConcurrentQueueName = Exclude<
QueueName,
QueueName.STORAGE_TEMPLATE_MIGRATION | QueueName.FACIAL_RECOGNITION
>;
export enum JobCommand {
START = 'start',
PAUSE = 'pause',
@@ -57,9 +63,10 @@ export enum JobName {
// facial recognition
PERSON_CLEANUP = 'person-cleanup',
PERSON_DELETE = 'person-delete',
QUEUE_RECOGNIZE_FACES = 'queue-recognize-faces',
RECOGNIZE_FACES = 'recognize-faces',
QUEUE_FACE_DETECTION = 'queue-face-detection',
FACE_DETECTION = 'face-detection',
QUEUE_FACIAL_RECOGNITION = 'queue-facial-recognition',
FACIAL_RECOGNITION = 'facial-recognition',
// library managment
LIBRARY_SCAN = 'library-refresh',
@@ -95,7 +102,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.DELETE_FILES]: QueueName.BACKGROUND_TASK,
[JobName.CLEAN_OLD_AUDIT_LOGS]: QueueName.BACKGROUND_TASK,
[JobName.PERSON_CLEANUP]: QueueName.BACKGROUND_TASK,
[JobName.PERSON_DELETE]: QueueName.BACKGROUND_TASK,
[JobName.USER_SYNC_USAGE]: QueueName.BACKGROUND_TASK,
// conversion
@@ -124,8 +130,10 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.MIGRATE_PERSON]: QueueName.MIGRATION,
// facial recognition
[JobName.QUEUE_RECOGNIZE_FACES]: QueueName.RECOGNIZE_FACES,
[JobName.RECOGNIZE_FACES]: QueueName.RECOGNIZE_FACES,
[JobName.QUEUE_FACE_DETECTION]: QueueName.FACE_DETECTION,
[JobName.FACE_DETECTION]: QueueName.FACE_DETECTION,
[JobName.QUEUE_FACIAL_RECOGNITION]: QueueName.FACIAL_RECOGNITION,
[JobName.FACIAL_RECOGNITION]: QueueName.FACIAL_RECOGNITION,
// clip
[JobName.QUEUE_ENCODE_CLIP]: QueueName.SMART_SEARCH,
server/src/domain/job/job.dto.ts
+4 -1
View File
@@ -75,7 +75,10 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
[QueueName.SEARCH]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.RECOGNIZE_FACES]!: JobStatusDto;
[QueueName.FACE_DETECTION]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.FACIAL_RECOGNITION]!: JobStatusDto;
@ApiProperty({ type: JobStatusDto })
[QueueName.SIDECAR]!: JobStatusDto;
server/src/domain/job/job.interface.ts
+4
View File
@@ -35,3 +35,7 @@ export interface ISidecarWriteJob extends IEntityJob {
latitude?: number;
longitude?: number;
}
export interface IDeferrableJob extends IEntityJob {
deferred?: boolean;
}
server/src/domain/job/job.service.spec.ts
+29 -11
View File
@@ -104,7 +104,8 @@ describe(JobService.name, () => {
[QueueName.MIGRATION]: expectedJobStatus,
[QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
[QueueName.FACE_DETECTION]: expectedJobStatus,
[QueueName.FACIAL_RECOGNITION]: expectedJobStatus,
[QueueName.SIDECAR]: expectedJobStatus,
[QueueName.LIBRARY]: expectedJobStatus,
});
@@ -189,12 +190,20 @@ describe(JobService.name, () => {
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
});
it('should handle a start recognize faces command', async () => {
it('should handle a start face detection command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false });
await sut.handleCommand(QueueName.FACE_DETECTION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_FACE_DETECTION, data: { force: false } });
});
it('should handle a start facial recognition command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.FACIAL_RECOGNITION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false } });
});
it('should throw a bad request when an invalid queue is used', async () => {
@@ -224,7 +233,7 @@ describe(JobService.name, () => {
[QueueName.BACKGROUND_TASK]: { concurrency: 10 },
[QueueName.SMART_SEARCH]: { concurrency: 10 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 10 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
[QueueName.FACE_DETECTION]: { concurrency: 10 },
[QueueName.SEARCH]: { concurrency: 10 },
[QueueName.SIDECAR]: { concurrency: 10 },
[QueueName.LIBRARY]: { concurrency: 10 },
@@ -237,7 +246,7 @@ describe(JobService.name, () => {
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.BACKGROUND_TASK, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SMART_SEARCH, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.FACE_DETECTION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
@@ -280,7 +289,7 @@ describe(JobService.name, () => {
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.FACE_DETECTION,
],
},
{
@@ -289,7 +298,7 @@ describe(JobService.name, () => {
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.FACE_DETECTION,
JobName.VIDEO_CONVERSION,
],
},
@@ -299,7 +308,7 @@ describe(JobService.name, () => {
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.FACE_DETECTION,
JobName.VIDEO_CONVERSION,
],
},
@@ -308,7 +317,11 @@ describe(JobService.name, () => {
jobs: [],
},
{
item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
item: { name: JobName.FACE_DETECTION, data: { id: 'asset-1' } },
jobs: [JobName.QUEUE_FACIAL_RECOGNITION],
},
{
item: { name: JobName.FACIAL_RECOGNITION, data: { id: 'asset-1' } },
jobs: [],
},
];
@@ -355,7 +368,12 @@ describe(JobService.name, () => {
configKey: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED,
},
{
queue: QueueName.RECOGNIZE_FACES,
queue: QueueName.FACE_DETECTION,
feature: FeatureFlag.FACIAL_RECOGNITION,
configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
},
{
queue: QueueName.FACIAL_RECOGNITION,
feature: FeatureFlag.FACIAL_RECOGNITION,
configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
},
server/src/domain/job/job.service.ts
+22 -7
View File
@@ -14,7 +14,7 @@ import {
QueueCleanType,
} from '../repositories';
import { FeatureFlag, SystemConfigCore } from '../system-config/system-config.core';
import { JobCommand, JobName, QueueName } from './job.constants';
import { ConcurrentQueueName, JobCommand, JobName, QueueName } from './job.constants';
import { AllJobStatusResponseDto, JobCommandDto, JobStatusDto } from './job.dto';
@Injectable()
@@ -108,9 +108,13 @@ export class JobService {
case QueueName.THUMBNAIL_GENERATION:
return this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force } });
case QueueName.RECOGNIZE_FACES:
case QueueName.FACE_DETECTION:
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
return this.jobRepository.queue({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force } });
return this.jobRepository.queue({ name: JobName.QUEUE_FACE_DETECTION, data: { force } });
case QueueName.FACIAL_RECOGNITION:
await this.configCore.requireFeature(FeatureFlag.FACIAL_RECOGNITION);
return this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force } });
case QueueName.LIBRARY:
return this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force } });
@@ -124,7 +128,8 @@ export class JobService {
const config = await this.configCore.getConfig();
for (const queueName of Object.values(QueueName)) {
let concurrency = 1;
if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
if (this.isConcurrentQueue(queueName)) {
concurrency = config.job[queueName].concurrency;
}
@@ -145,10 +150,10 @@ export class JobService {
}
this.configCore.config$.subscribe((config) => {
this.logger.log(`Updating queue concurrency settings`);
this.logger.debug(`Updating queue concurrency settings`);
for (const queueName of Object.values(QueueName)) {
let concurrency = 1;
if (queueName !== QueueName.STORAGE_TEMPLATE_MIGRATION) {
if (this.isConcurrentQueue(queueName)) {
concurrency = config.job[queueName].concurrency;
}
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
@@ -157,6 +162,10 @@ export class JobService {
});
}
private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
return ![QueueName.FACIAL_RECOGNITION, QueueName.STORAGE_TEMPLATE_MIGRATION].includes(name);
}
async handleNightlyJobs() {
await this.jobRepository.queueAll([
{ name: JobName.ASSET_DELETION_CHECK },
@@ -217,7 +226,7 @@ export class JobService {
{ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data },
{ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data },
{ name: JobName.ENCODE_CLIP, data: item.data },
{ name: JobName.RECOGNIZE_FACES, data: item.data },
{ name: JobName.FACE_DETECTION, data: item.data },
];
const [asset] = await this.assetRepository.getByIds([item.data.id]);
@@ -244,6 +253,12 @@ export class JobService {
if (asset && asset.isVisible) {
this.communicationRepository.send(ClientEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset));
}
break;
}
case JobName.FACE_DETECTION: {
await this.jobRepository.queue({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: item.data });
break;
}
}
}
server/src/domain/media/media.service.spec.ts
+29 -16
View File
@@ -70,7 +70,10 @@ describe(MediaService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([personStub.newThumbnail]);
personMock.getAll.mockResolvedValue({
items: [personStub.newThumbnail],
hasNextPage: false,
});
personMock.getFacesByIds.mockResolvedValue([faceStub.face1]);
await sut.handleQueueGenerateThumbnails({ force: true });
@@ -84,8 +87,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).not.toHaveBeenCalled();
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,
@@ -99,7 +101,10 @@ describe(MediaService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAllWithoutThumbnail.mockResolvedValue([personStub.noThumbnail]);
personMock.getAll.mockResolvedValue({
items: [personStub.noThumbnail],
hasNextPage: false,
});
personMock.getRandomFace.mockResolvedValue(faceStub.face1);
await sut.handleQueueGenerateThumbnails({ force: false });
@@ -107,8 +112,7 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
expect(personMock.getRandomFace).toHaveBeenCalled();
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
@@ -125,7 +129,10 @@ describe(MediaService.name, () => {
items: [assetStub.noResizePath],
hasNextPage: false,
});
personMock.getAllWithoutThumbnail.mockResolvedValue([]);
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
await sut.handleQueueGenerateThumbnails({ force: false });
@@ -138,8 +145,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
});
it('should queue all assets with missing webp path', async () => {
@@ -147,7 +153,10 @@ describe(MediaService.name, () => {
items: [assetStub.noWebpPath],
hasNextPage: false,
});
personMock.getAllWithoutThumbnail.mockResolvedValue([]);
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
await sut.handleQueueGenerateThumbnails({ force: false });
@@ -160,8 +169,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
});
it('should queue all assets with missing thumbhash', async () => {
@@ -169,7 +177,10 @@ describe(MediaService.name, () => {
items: [assetStub.noThumbhash],
hasNextPage: false,
});
personMock.getAllWithoutThumbnail.mockResolvedValue([]);
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
await sut.handleQueueGenerateThumbnails({ force: false });
@@ -182,8 +193,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).not.toHaveBeenCalled();
expect(personMock.getAllWithoutThumbnail).toHaveBeenCalled();
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
});
});
@@ -394,7 +404,10 @@ describe(MediaService.name, () => {
items: [assetStub.video],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([]);
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
await sut.handleQueueVideoConversion({ force: true });
server/src/domain/media/media.service.ts
+22 -13
View File
@@ -93,20 +93,24 @@ export class MediaService {
await this.jobRepository.queueAll(jobs);
}
const people = force ? await this.personRepository.getAll() : await this.personRepository.getAllWithoutThumbnail();
const jobs: JobItem[] = [];
for (const person of people) {
if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id);
if (!face) {
continue;
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination, { where: force ? undefined : { thumbnailPath: '' } }),
);
for await (const people of personPagination) {
for (const person of people) {
if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id);
if (!face) {
continue;
}
await this.personRepository.update({ id: person.id, faceAssetId: face.assetId });
}
await this.personRepository.update({ id: person.id, faceAssetId: face.assetId });
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
}
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
}
await this.jobRepository.queueAll(jobs);
@@ -131,11 +135,16 @@ export class MediaService {
);
}
const people = await this.personRepository.getAll();
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.MIGRATE_PERSON, data: { id: person.id } })),
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination),
);
for await (const people of personPagination) {
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.MIGRATE_PERSON, data: { id: person.id } })),
);
}
return true;
}
server/src/domain/person/person.service.spec.ts
+245 -89
View File
@@ -18,10 +18,12 @@ import {
newSystemConfigRepositoryMock,
personStub,
} from '@test';
import { IsNull } from 'typeorm';
import { BulkIdErrorReason } from '../asset';
import { CacheControl, ImmichFileResponse } from '../domain.util';
import { JobName } from '../job';
import {
FaceSearchResult,
IAssetRepository,
ICryptoRepository,
IJobRepository,
@@ -120,7 +122,7 @@ describe(PersonService.name, () => {
people: [responseDto],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
minimumFaceCount: 1,
minimumFaceCount: 3,
withHidden: false,
});
});
@@ -132,7 +134,7 @@ describe(PersonService.name, () => {
people: [responseDto],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
minimumFaceCount: 1,
minimumFaceCount: 3,
withHidden: false,
});
});
@@ -153,7 +155,7 @@ describe(PersonService.name, () => {
],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.user.id, {
minimumFaceCount: 1,
minimumFaceCount: 3,
withHidden: true,
});
});
@@ -516,51 +518,22 @@ describe(PersonService.name, () => {
});
});
describe('handlePersonDelete', () => {
it('should stop if a person has not be found', async () => {
personMock.getById.mockResolvedValue(null);
await expect(sut.handlePersonDelete({ id: 'person-1' })).resolves.toBe(false);
expect(personMock.update).not.toHaveBeenCalled();
expect(storageMock.unlink).not.toHaveBeenCalled();
});
it('should delete a person', async () => {
personMock.getById.mockResolvedValue(personStub.primaryPerson);
await expect(sut.handlePersonDelete({ id: 'person-1' })).resolves.toBe(true);
expect(personMock.delete).toHaveBeenCalledWith(personStub.primaryPerson);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.primaryPerson.thumbnailPath);
});
});
describe('handlePersonDelete', () => {
it('should delete person', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
await sut.handlePersonDelete({ id: personStub.withName.id });
expect(personMock.delete).toHaveBeenCalledWith(personStub.withName);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
});
});
describe('handlePersonCleanup', () => {
it('should delete people without faces', async () => {
personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
await sut.handlePersonCleanup();
expect(jobMock.queueAll).toHaveBeenCalledWith([
{ name: JobName.PERSON_DELETE, data: { id: personStub.noName.id } },
]);
expect(personMock.delete).toHaveBeenCalledWith([personStub.noName]);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.noName.thumbnailPath);
});
});
describe('handleQueueRecognizeFaces', () => {
describe('handleQueueDetectFaces', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
await expect(sut.handleQueueDetectFaces({})).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
@@ -571,12 +544,13 @@ describe(PersonService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({});
await sut.handleQueueDetectFaces({});
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.RECOGNIZE_FACES,
name: JobName.FACE_DETECTION,
data: { id: assetStub.image.id },
},
]);
@@ -587,39 +561,133 @@ describe(PersonService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([personStub.withName]);
personMock.deleteAll.mockResolvedValue(5);
personMock.getAll.mockResolvedValue({
items: [personStub.withName],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({ force: true });
await sut.handleQueueDetectFaces({ force: true });
expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.RECOGNIZE_FACES,
name: JobName.FACE_DETECTION,
data: { id: assetStub.image.id },
},
]);
});
it('should delete existing people and faces if forced', async () => {
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleQueueDetectFaces({ force: true });
expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.PERSON_DELETE,
data: { id: personStub.withName.id },
name: JobName.FACE_DETECTION,
data: { id: assetStub.image.id },
},
]);
expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
});
});
describe('handleRecognizeFaces', () => {
describe('handleQueueRecognizeFaces', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleRecognizeFaces({ id: 'foo' })).resolves.toBe(true);
await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
it('should queue missing assets', async () => {
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({});
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { personId: IsNull() } });
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
data: { id: faceStub.face1.id, deferred: false },
},
]);
});
it('should queue all assets', async () => {
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({ force: true });
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
data: { id: faceStub.face1.id, deferred: false },
},
]);
});
it('should delete existing people and faces if forced', async () => {
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({ force: true });
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
data: { id: faceStub.face1.id, deferred: false },
},
]);
expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
});
});
describe('handleDetectFaces', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleDetectFaces({ id: 'foo' })).resolves.toBe(true);
expect(assetMock.getByIds).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
it('should skip when no resize path', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
await sut.handleDetectFaces({ id: assetStub.noResizePath.id });
expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
});
@@ -636,7 +704,7 @@ describe(PersonService.name, () => {
],
},
]);
await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
await sut.handleDetectFaces({ id: assetStub.noResizePath.id });
expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
});
@@ -645,7 +713,7 @@ describe(PersonService.name, () => {
machineLearningMock.detectFaces.mockResolvedValue([]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
await sut.handleDetectFaces({ id: assetStub.image.id });
expect(machineLearningMock.detectFaces).toHaveBeenCalledWith(
'http://immich-machine-learning:3003',
{
@@ -655,7 +723,7 @@ describe(PersonService.name, () => {
enabled: true,
maxDistance: 0.6,
minScore: 0.7,
minFaces: 1,
minFaces: 3,
modelName: 'buffalo_l',
},
);
@@ -670,37 +738,13 @@ describe(PersonService.name, () => {
expect(assetMock.upsertJobStatus.mock.calls[0][0].facesRecognizedAt?.getTime()).toBeGreaterThan(start);
});
it('should match existing people', async () => {
it('should create a face with no person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
smartInfoMock.searchFaces.mockResolvedValue([faceStub.face1]);
smartInfoMock.searchFaces.mockResolvedValue([{ face: faceStub.face1, distance: 0.7 }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
await sut.handleDetectFaces({ id: assetStub.image.id });
expect(personMock.createFace).toHaveBeenCalledWith({
personId: 'person-1',
assetId: 'asset-id',
embedding: [1, 2, 3, 4],
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageHeight: 500,
imageWidth: 400,
});
});
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
smartInfoMock.searchFaces.mockResolvedValue([]);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
personMock.createFace.mockResolvedValue(faceStub.primaryFace1);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetStub.image.ownerId });
expect(personMock.createFace).toHaveBeenCalledWith({
personId: 'person-1',
assetId: 'asset-id',
embedding: [1, 2, 3, 4],
boundingBoxX1: 100,
@@ -710,8 +754,130 @@ describe(PersonService.name, () => {
imageHeight: 500,
imageWidth: 400,
});
expect(personMock.reassignFace).not.toHaveBeenCalled();
expect(personMock.reassignFaces).not.toHaveBeenCalled();
});
});
describe('handleRecognizeFaces', () => {
it('should return false if face does not exist', async () => {
personMock.getFaceByIdWithAssets.mockResolvedValue(null);
expect(await sut.handleRecognizeFaces({ id: faceStub.face1.id })).toBe(false);
expect(personMock.reassignFaces).not.toHaveBeenCalled();
expect(personMock.create).not.toHaveBeenCalled();
expect(personMock.createFace).not.toHaveBeenCalled();
});
it('should return true if face already has an assigned person', async () => {
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.face1);
expect(await sut.handleRecognizeFaces({ id: faceStub.face1.id })).toBe(true);
expect(personMock.reassignFaces).not.toHaveBeenCalled();
expect(personMock.create).not.toHaveBeenCalled();
expect(personMock.createFace).not.toHaveBeenCalled();
});
it('should match existing person', async () => {
if (!faceStub.primaryFace1.person) {
throw new Error('faceStub.primaryFace1.person is null');
}
const faces = [
{ face: faceStub.noPerson1, distance: 0.0 },
{ face: faceStub.primaryFace1, distance: 0.2 },
{ face: faceStub.noPerson2, distance: 0.3 },
{ face: faceStub.face1, distance: 0.4 },
] as FaceSearchResult[];
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 1 },
]);
smartInfoMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(faceStub.primaryFace1.person);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
expect(personMock.create).not.toHaveBeenCalled();
expect(personMock.reassignFaces).toHaveBeenCalledTimes(1);
expect(personMock.reassignFaces).toHaveBeenCalledWith({
faceIds: expect.arrayContaining([faceStub.noPerson1.id]),
newPersonId: faceStub.primaryFace1.person.id,
});
expect(personMock.reassignFaces).toHaveBeenCalledWith({
faceIds: expect.not.arrayContaining([faceStub.face1.id]),
newPersonId: faceStub.primaryFace1.person.id,
});
});
it('should create a new person if the face is a core point with no person', async () => {
const faces = [
{ face: faceStub.noPerson1, distance: 0.0 },
{ face: faceStub.noPerson2, distance: 0.3 },
] as FaceSearchResult[];
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 1 },
]);
smartInfoMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
expect(personMock.create).toHaveBeenCalledWith({
ownerId: faceStub.noPerson1.asset.ownerId,
faceAssetId: faceStub.noPerson1.id,
});
expect(personMock.reassignFaces).toHaveBeenCalledWith({
faceIds: [faceStub.noPerson1.id],
newPersonId: personStub.withName.id,
});
});
it('should defer non-core faces to end of queue', async () => {
const faces = [{ face: faceStub.noPerson1, distance: 0.0 }] as FaceSearchResult[];
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
]);
smartInfoMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.FACIAL_RECOGNITION,
data: { id: faceStub.noPerson1.id, deferred: true },
});
expect(smartInfoMock.searchFaces).toHaveBeenCalledTimes(1);
expect(personMock.create).not.toHaveBeenCalled();
expect(personMock.reassignFaces).not.toHaveBeenCalled();
});
it('should not assign person to non-core face with no matching person', async () => {
const faces = [{ face: faceStub.noPerson1, distance: 0.0 }] as FaceSearchResult[];
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_MIN_FACES, value: 2 },
]);
smartInfoMock.searchFaces.mockResolvedValueOnce(faces).mockResolvedValueOnce([]);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id, deferred: true });
expect(jobMock.queue).not.toHaveBeenCalled();
expect(smartInfoMock.searchFaces).toHaveBeenCalledTimes(2);
expect(personMock.create).not.toHaveBeenCalled();
expect(personMock.reassignFaces).not.toHaveBeenCalled();
});
});
describe('handleGeneratePersonThumbnail', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
@@ -822,7 +988,6 @@ describe(PersonService.name, () => {
it('should require person.write and person.merge permission', async () => {
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
personMock.delete.mockResolvedValue(personStub.mergePerson);
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).rejects.toBeInstanceOf(
BadRequestException,
@@ -837,7 +1002,6 @@ describe(PersonService.name, () => {
it('should merge two people without smart merge', async () => {
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
personMock.delete.mockResolvedValue(personStub.mergePerson);
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1']));
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-2']));
@@ -852,17 +1016,12 @@ describe(PersonService.name, () => {
expect(personMock.update).not.toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.PERSON_DELETE,
data: { id: personStub.mergePerson.id },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
it('should merge two people with smart merge', async () => {
personMock.getById.mockResolvedValueOnce(personStub.randomPerson);
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.delete.mockResolvedValue(personStub.primaryPerson);
personMock.update.mockResolvedValue({ ...personStub.randomPerson, name: personStub.primaryPerson.name });
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-3']));
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1']));
@@ -881,10 +1040,7 @@ describe(PersonService.name, () => {
name: personStub.primaryPerson.name,
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.PERSON_DELETE,
data: { id: personStub.primaryPerson.id },
});
expect(personMock.delete).toHaveBeenCalledWith([personStub.primaryPerson]);
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
@@ -954,7 +1110,7 @@ describe(PersonService.name, () => {
boundingBoxX2: 1,
boundingBoxY1: 0,
boundingBoxY2: 1,
id: 'assetFaceId',
id: faceStub.face1.id,
imageHeight: 1024,
imageWidth: 1024,
person: mapPerson(personStub.withName),
server/src/domain/person/person.service.ts
+137 -64
View File
@@ -2,12 +2,13 @@ import { PersonEntity } from '@app/infra/entities';
import { PersonPathType } from '@app/infra/entities/move.entity';
import { ImmichLogger } from '@app/infra/logger';
import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { AccessCore, Permission } from '../access';
import { AssetResponseDto, BulkIdErrorReason, BulkIdResponseDto, mapAsset } from '../asset';
import { AuthDto } from '../auth';
import { mimeTypes } from '../domain.constant';
import { CacheControl, ImmichFileResponse, usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { IBaseJob, IDeferrableJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
import { FACE_THUMBNAIL_SIZE } from '../media';
import {
CropOptions,
@@ -249,64 +250,63 @@ export class PersonService {
return results;
}
async handlePersonDelete({ id }: IEntityJob) {
const person = await this.repository.getById(id);
if (!person) {
return false;
}
private async delete(people: PersonEntity[]) {
await Promise.all(people.map((person) => this.storageRepository.unlink(person.thumbnailPath)));
await this.repository.delete(people);
this.logger.debug(`Deleted ${people.length} people`);
}
try {
await this.repository.delete(person);
await this.storageRepository.unlink(person.thumbnailPath);
} catch (error: Error | any) {
this.logger.error(`Unable to delete person: ${error}`, error?.stack);
}
private async deleteAllPeople() {
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.repository.getAll(pagination),
);
return true;
for await (const people of personPagination) {
await this.delete(people); // deletes thumbnails too
}
}
async handlePersonCleanup() {
const people = await this.repository.getAllWithoutFaces();
for (const person of people) {
this.logger.debug(`Person ${person.name || person.id} no longer has any faces, deleting.`);
}
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
);
await this.delete(people);
return true;
}
async handleQueueRecognizeFaces({ force }: IBaseJob) {
async handleQueueDetectFaces({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
return true;
}
if (force) {
await this.deleteAllPeople();
await this.repository.deleteAllFaces();
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination, { order: 'DESC' })
? this.assetRepository.getAll(pagination, {
order: 'DESC',
withFaces: true,
withPeople: false,
withSmartInfo: false,
withSmartSearch: false,
withExif: false,
withStacked: false,
})
: this.assetRepository.getWithout(pagination, WithoutProperty.FACES);
});
if (force) {
const people = await this.repository.getAll();
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.PERSON_DELETE, data: { id: person.id } })),
);
this.logger.debug(`Deleted ${people.length} people`);
}
for await (const assets of assetPagination) {
await this.jobRepository.queueAll(
assets.map((asset) => ({ name: JobName.RECOGNIZE_FACES, data: { id: asset.id } })),
assets.map((asset) => ({ name: JobName.FACE_DETECTION, data: { id: asset.id } })),
);
}
return true;
}
async handleRecognizeFaces({ id }: IEntityJob) {
async handleDetectFaces({ id }: IEntityJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
return true;
@@ -315,7 +315,7 @@ export class PersonService {
const relations = {
exifInfo: true,
faces: {
person: true,
person: false,
},
};
const [asset] = await this.assetRepository.getByIds([id], relations);
@@ -332,38 +332,19 @@ export class PersonService {
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `vector(${face.embedding.length})` })));
for (const { embedding, ...rest } of faces) {
const matches = await this.smartInfoRepository.searchFaces({
userIds: [asset.ownerId],
embedding,
numResults: 1,
maxDistance: machineLearning.facialRecognition.maxDistance,
});
let personId = matches[0]?.personId || null;
let newPerson: PersonEntity | null = null;
if (!personId) {
this.logger.debug('No matches, creating a new person.');
newPerson = await this.repository.create({ ownerId: asset.ownerId });
personId = newPerson.id;
}
const face = await this.repository.createFace({
for (const face of faces) {
const mappedFace = {
assetId: asset.id,
personId,
embedding,
imageHeight: rest.imageHeight,
imageWidth: rest.imageWidth,
boundingBoxX1: rest.boundingBox.x1,
boundingBoxX2: rest.boundingBox.x2,
boundingBoxY1: rest.boundingBox.y1,
boundingBoxY2: rest.boundingBox.y2,
});
embedding: face.embedding,
imageHeight: face.imageHeight,
imageWidth: face.imageWidth,
boundingBoxX1: face.boundingBox.x1,
boundingBoxX2: face.boundingBox.x2,
boundingBoxY1: face.boundingBox.y1,
boundingBoxY2: face.boundingBox.y2,
};
if (newPerson) {
await this.repository.update({ id: personId, faceAssetId: face.id });
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
}
await this.repository.createFace(mappedFace);
}
await this.assetRepository.upsertJobStatus({
@@ -374,6 +355,98 @@ export class PersonService {
return true;
}
async handleQueueRecognizeFaces({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
return true;
}
await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
if (force) {
await this.deleteAllPeople();
}
const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }),
);
for await (const page of facePagination) {
await this.jobRepository.queueAll(
page.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id, deferred: false } })),
);
}
return true;
}
async handleRecognizeFaces({ id, deferred }: IDeferrableJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
return true;
}
const face = await this.repository.getFaceByIdWithAssets(
id,
{ person: true, asset: true },
{ id: true, personId: true, embedding: true },
);
if (!face) {
this.logger.warn(`Face ${id} not found`);
return false;
}
if (face.personId) {
this.logger.debug(`Face ${id} already has a person assigned`);
return true;
}
const matches = await this.smartInfoRepository.searchFaces({
userIds: [face.asset.ownerId],
embedding: face.embedding,
maxDistance: machineLearning.facialRecognition.maxDistance,
numResults: machineLearning.facialRecognition.minFaces,
});
this.logger.debug(`Face ${id} has ${matches.length} match${matches.length != 1 ? 'es' : ''}`);
const isCore = matches.length >= machineLearning.facialRecognition.minFaces;
if (!isCore && !deferred) {
this.logger.debug(`Deferring non-core face ${id} for later processing`);
await this.jobRepository.queue({ name: JobName.FACIAL_RECOGNITION, data: { id, deferred: true } });
return true;
}
let personId = matches.find((match) => match.face.personId)?.face.personId; // `matches` also includes the face itself
if (!personId) {
const matchWithPerson = await this.smartInfoRepository.searchFaces({
userIds: [face.asset.ownerId],
embedding: face.embedding,
maxDistance: machineLearning.facialRecognition.maxDistance,
numResults: 1,
hasPerson: true,
});
if (matchWithPerson.length > 0) {
personId = matchWithPerson[0].face.personId;
}
}
if (isCore && !personId) {
this.logger.log(`Creating new person for face ${id}`);
const newPerson = await this.repository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id });
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
personId = newPerson.id;
}
if (personId) {
this.logger.debug(`Assigning face ${id} to person ${personId}`);
await this.repository.reassignFaces({ faceIds: [id], newPersonId: personId });
}
return true;
}
async handlePersonMigration({ id }: IEntityJob) {
const person = await this.repository.getById(id);
if (!person) {
@@ -499,7 +572,7 @@ export class PersonService {
this.logger.log(`Merging ${mergeName} into ${primaryName}`);
await this.repository.reassignFaces(mergeData);
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: mergePerson.id } });
await this.delete([mergePerson]);
this.logger.log(`Merged ${mergeName} into ${primaryName}`);
results.push({ id: mergeId, success: true });
server/src/domain/repositories/asset.repository.ts
+10 -2
View File
@@ -1,6 +1,6 @@
import { SearchExploreItem } from '@app/domain';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { FindOptionsRelations } from 'typeorm';
import { FindOptionsRelations, FindOptionsSelect } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
export type AssetStats = Record<AssetType, number>;
@@ -33,6 +33,9 @@ export interface AssetSearchOptions {
withStacked?: boolean;
withExif?: boolean;
withPeople?: boolean;
withSmartInfo?: boolean;
withSmartSearch?: boolean;
withFaces?: boolean;
createdBefore?: Date;
createdAfter?: Date;
@@ -93,6 +96,7 @@ export enum WithoutProperty {
CLIP_ENCODING = 'clip-embedding',
OBJECT_TAGS = 'object-tags',
FACES = 'faces',
PERSON = 'person',
SIDECAR = 'sidecar',
}
@@ -168,7 +172,11 @@ export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
create(asset: AssetCreate): Promise<AssetEntity>;
getByDate(ownerId: string, date: Date): Promise<AssetEntity[]>;
getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]>;
getByIds(
ids: string[],
relations?: FindOptionsRelations<AssetEntity>,
select?: FindOptionsSelect<AssetEntity>,
): Promise<AssetEntity[]>;
getByDayOfYear(ownerId: string, monthDay: MonthDay): Promise<AssetEntity[]>;
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
server/src/domain/repositories/job.repository.ts
+7 -4
View File
@@ -3,6 +3,7 @@ import { JobName, QueueName } from '../job/job.constants';
import {
IAssetDeletionJob,
IBaseJob,
IDeferrableJob,
IDeleteFilesJob,
IEntityJob,
ILibraryFileJob,
@@ -63,11 +64,12 @@ export type JobItem =
| { name: JobName.SIDECAR_SYNC; data: IEntityJob }
| { name: JobName.SIDECAR_WRITE; data: ISidecarWriteJob }
// Recognize Faces
| { name: JobName.QUEUE_RECOGNIZE_FACES; data: IBaseJob }
| { name: JobName.RECOGNIZE_FACES; data: IEntityJob }
// Facial Recognition
| { name: JobName.QUEUE_FACE_DETECTION; data: IBaseJob }
| { name: JobName.FACE_DETECTION; data: IEntityJob }
| { name: JobName.QUEUE_FACIAL_RECOGNITION; data: IBaseJob }
| { name: JobName.FACIAL_RECOGNITION; data: IDeferrableJob }
| { name: JobName.GENERATE_PERSON_THUMBNAIL; data: IEntityJob }
| { name: JobName.PERSON_DELETE; data: IEntityJob }
// Clip Embedding
| { name: JobName.QUEUE_ENCODE_CLIP; data: IBaseJob }
@@ -111,4 +113,5 @@ export interface IJobRepository {
clear(name: QueueName, type: QueueCleanType): Promise<string[]>;
getQueueStatus(name: QueueName): Promise<QueueStatus>;
getJobCounts(name: QueueName): Promise<JobCounts>;
waitForQueueCompletion(...queues: QueueName[]): Promise<void>;
}
server/src/domain/repositories/person.repository.ts
+20 -14
View File
@@ -1,4 +1,6 @@
import { AssetEntity, AssetFaceEntity, PersonEntity } from '@app/infra/entities';
import { FindManyOptions, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
export const IPersonRepository = 'IPersonRepository';
@@ -17,7 +19,8 @@ export interface AssetFaceId {
}
export interface UpdateFacesData {
oldPersonId: string;
oldPersonId?: string;
faceIds?: string[];
newPersonId: string;
}
@@ -26,8 +29,7 @@ export interface PersonStatistics {
}
export interface IPersonRepository {
getAll(): Promise<PersonEntity[]>;
getAllWithoutThumbnail(): Promise<PersonEntity[]>;
getAll(pagination: PaginationOptions, options?: FindManyOptions<PersonEntity>): Paginated<PersonEntity>;
getAllForUser(userId: string, options: PersonSearchOptions): Promise<PersonEntity[]>;
getAllWithoutFaces(): Promise<PersonEntity[]>;
getById(personId: string): Promise<PersonEntity | null>;
@@ -35,19 +37,23 @@ export interface IPersonRepository {
getAssets(personId: string): Promise<AssetEntity[]>;
reassignFaces(data: UpdateFacesData): Promise<number>;
create(entity: Partial<PersonEntity>): Promise<PersonEntity>;
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
delete(entity: PersonEntity): Promise<PersonEntity | null>;
deleteAll(): Promise<number>;
getStatistics(personId: string): Promise<PersonStatistics>;
getAllFaces(): Promise<AssetFaceEntity[]>;
createFace(entity: Partial<AssetFaceEntity>): Promise<void>;
delete(entities: PersonEntity[]): Promise<void>;
deleteAll(): Promise<void>;
deleteAllFaces(): Promise<void>;
getAllFaces(pagination: PaginationOptions, options?: FindManyOptions<AssetFaceEntity>): Paginated<AssetFaceEntity>;
getFaceById(id: string): Promise<AssetFaceEntity>;
getFaceByIdWithAssets(
id: string,
relations?: FindOptionsRelations<AssetFaceEntity>,
select?: FindOptionsSelect<AssetFaceEntity>,
): Promise<AssetFaceEntity | null>;
getFaces(assetId: string): Promise<AssetFaceEntity[]>;
getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]>;
getRandomFace(personId: string): Promise<AssetFaceEntity | null>;
createFace(entity: Partial<AssetFaceEntity>): Promise<AssetFaceEntity>;
getFaces(assetId: string): Promise<AssetFaceEntity[]>;
getStatistics(personId: string): Promise<PersonStatistics>;
reassignFace(assetFaceId: string, newPersonId: string): Promise<number>;
getFaceById(id: string): Promise<AssetFaceEntity>;
getFaceByIdWithAssets(id: string): Promise<AssetFaceEntity | null>;
reassignFaces(data: UpdateFacesData): Promise<number>;
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
}
server/src/domain/repositories/smart-info.repository.ts
+12 -3
View File
@@ -7,14 +7,23 @@ export type Embedding = number[];
export interface EmbeddingSearch {
userIds: string[];
embedding: Embedding;
numResults: number;
maxDistance?: number;
numResults?: number;
withArchived?: boolean;
}
export interface FaceEmbeddingSearch extends EmbeddingSearch {
maxDistance?: number;
hasPerson?: boolean;
}
export interface FaceSearchResult {
face: AssetFaceEntity;
distance: number;
}
export interface ISmartInfoRepository {
init(modelName: string): Promise<void>;
searchCLIP(search: EmbeddingSearch): Promise<AssetEntity[]>;
searchFaces(search: EmbeddingSearch): Promise<AssetFaceEntity[]>;
searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
}
server/src/domain/smart-info/dto/model-config.dto.ts
+2 -2
View File
@@ -30,14 +30,14 @@ export class RecognitionConfig extends ModelConfig {
@Min(0)
@Max(1)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
@ApiProperty({ type: 'number', format: 'float' })
minScore!: number;
@IsNumber()
@Min(0)
@Max(2)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
@ApiProperty({ type: 'number', format: 'float' })
maxDistance!: number;
@IsNumber()
server/src/domain/smart-info/smart-info.service.ts
+1 -8
View File
@@ -1,6 +1,5 @@
import { ImmichLogger } from '@app/infra/logger';
import { Inject, Injectable } from '@nestjs/common';
import { setTimeout } from 'timers/promises';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
import {
@@ -34,13 +33,7 @@ export class SmartInfoService {
async init() {
await this.jobRepository.pause(QueueName.SMART_SEARCH);
let { isActive } = await this.jobRepository.getQueueStatus(QueueName.SMART_SEARCH);
while (isActive) {
this.logger.verbose('Waiting for CLIP encoding queue to stop...');
await setTimeout(1000).then(async () => {
({ isActive } = await this.jobRepository.getQueueStatus(QueueName.SMART_SEARCH));
});
}
await this.jobRepository.waitForQueueCompletion(QueueName.SMART_SEARCH);
const { machineLearning } = await this.configCore.getConfig();
server/src/domain/system-config/dto/system-config-job.dto.ts
+3 -5
View File
@@ -1,7 +1,7 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
import { QueueName } from '../../job';
import { ConcurrentQueueName, QueueName } from '../../job';
export class JobSettingsDto {
@IsInt()
@@ -10,9 +10,7 @@ export class JobSettingsDto {
concurrency!: number;
}
export class SystemConfigJobDto
implements Record<Exclude<QueueName, QueueName.STORAGE_TEMPLATE_MIGRATION>, JobSettingsDto>
{
export class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto> {
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@@ -59,7 +57,7 @@ export class SystemConfigJobDto
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.RECOGNIZE_FACES]!: JobSettingsDto;
[QueueName.FACE_DETECTION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
server/src/domain/system-config/system-config.core.ts
+2 -2
View File
@@ -49,7 +49,7 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.SMART_SEARCH]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.FACE_DETECTION]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 5 },
@@ -73,7 +73,7 @@ export const defaults = Object.freeze<SystemConfig>({
modelName: 'buffalo_l',
minScore: 0.7,
maxDistance: 0.6,
minFaces: 1,
minFaces: 3,
},
},
map: {
server/src/domain/system-config/system-config.service.spec.ts
+2 -2
View File
@@ -30,7 +30,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.SMART_SEARCH]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.FACE_DETECTION]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.LIBRARY]: { concurrency: 5 },
@@ -73,7 +73,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
modelName: 'buffalo_l',
minScore: 0.7,
maxDistance: 0.6,
minFaces: 1,
minFaces: 3,
},
},
map: {