cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server): separate face clustering job (#5598)

Browse Source 
* separate facial clustering job

* update api

* fixed some tests

* invert clustering

* hdbscan

* update api

* remove commented code

* wip dbscan

* cleanup

removed cluster endpoint

remove commented code

* fixes

updated tests

minor fixes and formatting

fixed queuing

refinements

* scale search range based on library size

* defer non-core faces

* optimizations

removed unused query option

* assign faces individually for correctness

fixed unit tests

remove unused method

* don't select face embedding

update sql

linting

fixed ml typing

* updated job mock

* paginate people query

* select face embeddings because typeorm

* fix setting face detection concurrency

* update sql

formatting

linting

* simplify logic

remove unused imports

* more specific delete signature

* more accurate typing for face stubs

* add migration

formatting

* chore: better typing

* don't select embedding by default

remove unused import

* updated sql

* use normal try/catch

* stricter concurrency typing and enforcement

* update api

* update job concurrency panel to show disabled queues

formatting

* check jobId in queueAll

fix tests

* remove outdated comment

* better facial recognition icon

* wording

wording

formatting

* fixed tests

* fix

* formatting & sql

* try to fix sql check

* more detailed description

* update sql

* formatting

* wording

* update `minFaces` description

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Mert
2024-01-18 00:08:48 -05:00
committed by GitHub
parent 44873b4224
commit 68f52818ae
57 changed files with 1081 additions and 631 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/repositories/asset.repository.ts
+10 -2
View File
@@ -1,6 +1,6 @@
import { SearchExploreItem } from '@app/domain';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { FindOptionsRelations } from 'typeorm';
import { FindOptionsRelations, FindOptionsSelect } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
export type AssetStats = Record<AssetType, number>;
@@ -33,6 +33,9 @@ export interface AssetSearchOptions {
withStacked?: boolean;
withExif?: boolean;
withPeople?: boolean;
withSmartInfo?: boolean;
withSmartSearch?: boolean;
withFaces?: boolean;
createdBefore?: Date;
createdAfter?: Date;
@@ -93,6 +96,7 @@ export enum WithoutProperty {
CLIP_ENCODING = 'clip-embedding',
OBJECT_TAGS = 'object-tags',
FACES = 'faces',
PERSON = 'person',
SIDECAR = 'sidecar',
}
@@ -168,7 +172,11 @@ export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
create(asset: AssetCreate): Promise<AssetEntity>;
getByDate(ownerId: string, date: Date): Promise<AssetEntity[]>;
getByIds(ids: string[], relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity[]>;
getByIds(
ids: string[],
relations?: FindOptionsRelations<AssetEntity>,
select?: FindOptionsSelect<AssetEntity>,
): Promise<AssetEntity[]>;
getByDayOfYear(ownerId: string, monthDay: MonthDay): Promise<AssetEntity[]>;
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
server/src/domain/repositories/job.repository.ts
+7 -4
View File
@@ -3,6 +3,7 @@ import { JobName, QueueName } from '../job/job.constants';
import {
IAssetDeletionJob,
IBaseJob,
IDeferrableJob,
IDeleteFilesJob,
IEntityJob,
ILibraryFileJob,
@@ -63,11 +64,12 @@ export type JobItem =
| { name: JobName.SIDECAR_SYNC; data: IEntityJob }
| { name: JobName.SIDECAR_WRITE; data: ISidecarWriteJob }
// Recognize Faces
| { name: JobName.QUEUE_RECOGNIZE_FACES; data: IBaseJob }
| { name: JobName.RECOGNIZE_FACES; data: IEntityJob }
// Facial Recognition
| { name: JobName.QUEUE_FACE_DETECTION; data: IBaseJob }
| { name: JobName.FACE_DETECTION; data: IEntityJob }
| { name: JobName.QUEUE_FACIAL_RECOGNITION; data: IBaseJob }
| { name: JobName.FACIAL_RECOGNITION; data: IDeferrableJob }
| { name: JobName.GENERATE_PERSON_THUMBNAIL; data: IEntityJob }
| { name: JobName.PERSON_DELETE; data: IEntityJob }
// Clip Embedding
| { name: JobName.QUEUE_ENCODE_CLIP; data: IBaseJob }
@@ -111,4 +113,5 @@ export interface IJobRepository {
clear(name: QueueName, type: QueueCleanType): Promise<string[]>;
getQueueStatus(name: QueueName): Promise<QueueStatus>;
getJobCounts(name: QueueName): Promise<JobCounts>;
waitForQueueCompletion(...queues: QueueName[]): Promise<void>;
}
server/src/domain/repositories/person.repository.ts
+20 -14
View File
@@ -1,4 +1,6 @@
import { AssetEntity, AssetFaceEntity, PersonEntity } from '@app/infra/entities';
import { FindManyOptions, FindOptionsRelations, FindOptionsSelect } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
export const IPersonRepository = 'IPersonRepository';
@@ -17,7 +19,8 @@ export interface AssetFaceId {
}
export interface UpdateFacesData {
oldPersonId: string;
oldPersonId?: string;
faceIds?: string[];
newPersonId: string;
}
@@ -26,8 +29,7 @@ export interface PersonStatistics {
}
export interface IPersonRepository {
getAll(): Promise<PersonEntity[]>;
getAllWithoutThumbnail(): Promise<PersonEntity[]>;
getAll(pagination: PaginationOptions, options?: FindManyOptions<PersonEntity>): Paginated<PersonEntity>;
getAllForUser(userId: string, options: PersonSearchOptions): Promise<PersonEntity[]>;
getAllWithoutFaces(): Promise<PersonEntity[]>;
getById(personId: string): Promise<PersonEntity | null>;
@@ -35,19 +37,23 @@ export interface IPersonRepository {
getAssets(personId: string): Promise<AssetEntity[]>;
reassignFaces(data: UpdateFacesData): Promise<number>;
create(entity: Partial<PersonEntity>): Promise<PersonEntity>;
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
delete(entity: PersonEntity): Promise<PersonEntity | null>;
deleteAll(): Promise<number>;
getStatistics(personId: string): Promise<PersonStatistics>;
getAllFaces(): Promise<AssetFaceEntity[]>;
createFace(entity: Partial<AssetFaceEntity>): Promise<void>;
delete(entities: PersonEntity[]): Promise<void>;
deleteAll(): Promise<void>;
deleteAllFaces(): Promise<void>;
getAllFaces(pagination: PaginationOptions, options?: FindManyOptions<AssetFaceEntity>): Paginated<AssetFaceEntity>;
getFaceById(id: string): Promise<AssetFaceEntity>;
getFaceByIdWithAssets(
id: string,
relations?: FindOptionsRelations<AssetFaceEntity>,
select?: FindOptionsSelect<AssetFaceEntity>,
): Promise<AssetFaceEntity | null>;
getFaces(assetId: string): Promise<AssetFaceEntity[]>;
getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]>;
getRandomFace(personId: string): Promise<AssetFaceEntity | null>;
createFace(entity: Partial<AssetFaceEntity>): Promise<AssetFaceEntity>;
getFaces(assetId: string): Promise<AssetFaceEntity[]>;
getStatistics(personId: string): Promise<PersonStatistics>;
reassignFace(assetFaceId: string, newPersonId: string): Promise<number>;
getFaceById(id: string): Promise<AssetFaceEntity>;
getFaceByIdWithAssets(id: string): Promise<AssetFaceEntity | null>;
reassignFaces(data: UpdateFacesData): Promise<number>;
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
}
server/src/domain/repositories/smart-info.repository.ts
+12 -3
View File
@@ -7,14 +7,23 @@ export type Embedding = number[];
export interface EmbeddingSearch {
userIds: string[];
embedding: Embedding;
numResults: number;
maxDistance?: number;
numResults?: number;
withArchived?: boolean;
}
export interface FaceEmbeddingSearch extends EmbeddingSearch {
maxDistance?: number;
hasPerson?: boolean;
}
export interface FaceSearchResult {
face: AssetFaceEntity;
distance: number;
}
export interface ISmartInfoRepository {
init(modelName: string): Promise<void>;
searchCLIP(search: EmbeddingSearch): Promise<AssetEntity[]>;
searchFaces(search: EmbeddingSearch): Promise<AssetFaceEntity[]>;
searchFaces(search: FaceEmbeddingSearch): Promise<FaceSearchResult[]>;
upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
}