cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use pgvecto.rs (#3605)

Browse Source
This commit is contained in:
Jason Rasmussen
2023-12-08 11:15:46 -05:00
committed by GitHub
parent 429ad28810
commit 1e99ba8167
99 changed files with 1935 additions and 2583 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/person/person.service.spec.ts
+43 -48
View File
@@ -12,7 +12,7 @@ import {
newMediaRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newSmartInfoRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
personStub,
@@ -26,12 +26,12 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
import { PersonResponseDto, mapFaces } from './person.dto';
import { PersonResponseDto, mapFaces, mapPerson } from './person.dto';
import { PersonService } from './person.service';
const responseDto: PersonResponseDto = {
@@ -61,33 +61,6 @@ const detectFaceMock = {
score: 0.2,
};
const faceSearch = {
noMatch: {
total: 0,
count: 0,
page: 1,
items: [],
distances: [],
facets: [],
},
oneMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.1],
facets: [],
},
oneRemoteMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.8],
facets: [],
},
};
describe(PersonService.name, () => {
let accessMock: IAccessRepositoryMock;
let assetMock: jest.Mocked<IAssetRepository>;
@@ -97,8 +70,8 @@ describe(PersonService.name, () => {
let mediaMock: jest.Mocked<IMediaRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
let sut: PersonService;
beforeEach(async () => {
@@ -110,8 +83,8 @@ describe(PersonService.name, () => {
moveMock = newMoveRepositoryMock();
mediaMock = newMediaRepositoryMock();
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock();
sut = new PersonService(
accessMock,
assetMock,
@@ -119,10 +92,10 @@ describe(PersonService.name, () => {
moveMock,
mediaMock,
personMock,
searchMock,
configMock,
storageMock,
jobMock,
smartInfoMock,
);
mediaMock.crop.mockResolvedValue(croppedFace);
@@ -283,10 +256,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -320,10 +289,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -508,6 +473,17 @@ describe(PersonService.name, () => {
});
});
describe('handlePersonDelete', () => {
it('should delete person', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
await sut.handlePersonDelete({ id: personStub.withName.id });
expect(personMock.delete).toHaveBeenCalledWith(personStub.withName);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
});
});
describe('handlePersonCleanup', () => {
it('should delete people without faces', async () => {
personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
@@ -547,7 +523,7 @@ describe(PersonService.name, () => {
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([personStub.withName]);
searchMock.deleteAllFaces.mockResolvedValue(100);
personMock.deleteAll.mockResolvedValue(5);
await sut.handleQueueRecognizeFaces({ force: true });
@@ -626,7 +602,7 @@ describe(PersonService.name, () => {
it('should match existing people', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
smartInfoMock.searchFaces.mockResolvedValue([faceStub.face1]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
@@ -645,7 +621,7 @@ describe(PersonService.name, () => {
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
smartInfoMock.searchFaces.mockResolvedValue([]);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
personMock.createFace.mockResolvedValue(faceStub.primaryFace1);
@@ -664,10 +640,6 @@ describe(PersonService.name, () => {
imageHeight: 500,
imageWidth: 400,
});
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
[{ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'person-1' } }],
]);
});
});
describe('handleGeneratePersonThumbnail', () => {
@@ -873,4 +845,27 @@ describe(PersonService.name, () => {
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
});
describe('mapFace', () => {
it('should map a face', () => {
expect(mapFaces(faceStub.face1, personStub.withName.owner)).toEqual({
boundingBoxX1: 0,
boundingBoxX2: 1,
boundingBoxY1: 0,
boundingBoxY2: 1,
id: 'assetFaceId',
imageHeight: 1024,
imageWidth: 1024,
person: mapPerson(personStub.withName),
});
});
it('should not map person if person is null', () => {
expect(mapFaces({ ...faceStub.face1, person: null }, authStub.user1).person).toBeNull();
});
it('should not map person if person does not match auth user id', () => {
expect(mapFaces(faceStub.face1, authStub.user1).person).toBeNull();
});
});
});
server/src/domain/person/person.service.ts
+11 -35
View File
@@ -9,7 +9,6 @@ import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { FACE_THUMBNAIL_SIZE } from '../media';
import {
AssetFaceId,
CropOptions,
IAccessRepository,
IAssetRepository,
@@ -18,7 +17,7 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
@@ -56,10 +55,10 @@ export class PersonService {
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IPersonRepository) private repository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
@@ -198,11 +197,6 @@ export class PersonService {
if (name !== undefined || birthDate !== undefined || isHidden !== undefined) {
person = await this.repository.update({ id, name, birthDate, isHidden });
if (this.needsSearchIndexUpdate(dto)) {
const assets = await this.repository.getAssets(id);
const ids = assets.map((asset) => asset.id);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
}
}
if (assetId) {
@@ -281,8 +275,7 @@ export class PersonService {
for (const person of people) {
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: person.id } });
}
const faces = await this.searchRepository.deleteAllFaces();
this.logger.debug(`Deleted ${people} people and ${faces} faces`);
this.logger.debug(`Deleted ${people.length} people`);
}
for await (const assets of assetPagination) {
@@ -318,20 +311,17 @@ export class PersonService {
);
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `vector(${face.embedding.length})` })));
for (const { embedding, ...rest } of faces) {
const faceSearchResult = await this.searchRepository.searchFaces(embedding, { ownerId: asset.ownerId });
let personId: string | null = null;
// try to find a matching face and link to the associated person
// The closer to 0, the better the match. Range is from 0 to 2
if (faceSearchResult.total && faceSearchResult.distances[0] <= machineLearning.facialRecognition.maxDistance) {
this.logger.verbose(`Match face with distance ${faceSearchResult.distances[0]}`);
personId = faceSearchResult.items[0].personId;
}
const matches = await this.smartInfoRepository.searchFaces({
ownerId: asset.ownerId,
embedding,
numResults: 1,
maxDistance: machineLearning.facialRecognition.maxDistance,
});
let personId = matches[0]?.personId || null;
let newPerson: PersonEntity | null = null;
if (!personId) {
this.logger.debug('No matches, creating a new person.');
@@ -350,8 +340,6 @@ export class PersonService {
boundingBoxY1: rest.boundingBox.y1,
boundingBoxY2: rest.boundingBox.y2,
});
const faceId: AssetFaceId = { assetId: asset.id, personId };
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
if (newPerson) {
await this.repository.update({ id: personId, faceAssetId: face.id });
@@ -489,21 +477,9 @@ export class PersonService {
}
}
// Re-index all faces in typesense for up-to-date search results
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
return results;
}
/**
* Returns true if the given person update is going to require an update of the search index.
* @param dto the Person going to be updated
* @private
*/
private needsSearchIndexUpdate(dto: PersonUpdateDto): boolean {
return dto.name !== undefined || dto.isHidden !== undefined;
}
private async findOrFail(id: string) {
const person = await this.repository.getById(id);
if (!person) {