cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: use pgvecto.rs (#3605)

Browse Source
This commit is contained in:
Jason Rasmussen
2023-12-08 11:15:46 -05:00
committed by GitHub
parent 429ad28810
commit 1e99ba8167
99 changed files with 1935 additions and 2583 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/domain/album/album.service.spec.ts
-10
View File
@@ -12,7 +12,6 @@ import {
} from '@test';
import _ from 'lodash';
import { BulkIdErrorReason } from '../asset';
import { JobName } from '../job';
import { IAlbumRepository, IAssetRepository, IJobRepository, IUserRepository } from '../repositories';
import { AlbumService } from './album.service';
@@ -188,11 +187,6 @@ describe(AlbumService.name, () => {
assetIds: ['123'],
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ALBUM,
data: { ids: [albumStub.empty.id] },
});
expect(albumMock.create).toHaveBeenCalledWith({
ownerId: authStub.admin.id,
albumName: albumStub.empty.albumName,
@@ -270,10 +264,6 @@ describe(AlbumService.name, () => {
id: 'album-4',
albumName: 'new album name',
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ALBUM,
data: { ids: [albumStub.oneAsset.id] },
});
});
});
server/src/domain/album/album.service.ts
-5
View File
@@ -4,7 +4,6 @@ import { AccessCore, Permission } from '../access';
import { BulkIdErrorReason, BulkIdResponseDto, BulkIdsDto } from '../asset';
import { AuthUserDto } from '../auth';
import { setUnion } from '../domain.util';
import { JobName } from '../job';
import {
AlbumAssetCount,
AlbumInfoOptions,
@@ -131,7 +130,6 @@ export class AlbumService {
albumThumbnailAssetId: dto.assetIds?.[0] || null,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [album.id] } });
return mapAlbumWithAssets(album);
}
@@ -154,8 +152,6 @@ export class AlbumService {
isActivityEnabled: dto.isActivityEnabled,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [updatedAlbum.id] } });
return mapAlbumWithoutAssets(updatedAlbum);
}
@@ -165,7 +161,6 @@ export class AlbumService {
const album = await this.findOrFail(id, { withAssets: false });
await this.albumRepository.delete(album);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ALBUM, data: { ids: [id] } });
}
async addAssets(authUser: AuthUserDto, id: string, dto: BulkIdsDto): Promise<BulkIdResponseDto[]> {
server/src/domain/asset/asset.service.spec.ts
+3 -35
View File
@@ -794,14 +794,7 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: false });
expect(assetMock.softDeleteAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_REMOVE_ASSET,
data: { ids: ['asset1', 'asset2'] },
},
],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
@@ -820,14 +813,7 @@ describe(AssetService.name, () => {
await sut.restoreAll(authStub.user1, { ids: ['asset1', 'asset2'] });
expect(assetMock.restoreAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: ['asset1', 'asset2'] },
},
],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
@@ -853,19 +839,6 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetWithFace.id });
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_REMOVE_FACE,
data: { assetId: faceStub.face1.assetId, personId: faceStub.face1.personId },
},
],
[
{
name: JobName.SEARCH_REMOVE_FACE,
data: { assetId: faceStub.mergeFace1.assetId, personId: faceStub.mergeFace1.personId },
},
],
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetWithFace.id] } }],
[
{
name: JobName.DELETE_FILES,
@@ -907,9 +880,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.readOnly.id });
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.readOnly.id] } }],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
expect(assetMock.remove).toHaveBeenCalledWith(assetStub.readOnly);
});
@@ -934,7 +905,6 @@ describe(AssetService.name, () => {
expect(assetMock.remove).toHaveBeenCalledWith(assetStub.external);
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.external.id] } }],
[
{
name: JobName.DELETE_FILES,
@@ -955,9 +925,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.livePhotoStillAsset.id });
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.livePhotoStillAsset.id] } }],
[{ name: JobName.ASSET_DELETION, data: { id: assetStub.livePhotoMotionAsset.id } }],
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.livePhotoMotionAsset.id] } }],
[
{
name: JobName.DELETE_FILES,
server/src/domain/asset/asset.service.ts
+4 -16
View File
@@ -397,7 +397,6 @@ export class AssetService {
await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude });
const asset = await this.assetRepository.save({ id, ...rest });
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [id] } });
return mapAsset(asset);
}
@@ -426,7 +425,10 @@ export class AssetService {
await this.updateMetadata({ id, dateTimeOriginal, latitude, longitude });
}
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
for (const id of ids) {
await this.updateMetadata({ id, dateTimeOriginal, latitude, longitude });
}
await this.assetRepository.updateAll(ids, options);
this.communicationRepository.send(CommunicationEvent.ASSET_UPDATE, authUser.id, ids);
}
@@ -463,16 +465,6 @@ export class AssetService {
return false;
}
if (asset.faces) {
await Promise.all(
asset.faces.map(
({ assetId, personId }) =>
personId != null &&
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
),
);
}
// Replace the parent of the stack children with a new asset
if (asset.stack && asset.stack.length != 0) {
const stackIds = asset.stack.map((a) => a.id);
@@ -482,7 +474,6 @@ export class AssetService {
}
await this.assetRepository.remove(asset);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [asset.id] } });
this.communicationRepository.send(CommunicationEvent.ASSET_DELETE, asset.ownerId, id);
// TODO refactor this to use cascades
@@ -513,7 +504,6 @@ export class AssetService {
}
} else {
await this.assetRepository.softDeleteAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_TRASH, authUser.id, ids);
}
}
@@ -527,7 +517,6 @@ export class AssetService {
for await (const assets of assetPagination) {
const ids = assets.map((a) => a.id);
await this.assetRepository.restoreAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_RESTORE, authUser.id, ids);
}
return;
@@ -547,7 +536,6 @@ export class AssetService {
const { ids } = dto;
await this.access.requirePermission(authUser, Permission.ASSET_RESTORE, ids);
await this.assetRepository.restoreAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_RESTORE, authUser.id, ids);
}
server/src/domain/domain.config.ts
+1 -6
View File
@@ -13,16 +13,11 @@ export const immichAppConfig: ConfigModuleOptions = {
envFilePath: '.env',
isGlobal: true,
validationSchema: Joi.object({
NODE_ENV: Joi.string().required().valid('development', 'production', 'staging').default('development'),
NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
DB_USERNAME: WHEN_DB_URL_SET,
DB_PASSWORD: WHEN_DB_URL_SET,
DB_DATABASE_NAME: WHEN_DB_URL_SET,
DB_URL: Joi.string().optional(),
TYPESENSE_API_KEY: Joi.when('TYPESENSE_ENABLED', {
is: 'false',
then: Joi.string().optional(),
otherwise: Joi.string().required(),
}),
DISABLE_REVERSE_GEOCODING: Joi.boolean().optional().valid(true, false).default(false),
REVERSE_GEOCODING_PRECISION: Joi.number().optional().valid(0, 1, 2, 3).default(3),
LOG_LEVEL: Joi.string().optional().valid('simple', 'verbose', 'debug', 'log', 'warn', 'error').default('log'),
server/src/domain/domain.module.ts
+2 -8
View File
@@ -1,4 +1,4 @@
import { DynamicModule, Global, Module, ModuleMetadata, OnApplicationShutdown, Provider } from '@nestjs/common';
import { DynamicModule, Global, Module, ModuleMetadata, Provider } from '@nestjs/common';
import { ActivityService } from './activity';
import { AlbumService } from './album';
import { APIKeyService } from './api-key';
@@ -54,9 +54,7 @@ const providers: Provider[] = [
@Global()
@Module({})
export class DomainModule implements OnApplicationShutdown {
constructor(private searchService: SearchService) {}
export class DomainModule {
static register(options: Pick<ModuleMetadata, 'imports'>): DynamicModule {
return {
module: DomainModule,
@@ -65,8 +63,4 @@ export class DomainModule implements OnApplicationShutdown {
exports: [...providers],
};
}
onApplicationShutdown() {
this.searchService.teardown();
}
}
server/src/domain/job/job.constants.ts
-26
View File
@@ -78,17 +78,6 @@ export enum JobName {
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
// search
SEARCH_INDEX_ASSETS = 'search-index-assets',
SEARCH_INDEX_ASSET = 'search-index-asset',
SEARCH_INDEX_FACE = 'search-index-face',
SEARCH_INDEX_FACES = 'search-index-faces',
SEARCH_INDEX_ALBUMS = 'search-index-albums',
SEARCH_INDEX_ALBUM = 'search-index-album',
SEARCH_REMOVE_ALBUM = 'search-remove-album',
SEARCH_REMOVE_ASSET = 'search-remove-asset',
SEARCH_REMOVE_FACE = 'search-remove-face',
// clip
QUEUE_ENCODE_CLIP = 'queue-clip-encode',
ENCODE_CLIP = 'clip-encode',
@@ -151,21 +140,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_ENCODE_CLIP]: QueueName.CLIP_ENCODING,
[JobName.ENCODE_CLIP]: QueueName.CLIP_ENCODING,
// search - albums
[JobName.SEARCH_INDEX_ALBUMS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ALBUM]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ALBUM]: QueueName.SEARCH,
// search - assets
[JobName.SEARCH_INDEX_ASSETS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ASSET]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ASSET]: QueueName.SEARCH,
// search - faces
[JobName.SEARCH_INDEX_FACES]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_FACE]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_FACE]: QueueName.SEARCH,
// XMP sidecars
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,
server/src/domain/job/job.interface.ts
-5
View File
@@ -2,11 +2,6 @@ export interface IBaseJob {
force?: boolean;
}
export interface IAssetFaceJob extends IBaseJob {
assetId: string;
personId: string;
}
export interface IEntityJob extends IBaseJob {
id: string;
source?: 'upload' | 'sidecar-write';
server/src/domain/job/job.service.spec.ts
+37 -6
View File
@@ -1,4 +1,4 @@
import { SystemConfig } from '@app/infra/entities';
import { SystemConfig, SystemConfigKey } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import {
assetStub,
@@ -18,7 +18,7 @@ import {
JobHandler,
JobItem,
} from '../repositories';
import { SystemConfigCore } from '../system-config/system-config.core';
import { FeatureFlag, SystemConfigCore } from '../system-config/system-config.core';
import { JobCommand, JobName, QueueName } from './job.constants';
import { JobService } from './job.service';
@@ -271,7 +271,7 @@ describe(JobService.name, () => {
},
{
item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } },
jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET],
jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE],
},
{
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } },
@@ -281,6 +281,10 @@ describe(JobService.name, () => {
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } },
jobs: [],
},
{
item: { name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [],
},
{
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [
@@ -315,15 +319,15 @@ describe(JobService.name, () => {
},
{
item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
{
item: { name: JobName.ENCODE_CLIP, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
{
item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
];
@@ -357,5 +361,32 @@ describe(JobService.name, () => {
expect(jobMock.queue).not.toHaveBeenCalled();
});
}
const featureTests: Array<{ queue: QueueName; feature: FeatureFlag; configKey: SystemConfigKey }> = [
{
queue: QueueName.CLIP_ENCODING,
feature: FeatureFlag.CLIP_ENCODE,
configKey: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED,
},
{
queue: QueueName.OBJECT_TAGGING,
feature: FeatureFlag.TAG_IMAGE,
configKey: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED,
},
{
queue: QueueName.RECOGNIZE_FACES,
feature: FeatureFlag.FACIAL_RECOGNITION,
configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
},
];
for (const { queue, feature, configKey } of featureTests) {
it(`should throw an error if attempting to queue ${queue} when ${feature} is disabled`, async () => {
configMock.load.mockResolvedValue([{ key: configKey, value: false }]);
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await expect(sut.handleCommand(queue, { command: JobCommand.START, force: false })).rejects.toThrow();
});
}
});
});
server/src/domain/job/job.service.ts
-10
View File
@@ -236,15 +236,5 @@ export class JobService {
}
}
}
// In addition to the above jobs, all of these should queue `SEARCH_INDEX_ASSET`
switch (item.name) {
case JobName.CLASSIFY_IMAGE:
case JobName.ENCODE_CLIP:
case JobName.RECOGNIZE_FACES:
case JobName.LINK_LIVE_PHOTOS:
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } });
break;
}
}
}
server/src/domain/person/person.service.spec.ts
+43 -48
View File
@@ -12,7 +12,7 @@ import {
newMediaRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newSmartInfoRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
personStub,
@@ -26,12 +26,12 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
import { PersonResponseDto, mapFaces } from './person.dto';
import { PersonResponseDto, mapFaces, mapPerson } from './person.dto';
import { PersonService } from './person.service';
const responseDto: PersonResponseDto = {
@@ -61,33 +61,6 @@ const detectFaceMock = {
score: 0.2,
};
const faceSearch = {
noMatch: {
total: 0,
count: 0,
page: 1,
items: [],
distances: [],
facets: [],
},
oneMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.1],
facets: [],
},
oneRemoteMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.8],
facets: [],
},
};
describe(PersonService.name, () => {
let accessMock: IAccessRepositoryMock;
let assetMock: jest.Mocked<IAssetRepository>;
@@ -97,8 +70,8 @@ describe(PersonService.name, () => {
let mediaMock: jest.Mocked<IMediaRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
let sut: PersonService;
beforeEach(async () => {
@@ -110,8 +83,8 @@ describe(PersonService.name, () => {
moveMock = newMoveRepositoryMock();
mediaMock = newMediaRepositoryMock();
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock();
sut = new PersonService(
accessMock,
assetMock,
@@ -119,10 +92,10 @@ describe(PersonService.name, () => {
moveMock,
mediaMock,
personMock,
searchMock,
configMock,
storageMock,
jobMock,
smartInfoMock,
);
mediaMock.crop.mockResolvedValue(croppedFace);
@@ -283,10 +256,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -320,10 +289,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -508,6 +473,17 @@ describe(PersonService.name, () => {
});
});
describe('handlePersonDelete', () => {
it('should delete person', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
await sut.handlePersonDelete({ id: personStub.withName.id });
expect(personMock.delete).toHaveBeenCalledWith(personStub.withName);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
});
});
describe('handlePersonCleanup', () => {
it('should delete people without faces', async () => {
personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
@@ -547,7 +523,7 @@ describe(PersonService.name, () => {
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([personStub.withName]);
searchMock.deleteAllFaces.mockResolvedValue(100);
personMock.deleteAll.mockResolvedValue(5);
await sut.handleQueueRecognizeFaces({ force: true });
@@ -626,7 +602,7 @@ describe(PersonService.name, () => {
it('should match existing people', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
smartInfoMock.searchFaces.mockResolvedValue([faceStub.face1]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
@@ -645,7 +621,7 @@ describe(PersonService.name, () => {
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
smartInfoMock.searchFaces.mockResolvedValue([]);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
personMock.createFace.mockResolvedValue(faceStub.primaryFace1);
@@ -664,10 +640,6 @@ describe(PersonService.name, () => {
imageHeight: 500,
imageWidth: 400,
});
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
[{ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'person-1' } }],
]);
});
});
describe('handleGeneratePersonThumbnail', () => {
@@ -873,4 +845,27 @@ describe(PersonService.name, () => {
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
});
describe('mapFace', () => {
it('should map a face', () => {
expect(mapFaces(faceStub.face1, personStub.withName.owner)).toEqual({
boundingBoxX1: 0,
boundingBoxX2: 1,
boundingBoxY1: 0,
boundingBoxY2: 1,
id: 'assetFaceId',
imageHeight: 1024,
imageWidth: 1024,
person: mapPerson(personStub.withName),
});
});
it('should not map person if person is null', () => {
expect(mapFaces({ ...faceStub.face1, person: null }, authStub.user1).person).toBeNull();
});
it('should not map person if person does not match auth user id', () => {
expect(mapFaces(faceStub.face1, authStub.user1).person).toBeNull();
});
});
});
server/src/domain/person/person.service.ts
+11 -35
View File
@@ -9,7 +9,6 @@ import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { FACE_THUMBNAIL_SIZE } from '../media';
import {
AssetFaceId,
CropOptions,
IAccessRepository,
IAssetRepository,
@@ -18,7 +17,7 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
@@ -56,10 +55,10 @@ export class PersonService {
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IPersonRepository) private repository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
@@ -198,11 +197,6 @@ export class PersonService {
if (name !== undefined || birthDate !== undefined || isHidden !== undefined) {
person = await this.repository.update({ id, name, birthDate, isHidden });
if (this.needsSearchIndexUpdate(dto)) {
const assets = await this.repository.getAssets(id);
const ids = assets.map((asset) => asset.id);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
}
}
if (assetId) {
@@ -281,8 +275,7 @@ export class PersonService {
for (const person of people) {
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: person.id } });
}
const faces = await this.searchRepository.deleteAllFaces();
this.logger.debug(`Deleted ${people} people and ${faces} faces`);
this.logger.debug(`Deleted ${people.length} people`);
}
for await (const assets of assetPagination) {
@@ -318,20 +311,17 @@ export class PersonService {
);
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `vector(${face.embedding.length})` })));
for (const { embedding, ...rest } of faces) {
const faceSearchResult = await this.searchRepository.searchFaces(embedding, { ownerId: asset.ownerId });
let personId: string | null = null;
// try to find a matching face and link to the associated person
// The closer to 0, the better the match. Range is from 0 to 2
if (faceSearchResult.total && faceSearchResult.distances[0] <= machineLearning.facialRecognition.maxDistance) {
this.logger.verbose(`Match face with distance ${faceSearchResult.distances[0]}`);
personId = faceSearchResult.items[0].personId;
}
const matches = await this.smartInfoRepository.searchFaces({
ownerId: asset.ownerId,
embedding,
numResults: 1,
maxDistance: machineLearning.facialRecognition.maxDistance,
});
let personId = matches[0]?.personId || null;
let newPerson: PersonEntity | null = null;
if (!personId) {
this.logger.debug('No matches, creating a new person.');
@@ -350,8 +340,6 @@ export class PersonService {
boundingBoxY1: rest.boundingBox.y1,
boundingBoxY2: rest.boundingBox.y2,
});
const faceId: AssetFaceId = { assetId: asset.id, personId };
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
if (newPerson) {
await this.repository.update({ id: personId, faceAssetId: face.id });
@@ -489,21 +477,9 @@ export class PersonService {
}
}
// Re-index all faces in typesense for up-to-date search results
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
return results;
}
/**
* Returns true if the given person update is going to require an update of the search index.
* @param dto the Person going to be updated
* @private
*/
private needsSearchIndexUpdate(dto: PersonUpdateDto): boolean {
return dto.name !== undefined || dto.isHidden !== undefined;
}
private async findOrFail(id: string) {
const person = await this.repository.getById(id);
if (!person) {
server/src/domain/repositories/asset.repository.ts
+27 -3
View File
@@ -1,3 +1,4 @@
import { SearchExploreItem } from '@app/domain';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { FindOptionsRelations } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
@@ -105,8 +106,7 @@ export enum TimeBucketSize {
MONTH = 'MONTH',
}
export interface TimeBucketOptions {
size: TimeBucketSize;
export interface AssetBuilderOptions {
isArchived?: boolean;
isFavorite?: boolean;
isTrashed?: boolean;
@@ -114,6 +114,12 @@ export interface TimeBucketOptions {
personId?: string;
userIds?: string[];
withStacked?: boolean;
exifInfo?: boolean;
assetType?: AssetType;
}
export interface TimeBucketOptions extends AssetBuilderOptions {
size: TimeBucketSize;
}
export interface TimeBucketItem {
@@ -142,6 +148,21 @@ export interface MonthDay {
month: number;
}
export interface AssetExploreFieldOptions {
maxFields: number;
minAssetsPerField: number;
}
export interface AssetExploreOptions extends AssetExploreFieldOptions {
relation: keyof AssetEntity;
relatedField: string;
unnest?: boolean;
}
export interface MetadataSearchOptions {
numResults: number;
}
export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
@@ -152,7 +173,7 @@ export interface IAssetRepository {
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
getById(id: string): Promise<AssetEntity | null>;
getById(id: string, relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null>;
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getWith(pagination: PaginationOptions, property: WithProperty, libraryId?: string): Paginated<AssetEntity>;
getRandom(userId: string, count: number): Promise<AssetEntity[]>;
@@ -176,4 +197,7 @@ export interface IAssetRepository {
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void>;
search(options: AssetSearchOptions): Promise<AssetEntity[]>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
searchMetadata(query: string, userId: string, options: MetadataSearchOptions): Promise<AssetEntity[]>;
}
server/src/domain/repositories/job.repository.ts
+1 -14
View File
@@ -2,9 +2,7 @@ import { JobName, QueueName } from '../job/job.constants';
import {
IAssetDeletionJob,
IAssetFaceJob,
IBaseJob,
IBulkEntityJob,
IDeleteFilesJob,
IEntityJob,
ILibraryFileJob,
@@ -96,18 +94,7 @@ export type JobItem =
| { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
| { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_INDEX_FACES; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob }
| { name: JobName.SEARCH_INDEX_ALBUMS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob };
export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
export type JobItemHandler = (item: JobItem) => Promise<void>;
server/src/domain/repositories/person.repository.ts
-2
View File
@@ -41,9 +41,7 @@ export interface IPersonRepository {
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
delete(entity: PersonEntity): Promise<PersonEntity | null>;
deleteAll(): Promise<number>;
getStatistics(personId: string): Promise<PersonStatistics>;
getAllFaces(): Promise<AssetFaceEntity[]>;
getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]>;
getRandomFace(personId: string): Promise<AssetFaceEntity | null>;
server/src/domain/repositories/search.repository.ts
+7 -48
View File
@@ -1,20 +1,10 @@
import { AlbumEntity, AssetEntity, AssetFaceEntity, AssetType } from '@app/infra/entities';
export enum SearchCollection {
ASSETS = 'assets',
ALBUMS = 'albums',
FACES = 'faces',
}
import { AssetType } from '@app/infra/entities';
export enum SearchStrategy {
CLIP = 'CLIP',
TEXT = 'TEXT',
}
export interface SearchFaceFilter {
ownerId: string;
}
export interface SearchFilter {
id?: string;
userId: string;
@@ -55,43 +45,12 @@ export interface SearchFacet {
}>;
}
export type SearchExploreItemSet<T> = Array<{
value: string;
data: T;
}>;
export interface SearchExploreItem<T> {
fieldName: string;
items: Array<{
value: string;
data: T;
}>;
}
export type OwnedFaceEntity = Pick<AssetFaceEntity, 'assetId' | 'personId' | 'embedding'> & {
/** computed as assetId|personId */
id: string;
/** copied from asset.id */
ownerId: string;
};
export type SearchCollectionIndexStatus = Record<SearchCollection, boolean>;
export const ISearchRepository = 'ISearchRepository';
export interface ISearchRepository {
setup(): Promise<void>;
checkMigrationStatus(): Promise<SearchCollectionIndexStatus>;
importAlbums(items: AlbumEntity[], done: boolean): Promise<void>;
importAssets(items: AssetEntity[], done: boolean): Promise<void>;
importFaces(items: OwnedFaceEntity[], done: boolean): Promise<void>;
deleteAlbums(ids: string[]): Promise<void>;
deleteAssets(ids: string[]): Promise<void>;
deleteFaces(ids: string[]): Promise<void>;
deleteAllFaces(): Promise<number>;
updateCLIPField(num_dim: number): Promise<void>;
searchAlbums(query: string, filters: SearchFilter): Promise<SearchResult<AlbumEntity>>;
searchAssets(query: string, filters: SearchFilter): Promise<SearchResult<AssetEntity>>;
vectorSearch(query: number[], filters: SearchFilter): Promise<SearchResult<AssetEntity>>;
searchFaces(query: number[], filters: SearchFaceFilter): Promise<SearchResult<AssetFaceEntity>>;
explore(userId: string): Promise<SearchExploreItem<AssetEntity>[]>;
items: SearchExploreItemSet<T>;
}
server/src/domain/repositories/smart-info.repository.ts
+15 -3
View File
@@ -1,7 +1,19 @@
import { SmartInfoEntity } from '@app/infra/entities';
import { AssetEntity, AssetFaceEntity, SmartInfoEntity } from '@app/infra/entities';
export const ISmartInfoRepository = 'ISmartInfoRepository';
export interface ISmartInfoRepository {
upsert(info: Partial<SmartInfoEntity>): Promise<void>;
export type Embedding = number[];
export interface EmbeddingSearch {
ownerId: string;
embedding: Embedding;
numResults: number;
maxDistance?: number;
}
export interface ISmartInfoRepository {
init(modelName: string): Promise<void>;
searchCLIP(search: EmbeddingSearch): Promise<AssetEntity[]>;
searchFaces(search: EmbeddingSearch): Promise<AssetFaceEntity[]>;
upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
}
server/src/domain/search/dto/search.dto.ts
+1 -53
View File
@@ -1,6 +1,6 @@
import { AssetType } from '@app/infra/entities';
import { Transform } from 'class-transformer';
import { IsArray, IsBoolean, IsEnum, IsNotEmpty, IsString } from 'class-validator';
import { IsBoolean, IsEnum, IsNotEmpty, IsString } from 'class-validator';
import { Optional, toBoolean } from '../../domain.util';
export class SearchDto {
@@ -23,58 +23,6 @@ export class SearchDto {
@Optional()
type?: AssetType;
@IsBoolean()
@Optional()
@Transform(toBoolean)
isFavorite?: boolean;
@IsBoolean()
@Optional()
@Transform(toBoolean)
isArchived?: boolean;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.city'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.state'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.country'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.make'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.model'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.projectionType'?: string;
@IsString({ each: true })
@IsArray()
@Optional()
@Transform(({ value }) => value.split(','))
'smartInfo.objects'?: string[];
@IsString({ each: true })
@IsArray()
@Optional()
@Transform(({ value }) => value.split(','))
'smartInfo.tags'?: string[];
@IsBoolean()
@Optional()
@Transform(toBoolean)
server/src/domain/search/search.service.spec.ts
+67 -351
View File
@@ -1,29 +1,20 @@
import { BadRequestException } from '@nestjs/common';
import { SystemConfigKey } from '@app/infra/entities';
import {
albumStub,
assetStub,
asyncTick,
authStub,
faceStub,
newAlbumRepositoryMock,
newAssetRepositoryMock,
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newSmartInfoRepositoryMock,
newSystemConfigRepositoryMock,
searchStub,
personStub,
} from '@test';
import { plainToInstance } from 'class-transformer';
import { mapAsset } from '../asset';
import { JobName } from '../job';
import {
IAlbumRepository,
IAssetRepository,
IJobRepository,
IMachineLearningRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { SearchDto } from './dto';
@@ -33,401 +24,126 @@ jest.useFakeTimers();
describe(SearchService.name, () => {
let sut: SearchService;
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
beforeEach(async () => {
albumMock = newAlbumRepositoryMock();
beforeEach(() => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
personMock = newPersonRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
searchMock = newSearchRepositoryMock();
sut = new SearchService(albumMock, assetMock, jobMock, machineMock, personMock, searchMock, configMock);
searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
delete process.env.TYPESENSE_ENABLED;
await sut.init();
});
const disableSearch = () => {
searchMock.setup.mockClear();
searchMock.checkMigrationStatus.mockClear();
jobMock.queue.mockClear();
process.env.TYPESENSE_ENABLED = 'false';
};
afterEach(() => {
sut.teardown();
personMock = newPersonRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock();
sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('request dto', () => {
it('should convert smartInfo.tags to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.tags': 'a,b,c' });
expect(instance['smartInfo.tags']).toEqual(['a', 'b', 'c']);
});
describe('searchPerson', () => {
it('should pass options to search', async () => {
const { name } = personStub.withName;
it('should handle empty smartInfo.tags', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.tags']).toBeUndefined();
});
await sut.searchPerson(authStub.user1, { name, withHidden: false });
it('should convert smartInfo.objects to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.objects': 'a,b,c' });
expect(instance['smartInfo.objects']).toEqual(['a', 'b', 'c']);
});
expect(personMock.getByName).toHaveBeenCalledWith(authStub.user1.id, name, { withHidden: false });
it('should handle empty smartInfo.objects', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.objects']).toBeUndefined();
});
});
await sut.searchPerson(authStub.user1, { name, withHidden: true });
describe(`init`, () => {
it('should skip when search is disabled', async () => {
disableSearch();
await sut.init();
expect(searchMock.setup).not.toHaveBeenCalled();
expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should skip schema migration if not needed', async () => {
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should do schema migration if needed', async () => {
searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true, faces: true });
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_ASSETS }],
[{ name: JobName.SEARCH_INDEX_ALBUMS }],
[{ name: JobName.SEARCH_INDEX_FACES }],
]);
expect(personMock.getByName).toHaveBeenCalledWith(authStub.user1.id, name, { withHidden: true });
});
});
describe('getExploreData', () => {
it('should throw bad request exception if search is disabled', async () => {
disableSearch();
await expect(sut.getExploreData(authStub.admin)).rejects.toBeInstanceOf(BadRequestException);
expect(searchMock.explore).not.toHaveBeenCalled();
});
it('should get assets by city and tag', async () => {
assetMock.getAssetIdByCity.mockResolvedValueOnce({
fieldName: 'exifInfo.city',
items: [{ value: 'Paris', data: assetStub.image.id }],
});
assetMock.getAssetIdByTag.mockResolvedValueOnce({
fieldName: 'smartInfo.tags',
items: [{ value: 'train', data: assetStub.imageFrom2015.id }],
});
assetMock.getByIds.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
const expectedResponse = [
{ fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
{ fieldName: 'smartInfo.tags', items: [{ value: 'train', data: mapAsset(assetStub.imageFrom2015) }] },
];
it('should return explore data if feature flag SEARCH is set', async () => {
searchMock.explore.mockResolvedValue([{ fieldName: 'name', items: [{ value: 'image', data: assetStub.image }] }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
const result = await sut.getExploreData(authStub.user1);
await expect(sut.getExploreData(authStub.admin)).resolves.toEqual([
{
fieldName: 'name',
items: [{ value: 'image', data: mapAsset(assetStub.image) }],
},
]);
expect(searchMock.explore).toHaveBeenCalledWith(authStub.admin.id);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(result).toEqual(expectedResponse);
});
});
describe('search', () => {
// it('should throw an error is search is disabled', async () => {
// sut['enabled'] = false;
it('should throw an error if query is missing', async () => {
await expect(sut.search(authStub.user1, { q: '' })).rejects.toThrow('Missing query');
});
// await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
// expect(searchMock.searchAlbums).not.toHaveBeenCalled();
// expect(searchMock.searchAssets).not.toHaveBeenCalled();
// });
it('should search assets and albums using text search', async () => {
searchMock.searchAssets.mockResolvedValue(searchStub.withImage);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await expect(sut.search(authStub.admin, {})).resolves.toEqual({
it('should search by metadata if `clip` option is false', async () => {
const dto: SearchDto = { q: 'test query', clip: false };
assetMock.searchMetadata.mockResolvedValueOnce([assetStub.image]);
const expectedResponse = {
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 1,
count: 1,
page: 1,
items: [mapAsset(assetStub.image)],
facets: [],
distances: [],
},
});
};
// expect(searchMock.searchAssets).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
expect(searchMock.searchAlbums).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
const result = await sut.search(authStub.user1, dto);
expect(result).toEqual(expectedResponse);
expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, authStub.user1.id, { numResults: 250 });
expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled();
});
it('should search assets and albums using vector search', async () => {
searchMock.vectorSearch.mockResolvedValue(searchStub.emptyResults);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
machineMock.encodeText.mockResolvedValue([123]);
await expect(sut.search(authStub.admin, { clip: true, query: 'foo' })).resolves.toEqual({
it('should search by CLIP if `clip` option is true', async () => {
const dto: SearchDto = { q: 'test query', clip: true };
const embedding = [1, 2, 3];
smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
machineMock.encodeText.mockResolvedValueOnce(embedding);
const expectedResponse = {
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 0,
count: 0,
page: 1,
items: [],
total: 1,
count: 1,
items: [mapAsset(assetStub.image)],
facets: [],
distances: [],
},
});
};
expect(machineMock.encodeText).toHaveBeenCalledWith(expect.any(String), { text: 'foo' }, expect.any(Object));
expect(searchMock.vectorSearch).toHaveBeenCalledWith([123], {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
expect(searchMock.searchAlbums).toHaveBeenCalledWith('foo', {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
});
});
const result = await sut.search(authStub.user1, dto);
describe('handleIndexAssets', () => {
it('should call done, even when there are no assets', async () => {
await sut.handleIndexAssets();
expect(searchMock.importAssets).toHaveBeenCalledWith([], true);
expect(result).toEqual(expectedResponse);
expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({ ownerId: authStub.user1.id, embedding, numResults: 100 });
expect(assetMock.searchMetadata).not.toHaveBeenCalled();
});
it('should index all the assets', async () => {
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
it('should throw an error if clip is requested but disabled', async () => {
const dto: SearchDto = { q: 'test query', clip: true };
configMock.load
.mockResolvedValueOnce([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }])
.mockResolvedValueOnce([{ key: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED, value: false }]);
await sut.handleIndexAssets();
expect(searchMock.importAssets.mock.calls).toEqual([
[[assetStub.image], false],
[[], true],
]);
});
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexAssets();
expect(searchMock.importAssets).not.toHaveBeenCalled();
expect(searchMock.importAlbums).not.toHaveBeenCalled();
});
});
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
it('should index the asset', () => {
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
});
describe('handleIndexAlbums', () => {
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexAlbums();
});
it('should index all the albums', async () => {
albumMock.getAll.mockResolvedValue([albumStub.empty]);
await sut.handleIndexAlbums();
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], true);
});
});
describe('handleIndexAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
it('should index the album', () => {
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
});
describe('handleRemoveAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAlbum({ ids: ['album1'] });
});
it('should remove the album', () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
});
});
describe('handleRemoveAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAsset({ ids: ['asset1'] });
});
it('should remove the asset', () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
});
});
describe('handleIndexFaces', () => {
it('should call done, even when there are no faces', async () => {
personMock.getAllFaces.mockResolvedValue([]);
await sut.handleIndexFaces();
expect(searchMock.importFaces).toHaveBeenCalledWith([], true);
});
it('should index all the faces', async () => {
personMock.getAllFaces.mockResolvedValue([faceStub.face1]);
await sut.handleIndexFaces();
expect(searchMock.importFaces.mock.calls).toEqual([
[
[
{
id: 'asset-id|person-1',
ownerId: 'user-id',
assetId: 'asset-id',
personId: 'person-1',
embedding: [1, 2, 3, 4],
},
],
false,
],
[[], true],
]);
});
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexFaces();
expect(searchMock.importFaces).not.toHaveBeenCalled();
});
});
describe('handleIndexAsset', () => {
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(searchMock.importFaces).not.toHaveBeenCalled();
expect(personMock.getFacesByIds).not.toHaveBeenCalled();
});
it('should index the face', async () => {
personMock.getFacesByIds.mockResolvedValue([faceStub.face1]);
await sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(personMock.getFacesByIds).toHaveBeenCalledWith([{ assetId: 'asset-1', personId: 'person-1' }]);
});
});
describe('handleRemoveFace', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
it('should remove the face', () => {
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
});
describe('flush', () => {
it('should flush queued album updates', async () => {
albumMock.getByIds.mockResolvedValue([albumStub.empty]);
sut.handleIndexAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(albumMock.getByIds).toHaveBeenCalledWith(['album1']);
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], false);
});
it('should flush queued album deletes', async () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(searchMock.deleteAlbums).toHaveBeenCalledWith(['album1']);
});
it('should flush queued asset updates', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
sut.handleIndexAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset1']);
expect(searchMock.importAssets).toHaveBeenCalledWith([assetStub.image], false);
});
it('should flush queued asset deletes', async () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(searchMock.deleteAssets).toHaveBeenCalledWith(['asset1']);
await expect(sut.search(authStub.user1, dto)).rejects.toThrow('CLIP is not enabled');
await expect(sut.search(authStub.user1, dto)).rejects.toThrow('CLIP is not enabled');
});
});
});
server/src/domain/search/search.service.ts
+44 -341
View File
@@ -1,396 +1,99 @@
import { AlbumEntity, AssetEntity, AssetFaceEntity } from '@app/infra/entities';
import { AssetEntity } from '@app/infra/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { mapAlbumWithAssets } from '../album';
import { AssetResponseDto, mapAsset } from '../asset';
import { AuthUserDto } from '../auth';
import { usePagination } from '../domain.util';
import { IAssetFaceJob, IBulkEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { PersonResponseDto } from '../person/person.dto';
import { PersonResponseDto } from '../person';
import {
AssetFaceId,
IAlbumRepository,
IAssetRepository,
IJobRepository,
IMachineLearningRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
ISystemConfigRepository,
OwnedFaceEntity,
SearchCollection,
SearchExploreItem,
SearchResult,
SearchStrategy,
} from '../repositories';
import { FeatureFlag, SystemConfigCore } from '../system-config';
import { SearchDto, SearchPeopleDto } from './dto';
import { SearchResponseDto } from './response-dto';
interface SyncQueue {
upsert: Set<string>;
delete: Set<string>;
}
@Injectable()
export class SearchService {
private logger = new Logger(SearchService.name);
private enabled = false;
private timer: NodeJS.Timeout | null = null;
private configCore: SystemConfigCore;
private albumQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private assetQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private faceQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
constructor(
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
@Inject(IPersonRepository) private personRepository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
}
teardown() {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
}
async init() {
this.enabled = await this.configCore.hasFeature(FeatureFlag.SEARCH);
if (!this.enabled) {
return;
}
this.logger.log('Running bootstrap');
await this.searchRepository.setup();
const migrationStatus = await this.searchRepository.checkMigrationStatus();
if (migrationStatus[SearchCollection.ASSETS]) {
this.logger.debug('Queueing job to re-index all assets');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSETS });
}
if (migrationStatus[SearchCollection.ALBUMS]) {
this.logger.debug('Queueing job to re-index all albums');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUMS });
}
if (migrationStatus[SearchCollection.FACES]) {
this.logger.debug('Queueing job to re-index all faces');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
}
this.timer = setInterval(() => this.flush(), 5_000);
async searchPerson(authUser: AuthUserDto, dto: SearchPeopleDto): Promise<PersonResponseDto[]> {
return this.personRepository.getByName(authUser.id, dto.name, { withHidden: dto.withHidden });
}
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const results = await this.searchRepository.explore(authUser.id);
const lookup = await this.getLookupMap(
results.reduce(
(ids: string[], result: SearchExploreItem<AssetEntity>) => [
...ids,
...result.items.map((item) => item.data.id),
],
[],
),
);
const options = { maxFields: 12, minAssetsPerField: 5 };
const results = await Promise.all([
this.assetRepository.getAssetIdByCity(authUser.id, options),
this.assetRepository.getAssetIdByTag(authUser.id, options),
]);
const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data)));
const assets = await this.assetRepository.getByIds(Array.from(assetIds));
const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
return results.map(({ fieldName, items }) => ({
fieldName,
items: items
.map(({ value, data }) => ({ value, data: lookup[data.id] }))
.filter(({ data }) => !!data)
.map(({ value, data }) => ({ value, data: mapAsset(data) })),
items: items.map(({ value, data }) => ({ value, data: assetMap.get(data) as AssetResponseDto })),
}));
}
async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {
const { machineLearning } = await this.configCore.getConfig();
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const query = dto.q || dto.query || '*';
const query = dto.q || dto.query;
if (!query) {
throw new Error('Missing query');
}
const hasClip = machineLearning.enabled && machineLearning.clip.enabled;
const strategy = dto.clip && hasClip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const filters = { userId: authUser.id, ...dto };
if (dto.clip && !hasClip) {
throw new Error('CLIP is not enabled');
}
const strategy = dto.clip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
let assets: AssetEntity[] = [];
let assets: SearchResult<AssetEntity>;
switch (strategy) {
case SearchStrategy.CLIP:
const {
machineLearning: { clip },
} = await this.configCore.getConfig();
const embedding = await this.machineLearning.encodeText(machineLearning.url, { text: query }, clip);
assets = await this.searchRepository.vectorSearch(embedding, filters);
const embedding = await this.machineLearning.encodeText(
machineLearning.url,
{ text: query },
machineLearning.clip,
);
assets = await this.smartInfoRepository.searchCLIP({ ownerId: authUser.id, embedding, numResults: 100 });
break;
case SearchStrategy.TEXT:
assets = await this.assetRepository.searchMetadata(query, authUser.id, { numResults: 250 });
default:
assets = await this.searchRepository.searchAssets(query, filters);
break;
}
const albums = await this.searchRepository.searchAlbums(query, filters);
const lookup = await this.getLookupMap(assets.items.map((asset) => asset.id));
return {
albums: { ...albums, items: albums.items.map(mapAlbumWithAssets) },
albums: {
total: 0,
count: 0,
items: [],
facets: [],
},
assets: {
...assets,
items: assets.items
.map((item) => lookup[item.id])
.filter((item) => !!item)
.map((asset) => mapAsset(asset)),
total: assets.length,
count: assets.length,
items: assets.map((asset) => mapAsset(asset)),
facets: [],
},
};
}
searchPerson(authUser: AuthUserDto, dto: SearchPeopleDto): Promise<PersonResponseDto[]> {
return this.personRepository.getByName(authUser.id, dto.name, { withHidden: dto.withHidden });
}
async handleIndexAlbums() {
if (!this.enabled) {
return false;
}
const albums = this.patchAlbums(await this.albumRepository.getAll());
this.logger.log(`Indexing ${albums.length} albums`);
await this.searchRepository.importAlbums(albums, true);
return true;
}
async handleIndexAssets() {
if (!this.enabled) {
return false;
}
// TODO: do this in batches based on searchIndexVersion
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { isVisible: true }),
);
for await (const assets of assetPagination) {
this.logger.debug(`Indexing ${assets.length} assets`);
const patchedAssets = this.patchAssets(assets);
await this.searchRepository.importAssets(patchedAssets, false);
}
await this.searchRepository.importAssets([], true);
this.logger.debug('Finished re-indexing all assets');
return false;
}
async handleIndexFaces() {
if (!this.enabled) {
return false;
}
await this.searchRepository.deleteAllFaces();
// TODO: do this in batches based on searchIndexVersion
const faces = this.patchFaces(await this.personRepository.getAllFaces());
this.logger.log(`Indexing ${faces.length} faces`);
const chunkSize = 1000;
for (let i = 0; i < faces.length; i += chunkSize) {
await this.searchRepository.importFaces(faces.slice(i, i + chunkSize), false);
}
await this.searchRepository.importFaces([], true);
this.logger.debug('Finished re-indexing all faces');
return true;
}
handleIndexAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.upsert.add(id);
}
return true;
}
handleIndexAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.upsert.add(id);
}
return true;
}
async handleIndexFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
// immediately push to typesense
await this.searchRepository.importFaces(await this.idsToFaces([{ assetId, personId }]), false);
return true;
}
handleRemoveAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.delete.add(id);
}
return true;
}
handleRemoveAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.delete.add(id);
}
return true;
}
handleRemoveFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
this.faceQueue.delete.add(this.asKey({ assetId, personId }));
return true;
}
private async flush() {
if (this.albumQueue.upsert.size > 0) {
const ids = [...this.albumQueue.upsert.keys()];
const items = await this.idsToAlbums(ids);
this.logger.debug(`Flushing ${items.length} album upserts`);
await this.searchRepository.importAlbums(items, false);
this.albumQueue.upsert.clear();
}
if (this.albumQueue.delete.size > 0) {
const ids = [...this.albumQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} album deletes`);
await this.searchRepository.deleteAlbums(ids);
this.albumQueue.delete.clear();
}
if (this.assetQueue.upsert.size > 0) {
const ids = [...this.assetQueue.upsert.keys()];
const items = await this.idsToAssets(ids);
this.logger.debug(`Flushing ${items.length} asset upserts`);
await this.searchRepository.importAssets(items, false);
this.assetQueue.upsert.clear();
}
if (this.assetQueue.delete.size > 0) {
const ids = [...this.assetQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} asset deletes`);
await this.searchRepository.deleteAssets(ids);
this.assetQueue.delete.clear();
}
if (this.faceQueue.upsert.size > 0) {
const ids = [...this.faceQueue.upsert.keys()].map((key) => this.asParts(key));
const items = await this.idsToFaces(ids);
this.logger.debug(`Flushing ${items.length} face upserts`);
await this.searchRepository.importFaces(items, false);
this.faceQueue.upsert.clear();
}
if (this.faceQueue.delete.size > 0) {
const ids = [...this.faceQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} face deletes`);
await this.searchRepository.deleteFaces(ids);
this.faceQueue.delete.clear();
}
}
private async idsToAlbums(ids: string[]): Promise<AlbumEntity[]> {
const entities = await this.albumRepository.getByIds(ids);
return this.patchAlbums(entities);
}
private async idsToAssets(ids: string[]): Promise<AssetEntity[]> {
const entities = await this.assetRepository.getByIds(ids);
return this.patchAssets(entities.filter((entity) => entity.isVisible));
}
private async idsToFaces(ids: AssetFaceId[]): Promise<OwnedFaceEntity[]> {
return this.patchFaces(await this.personRepository.getFacesByIds(ids));
}
private patchAssets(assets: AssetEntity[]): AssetEntity[] {
return assets;
}
private patchAlbums(albums: AlbumEntity[]): AlbumEntity[] {
return albums.map((entity) => ({ ...entity, assets: [] }));
}
private patchFaces(faces: AssetFaceEntity[]): OwnedFaceEntity[] {
const results: OwnedFaceEntity[] = [];
for (const face of faces) {
if (face.personId) {
results.push({
id: this.asKey(face as AssetFaceId),
ownerId: face.asset.ownerId,
assetId: face.assetId,
personId: face.personId,
embedding: face.embedding,
});
}
}
return results;
}
private asKey(face: AssetFaceId): string {
return `${face.assetId}|${face.personId}`;
}
private asParts(key: string): AssetFaceId {
const [assetId, personId] = key.split('|');
return { assetId, personId };
}
private async getLookupMap(assetIds: string[]) {
const assets = await this.assetRepository.getByIds(assetIds);
const lookup: Record<string, AssetEntity> = {};
for (const asset of assets) {
lookup[asset.id] = asset;
}
return lookup;
}
}
server/src/domain/smart-info/smart-info.constant.ts
+107
View File
@@ -0,0 +1,107 @@
export type ModelInfo = {
dimSize: number;
};
export const CLIP_MODEL_INFO: Record<string, ModelInfo> = {
RN50__openai: {
dimSize: 1024,
},
RN50__yfcc15m: {
dimSize: 1024,
},
RN50__cc12m: {
dimSize: 1024,
},
RN101__openai: {
dimSize: 512,
},
RN101__yfcc15m: {
dimSize: 512,
},
RN50x4__openai: {
dimSize: 640,
},
RN50x16__openai: {
dimSize: 768,
},
RN50x64__openai: {
dimSize: 1024,
},
'ViT-B-32__openai': {
dimSize: 512,
},
'ViT-B-32__laion2b_e16': {
dimSize: 512,
},
'ViT-B-32__laion400m_e31': {
dimSize: 512,
},
'ViT-B-32__laion400m_e32': {
dimSize: 512,
},
'ViT-B-32__laion2b-s34b-b79k': {
dimSize: 512,
},
'ViT-B-16__openai': {
dimSize: 512,
},
'ViT-B-16__laion400m_e31': {
dimSize: 512,
},
'ViT-B-16__laion400m_e32': {
dimSize: 512,
},
'ViT-B-16-plus-240__laion400m_e31': {
dimSize: 640,
},
'ViT-B-16-plus-240__laion400m_e32': {
dimSize: 640,
},
'ViT-L-14__openai': {
dimSize: 768,
},
'ViT-L-14__laion400m_e31': {
dimSize: 768,
},
'ViT-L-14__laion400m_e32': {
dimSize: 768,
},
'ViT-L-14__laion2b-s32b-b82k': {
dimSize: 768,
},
'ViT-L-14-336__openai': {
dimSize: 768,
},
'ViT-H-14__laion2b-s32b-b79k': {
dimSize: 1024,
},
'ViT-g-14__laion2b-s12b-b42k': {
dimSize: 1024,
},
'LABSE-Vit-L-14': {
dimSize: 768,
},
'XLM-Roberta-Large-Vit-B-32': {
dimSize: 512,
},
'XLM-Roberta-Large-Vit-B-16Plus': {
dimSize: 640,
},
'XLM-Roberta-Large-Vit-L-14': {
dimSize: 768,
},
};
export function cleanModelName(modelName: string): string {
const tokens = modelName.split('/');
return tokens[tokens.length - 1].replace(/:/g, '_');
}
export function getCLIPModelInfo(modelName: string): ModelInfo {
const modelInfo = CLIP_MODEL_INFO[cleanModelName(modelName)];
if (!modelInfo) {
throw new Error(`Unknown CLIP model: ${modelName}`);
}
return modelInfo;
}
server/src/domain/smart-info/smart-info.service.spec.ts
+24 -4
View File
@@ -16,6 +16,7 @@ import {
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
import { cleanModelName, getCLIPModelInfo } from './smart-info.constant';
import { SmartInfoService } from './smart-info.service';
const asset = {
@@ -195,10 +196,29 @@ describe(SmartInfoService.name, () => {
{ imagePath: 'path/to/resize.ext' },
{ enabled: true, modelName: 'ViT-B-32__openai' },
);
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
clipEmbedding: [0.01, 0.02, 0.03],
});
expect(smartMock.upsert).toHaveBeenCalledWith(
{
assetId: 'asset-1',
},
[0.01, 0.02, 0.03],
);
});
});
describe('cleanModelName', () => {
it('should clean name', () => {
expect(cleanModelName('ViT-B-32::openai')).toEqual('ViT-B-32__openai');
expect(cleanModelName('M-CLIP/XLM-Roberta-Large-Vit-L-14')).toEqual('XLM-Roberta-Large-Vit-L-14');
});
});
describe('getCLIPModelInfo', () => {
it('should return the model info', () => {
expect(getCLIPModelInfo('ViT-B-32__openai')).toEqual({ dimSize: 512 });
});
it('should throw an error if the model is not present', () => {
expect(() => getCLIPModelInfo('test-model')).toThrow('Unknown CLIP model: test-model');
});
});
});
server/src/domain/smart-info/smart-info.service.ts
+23 -3
View File
@@ -1,6 +1,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { setTimeout } from 'timers/promises';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
import {
IAssetRepository,
IJobRepository,
@@ -14,6 +15,7 @@ import { SystemConfigCore } from '../system-config';
@Injectable()
export class SmartInfoService {
private configCore: SystemConfigCore;
private logger = new Logger(SmartInfoService.name);
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -25,6 +27,24 @@ export class SmartInfoService {
this.configCore = SystemConfigCore.create(configRepository);
}
async init() {
await this.jobRepository.pause(QueueName.CLIP_ENCODING);
let { isActive } = await this.jobRepository.getQueueStatus(QueueName.CLIP_ENCODING);
while (isActive) {
this.logger.verbose('Waiting for CLIP encoding queue to stop...');
await setTimeout(1000).then(async () => {
({ isActive } = await this.jobRepository.getQueueStatus(QueueName.CLIP_ENCODING));
});
}
const { machineLearning } = await this.configCore.getConfig();
await this.repository.init(machineLearning.clip.modelName);
await this.jobRepository.resume(QueueName.CLIP_ENCODING);
}
async handleQueueObjectTagging({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.classification.enabled) {
@@ -105,7 +125,7 @@ export class SmartInfoService {
machineLearning.clip,
);
await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding });
await this.repository.upsert({ assetId: asset.id }, clipEmbedding);
return true;
}
server/src/domain/system-config/system-config.core.ts
+1 -1
View File
@@ -210,7 +210,7 @@ export class SystemConfigCore {
[FeatureFlag.MAP]: config.map.enabled,
[FeatureFlag.REVERSE_GEOCODING]: config.reverseGeocoding.enabled,
[FeatureFlag.SIDECAR]: true,
[FeatureFlag.SEARCH]: process.env.TYPESENSE_ENABLED !== 'false',
[FeatureFlag.SEARCH]: true,
[FeatureFlag.TRASH]: config.trash.enabled,
// TODO: use these instead of `POST oauth/config`
server/src/domain/system-config/system-config.service.spec.ts
+8 -2
View File
@@ -13,7 +13,12 @@ import {
import { BadRequestException } from '@nestjs/common';
import { newCommunicationRepositoryMock, newJobRepositoryMock, newSystemConfigRepositoryMock } from '@test';
import { JobName, QueueName } from '../job';
import { ICommunicationRepository, IJobRepository, ISystemConfigRepository } from '../repositories';
import {
ICommunicationRepository,
IJobRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { defaults, SystemConfigValidator } from './system-config.core';
import { SystemConfigService } from './system-config.service';
@@ -133,13 +138,14 @@ describe(SystemConfigService.name, () => {
let configMock: jest.Mocked<ISystemConfigRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
beforeEach(async () => {
delete process.env.IMMICH_CONFIG_FILE;
configMock = newSystemConfigRepositoryMock();
communicationMock = newCommunicationRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new SystemConfigService(configMock, communicationMock, jobMock);
sut = new SystemConfigService(configMock, communicationMock, jobMock, smartInfoMock);
});
it('should work', () => {
server/src/domain/system-config/system-config.service.ts
+14 -3
View File
@@ -1,6 +1,12 @@
import { Inject, Injectable } from '@nestjs/common';
import { JobName } from '../job';
import { CommunicationEvent, ICommunicationRepository, IJobRepository, ISystemConfigRepository } from '../repositories';
import {
CommunicationEvent,
ICommunicationRepository,
IJobRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { SystemConfigDto, mapConfig } from './dto/system-config.dto';
import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
import {
@@ -22,6 +28,7 @@ export class SystemConfigService {
@Inject(ISystemConfigRepository) private repository: ISystemConfigRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
) {
this.core = SystemConfigCore.create(repository);
}
@@ -41,10 +48,14 @@ export class SystemConfigService {
}
async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
const config = await this.core.updateConfig(dto);
const oldConfig = await this.core.getConfig();
const newConfig = await this.core.updateConfig(dto);
await this.jobRepository.queue({ name: JobName.SYSTEM_CONFIG_CHANGE });
this.communicationRepository.broadcast(CommunicationEvent.CONFIG_UPDATE, {});
return mapConfig(config);
if (oldConfig.machineLearning.clip.modelName !== newConfig.machineLearning.clip.modelName) {
await this.smartInfoRepository.init(newConfig.machineLearning.clip.modelName);
}
return mapConfig(newConfig);
}
async refreshConfig() {