cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(server, web): smart search filtering and pagination (#6525)

Browse Source 
* initial pagination impl

* use limit + offset instead of take + skip

* wip web pagination

* working infinite scroll

* update api

* formatting

* fix rebase

* search refactor

* re-add runtime config for vector search

* fix rebase

* fixes

* useless omitBy

* unnecessary handling

* add sql decorator for `searchAssets`

* fixed search builder

* fixed sql

* remove mock method

* linting

* fixed pagination

* fixed unit tests

* formatting

* fix e2e tests

* re-flatten search builder

* refactor endpoints

* clean up dto

* refinements

* don't break everything just yet

* update openapi spec & sql

* update api

* linting

* update sql

* fixes

* optimize web code

* fix typing

* add page limit

* make limit based on asset count

* increase limit

* simpler import
This commit is contained in:
Mert
2024-02-12 20:50:47 -05:00
committed by GitHub
parent f1e4fdf175
commit e334443919
54 changed files with 3993 additions and 790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/infra/repositories/api-key.repository.ts
+1 -1
View File
@@ -42,7 +42,7 @@ export class ApiKeyRepository implements IKeyRepository {
return this.repository.findOne({ where: { userId, id } });
}
@GenerateSql({ params: [DummyValue.STRING] })
@GenerateSql({ params: [DummyValue.UUID] })
getByUserId(userId: string): Promise<APIKeyEntity[]> {
return this.repository.find({ where: { userId }, order: { createdAt: 'DESC' } });
}
server/src/infra/repositories/asset.repository.ts
+11 -172
View File
@@ -12,6 +12,7 @@ import {
MetadataSearchOptions,
MonthDay,
Paginated,
PaginationMode,
PaginationOptions,
SearchExploreItem,
TimeBucketItem,
@@ -22,26 +23,21 @@ import {
} from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import _ from 'lodash';
import { DateTime } from 'luxon';
import path from 'node:path';
import {
And,
Brackets,
FindOptionsRelations,
FindOptionsSelect,
FindOptionsWhere,
In,
IsNull,
LessThan,
Not,
Repository,
} from 'typeorm';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
const DEFAULT_SEARCH_SIZE = 250;
import { Chunked, ChunkedArray, OptionalBetween, paginate, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
const truncateMap: Record<TimeBucketSize, string> = {
[TimeBucketSize.DAY]: 'day',
@@ -70,142 +66,6 @@ export class AssetRepository implements IAssetRepository {
await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
}
search(options: AssetSearchOptions): Promise<AssetEntity[]> {
const {
id,
libraryId,
deviceAssetId,
type,
checksum,
ownerId,
isVisible,
isFavorite,
isExternal,
isReadOnly,
isOffline,
isArchived,
isMotion,
isEncoded,
createdBefore,
createdAfter,
updatedBefore,
updatedAfter,
trashedBefore,
trashedAfter,
takenBefore,
takenAfter,
originalFileName,
originalPath,
resizePath,
webpPath,
encodedVideoPath,
city,
state,
country,
make,
model,
lensModel,
withDeleted: _withDeleted,
withExif: _withExif,
withStacked,
withPeople,
withSmartInfo,
order,
} = options;
const withDeleted = _withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
const page = Math.max(options.page || 1, 1);
const size = Math.min(options.size || DEFAULT_SEARCH_SIZE, DEFAULT_SEARCH_SIZE);
const exifWhere = _.omitBy(
{
city,
state,
country,
make,
model,
lensModel,
},
_.isUndefined,
);
const withExif = Object.keys(exifWhere).length > 0 || _withExif;
const where: FindOptionsWhere<AssetEntity> = _.omitBy(
{
ownerId,
id,
libraryId,
deviceAssetId,
type,
checksum,
isVisible,
isFavorite,
isExternal,
isReadOnly,
isOffline,
isArchived,
livePhotoVideoId: isMotion && Not(IsNull()),
originalFileName,
originalPath,
resizePath,
webpPath,
encodedVideoPath: encodedVideoPath ?? (isEncoded && Not(IsNull())),
createdAt: OptionalBetween(createdAfter, createdBefore),
updatedAt: OptionalBetween(updatedAfter, updatedBefore),
deletedAt: OptionalBetween(trashedAfter, trashedBefore),
fileCreatedAt: OptionalBetween(takenAfter, takenBefore),
exifInfo: Object.keys(exifWhere).length > 0 ? exifWhere : undefined,
},
_.isUndefined,
);
const builder = this.repository.createQueryBuilder('asset');
if (withExif) {
if (_withExif) {
builder.leftJoinAndSelect('asset.exifInfo', 'exifInfo');
} else {
builder.leftJoin('asset.exifInfo', 'exifInfo');
}
}
if (withPeople) {
builder.leftJoinAndSelect('asset.faces', 'faces');
builder.leftJoinAndSelect('faces.person', 'person');
}
if (withSmartInfo) {
builder.leftJoinAndSelect('asset.smartInfo', 'smartInfo');
}
if (withDeleted) {
builder.withDeleted();
}
builder.where(where);
if (withStacked) {
builder
.leftJoinAndSelect('asset.stack', 'stack')
.leftJoinAndSelect('stack.assets', 'stackedAssets')
.andWhere(new Brackets((qb) => qb.where('stack.primaryAssetId = asset.id').orWhere('asset.stackId IS NULL')));
}
return builder
.skip(size * (page - 1))
.take(size)
.orderBy('asset.fileCreatedAt', order ?? 'DESC')
.getMany();
}
create(asset: AssetCreate): Promise<AssetEntity> {
return this.repository.save(asset);
}
@@ -316,17 +176,7 @@ export class AssetRepository implements IAssetRepository {
}
getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
return paginate(this.repository, pagination, {
where: {
ownerId: userId,
isVisible: options.isVisible,
deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
},
relations: {
exifInfo: true,
},
withDeleted: !!options.trashedBefore,
});
return this.getAll(pagination, { ...options, id: userId });
}
@GenerateSql({ params: [[DummyValue.UUID]] })
@@ -345,24 +195,13 @@ export class AssetRepository implements IAssetRepository {
}
getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
return paginate(this.repository, pagination, {
where: {
isVisible: options.isVisible,
type: options.type,
deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
},
relations: {
exifInfo: options.withExif !== false,
smartInfo: options.withSmartInfo !== false,
tags: options.withSmartInfo !== false,
faces: options.withFaces !== false,
smartSearch: options.withSmartInfo === true,
},
withDeleted: options.withDeleted ?? !!options.trashedBefore,
order: {
// Ensures correct order when paginating
createdAt: options.order ?? 'ASC',
},
let builder = this.repository.createQueryBuilder('asset');
builder = searchAssetBuilder(builder, options);
builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE,
skip: pagination.skip,
take: pagination.take,
});
}
@@ -435,7 +274,7 @@ export class AssetRepository implements IAssetRepository {
await this.repository.remove(asset);
}
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.BUFFER] })
@GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null> {
return this.repository.findOne({ where: { ownerId: userId, checksum } });
}
server/src/infra/repositories/index.ts
+1 -1
View File
@@ -17,9 +17,9 @@ export * from './metadata.repository';
export * from './move.repository';
export * from './partner.repository';
export * from './person.repository';
export * from './search.repository';
export * from './server-info.repository';
export * from './shared-link.repository';
export * from './smart-info.repository';
export * from './system-config.repository';
export * from './system-metadata.repository';
export * from './tag.repository';
server/src/infra/repositories/smart-info.repository.ts → server/src/infra/repositories/search.repository.ts
+75 -45
View File
@@ -1,10 +1,15 @@
import {
AssetSearchOptions,
DatabaseExtension,
Embedding,
EmbeddingSearch,
FaceEmbeddingSearch,
FaceSearchResult,
ISmartInfoRepository,
ISearchRepository,
Paginated,
PaginationMode,
PaginationResult,
SearchPaginationOptions,
SmartSearchOptions,
} from '@app/domain';
import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
import { AssetEntity, AssetFaceEntity, SmartInfoEntity, SmartSearchEntity } from '@app/infra/entities';
@@ -14,11 +19,11 @@ import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { vectorExt } from '../database.config';
import { DummyValue, GenerateSql } from '../infra.util';
import { asVector, isValidInteger } from '../infra.utils';
import { asVector, isValidInteger, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
@Injectable()
export class SmartInfoRepository implements ISmartInfoRepository {
private logger = new ImmichLogger(SmartInfoRepository.name);
export class SearchRepository implements ISearchRepository {
private logger = new ImmichLogger(SearchRepository.name);
private faceColumns: string[];
constructor(
@@ -35,48 +40,74 @@ export class SmartInfoRepository implements ISmartInfoRepository {
async init(modelName: string): Promise<void> {
const { dimSize } = getCLIPModelInfo(modelName);
if (dimSize == null) {
throw new Error(`Invalid CLIP model name: ${modelName}`);
}
const curDimSize = await this.getDimSize();
this.logger.verbose(`Current database CLIP dimension size is ${curDimSize}`);
const currentDimSize = await this.getDimSize();
this.logger.verbose(`Current database CLIP dimension size is ${currentDimSize}`);
if (dimSize != currentDimSize) {
this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${currentDimSize}.`);
if (dimSize != curDimSize) {
this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${curDimSize}.`);
await this.updateDimSize(dimSize);
}
}
@GenerateSql({
params: [{ userIds: [DummyValue.UUID], embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
params: [
{ page: 1, size: 100 },
{
takenAfter: DummyValue.DATE,
lensModel: DummyValue.STRING,
ownerId: DummyValue.UUID,
withStacked: true,
isFavorite: true,
},
],
})
async searchCLIP({ userIds, embedding, numResults, withArchived }: EmbeddingSearch): Promise<AssetEntity[]> {
if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`);
}
async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> {
let builder = this.assetRepository.createQueryBuilder('asset');
builder = searchAssetBuilder(builder, options);
// setting this too low messes with prefilter recall
numResults = Math.max(numResults, 64);
builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE,
skip: (pagination.page - 1) * pagination.size,
take: pagination.size,
});
}
@GenerateSql({
params: [
{ page: 1, size: 100 },
{
takenAfter: DummyValue.DATE,
embedding: Array.from({ length: 512 }, Math.random),
lensModel: DummyValue.STRING,
withStacked: true,
isFavorite: true,
userIds: [DummyValue.UUID],
},
],
})
async searchSmart(
pagination: SearchPaginationOptions,
{ embedding, userIds, ...options }: SmartSearchOptions,
): Paginated<AssetEntity> {
let results: PaginationResult<AssetEntity> = { items: [], hasNextPage: false };
let results: AssetEntity[] = [];
await this.assetRepository.manager.transaction(async (manager) => {
const query = manager
.createQueryBuilder(AssetEntity, 'a')
.innerJoin('a.smartSearch', 's')
.leftJoinAndSelect('a.exifInfo', 'e')
.where('a.ownerId IN (:...userIds )')
.orderBy('s.embedding <=> :embedding')
let builder = manager.createQueryBuilder(AssetEntity, 'asset');
builder = searchAssetBuilder(builder, options);
builder
.innerJoin('asset.smartSearch', 'search')
.andWhere('asset.ownerId IN (:...userIds )')
.orderBy('search.embedding <=> :embedding')
.setParameters({ userIds, embedding: asVector(embedding) });
if (!withArchived) {
query.andWhere('a.isArchived = false');
}
query.andWhere('a.isVisible = true').andWhere('a.fileCreatedAt < NOW()');
query.limit(numResults);
await manager.query(this.getRuntimeConfig(numResults));
results = await query.getMany();
await manager.query(this.getRuntimeConfig(pagination.size));
results = await paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.LIMIT_OFFSET,
skip: (pagination.page - 1) * pagination.size,
take: pagination.size,
});
});
return results;
@@ -135,7 +166,6 @@ export class SmartInfoRepository implements ISmartInfoRepository {
.where('res.distance <= :maxDistance', { maxDistance })
.getRawMany();
});
return results.map((row) => ({
face: this.assetFaceRepository.create(row),
distance: row.distance,
@@ -163,17 +193,14 @@ export class SmartInfoRepository implements ISmartInfoRepository {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
const currentDimSize = await this.getDimSize();
if (currentDimSize === dimSize) {
const curDimSize = await this.getDimSize();
if (curDimSize === dimSize) {
return;
}
this.logger.log(`Updating database CLIP dimension size to ${dimSize}.`);
await this.smartSearchRepository.manager.transaction(async (manager) => {
if (vectorExt === DatabaseExtension.VECTORS) {
await manager.query(`SET vectors.pgvector_compatibility=on`);
}
await manager.query(`DROP TABLE smart_search`);
await manager.query(`
@@ -182,12 +209,15 @@ export class SmartInfoRepository implements ISmartInfoRepository {
embedding vector(${dimSize}) NOT NULL )`);
await manager.query(`
CREATE INDEX IF NOT EXISTS clip_index ON smart_search
USING hnsw (embedding vector_cosine_ops)
WITH (ef_construction = 300, m = 16)`);
CREATE INDEX clip_index ON smart_search
USING vectors (embedding vector_cos_ops) WITH (options = $$
[indexing.hnsw]
m = 16
ef_construction = 300
$$)`);
});
this.logger.log(`Successfully updated database CLIP dimension size from ${currentDimSize} to ${dimSize}.`);
this.logger.log(`Successfully updated database CLIP dimension size from ${curDimSize} to ${dimSize}.`);
}
private async getDimSize(): Promise<number> {