feat(server, web): smart search filtering and pagination (#6525)
* initial pagination impl * use limit + offset instead of take + skip * wip web pagination * working infinite scroll * update api * formatting * fix rebase * search refactor * re-add runtime config for vector search * fix rebase * fixes * useless omitBy * unnecessary handling * add sql decorator for `searchAssets` * fixed search builder * fixed sql * remove mock method * linting * fixed pagination * fixed unit tests * formatting * fix e2e tests * re-flatten search builder * refactor endpoints * clean up dto * refinements * don't break everything just yet * update openapi spec & sql * update api * linting * update sql * fixes * optimize web code * fix typing * add page limit * make limit based on asset count * increase limit * simpler import
This commit is contained in:
Mert
@@ -12,6 +12,7 @@ import {
|
||||
MetadataSearchOptions,
|
||||
MonthDay,
|
||||
Paginated,
|
||||
PaginationMode,
|
||||
PaginationOptions,
|
||||
SearchExploreItem,
|
||||
TimeBucketItem,
|
||||
@@ -22,26 +23,21 @@ import {
|
||||
} from '@app/domain';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import _ from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
And,
|
||||
Brackets,
|
||||
FindOptionsRelations,
|
||||
FindOptionsSelect,
|
||||
FindOptionsWhere,
|
||||
In,
|
||||
IsNull,
|
||||
LessThan,
|
||||
Not,
|
||||
Repository,
|
||||
} from 'typeorm';
|
||||
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
|
||||
import { DummyValue, GenerateSql } from '../infra.util';
|
||||
import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
|
||||
|
||||
const DEFAULT_SEARCH_SIZE = 250;
|
||||
import { Chunked, ChunkedArray, OptionalBetween, paginate, paginatedBuilder, searchAssetBuilder } from '../infra.utils';
|
||||
|
||||
const truncateMap: Record<TimeBucketSize, string> = {
|
||||
[TimeBucketSize.DAY]: 'day',
|
||||
@@ -70,142 +66,6 @@ export class AssetRepository implements IAssetRepository {
|
||||
await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
|
||||
}
|
||||
|
||||
search(options: AssetSearchOptions): Promise<AssetEntity[]> {
|
||||
const {
|
||||
id,
|
||||
libraryId,
|
||||
deviceAssetId,
|
||||
type,
|
||||
checksum,
|
||||
ownerId,
|
||||
|
||||
isVisible,
|
||||
isFavorite,
|
||||
isExternal,
|
||||
isReadOnly,
|
||||
isOffline,
|
||||
isArchived,
|
||||
isMotion,
|
||||
isEncoded,
|
||||
|
||||
createdBefore,
|
||||
createdAfter,
|
||||
updatedBefore,
|
||||
updatedAfter,
|
||||
trashedBefore,
|
||||
trashedAfter,
|
||||
takenBefore,
|
||||
takenAfter,
|
||||
|
||||
originalFileName,
|
||||
originalPath,
|
||||
resizePath,
|
||||
webpPath,
|
||||
encodedVideoPath,
|
||||
|
||||
city,
|
||||
state,
|
||||
country,
|
||||
make,
|
||||
model,
|
||||
lensModel,
|
||||
|
||||
withDeleted: _withDeleted,
|
||||
withExif: _withExif,
|
||||
withStacked,
|
||||
withPeople,
|
||||
withSmartInfo,
|
||||
|
||||
order,
|
||||
} = options;
|
||||
|
||||
const withDeleted = _withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
|
||||
|
||||
const page = Math.max(options.page || 1, 1);
|
||||
const size = Math.min(options.size || DEFAULT_SEARCH_SIZE, DEFAULT_SEARCH_SIZE);
|
||||
|
||||
const exifWhere = _.omitBy(
|
||||
{
|
||||
city,
|
||||
state,
|
||||
country,
|
||||
make,
|
||||
model,
|
||||
lensModel,
|
||||
},
|
||||
_.isUndefined,
|
||||
);
|
||||
|
||||
const withExif = Object.keys(exifWhere).length > 0 || _withExif;
|
||||
|
||||
const where: FindOptionsWhere<AssetEntity> = _.omitBy(
|
||||
{
|
||||
ownerId,
|
||||
id,
|
||||
libraryId,
|
||||
deviceAssetId,
|
||||
type,
|
||||
checksum,
|
||||
isVisible,
|
||||
isFavorite,
|
||||
isExternal,
|
||||
isReadOnly,
|
||||
isOffline,
|
||||
isArchived,
|
||||
livePhotoVideoId: isMotion && Not(IsNull()),
|
||||
originalFileName,
|
||||
originalPath,
|
||||
resizePath,
|
||||
webpPath,
|
||||
encodedVideoPath: encodedVideoPath ?? (isEncoded && Not(IsNull())),
|
||||
createdAt: OptionalBetween(createdAfter, createdBefore),
|
||||
updatedAt: OptionalBetween(updatedAfter, updatedBefore),
|
||||
deletedAt: OptionalBetween(trashedAfter, trashedBefore),
|
||||
fileCreatedAt: OptionalBetween(takenAfter, takenBefore),
|
||||
exifInfo: Object.keys(exifWhere).length > 0 ? exifWhere : undefined,
|
||||
},
|
||||
_.isUndefined,
|
||||
);
|
||||
|
||||
const builder = this.repository.createQueryBuilder('asset');
|
||||
|
||||
if (withExif) {
|
||||
if (_withExif) {
|
||||
builder.leftJoinAndSelect('asset.exifInfo', 'exifInfo');
|
||||
} else {
|
||||
builder.leftJoin('asset.exifInfo', 'exifInfo');
|
||||
}
|
||||
}
|
||||
|
||||
if (withPeople) {
|
||||
builder.leftJoinAndSelect('asset.faces', 'faces');
|
||||
builder.leftJoinAndSelect('faces.person', 'person');
|
||||
}
|
||||
|
||||
if (withSmartInfo) {
|
||||
builder.leftJoinAndSelect('asset.smartInfo', 'smartInfo');
|
||||
}
|
||||
|
||||
if (withDeleted) {
|
||||
builder.withDeleted();
|
||||
}
|
||||
|
||||
builder.where(where);
|
||||
|
||||
if (withStacked) {
|
||||
builder
|
||||
.leftJoinAndSelect('asset.stack', 'stack')
|
||||
.leftJoinAndSelect('stack.assets', 'stackedAssets')
|
||||
.andWhere(new Brackets((qb) => qb.where('stack.primaryAssetId = asset.id').orWhere('asset.stackId IS NULL')));
|
||||
}
|
||||
|
||||
return builder
|
||||
.skip(size * (page - 1))
|
||||
.take(size)
|
||||
.orderBy('asset.fileCreatedAt', order ?? 'DESC')
|
||||
.getMany();
|
||||
}
|
||||
|
||||
create(asset: AssetCreate): Promise<AssetEntity> {
|
||||
return this.repository.save(asset);
|
||||
}
|
||||
@@ -316,17 +176,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
|
||||
return paginate(this.repository, pagination, {
|
||||
where: {
|
||||
ownerId: userId,
|
||||
isVisible: options.isVisible,
|
||||
deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
|
||||
},
|
||||
relations: {
|
||||
exifInfo: true,
|
||||
},
|
||||
withDeleted: !!options.trashedBefore,
|
||||
});
|
||||
return this.getAll(pagination, { ...options, id: userId });
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID]] })
|
||||
@@ -345,24 +195,13 @@ export class AssetRepository implements IAssetRepository {
|
||||
}
|
||||
|
||||
getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
|
||||
return paginate(this.repository, pagination, {
|
||||
where: {
|
||||
isVisible: options.isVisible,
|
||||
type: options.type,
|
||||
deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
|
||||
},
|
||||
relations: {
|
||||
exifInfo: options.withExif !== false,
|
||||
smartInfo: options.withSmartInfo !== false,
|
||||
tags: options.withSmartInfo !== false,
|
||||
faces: options.withFaces !== false,
|
||||
smartSearch: options.withSmartInfo === true,
|
||||
},
|
||||
withDeleted: options.withDeleted ?? !!options.trashedBefore,
|
||||
order: {
|
||||
// Ensures correct order when paginating
|
||||
createdAt: options.order ?? 'ASC',
|
||||
},
|
||||
let builder = this.repository.createQueryBuilder('asset');
|
||||
builder = searchAssetBuilder(builder, options);
|
||||
builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
|
||||
return paginatedBuilder<AssetEntity>(builder, {
|
||||
mode: PaginationMode.SKIP_TAKE,
|
||||
skip: pagination.skip,
|
||||
take: pagination.take,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -435,7 +274,7 @@ export class AssetRepository implements IAssetRepository {
|
||||
await this.repository.remove(asset);
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [[DummyValue.UUID], DummyValue.BUFFER] })
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
|
||||
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null> {
|
||||
return this.repository.findOne({ where: { ownerId: userId, checksum } });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user