feat(server, web): smart search filtering and pagination (#6525)

* initial pagination impl

* use limit + offset instead of take + skip

* wip web pagination

* working infinite scroll

* update api

* formatting

* fix rebase

* search refactor

* re-add runtime config for vector search

* fix rebase

* fixes

* useless omitBy

* unnecessary handling

* add sql decorator for `searchAssets`

* fixed search builder

* fixed sql

* remove mock method

* linting

* fixed pagination

* fixed unit tests

* formatting

* fix e2e tests

* re-flatten search builder

* refactor endpoints

* clean up dto

* refinements

* don't break everything just yet

* update openapi spec & sql

* update api

* linting

* update sql

* fixes

* optimize web code

* fix typing

* add page limit

* make limit based on asset count

* increase limit

* simpler import

server/src/domain/search/dto/search.dto.ts

@@ -1,8 +1,184 @@
+import { AssetOrder } from '@app/domain/asset/dto/asset.dto';
 import { AssetType } from '@app/infra/entities';
-import { Transform } from 'class-transformer';
-import { IsBoolean, IsEnum, IsNotEmpty, IsString } from 'class-validator';
-import { Optional, toBoolean } from '../../domain.util';
+import { ApiProperty } from '@nestjs/swagger';
+import { Transform, Type } from 'class-transformer';
+import { IsBoolean, IsEnum, IsInt, IsNotEmpty, IsString, Max, Min } from 'class-validator';
+import { Optional, QueryBoolean, QueryDate, ValidateUUID, toBoolean } from '../../domain.util';
 
+class BaseSearchDto {
+  @ValidateUUID({ optional: true })
+  libraryId?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  deviceId?: string;
+
+  @IsEnum(AssetType)
+  @Optional()
+  @ApiProperty({ enumName: 'AssetTypeEnum', enum: AssetType })
+  type?: AssetType;
+
+  @QueryBoolean({ optional: true })
+  isArchived?: boolean;
+
+  @QueryBoolean({ optional: true })
+  withArchived?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isEncoded?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isExternal?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isFavorite?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isMotion?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isOffline?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isReadOnly?: boolean;
+
+  @QueryBoolean({ optional: true })
+  isVisible?: boolean;
+
+  @QueryBoolean({ optional: true })
+  withDeleted?: boolean;
+
+  @QueryBoolean({ optional: true })
+  withExif?: boolean;
+
+  @QueryDate({ optional: true })
+  createdBefore?: Date;
+
+  @QueryDate({ optional: true })
+  createdAfter?: Date;
+
+  @QueryDate({ optional: true })
+  updatedBefore?: Date;
+
+  @QueryDate({ optional: true })
+  updatedAfter?: Date;
+
+  @QueryDate({ optional: true })
+  trashedBefore?: Date;
+
+  @QueryDate({ optional: true })
+  trashedAfter?: Date;
+
+  @QueryDate({ optional: true })
+  takenBefore?: Date;
+
+  @QueryDate({ optional: true })
+  takenAfter?: Date;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  city?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  state?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  country?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  make?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  model?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  lensModel?: string;
+
+  @IsInt()
+  @Min(1)
+  @Type(() => Number)
+  @Optional()
+  page?: number;
+
+  @IsInt()
+  @Min(1)
+  @Max(1000)
+  @Type(() => Number)
+  @Optional()
+  size?: number;
+}
+
+export class MetadataSearchDto extends BaseSearchDto {
+  @ValidateUUID({ optional: true })
+  id?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  deviceAssetId?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  checksum?: string;
+
+  @QueryBoolean({ optional: true })
+  withStacked?: boolean;
+
+  @QueryBoolean({ optional: true })
+  withPeople?: boolean;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  originalFileName?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  originalPath?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  resizePath?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  webpPath?: string;
+
+  @IsString()
+  @IsNotEmpty()
+  @Optional()
+  encodedVideoPath?: string;
+
+  @IsEnum(AssetOrder)
+  @Optional()
+  @ApiProperty({ enumName: 'AssetOrder', enum: AssetOrder })
+  order?: AssetOrder;
+}
+
+export class SmartSearchDto extends BaseSearchDto {
+  @IsString()
+  @IsNotEmpty()
+  query!: string;
+}
+
+// TODO: remove after implementing new search filters
+/** @deprecated */
 export class SearchDto {
   @IsString()
   @IsNotEmpty()
@@ -43,6 +219,19 @@ export class SearchDto {
   @Optional()
   @Transform(toBoolean)
   withArchived?: boolean;
+
+  @IsInt()
+  @Min(1)
+  @Type(() => Number)
+  @Optional()
+  page?: number;
+
+  @IsInt()
+  @Min(1)
+  @Max(1000)
+  @Type(() => Number)
+  @Optional()
+  size?: number;
 }
 
 export class SearchPeopleDto {

server/src/domain/search/response-dto/search-response.dto.ts

@@ -29,6 +29,7 @@ class SearchAssetResponseDto {
   count!: number;
   items!: AssetResponseDto[];
   facets!: SearchFacetResponseDto[];
+  nextPage!: string | null;
 }
 
 export class SearchResponseDto {

server/src/domain/search/search.service.spec.ts

@@ -6,7 +6,7 @@ import {
   newMachineLearningRepositoryMock,
   newPartnerRepositoryMock,
   newPersonRepositoryMock,
-  newSmartInfoRepositoryMock,
+  newSearchRepositoryMock,
   newSystemConfigRepositoryMock,
   personStub,
 } from '@test';
@@ -16,7 +16,7 @@ import {
   IMachineLearningRepository,
   IPartnerRepository,
   IPersonRepository,
-  ISmartInfoRepository,
+  ISearchRepository,
   ISystemConfigRepository,
 } from '../repositories';
 import { SearchDto } from './dto';
@@ -30,7 +30,7 @@ describe(SearchService.name, () => {
   let configMock: jest.Mocked<ISystemConfigRepository>;
   let machineMock: jest.Mocked<IMachineLearningRepository>;
   let personMock: jest.Mocked<IPersonRepository>;
-  let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
+  let searchMock: jest.Mocked<ISearchRepository>;
   let partnerMock: jest.Mocked<IPartnerRepository>;
 
   beforeEach(() => {
@@ -38,9 +38,9 @@ describe(SearchService.name, () => {
     configMock = newSystemConfigRepositoryMock();
     machineMock = newMachineLearningRepositoryMock();
     personMock = newPersonRepositoryMock();
-    smartInfoMock = newSmartInfoRepositoryMock();
+    searchMock = newSearchRepositoryMock();
     partnerMock = newPartnerRepositoryMock();
-    sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock, partnerMock);
+    sut = new SearchService(configMock, machineMock, personMock, searchMock, assetMock, partnerMock);
   });
 
   it('should work', () => {
@@ -104,6 +104,7 @@ describe(SearchService.name, () => {
           count: 1,
           items: [mapAsset(assetStub.image)],
           facets: [],
+          nextPage: null,
         },
       };
 
@@ -111,13 +112,13 @@ describe(SearchService.name, () => {
 
       expect(result).toEqual(expectedResponse);
       expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, [authStub.user1.user.id], { numResults: 250 });
-      expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled();
+      expect(searchMock.searchSmart).not.toHaveBeenCalled();
     });
 
     it('should search archived photos if `withArchived` option is true', async () => {
       const dto: SearchDto = { q: 'test query', clip: true, withArchived: true };
       const embedding = [1, 2, 3];
-      smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
+      searchMock.searchSmart.mockResolvedValueOnce({ items: [assetStub.image], hasNextPage: false });
       machineMock.encodeText.mockResolvedValueOnce(embedding);
       partnerMock.getAll.mockResolvedValueOnce([]);
       const expectedResponse = {
@@ -132,25 +133,28 @@ describe(SearchService.name, () => {
           count: 1,
           items: [mapAsset(assetStub.image)],
           facets: [],
+          nextPage: null,
         },
       };
 
       const result = await sut.search(authStub.user1, dto);
 
       expect(result).toEqual(expectedResponse);
-      expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({
-        userIds: [authStub.user1.user.id],
-        embedding,
-        numResults: 100,
-        withArchived: true,
-      });
+      expect(searchMock.searchSmart).toHaveBeenCalledWith(
+        { page: 1, size: 100 },
+        {
+          userIds: [authStub.user1.user.id],
+          embedding,
+          withArchived: true,
+        },
+      );
       expect(assetMock.searchMetadata).not.toHaveBeenCalled();
     });
 
     it('should search by CLIP if `clip` option is true', async () => {
       const dto: SearchDto = { q: 'test query', clip: true };
       const embedding = [1, 2, 3];
-      smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
+      searchMock.searchSmart.mockResolvedValueOnce({ items: [assetStub.image], hasNextPage: false });
       machineMock.encodeText.mockResolvedValueOnce(embedding);
       partnerMock.getAll.mockResolvedValueOnce([]);
       const expectedResponse = {
@@ -165,18 +169,21 @@ describe(SearchService.name, () => {
           count: 1,
           items: [mapAsset(assetStub.image)],
           facets: [],
+          nextPage: null,
         },
       };
 
       const result = await sut.search(authStub.user1, dto);
 
       expect(result).toEqual(expectedResponse);
-      expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({
-        userIds: [authStub.user1.user.id],
-        embedding,
-        numResults: 100,
-        withArchived: false,
-      });
+      expect(searchMock.searchSmart).toHaveBeenCalledWith(
+        { page: 1, size: 100 },
+        {
+          userIds: [authStub.user1.user.id],
+          embedding,
+          withArchived: false,
+        },
+      );
       expect(assetMock.searchMetadata).not.toHaveBeenCalled();
     });
 

server/src/domain/search/search.service.ts

@@ -1,7 +1,7 @@
 import { AssetEntity } from '@app/infra/entities';
 import { ImmichLogger } from '@app/infra/logger';
 import { Inject, Injectable } from '@nestjs/common';
-import { AssetResponseDto, mapAsset } from '../asset';
+import { AssetOrder, AssetResponseDto, mapAsset } from '../asset';
 import { AuthDto } from '../auth';
 import { PersonResponseDto } from '../person';
 import {
@@ -9,13 +9,13 @@ import {
   IMachineLearningRepository,
   IPartnerRepository,
   IPersonRepository,
-  ISmartInfoRepository,
+  ISearchRepository,
   ISystemConfigRepository,
   SearchExploreItem,
   SearchStrategy,
 } from '../repositories';
 import { FeatureFlag, SystemConfigCore } from '../system-config';
-import { SearchDto, SearchPeopleDto } from './dto';
+import { MetadataSearchDto, SearchDto, SearchPeopleDto, SmartSearchDto } from './dto';
 import { SearchResponseDto } from './response-dto';
 
 @Injectable()
@@ -27,7 +27,7 @@ export class SearchService {
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
     @Inject(IPersonRepository) private personRepository: IPersonRepository,
-    @Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
+    @Inject(ISearchRepository) private searchRepository: ISearchRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
     @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
   ) {
@@ -55,6 +55,53 @@ export class SearchService {
     }));
   }
 
+  async searchMetadata(auth: AuthDto, dto: MetadataSearchDto): Promise<SearchResponseDto> {
+    let checksum: Buffer | undefined;
+
+    if (dto.checksum) {
+      const encoding = dto.checksum.length === 28 ? 'base64' : 'hex';
+      checksum = Buffer.from(dto.checksum, encoding);
+    }
+
+    const page = dto.page ?? 1;
+    const size = dto.size || 250;
+    const enumToOrder = { [AssetOrder.ASC]: 'ASC', [AssetOrder.DESC]: 'DESC' } as const;
+    const { hasNextPage, items } = await this.searchRepository.searchMetadata(
+      { page, size },
+      {
+        ...dto,
+        checksum,
+        ownerId: auth.user.id,
+        orderDirection: dto.order ? enumToOrder[dto.order] : 'DESC',
+      },
+    );
+
+    return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null);
+  }
+
+  async searchSmart(auth: AuthDto, dto: SmartSearchDto): Promise<SearchResponseDto> {
+    await this.configCore.requireFeature(FeatureFlag.SMART_SEARCH);
+    const { machineLearning } = await this.configCore.getConfig();
+    const userIds = await this.getUserIdsToSearch(auth);
+
+    const embedding = await this.machineLearning.encodeText(
+      machineLearning.url,
+      { text: dto.query },
+      machineLearning.clip,
+    );
+
+    const page = dto.page ?? 1;
+    const size = dto.size || 100;
+    const { hasNextPage, items } = await this.searchRepository.searchSmart(
+      { page, size },
+      { ...dto, userIds, embedding },
+    );
+
+    return this.mapResponse(items, hasNextPage ? (page + 1).toString() : null);
+  }
+
+  // TODO: remove after implementing new search filters
+  /** @deprecated */
   async search(auth: AuthDto, dto: SearchDto): Promise<SearchResponseDto> {
     await this.configCore.requireFeature(FeatureFlag.SEARCH);
     const { machineLearning } = await this.configCore.getConfig();
@@ -70,10 +117,10 @@ export class SearchService {
     }
 
     const userIds = await this.getUserIdsToSearch(auth);
-    const withArchived = dto.withArchived || false;
+    const page = dto.page ?? 1;
 
+    let nextPage: string | null = null;
     let assets: AssetEntity[] = [];
-
     switch (strategy) {
       case SearchStrategy.SMART: {
         const embedding = await this.machineLearning.encodeText(
@@ -81,36 +128,30 @@ export class SearchService {
           { text: query },
           machineLearning.clip,
         );
-        assets = await this.smartInfoRepository.searchCLIP({
-          userIds: userIds,
-          embedding,
-          numResults: 100,
-          withArchived,
-        });
+
+        const { hasNextPage, items } = await this.searchRepository.searchSmart(
+          { page, size: dto.size || 100 },
+          {
+            userIds,
+            embedding,
+            withArchived: !!dto.withArchived,
+          },
+        );
+        if (hasNextPage) {
+          nextPage = (page + 1).toString();
+        }
+        assets = items;
         break;
       }
       case SearchStrategy.TEXT: {
-        assets = await this.assetRepository.searchMetadata(query, userIds, { numResults: 250 });
+        assets = await this.assetRepository.searchMetadata(query, userIds, { numResults: dto.size || 250 });
       }
       default: {
         break;
       }
     }
 
-    return {
-      albums: {
-        total: 0,
-        count: 0,
-        items: [],
-        facets: [],
-      },
-      assets: {
-        total: assets.length,
-        count: assets.length,
-        items: assets.map((asset) => mapAsset(asset)),
-        facets: [],
-      },
-    };
+    return this.mapResponse(assets, nextPage);
   }
 
   private async getUserIdsToSearch(auth: AuthDto): Promise<string[]> {
@@ -122,4 +163,17 @@ export class SearchService {
     userIds.push(...partnersIds);
     return userIds;
   }
+
+  private async mapResponse(assets: AssetEntity[], nextPage: string | null): Promise<SearchResponseDto> {
+    return {
+      albums: { total: 0, count: 0, items: [], facets: [] },
+      assets: {
+        total: assets.length,
+        count: assets.length,
+        items: assets.map((asset) => mapAsset(asset)),
+        facets: [],
+        nextPage,
+      },
+    };
+  }
 }