feat: use pgvecto.rs (#3605)

server/test/e2e/asset.e2e-spec.ts

@@ -10,19 +10,15 @@ import {
   usePagination,
 } from '@app/domain';
 import { AssetController } from '@app/immich';
-import { AssetEntity, AssetType, LibraryType, SharedLinkType } from '@app/infra/entities';
+import { AssetEntity, AssetType, SharedLinkType } from '@app/infra/entities';
 import { AssetRepository } from '@app/infra/repositories';
 import { INestApplication } from '@nestjs/common';
 import { api } from '@test/api';
 import { errorStub, userDto, uuidStub } from '@test/fixtures';
-import { testApp } from '@test/test-utils';
+import { generateAsset, testApp, today, yesterday } from '@test/test-utils';
 import { randomBytes } from 'crypto';
-import { DateTime } from 'luxon';
 import request from 'supertest';
 
-const today = DateTime.fromObject({ year: 2023, month: 11, day: 3 });
-const yesterday = today.minus({ days: 1 });
-
 const makeUploadDto = (options?: { omit: string }): Record<string, any> => {
   const dto: Record<string, any> = {
     deviceAssetId: 'example-image',
@@ -54,30 +50,14 @@ describe(`${AssetController.name} (e2e)`, () => {
   let asset4: AssetResponseDto;
   let asset5: AssetResponseDto;
 
-  let assetCount = 0;
-  const createAsset = async (loginResponse: LoginResponseDto, createdAt: Date, other: Partial<AssetEntity> = {}) => {
-    const id = assetCount++;
-    const asset = await assetRepository.create({
-      createdAt: today.toJSDate(),
-      updatedAt: today.toJSDate(),
-      ownerId: loginResponse.userId,
-      checksum: randomBytes(20),
-      originalPath: `/tests/test_${id}`,
-      deviceAssetId: `test_${id}`,
-      deviceId: 'e2e-test',
-      libraryId: (
-        libraries.find(
-          ({ ownerId, type }) => ownerId === loginResponse.userId && type === LibraryType.UPLOAD,
-        ) as LibraryResponseDto
-      ).id,
-      isVisible: true,
-      fileCreatedAt: createdAt,
-      fileModifiedAt: new Date(),
-      localDateTime: createdAt,
-      type: AssetType.IMAGE,
-      originalFileName: `test_${id}`,
-      ...other,
-    });
+  const createAsset = async (
+    loginResponse: LoginResponseDto,
+    fileCreatedAt: Date,
+    other: Partial<AssetEntity> = {},
+  ) => {
+    const asset = await assetRepository.create(
+      generateAsset(loginResponse.userId, libraries, { fileCreatedAt, ...other }),
+    );
 
     return mapAsset(asset);
   };
@@ -764,7 +744,11 @@ describe(`${AssetController.name} (e2e)`, () => {
       const personRepository = app.get<IPersonRepository>(IPersonRepository);
       const person = await personRepository.create({ ownerId: asset1.ownerId, name: 'Test Person' });
 
-      await personRepository.createFace({ assetId: asset1.id, personId: person.id });
+      await personRepository.createFace({
+        assetId: asset1.id,
+        personId: person.id,
+        embedding: Array.from({ length: 512 }, Math.random),
+      });
 
       const { status, body } = await request(server)
         .put(`/asset/${asset1.id}`)
@@ -1339,7 +1323,11 @@ describe(`${AssetController.name} (e2e)`, () => {
           beforeEach(async () => {
             const personRepository = app.get<IPersonRepository>(IPersonRepository);
             const person = await personRepository.create({ ownerId: asset1.ownerId, name: 'Test Person' });
-            await personRepository.createFace({ assetId: asset1.id, personId: person.id });
+            await personRepository.createFace({
+              assetId: asset1.id,
+              personId: person.id,
+              embedding: Array.from({ length: 512 }, Math.random),
+            });
           });
 
           it('should not return asset with facesRecognizedAt unset', async () => {

server/test/e2e/person.e2e-spec.ts

@@ -37,7 +37,11 @@ describe(`${PersonController.name}`, () => {
       name: 'visible_person',
       thumbnailPath: '/thumbnail/face_asset',
     });
-    await personRepository.createFace({ assetId: faceAsset.id, personId: visiblePerson.id });
+    await personRepository.createFace({
+      assetId: faceAsset.id,
+      personId: visiblePerson.id,
+      embedding: Array.from({ length: 512 }, Math.random),
+    });
 
     hiddenPerson = await personRepository.create({
       ownerId: loginResponse.userId,
@@ -45,7 +49,11 @@ describe(`${PersonController.name}`, () => {
       isHidden: true,
       thumbnailPath: '/thumbnail/face_asset',
     });
-    await personRepository.createFace({ assetId: faceAsset.id, personId: hiddenPerson.id });
+    await personRepository.createFace({
+      assetId: faceAsset.id,
+      personId: hiddenPerson.id,
+      embedding: Array.from({ length: 512 }, Math.random),
+    });
   });
 
   describe('GET /person', () => {

server/test/e2e/search.e2e-spec.ts

@@ -0,0 +1,213 @@
+import {
+  AssetResponseDto,
+  IAssetRepository,
+  ISmartInfoRepository,
+  LibraryResponseDto,
+  LoginResponseDto,
+  mapAsset,
+} from '@app/domain';
+import { SearchController } from '@app/immich';
+import { INestApplication } from '@nestjs/common';
+import { api } from '@test/api';
+import { errorStub } from '@test/fixtures';
+import { generateAsset, testApp } from '@test/test-utils';
+import request from 'supertest';
+
+describe(`${SearchController.name}`, () => {
+  let app: INestApplication;
+  let server: any;
+  let loginResponse: LoginResponseDto;
+  let accessToken: string;
+  let libraries: LibraryResponseDto[];
+  let assetRepository: IAssetRepository;
+  let smartInfoRepository: ISmartInfoRepository;
+  let asset1: AssetResponseDto;
+
+  beforeAll(async () => {
+    [server, app] = await testApp.create();
+    assetRepository = app.get<IAssetRepository>(IAssetRepository);
+    smartInfoRepository = app.get<ISmartInfoRepository>(ISmartInfoRepository);
+  });
+
+  afterAll(async () => {
+    await testApp.teardown();
+  });
+
+  beforeEach(async () => {
+    await testApp.reset();
+    await api.authApi.adminSignUp(server);
+    loginResponse = await api.authApi.adminLogin(server);
+    accessToken = loginResponse.accessToken;
+    libraries = await api.libraryApi.getAll(server, accessToken);
+
+    const assetId = (await assetRepository.create(generateAsset(loginResponse.userId, libraries))).id;
+    await assetRepository.upsertExif({
+      assetId,
+      latitude: 90,
+      longitude: 90,
+      city: 'Immich',
+      state: 'Nebraska',
+      country: 'United States',
+      make: 'Canon',
+      model: 'EOS Rebel T7',
+      lensModel: 'Fancy lens',
+    });
+    await smartInfoRepository.upsert(
+      { assetId, objects: ['car', 'tree'], tags: ['accident'] },
+      Array.from({ length: 512 }, Math.random),
+    );
+    const assetWithMetadata = await assetRepository.getById(assetId, { exifInfo: true, smartInfo: true });
+    if (!assetWithMetadata) {
+      throw new Error('Asset not found');
+    }
+    asset1 = mapAsset(assetWithMetadata);
+  });
+
+  describe('GET /search', () => {
+    beforeEach(async () => {});
+
+    it('should require authentication', async () => {
+      const { status, body } = await request(server).get('/search');
+
+      expect(status).toBe(401);
+      expect(body).toEqual(errorStub.unauthorized);
+    });
+
+    it('should return assets when searching by exif', async () => {
+      if (!asset1?.exifInfo?.make) {
+        throw new Error('Asset 1 does not have exif info');
+      }
+
+      const { status, body } = await request(server)
+        .get('/search')
+        .set('Authorization', `Bearer ${accessToken}`)
+        .query({ q: asset1.exifInfo.make });
+
+      expect(status).toBe(200);
+      expect(body).toMatchObject({
+        albums: {
+          total: 0,
+          count: 0,
+          items: [],
+          facets: [],
+        },
+        assets: {
+          total: 1,
+          count: 1,
+          items: [
+            {
+              id: asset1.id,
+              exifInfo: {
+                make: asset1.exifInfo.make,
+              },
+            },
+          ],
+          facets: [],
+        },
+      });
+    });
+
+    it('should be case-insensitive for metadata search', async () => {
+      if (!asset1?.exifInfo?.make) {
+        throw new Error('Asset 1 does not have exif info');
+      }
+
+      const { status, body } = await request(server)
+        .get('/search')
+        .set('Authorization', `Bearer ${accessToken}`)
+        .query({ q: asset1.exifInfo.make.toLowerCase() });
+
+      expect(status).toBe(200);
+      expect(body).toMatchObject({
+        albums: {
+          total: 0,
+          count: 0,
+          items: [],
+          facets: [],
+        },
+        assets: {
+          total: 1,
+          count: 1,
+          items: [
+            {
+              id: asset1.id,
+              exifInfo: {
+                make: asset1.exifInfo.make,
+              },
+            },
+          ],
+          facets: [],
+        },
+      });
+    });
+
+    it('should be whitespace-insensitive for metadata search', async () => {
+      if (!asset1?.exifInfo?.make) {
+        throw new Error('Asset 1 does not have exif info');
+      }
+
+      const { status, body } = await request(server)
+        .get('/search')
+        .set('Authorization', `Bearer ${accessToken}`)
+        .query({ q: ` ${asset1.exifInfo.make}  ` });
+
+      expect(status).toBe(200);
+      expect(body).toMatchObject({
+        albums: {
+          total: 0,
+          count: 0,
+          items: [],
+          facets: [],
+        },
+        assets: {
+          total: 1,
+          count: 1,
+          items: [
+            {
+              id: asset1.id,
+              exifInfo: {
+                make: asset1.exifInfo.make,
+              },
+            },
+          ],
+          facets: [],
+        },
+      });
+    });
+
+    it('should return assets when searching by object', async () => {
+      if (!asset1?.smartInfo?.objects) {
+        throw new Error('Asset 1 does not have smart info');
+      }
+
+      const { status, body } = await request(server)
+        .get('/search')
+        .set('Authorization', `Bearer ${accessToken}`)
+        .query({ q: asset1.smartInfo.objects[0] });
+
+      expect(status).toBe(200);
+      expect(body).toMatchObject({
+        albums: {
+          total: 0,
+          count: 0,
+          items: [],
+          facets: [],
+        },
+        assets: {
+          total: 1,
+          count: 1,
+          items: [
+            {
+              id: asset1.id,
+              smartInfo: {
+                objects: asset1.smartInfo.objects,
+                tags: asset1.smartInfo.tags,
+              },
+            },
+          ],
+          facets: [],
+        },
+      });
+    });
+  });
+});

server/test/e2e/server-info.e2e-spec.ts

@@ -81,7 +81,7 @@ describe(`${ServerInfoController.name} (e2e)`, () => {
         oauth: false,
         oauthAutoLaunch: false,
         passwordLogin: true,
-        search: false,
+        search: true,
         sidecar: true,
         tagImage: false,
         trash: true,

server/test/e2e/setup.ts

@@ -35,7 +35,7 @@ export default async () => {
 
   if (process.env.DB_HOSTNAME === undefined) {
     // DB hostname not set which likely means we're not running e2e through docker compose. Start a local postgres container.
-    const pg = await new PostgreSqlContainer('postgres')
+    const pg = await new PostgreSqlContainer('tensorchord/pgvecto-rs:pg14-v0.1.11')
       .withExposedPorts(5432)
       .withDatabase('immich')
       .withUsername('postgres')
@@ -47,7 +47,6 @@ export default async () => {
   }
 
   process.env.NODE_ENV = 'development';
-  process.env.TYPESENSE_ENABLED = 'false';
   process.env.IMMICH_MACHINE_LEARNING_ENABLED = 'false';
   process.env.IMMICH_TEST_ENV = 'true';
   process.env.TZ = 'Z';

server/test/fixtures/shared-link.stub.ts

@@ -209,7 +209,6 @@ export const sharedLinkStub = {
             tags: [],
             objects: ['a', 'b', 'c'],
             asset: null as any,
-            clipEmbedding: [0.12, 0.13, 0.14],
           },
           webpPath: '',
           thumbhash: null,

server/test/repositories/asset.repository.mock.ts

@@ -33,5 +33,8 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
     restoreAll: jest.fn(),
     softDeleteAll: jest.fn(),
     search: jest.fn(),
+    getAssetIdByCity: jest.fn(),
+    getAssetIdByTag: jest.fn(),
+    searchMetadata: jest.fn(),
   };
 };

server/test/repositories/index.ts

@@ -13,7 +13,6 @@ export * from './metadata.repository.mock';
 export * from './move.repository.mock';
 export * from './partner.repository.mock';
 export * from './person.repository.mock';
-export * from './search.repository.mock';
 export * from './shared-link.repository.mock';
 export * from './smart-info.repository.mock';
 export * from './storage.repository.mock';

server/test/repositories/search.repository.mock.ts

@@ -1,21 +0,0 @@
-import { ISearchRepository } from '@app/domain';
-
-export const newSearchRepositoryMock = (): jest.Mocked<ISearchRepository> => {
-  return {
-    setup: jest.fn(),
-    checkMigrationStatus: jest.fn(),
-    importAssets: jest.fn(),
-    importAlbums: jest.fn(),
-    importFaces: jest.fn(),
-    deleteAlbums: jest.fn(),
-    deleteAssets: jest.fn(),
-    deleteFaces: jest.fn(),
-    deleteAllFaces: jest.fn(),
-    updateCLIPField: jest.fn(),
-    searchAssets: jest.fn(),
-    searchAlbums: jest.fn(),
-    vectorSearch: jest.fn(),
-    explore: jest.fn(),
-    searchFaces: jest.fn(),
-  };
-};

server/test/repositories/smart-info.repository.mock.ts

@@ -2,6 +2,9 @@ import { ISmartInfoRepository } from '@app/domain';
 
 export const newSmartInfoRepositoryMock = (): jest.Mocked<ISmartInfoRepository> => {
   return {
+    init: jest.fn(),
+    searchCLIP: jest.fn(),
+    searchFaces: jest.fn(),
     upsert: jest.fn(),
   };
 };

server/test/test-utils.ts

@@ -1,9 +1,12 @@
-import { IJobRepository, JobItem, JobItemHandler, QueueName } from '@app/domain';
+import { AssetCreate, IJobRepository, JobItem, JobItemHandler, LibraryResponseDto, QueueName } from '@app/domain';
 import { AppModule } from '@app/immich';
 import { dataSource } from '@app/infra';
+import { AssetEntity, AssetType, LibraryType } from '@app/infra/entities';
 import { INestApplication } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
+import { randomBytes } from 'crypto';
 import * as fs from 'fs';
+import { DateTime } from 'luxon';
 import path from 'path';
 import { EntityTarget, ObjectLiteral } from 'typeorm';
 import { AppService } from '../src/microservices/app.service';
@@ -11,6 +14,9 @@ import { AppService } from '../src/microservices/app.service';
 export const IMMICH_TEST_ASSET_PATH = process.env.IMMICH_TEST_ASSET_PATH;
 export const IMMICH_TEST_ASSET_TEMP_PATH = path.normalize(`${IMMICH_TEST_ASSET_PATH}/temp/`);
 
+export const today = DateTime.fromObject({ year: 2023, month: 11, day: 3 });
+export const yesterday = today.minus({ days: 1 });
+
 export interface ResetOptions {
   entities?: EntityTarget<ObjectLiteral>[];
 }
@@ -20,6 +26,7 @@ export const db = {
       await dataSource.initialize();
     }
 
+    await dataSource.query(`SET vectors.enable_prefilter = on`);
     await dataSource.transaction(async (em) => {
       const entities = options?.entities || [];
       const tableNames =
@@ -114,3 +121,37 @@ export async function restoreTempFolder(): Promise<void> {
   // Create temp folder
   await fs.promises.mkdir(IMMICH_TEST_ASSET_TEMP_PATH);
 }
+
+function randomDate(start: Date, end: Date): Date {
+  return new Date(start.getTime() + Math.random() * (end.getTime() - start.getTime()));
+}
+
+let assetCount = 0;
+export function generateAsset(
+  userId: string,
+  libraries: LibraryResponseDto[],
+  other: Partial<AssetEntity> = {},
+): AssetCreate {
+  const id = assetCount++;
+  const { fileCreatedAt = randomDate(new Date(1970, 1, 1), new Date(2023, 1, 1)) } = other;
+
+  return {
+    createdAt: today.toJSDate(),
+    updatedAt: today.toJSDate(),
+    ownerId: userId,
+    checksum: randomBytes(20),
+    originalPath: `/tests/test_${id}`,
+    deviceAssetId: `test_${id}`,
+    deviceId: 'e2e-test',
+    libraryId: (
+      libraries.find(({ ownerId, type }) => ownerId === userId && type === LibraryType.UPLOAD) as LibraryResponseDto
+    ).id,
+    isVisible: true,
+    fileCreatedAt,
+    fileModifiedAt: new Date(),
+    localDateTime: fileCreatedAt,
+    type: AssetType.IMAGE,
+    originalFileName: `test_${id}`,
+    ...other,
+  };
+}