feat(server, web): smart search filtering and pagination (#6525)

* initial pagination impl * use limit + offset instead of take + skip * wip web pagination * working infinite scroll * update api * formatting * fix rebase * search refactor * re-add runtime config for vector search * fix rebase * fixes * useless omitBy * unnecessary handling * add sql decorator for `searchAssets` * fixed search builder * fixed sql * remove mock method * linting * fixed pagination * fixed unit tests * formatting * fix e2e tests * re-flatten search builder * refactor endpoints * clean up dto * refinements * don't break everything just yet * update openapi spec & sql * update api * linting * update sql * fixes * optimize web code * fix typing * add page limit * make limit based on asset count * increase limit * simpler import
2024-02-12 20:50:47 -05:00
parent f1e4fdf175
commit e334443919
54 changed files with 3993 additions and 790 deletions
@@ -17,9 +17,9 @@ import {
  IMoveRepository,
  IPartnerRepository,
  IPersonRepository,
+  ISearchRepository,
  IServerInfoRepository,
  ISharedLinkRepository,
-  ISmartInfoRepository,
  IStorageRepository,
  ISystemConfigRepository,
  ISystemMetadataRepository,
@@ -56,9 +56,9 @@ import {
  MoveRepository,
  PartnerRepository,
  PersonRepository,
+  SearchRepository,
  ServerInfoRepository,
  SharedLinkRepository,
-  SmartInfoRepository,
  SystemConfigRepository,
  SystemMetadataRepository,
  TagRepository,
@@ -86,7 +86,7 @@ const providers: Provider[] = [
  { provide: IPersonRepository, useClass: PersonRepository },
  { provide: IServerInfoRepository, useClass: ServerInfoRepository },
  { provide: ISharedLinkRepository, useClass: SharedLinkRepository },
-  { provide: ISmartInfoRepository, useClass: SmartInfoRepository },
+  { provide: ISearchRepository, useClass: SearchRepository },
  { provide: IStorageRepository, useClass: FilesystemProvider },
  { provide: ISystemConfigRepository, useClass: SystemConfigRepository },
  { provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
@@ -1,7 +1,19 @@
-import { Paginated, PaginationOptions } from '@app/domain';
+import { AssetSearchBuilderOptions, Paginated, PaginationOptions } from '@app/domain';
 import _ from 'lodash';
-import { Between, FindManyOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
-import { chunks, setUnion } from '../domain/domain.util';
+import {
+  Between,
+  Brackets,
+  FindManyOptions,
+  IsNull,
+  LessThanOrEqual,
+  MoreThanOrEqual,
+  Not,
+  ObjectLiteral,
+  Repository,
+  SelectQueryBuilder,
+} from 'typeorm';
+import { PaginatedBuilderOptions, PaginationMode, PaginationResult, chunks, setUnion } from '../domain/domain.util';
+import { AssetEntity } from './entities';
 import { DATABASE_PARAMETER_CHUNK_SIZE } from './infra.util';

 /**
@@ -18,9 +30,21 @@ export function OptionalBetween<T>(from?: T, to?: T) {
  }
 }

+export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
+  const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
+  return Number.isInteger(value) && value >= min && value <= max;
+};
+
+function paginationHelper<Entity extends ObjectLiteral>(items: Entity[], take: number): PaginationResult<Entity> {
+  const hasNextPage = items.length > take;
+  items.splice(take);
+
+  return { items, hasNextPage };
+}
+
 export async function paginate<Entity extends ObjectLiteral>(
  repository: Repository<Entity>,
-  paginationOptions: PaginationOptions,
+  { take, skip }: PaginationOptions,
  searchOptions?: FindManyOptions<Entity>,
 ): Paginated<Entity> {
  const items = await repository.find(
@@ -28,27 +52,33 @@ export async function paginate<Entity extends ObjectLiteral>(
      {
        ...searchOptions,
        // Take one more item to check if there's a next page
-        take: paginationOptions.take + 1,
-        skip: paginationOptions.skip,
+        take: take + 1,
+        skip,
      },
      _.isUndefined,
    ),
  );

-  const hasNextPage = items.length > paginationOptions.take;
-  items.splice(paginationOptions.take);
+  return paginationHelper(items, take);
+}

-  return { items, hasNextPage };
+export async function paginatedBuilder<Entity extends ObjectLiteral>(
+  qb: SelectQueryBuilder<Entity>,
+  { take, skip, mode }: PaginatedBuilderOptions,
+): Paginated<Entity> {
+  if (mode === PaginationMode.LIMIT_OFFSET) {
+    qb.limit(take + 1).offset(skip);
+  } else {
+    qb.take(take + 1).skip(skip);
+  }
+
+  const items = await qb.getMany();
+  return paginationHelper(items, take);
 }

 export const asVector = (embedding: number[], quote = false) =>
  quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;

-export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
-  const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
-  return Number.isInteger(value) && value >= min && value <= max;
-};
-
 /**
 * Wraps a method that takes a collection of parameters and sequentially calls it with chunks of the collection,
 * to overcome the maximum number of parameters allowed by the database driver.
@@ -91,3 +121,79 @@ export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator
 export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator {
  return Chunked({ ...options, mergeFn: setUnion });
 }
+
+export function searchAssetBuilder(
+  builder: SelectQueryBuilder<AssetEntity>,
+  options: AssetSearchBuilderOptions,
+): SelectQueryBuilder<AssetEntity> {
+  builder.andWhere(
+    _.omitBy(
+      {
+        createdAt: OptionalBetween(options.createdAfter, options.createdBefore),
+        updatedAt: OptionalBetween(options.updatedAfter, options.updatedBefore),
+        deletedAt: OptionalBetween(options.trashedAfter, options.trashedBefore),
+        fileCreatedAt: OptionalBetween(options.takenAfter, options.takenBefore),
+      },
+      _.isUndefined,
+    ),
+  );
+
+  const exifInfo = _.omitBy(_.pick(options, ['city', 'country', 'lensModel', 'make', 'model', 'state']), _.isUndefined);
+  if (Object.keys(exifInfo).length > 0) {
+    builder.leftJoin(`${builder.alias}.exifInfo`, 'exifInfo');
+    builder.andWhere({ exifInfo });
+  }
+
+  const id = _.pick(options, ['checksum', 'deviceAssetId', 'deviceId', 'id', 'libraryId', 'ownerId']);
+  builder.andWhere(_.omitBy(id, _.isUndefined));
+
+  const path = _.pick(options, ['encodedVideoPath', 'originalFileName', 'originalPath', 'resizePath', 'webpPath']);
+  builder.andWhere(_.omitBy(path, _.isUndefined));
+
+  const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']);
+  const { isArchived, isEncoded, isMotion, withArchived } = options;
+  builder.andWhere(
+    _.omitBy(
+      {
+        ...status,
+        isArchived: isArchived ?? withArchived,
+        encodedVideoPath: isEncoded ? Not(IsNull()) : undefined,
+        livePhotoVideoId: isMotion ? Not(IsNull()) : undefined,
+      },
+      _.isUndefined,
+    ),
+  );
+
+  if (options.withExif) {
+    builder.leftJoinAndSelect(`${builder.alias}.exifInfo`, 'exifInfo');
+  }
+
+  if (options.withFaces || options.withPeople) {
+    builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces');
+  }
+
+  if (options.withPeople) {
+    builder.leftJoinAndSelect(`${builder.alias}.person`, 'person');
+  }
+
+  if (options.withSmartInfo) {
+    builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo');
+  }
+
+  if (options.withStacked) {
+    builder
+      .leftJoinAndSelect(`${builder.alias}.stack`, 'stack')
+      .leftJoinAndSelect('stack.assets', 'stackedAssets')
+      .andWhere(
+        new Brackets((qb) => qb.where(`stack.primaryAssetId = ${builder.alias}.id`).orWhere('asset.stackId IS NULL')),
+      );
+  }
+
+  const withDeleted =
+    options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined);
+  if (withDeleted) {
+    builder.withDeleted();
+  }
+
+  return builder;
+}
@@ -42,7 +42,7 @@ export class ApiKeyRepository implements IKeyRepository {
    return this.repository.findOne({ where: { userId, id } });
  }

-  @GenerateSql({ params: [DummyValue.STRING] })
+  @GenerateSql({ params: [DummyValue.UUID] })
  getByUserId(userId: string): Promise<APIKeyEntity[]> {
    return this.repository.find({ where: { userId }, order: { createdAt: 'DESC' } });
  }
@@ -12,6 +12,7 @@ import {
  MetadataSearchOptions,
  MonthDay,
  Paginated,
+  PaginationMode,
  PaginationOptions,
  SearchExploreItem,
  TimeBucketItem,
@@ -22,26 +23,21 @@ import {
 } from '@app/domain';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import _ from 'lodash';
 import { DateTime } from 'luxon';
 import path from 'node:path';
 import {
-  And,
  Brackets,
  FindOptionsRelations,
  FindOptionsSelect,
  FindOptionsWhere,
  In,
  IsNull,
-  LessThan,
  Not,
  Repository,
 } from 'typeorm';
 import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
 import { DummyValue, GenerateSql } from '../infra.util';
-import { Chunked, ChunkedArray, OptionalBetween, paginate } from '../infra.utils';
-
-const DEFAULT_SEARCH_SIZE = 250;
+import { Chunked, ChunkedArray, OptionalBetween, paginate, paginatedBuilder, searchAssetBuilder } from '../infra.utils';

 const truncateMap: Record<TimeBucketSize, string> = {
  [TimeBucketSize.DAY]: 'day',
@@ -70,142 +66,6 @@ export class AssetRepository implements IAssetRepository {
    await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
  }

-  search(options: AssetSearchOptions): Promise<AssetEntity[]> {
-    const {
-      id,
-      libraryId,
-      deviceAssetId,
-      type,
-      checksum,
-      ownerId,
-
-      isVisible,
-      isFavorite,
-      isExternal,
-      isReadOnly,
-      isOffline,
-      isArchived,
-      isMotion,
-      isEncoded,
-
-      createdBefore,
-      createdAfter,
-      updatedBefore,
-      updatedAfter,
-      trashedBefore,
-      trashedAfter,
-      takenBefore,
-      takenAfter,
-
-      originalFileName,
-      originalPath,
-      resizePath,
-      webpPath,
-      encodedVideoPath,
-
-      city,
-      state,
-      country,
-      make,
-      model,
-      lensModel,
-
-      withDeleted: _withDeleted,
-      withExif: _withExif,
-      withStacked,
-      withPeople,
-      withSmartInfo,
-
-      order,
-    } = options;
-
-    const withDeleted = _withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined);
-
-    const page = Math.max(options.page || 1, 1);
-    const size = Math.min(options.size || DEFAULT_SEARCH_SIZE, DEFAULT_SEARCH_SIZE);
-
-    const exifWhere = _.omitBy(
-      {
-        city,
-        state,
-        country,
-        make,
-        model,
-        lensModel,
-      },
-      _.isUndefined,
-    );
-
-    const withExif = Object.keys(exifWhere).length > 0 || _withExif;
-
-    const where: FindOptionsWhere<AssetEntity> = _.omitBy(
-      {
-        ownerId,
-        id,
-        libraryId,
-        deviceAssetId,
-        type,
-        checksum,
-        isVisible,
-        isFavorite,
-        isExternal,
-        isReadOnly,
-        isOffline,
-        isArchived,
-        livePhotoVideoId: isMotion && Not(IsNull()),
-        originalFileName,
-        originalPath,
-        resizePath,
-        webpPath,
-        encodedVideoPath: encodedVideoPath ?? (isEncoded && Not(IsNull())),
-        createdAt: OptionalBetween(createdAfter, createdBefore),
-        updatedAt: OptionalBetween(updatedAfter, updatedBefore),
-        deletedAt: OptionalBetween(trashedAfter, trashedBefore),
-        fileCreatedAt: OptionalBetween(takenAfter, takenBefore),
-        exifInfo: Object.keys(exifWhere).length > 0 ? exifWhere : undefined,
-      },
-      _.isUndefined,
-    );
-
-    const builder = this.repository.createQueryBuilder('asset');
-
-    if (withExif) {
-      if (_withExif) {
-        builder.leftJoinAndSelect('asset.exifInfo', 'exifInfo');
-      } else {
-        builder.leftJoin('asset.exifInfo', 'exifInfo');
-      }
-    }
-
-    if (withPeople) {
-      builder.leftJoinAndSelect('asset.faces', 'faces');
-      builder.leftJoinAndSelect('faces.person', 'person');
-    }
-
-    if (withSmartInfo) {
-      builder.leftJoinAndSelect('asset.smartInfo', 'smartInfo');
-    }
-
-    if (withDeleted) {
-      builder.withDeleted();
-    }
-
-    builder.where(where);
-
-    if (withStacked) {
-      builder
-        .leftJoinAndSelect('asset.stack', 'stack')
-        .leftJoinAndSelect('stack.assets', 'stackedAssets')
-        .andWhere(new Brackets((qb) => qb.where('stack.primaryAssetId = asset.id').orWhere('asset.stackId IS NULL')));
-    }
-
-    return builder
-      .skip(size * (page - 1))
-      .take(size)
-      .orderBy('asset.fileCreatedAt', order ?? 'DESC')
-      .getMany();
-  }
-
  create(asset: AssetCreate): Promise<AssetEntity> {
    return this.repository.save(asset);
  }
@@ -316,17 +176,7 @@ export class AssetRepository implements IAssetRepository {
  }

  getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
-    return paginate(this.repository, pagination, {
-      where: {
-        ownerId: userId,
-        isVisible: options.isVisible,
-        deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
-      },
-      relations: {
-        exifInfo: true,
-      },
-      withDeleted: !!options.trashedBefore,
-    });
+    return this.getAll(pagination, { ...options, id: userId });
  }

  @GenerateSql({ params: [[DummyValue.UUID]] })
@@ -345,24 +195,13 @@ export class AssetRepository implements IAssetRepository {
  }

  getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
-    return paginate(this.repository, pagination, {
-      where: {
-        isVisible: options.isVisible,
-        type: options.type,
-        deletedAt: options.trashedBefore ? And(Not(IsNull()), LessThan(options.trashedBefore)) : undefined,
-      },
-      relations: {
-        exifInfo: options.withExif !== false,
-        smartInfo: options.withSmartInfo !== false,
-        tags: options.withSmartInfo !== false,
-        faces: options.withFaces !== false,
-        smartSearch: options.withSmartInfo === true,
-      },
-      withDeleted: options.withDeleted ?? !!options.trashedBefore,
-      order: {
-        // Ensures correct order when paginating
-        createdAt: options.order ?? 'ASC',
-      },
+    let builder = this.repository.createQueryBuilder('asset');
+    builder = searchAssetBuilder(builder, options);
+    builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
+    return paginatedBuilder<AssetEntity>(builder, {
+      mode: PaginationMode.SKIP_TAKE,
+      skip: pagination.skip,
+      take: pagination.take,
    });
  }

@@ -435,7 +274,7 @@ export class AssetRepository implements IAssetRepository {
    await this.repository.remove(asset);
  }

-  @GenerateSql({ params: [[DummyValue.UUID], DummyValue.BUFFER] })
+  @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
  getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null> {
    return this.repository.findOne({ where: { ownerId: userId, checksum } });
  }
@@ -17,9 +17,9 @@ export * from './metadata.repository';
 export * from './move.repository';
 export * from './partner.repository';
 export * from './person.repository';
+export * from './search.repository';
 export * from './server-info.repository';
 export * from './shared-link.repository';
-export * from './smart-info.repository';
 export * from './system-config.repository';
 export * from './system-metadata.repository';
 export * from './tag.repository';
@@ -1,10 +1,15 @@
 import {
+  AssetSearchOptions,
  DatabaseExtension,
  Embedding,
-  EmbeddingSearch,
  FaceEmbeddingSearch,
  FaceSearchResult,
-  ISmartInfoRepository,
+  ISearchRepository,
+  Paginated,
+  PaginationMode,
+  PaginationResult,
+  SearchPaginationOptions,
+  SmartSearchOptions,
 } from '@app/domain';
 import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
 import { AssetEntity, AssetFaceEntity, SmartInfoEntity, SmartSearchEntity } from '@app/infra/entities';
@@ -14,11 +19,11 @@ import { InjectRepository } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { vectorExt } from '../database.config';
 import { DummyValue, GenerateSql } from '../infra.util';
-import { asVector, isValidInteger } from '../infra.utils';
+import { asVector, isValidInteger, paginatedBuilder, searchAssetBuilder } from '../infra.utils';

@Injectable()
-export class SmartInfoRepository implements ISmartInfoRepository {
-  private logger = new ImmichLogger(SmartInfoRepository.name);
+export class SearchRepository implements ISearchRepository {
+  private logger = new ImmichLogger(SearchRepository.name);
  private faceColumns: string[];

  constructor(
@@ -35,48 +40,74 @@ export class SmartInfoRepository implements ISmartInfoRepository {

  async init(modelName: string): Promise<void> {
    const { dimSize } = getCLIPModelInfo(modelName);
-    if (dimSize == null) {
-      throw new Error(`Invalid CLIP model name: ${modelName}`);
-    }
+    const curDimSize = await this.getDimSize();
+    this.logger.verbose(`Current database CLIP dimension size is ${curDimSize}`);

-    const currentDimSize = await this.getDimSize();
-    this.logger.verbose(`Current database CLIP dimension size is ${currentDimSize}`);
-
-    if (dimSize != currentDimSize) {
-      this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${currentDimSize}.`);
+    if (dimSize != curDimSize) {
+      this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${curDimSize}.`);
      await this.updateDimSize(dimSize);
    }
  }

  @GenerateSql({
-    params: [{ userIds: [DummyValue.UUID], embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
+    params: [
+      { page: 1, size: 100 },
+      {
+        takenAfter: DummyValue.DATE,
+        lensModel: DummyValue.STRING,
+        ownerId: DummyValue.UUID,
+        withStacked: true,
+        isFavorite: true,
+      },
+    ],
  })
-  async searchCLIP({ userIds, embedding, numResults, withArchived }: EmbeddingSearch): Promise<AssetEntity[]> {
-    if (!isValidInteger(numResults, { min: 1 })) {
-      throw new Error(`Invalid value for 'numResults': ${numResults}`);
-    }
+  async searchMetadata(pagination: SearchPaginationOptions, options: AssetSearchOptions): Paginated<AssetEntity> {
+    let builder = this.assetRepository.createQueryBuilder('asset');
+    builder = searchAssetBuilder(builder, options);

-    // setting this too low messes with prefilter recall
-    numResults = Math.max(numResults, 64);
+    builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
+
+    return paginatedBuilder<AssetEntity>(builder, {
+      mode: PaginationMode.SKIP_TAKE,
+      skip: (pagination.page - 1) * pagination.size,
+      take: pagination.size,
+    });
+  }
+
+  @GenerateSql({
+    params: [
+      { page: 1, size: 100 },
+      {
+        takenAfter: DummyValue.DATE,
+        embedding: Array.from({ length: 512 }, Math.random),
+        lensModel: DummyValue.STRING,
+        withStacked: true,
+        isFavorite: true,
+        userIds: [DummyValue.UUID],
+      },
+    ],
+  })
+  async searchSmart(
+    pagination: SearchPaginationOptions,
+    { embedding, userIds, ...options }: SmartSearchOptions,
+  ): Paginated<AssetEntity> {
+    let results: PaginationResult<AssetEntity> = { items: [], hasNextPage: false };

-    let results: AssetEntity[] = [];
    await this.assetRepository.manager.transaction(async (manager) => {
-      const query = manager
-        .createQueryBuilder(AssetEntity, 'a')
-        .innerJoin('a.smartSearch', 's')
-        .leftJoinAndSelect('a.exifInfo', 'e')
-        .where('a.ownerId IN (:...userIds )')
-        .orderBy('s.embedding <=> :embedding')
+      let builder = manager.createQueryBuilder(AssetEntity, 'asset');
+      builder = searchAssetBuilder(builder, options);
+      builder
+        .innerJoin('asset.smartSearch', 'search')
+        .andWhere('asset.ownerId IN (:...userIds )')
+        .orderBy('search.embedding <=> :embedding')
        .setParameters({ userIds, embedding: asVector(embedding) });

-      if (!withArchived) {
-        query.andWhere('a.isArchived = false');
-      }
-      query.andWhere('a.isVisible = true').andWhere('a.fileCreatedAt < NOW()');
-      query.limit(numResults);
-
-      await manager.query(this.getRuntimeConfig(numResults));
-      results = await query.getMany();
+      await manager.query(this.getRuntimeConfig(pagination.size));
+      results = await paginatedBuilder<AssetEntity>(builder, {
+        mode: PaginationMode.LIMIT_OFFSET,
+        skip: (pagination.page - 1) * pagination.size,
+        take: pagination.size,
+      });
    });

    return results;
@@ -135,7 +166,6 @@ export class SmartInfoRepository implements ISmartInfoRepository {
        .where('res.distance <= :maxDistance', { maxDistance })
        .getRawMany();
    });
-
    return results.map((row) => ({
      face: this.assetFaceRepository.create(row),
      distance: row.distance,
@@ -163,17 +193,14 @@ export class SmartInfoRepository implements ISmartInfoRepository {
      throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
    }

-    const currentDimSize = await this.getDimSize();
-    if (currentDimSize === dimSize) {
+    const curDimSize = await this.getDimSize();
+    if (curDimSize === dimSize) {
      return;
    }

    this.logger.log(`Updating database CLIP dimension size to ${dimSize}.`);

    await this.smartSearchRepository.manager.transaction(async (manager) => {
-      if (vectorExt === DatabaseExtension.VECTORS) {
-        await manager.query(`SET vectors.pgvector_compatibility=on`);
-      }
      await manager.query(`DROP TABLE smart_search`);

      await manager.query(`
@@ -182,12 +209,15 @@ export class SmartInfoRepository implements ISmartInfoRepository {
          embedding  vector(${dimSize}) NOT NULL )`);

      await manager.query(`
-        CREATE INDEX IF NOT EXISTS clip_index ON smart_search
-        USING hnsw (embedding vector_cosine_ops)
-        WITH (ef_construction = 300, m = 16)`);
+        CREATE INDEX clip_index ON smart_search
+          USING vectors (embedding vector_cos_ops) WITH (options = $$
+          [indexing.hnsw]
+          m = 16
+          ef_construction = 300
+          $$)`);
    });

-    this.logger.log(`Successfully updated database CLIP dimension size from ${currentDimSize} to ${dimSize}.`);
+    this.logger.log(`Successfully updated database CLIP dimension size from ${curDimSize} to ${dimSize}.`);
  }

  private async getDimSize(): Promise<number> {
@@ -19,8 +19,8 @@ import {
  MoveRepository,
  PartnerRepository,
  PersonRepository,
+  SearchRepository,
  SharedLinkRepository,
-  SmartInfoRepository,
  SystemConfigRepository,
  SystemMetadataRepository,
  TagRepository,
@@ -41,7 +41,7 @@ const repositories = [
  PartnerRepository,
  PersonRepository,
  SharedLinkRepository,
-  SmartInfoRepository,
+  SearchRepository,
  SystemConfigRepository,
  SystemMetadataRepository,
  TagRepository,
@@ -142,7 +142,7 @@ class SqlGenerator {
        this.sqlLogger.clear();

        // errors still generate sql, which is all we care about
-        await target.apply(instance, params).catch(() => null);
+        await target.apply(instance, params).catch((error: Error) => console.error(`${queryLabel} error: ${error}`));

        if (this.sqlLogger.queries.length === 0) {
          console.warn(`No queries recorded for ${queryLabel}`);
@@ -0,0 +1,234 @@
+-- NOTE: This file is auto generated by ./sql-generator
+
+-- SearchRepository.searchMetadata
+SELECT DISTINCT
+  "distinctAlias"."asset_id" AS "ids_asset_id",
+  "distinctAlias"."asset_fileCreatedAt"
+FROM
+  (
+    SELECT
+      "asset"."id" AS "asset_id",
+      "asset"."deviceAssetId" AS "asset_deviceAssetId",
+      "asset"."ownerId" AS "asset_ownerId",
+      "asset"."libraryId" AS "asset_libraryId",
+      "asset"."deviceId" AS "asset_deviceId",
+      "asset"."type" AS "asset_type",
+      "asset"."originalPath" AS "asset_originalPath",
+      "asset"."resizePath" AS "asset_resizePath",
+      "asset"."webpPath" AS "asset_webpPath",
+      "asset"."thumbhash" AS "asset_thumbhash",
+      "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+      "asset"."createdAt" AS "asset_createdAt",
+      "asset"."updatedAt" AS "asset_updatedAt",
+      "asset"."deletedAt" AS "asset_deletedAt",
+      "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+      "asset"."localDateTime" AS "asset_localDateTime",
+      "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+      "asset"."isFavorite" AS "asset_isFavorite",
+      "asset"."isArchived" AS "asset_isArchived",
+      "asset"."isExternal" AS "asset_isExternal",
+      "asset"."isReadOnly" AS "asset_isReadOnly",
+      "asset"."isOffline" AS "asset_isOffline",
+      "asset"."checksum" AS "asset_checksum",
+      "asset"."duration" AS "asset_duration",
+      "asset"."isVisible" AS "asset_isVisible",
+      "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+      "asset"."originalFileName" AS "asset_originalFileName",
+      "asset"."sidecarPath" AS "asset_sidecarPath",
+      "asset"."stackId" AS "asset_stackId",
+      "stack"."id" AS "stack_id",
+      "stack"."primaryAssetId" AS "stack_primaryAssetId",
+      "stackedAssets"."id" AS "stackedAssets_id",
+      "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
+      "stackedAssets"."ownerId" AS "stackedAssets_ownerId",
+      "stackedAssets"."libraryId" AS "stackedAssets_libraryId",
+      "stackedAssets"."deviceId" AS "stackedAssets_deviceId",
+      "stackedAssets"."type" AS "stackedAssets_type",
+      "stackedAssets"."originalPath" AS "stackedAssets_originalPath",
+      "stackedAssets"."resizePath" AS "stackedAssets_resizePath",
+      "stackedAssets"."webpPath" AS "stackedAssets_webpPath",
+      "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
+      "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
+      "stackedAssets"."createdAt" AS "stackedAssets_createdAt",
+      "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
+      "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
+      "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
+      "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
+      "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
+      "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
+      "stackedAssets"."isArchived" AS "stackedAssets_isArchived",
+      "stackedAssets"."isExternal" AS "stackedAssets_isExternal",
+      "stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly",
+      "stackedAssets"."isOffline" AS "stackedAssets_isOffline",
+      "stackedAssets"."checksum" AS "stackedAssets_checksum",
+      "stackedAssets"."duration" AS "stackedAssets_duration",
+      "stackedAssets"."isVisible" AS "stackedAssets_isVisible",
+      "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
+      "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
+      "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
+      "stackedAssets"."stackId" AS "stackedAssets_stackId"
+    FROM
+      "assets" "asset"
+      LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
+      LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
+      LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
+      AND ("stackedAssets"."deletedAt" IS NULL)
+    WHERE
+      (
+        "asset"."fileCreatedAt" >= $1
+        AND "exifInfo"."lensModel" = $2
+        AND "asset"."ownerId" = $3
+        AND 1 = 1
+        AND "asset"."isFavorite" = $4
+        AND (
+          "stack"."primaryAssetId" = "asset"."id"
+          OR "asset"."stackId" IS NULL
+        )
+      )
+      AND ("asset"."deletedAt" IS NULL)
+  ) "distinctAlias"
+ORDER BY
+  "distinctAlias"."asset_fileCreatedAt" DESC,
+  "asset_id" ASC
+LIMIT
+  101
+
+-- SearchRepository.searchSmart
+START TRANSACTION
+SET
+  LOCAL vectors.enable_prefilter = on;
+
+SET
+  LOCAL vectors.search_mode = vbase;
+
+SET
+  LOCAL vectors.hnsw_ef_search = 100;
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."resizePath" AS "asset_resizePath",
+  "asset"."webpPath" AS "asset_webpPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId",
+  "stack"."id" AS "stack_id",
+  "stack"."primaryAssetId" AS "stack_primaryAssetId",
+  "stackedAssets"."id" AS "stackedAssets_id",
+  "stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
+  "stackedAssets"."ownerId" AS "stackedAssets_ownerId",
+  "stackedAssets"."libraryId" AS "stackedAssets_libraryId",
+  "stackedAssets"."deviceId" AS "stackedAssets_deviceId",
+  "stackedAssets"."type" AS "stackedAssets_type",
+  "stackedAssets"."originalPath" AS "stackedAssets_originalPath",
+  "stackedAssets"."resizePath" AS "stackedAssets_resizePath",
+  "stackedAssets"."webpPath" AS "stackedAssets_webpPath",
+  "stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
+  "stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
+  "stackedAssets"."createdAt" AS "stackedAssets_createdAt",
+  "stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
+  "stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
+  "stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
+  "stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
+  "stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
+  "stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
+  "stackedAssets"."isArchived" AS "stackedAssets_isArchived",
+  "stackedAssets"."isExternal" AS "stackedAssets_isExternal",
+  "stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly",
+  "stackedAssets"."isOffline" AS "stackedAssets_isOffline",
+  "stackedAssets"."checksum" AS "stackedAssets_checksum",
+  "stackedAssets"."duration" AS "stackedAssets_duration",
+  "stackedAssets"."isVisible" AS "stackedAssets_isVisible",
+  "stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
+  "stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
+  "stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
+  "stackedAssets"."stackId" AS "stackedAssets_stackId"
+FROM
+  "assets" "asset"
+  LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
+  LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
+  LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
+  AND ("stackedAssets"."deletedAt" IS NULL)
+  INNER JOIN "smart_search" "search" ON "search"."assetId" = "asset"."id"
+WHERE
+  (
+    "asset"."fileCreatedAt" >= $1
+    AND "exifInfo"."lensModel" = $2
+    AND 1 = 1
+    AND 1 = 1
+    AND "asset"."isFavorite" = $3
+    AND (
+      "stack"."primaryAssetId" = "asset"."id"
+      OR "asset"."stackId" IS NULL
+    )
+    AND "asset"."ownerId" IN ($4)
+  )
+  AND ("asset"."deletedAt" IS NULL)
+ORDER BY
+  "search"."embedding" <= > $5 ASC
+LIMIT
+  101
+COMMIT
+
+-- SearchRepository.searchFaces
+START TRANSACTION
+SET
+  LOCAL vectors.enable_prefilter = on;
+
+SET
+  LOCAL vectors.search_mode = vbase;
+
+SET
+  LOCAL vectors.hnsw_ef_search = 100;
+WITH
+  "cte" AS (
+    SELECT
+      "faces"."id" AS "id",
+      "faces"."assetId" AS "assetId",
+      "faces"."personId" AS "personId",
+      "faces"."imageWidth" AS "imageWidth",
+      "faces"."imageHeight" AS "imageHeight",
+      "faces"."boundingBoxX1" AS "boundingBoxX1",
+      "faces"."boundingBoxY1" AS "boundingBoxY1",
+      "faces"."boundingBoxX2" AS "boundingBoxX2",
+      "faces"."boundingBoxY2" AS "boundingBoxY2",
+      "faces"."embedding" <= > $1 AS "distance"
+    FROM
+      "asset_faces" "faces"
+      INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
+      AND ("asset"."deletedAt" IS NULL)
+    WHERE
+      "asset"."ownerId" IN ($2)
+    ORDER BY
+      "faces"."embedding" <= > $1 ASC
+    LIMIT
+      100
+  )
+SELECT
+  res.*
+FROM
+  "cte" "res"
+WHERE
+  res.distance <= $3
+COMMIT
@@ -1,129 +0,0 @@
-- NOTE: This file is auto generated by ./sql-generator
-
-- SmartInfoRepository.searchCLIP
-START TRANSACTION
-SET
-  LOCAL vectors.enable_prefilter = on;
-
-SET
-  LOCAL vectors.search_mode = vbase;
-
-SET
-  LOCAL vectors.hnsw_ef_search = 100;
-SELECT
-  "a"."id" AS "a_id",
-  "a"."deviceAssetId" AS "a_deviceAssetId",
-  "a"."ownerId" AS "a_ownerId",
-  "a"."libraryId" AS "a_libraryId",
-  "a"."deviceId" AS "a_deviceId",
-  "a"."type" AS "a_type",
-  "a"."originalPath" AS "a_originalPath",
-  "a"."resizePath" AS "a_resizePath",
-  "a"."webpPath" AS "a_webpPath",
-  "a"."thumbhash" AS "a_thumbhash",
-  "a"."encodedVideoPath" AS "a_encodedVideoPath",
-  "a"."createdAt" AS "a_createdAt",
-  "a"."updatedAt" AS "a_updatedAt",
-  "a"."deletedAt" AS "a_deletedAt",
-  "a"."fileCreatedAt" AS "a_fileCreatedAt",
-  "a"."localDateTime" AS "a_localDateTime",
-  "a"."fileModifiedAt" AS "a_fileModifiedAt",
-  "a"."isFavorite" AS "a_isFavorite",
-  "a"."isArchived" AS "a_isArchived",
-  "a"."isExternal" AS "a_isExternal",
-  "a"."isReadOnly" AS "a_isReadOnly",
-  "a"."isOffline" AS "a_isOffline",
-  "a"."checksum" AS "a_checksum",
-  "a"."duration" AS "a_duration",
-  "a"."isVisible" AS "a_isVisible",
-  "a"."livePhotoVideoId" AS "a_livePhotoVideoId",
-  "a"."originalFileName" AS "a_originalFileName",
-  "a"."sidecarPath" AS "a_sidecarPath",
-  "a"."stackId" AS "a_stackId",
-  "e"."assetId" AS "e_assetId",
-  "e"."description" AS "e_description",
-  "e"."exifImageWidth" AS "e_exifImageWidth",
-  "e"."exifImageHeight" AS "e_exifImageHeight",
-  "e"."fileSizeInByte" AS "e_fileSizeInByte",
-  "e"."orientation" AS "e_orientation",
-  "e"."dateTimeOriginal" AS "e_dateTimeOriginal",
-  "e"."modifyDate" AS "e_modifyDate",
-  "e"."timeZone" AS "e_timeZone",
-  "e"."latitude" AS "e_latitude",
-  "e"."longitude" AS "e_longitude",
-  "e"."projectionType" AS "e_projectionType",
-  "e"."city" AS "e_city",
-  "e"."livePhotoCID" AS "e_livePhotoCID",
-  "e"."autoStackId" AS "e_autoStackId",
-  "e"."state" AS "e_state",
-  "e"."country" AS "e_country",
-  "e"."make" AS "e_make",
-  "e"."model" AS "e_model",
-  "e"."lensModel" AS "e_lensModel",
-  "e"."fNumber" AS "e_fNumber",
-  "e"."focalLength" AS "e_focalLength",
-  "e"."iso" AS "e_iso",
-  "e"."exposureTime" AS "e_exposureTime",
-  "e"."profileDescription" AS "e_profileDescription",
-  "e"."colorspace" AS "e_colorspace",
-  "e"."bitsPerSample" AS "e_bitsPerSample",
-  "e"."fps" AS "e_fps"
-FROM
-  "assets" "a"
-  INNER JOIN "smart_search" "s" ON "s"."assetId" = "a"."id"
-  LEFT JOIN "exif" "e" ON "e"."assetId" = "a"."id"
-WHERE
-  (
-    "a"."ownerId" IN ($1)
-    AND "a"."isArchived" = false
-    AND "a"."isVisible" = true
-    AND "a"."fileCreatedAt" < NOW()
-  )
-  AND ("a"."deletedAt" IS NULL)
-ORDER BY
-  "s"."embedding" <= > $2 ASC
-LIMIT
-  100
-COMMIT
-
-- SmartInfoRepository.searchFaces
-START TRANSACTION
-SET
-  LOCAL vectors.enable_prefilter = on;
-
-SET
-  LOCAL vectors.search_mode = vbase;
-
-SET
-  LOCAL vectors.hnsw_ef_search = 100;
-WITH
-  "cte" AS (
-    SELECT
-      "faces"."id" AS "id",
-      "faces"."assetId" AS "assetId",
-      "faces"."personId" AS "personId",
-      "faces"."imageWidth" AS "imageWidth",
-      "faces"."imageHeight" AS "imageHeight",
-      "faces"."boundingBoxX1" AS "boundingBoxX1",
-      "faces"."boundingBoxY1" AS "boundingBoxY1",
-      "faces"."boundingBoxX2" AS "boundingBoxX2",
-      "faces"."boundingBoxY2" AS "boundingBoxY2",
-      "faces"."embedding" <= > $1 AS "distance"
-    FROM
-      "asset_faces" "faces"
-      INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
-      AND ("asset"."deletedAt" IS NULL)
-    WHERE
-      "asset"."ownerId" IN ($2)
-    ORDER BY
-      "faces"."embedding" <= > $1 ASC
-    LIMIT
-      100
-  )
-SELECT
-  res.*
-FROM
-  "cte" "res"
-WHERE
-  res.distance <= $3
-COMMIT