feat(server): optimize get asset query (#7176)

* faster query

* add index

* remove legacy code

* update mock

* remove unused imports

* add relations

* add stack

* formatting

* remove stack relation

* remove unused import

* increase chunk size

* generate sql

* linting

* fix typing

* formatting

server/src/infra/entities/asset.entity.ts

@@ -85,6 +85,7 @@ export class AssetEntity {
   @DeleteDateColumn({ type: 'timestamptz', nullable: true })
   deletedAt!: Date | null;
 
+  @Index('idx_asset_file_created_at')
   @Column({ type: 'timestamptz' })
   fileCreatedAt!: Date;
 

server/src/infra/migrations/1708227417898-AddFileCreatedAtIndex.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddFileCreatedAtIndex1708227417898 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX idx_asset_file_created_at ON assets ("fileCreatedAt")`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX idx_asset_file_created_at`);
+    }
+}

server/src/infra/repositories/asset.repository.ts

@@ -2,6 +2,7 @@ import {
   AssetBuilderOptions,
   AssetCreate,
   AssetExploreFieldOptions,
+  AssetSearchOneToOneRelationOptions,
   AssetSearchOptions,
   AssetStats,
   AssetStatsOptions,
@@ -175,8 +176,12 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
-  getByUserId(pagination: PaginationOptions, userId: string, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
-    return this.getAll(pagination, { ...options, ownerId: userId });
+  getByUserId(
+    pagination: PaginationOptions,
+    userId: string,
+    options: Omit<AssetSearchOptions, 'userIds'> = {},
+  ): Paginated<AssetEntity> {
+    return this.getAll(pagination, { ...options, userIds: [userId] });
   }
 
   @GenerateSql({ params: [[DummyValue.UUID]] })
@@ -205,6 +210,29 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  @GenerateSql({
+    params: [
+      { skip: 20_000, take: 10_000 },
+      {
+        takenBefore: DummyValue.DATE,
+        userIds: [DummyValue.UUID],
+      },
+    ],
+  })
+  getAllByFileCreationDate(
+    pagination: PaginationOptions,
+    options: AssetSearchOneToOneRelationOptions = {},
+  ): Paginated<AssetEntity> {
+    let builder = this.repository.createQueryBuilder('asset');
+    builder = searchAssetBuilder(builder, options);
+    builder.orderBy('asset.fileCreatedAt', options.orderDirection ?? 'DESC');
+    return paginatedBuilder<AssetEntity>(builder, {
+      mode: PaginationMode.LIMIT_OFFSET,
+      skip: pagination.skip,
+      take: pagination.take,
+    });
+  }
+
   /**
    * Get assets by device's Id on the database
    * @param ownerId

server/src/infra/sql/asset.repository.sql

@@ -395,6 +395,55 @@ ORDER BY
 LIMIT
   1
 
+-- AssetRepository.getAllByFileCreationDate
+SELECT
+  "asset"."id" AS "asset_id",
+  "asset"."deviceAssetId" AS "asset_deviceAssetId",
+  "asset"."ownerId" AS "asset_ownerId",
+  "asset"."libraryId" AS "asset_libraryId",
+  "asset"."deviceId" AS "asset_deviceId",
+  "asset"."type" AS "asset_type",
+  "asset"."originalPath" AS "asset_originalPath",
+  "asset"."resizePath" AS "asset_resizePath",
+  "asset"."webpPath" AS "asset_webpPath",
+  "asset"."thumbhash" AS "asset_thumbhash",
+  "asset"."encodedVideoPath" AS "asset_encodedVideoPath",
+  "asset"."createdAt" AS "asset_createdAt",
+  "asset"."updatedAt" AS "asset_updatedAt",
+  "asset"."deletedAt" AS "asset_deletedAt",
+  "asset"."fileCreatedAt" AS "asset_fileCreatedAt",
+  "asset"."localDateTime" AS "asset_localDateTime",
+  "asset"."fileModifiedAt" AS "asset_fileModifiedAt",
+  "asset"."isFavorite" AS "asset_isFavorite",
+  "asset"."isArchived" AS "asset_isArchived",
+  "asset"."isExternal" AS "asset_isExternal",
+  "asset"."isReadOnly" AS "asset_isReadOnly",
+  "asset"."isOffline" AS "asset_isOffline",
+  "asset"."checksum" AS "asset_checksum",
+  "asset"."duration" AS "asset_duration",
+  "asset"."isVisible" AS "asset_isVisible",
+  "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
+  "asset"."originalFileName" AS "asset_originalFileName",
+  "asset"."sidecarPath" AS "asset_sidecarPath",
+  "asset"."stackId" AS "asset_stackId"
+FROM
+  "assets" "asset"
+WHERE
+  (
+    "asset"."fileCreatedAt" <= $1
+    AND 1 = 1
+    AND "asset"."ownerId" IN ($2)
+    AND 1 = 1
+    AND 1 = 1
+  )
+  AND ("asset"."deletedAt" IS NULL)
+ORDER BY
+  "asset"."fileCreatedAt" DESC
+LIMIT
+  10001
+OFFSET
+  20000
+
 -- AssetRepository.getAllByDeviceId
 SELECT
   "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",