openapi

docker/docker-compose.prod.yml

@@ -77,22 +77,12 @@ services:
       - 5432:5432
     healthcheck:
       test: >-
-        pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
+        pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
-        Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
-        --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
-        echo "checksum failure count is $$Chksum";
-        [ "$$Chksum" = '0' ] || exit 1
       interval: 5m
       start_interval: 30s
       start_period: 5m
     command: >-
-      postgres
+      postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
-      -c shared_preload_libraries=vectors.so
-      -c 'search_path="$$user", public, vectors'
-      -c logging_collector=on
-      -c max_wal_size=2GB
-      -c shared_buffers=512MB
-      -c wal_compression=on
     restart: always
 
   # set IMMICH_TELEMETRY_INCLUDE=all in .env to enable metrics

docker/docker-compose.yml

@@ -67,22 +67,12 @@ services:
       - ${DB_DATA_LOCATION}:/var/lib/postgresql/data
     healthcheck:
       test: >-
-        pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1;
+        pg_isready --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" || exit 1; Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
-        Chksum="$$(psql --dbname="$${POSTGRES_DB}" --username="$${POSTGRES_USER}" --tuples-only --no-align
-        --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')";
-        echo "checksum failure count is $$Chksum";
-        [ "$$Chksum" = '0' ] || exit 1
       interval: 5m
       start_interval: 30s
       start_period: 5m
     command: >-
-      postgres
+      postgres -c shared_preload_libraries=vectors.so -c 'search_path="$$user", public, vectors' -c logging_collector=on -c max_wal_size=2GB -c shared_buffers=512MB -c wal_compression=on
-      -c shared_preload_libraries=vectors.so
-      -c 'search_path="$$user", public, vectors'
-      -c logging_collector=on
-      -c max_wal_size=2GB
-      -c shared_buffers=512MB
-      -c wal_compression=on
     restart: always
 
 volumes:

mobile/openapi/README.md

@@ -122,7 +122,9 @@ Class | Method | HTTP request | Description
 *DeprecatedApi* | [**getRandom**](doc//DeprecatedApi.md#getrandom) | **GET** /assets/random | 
 *DownloadApi* | [**downloadArchive**](doc//DownloadApi.md#downloadarchive) | **POST** /download/archive | 
 *DownloadApi* | [**getDownloadInfo**](doc//DownloadApi.md#getdownloadinfo) | **POST** /download/info | 
+*DuplicatesApi* | [**deduplicateAll**](doc//DuplicatesApi.md#deduplicateall) | **POST** /duplicates/bulk/deduplicate | 
 *DuplicatesApi* | [**getAssetDuplicates**](doc//DuplicatesApi.md#getassetduplicates) | **GET** /duplicates | 
+*DuplicatesApi* | [**keepAll**](doc//DuplicatesApi.md#keepall) | **POST** /duplicates/bulk/keep | 
 *FacesApi* | [**createFace**](doc//FacesApi.md#createface) | **POST** /faces | 
 *FacesApi* | [**deleteFace**](doc//FacesApi.md#deleteface) | **DELETE** /faces/{id} | 
 *FacesApi* | [**getFaces**](doc//FacesApi.md#getfaces) | **GET** /faces | 
@@ -327,6 +329,7 @@ Class | Method | HTTP request | Description
  - [CreateLibraryDto](doc//CreateLibraryDto.md)
  - [CreateProfileImageResponseDto](doc//CreateProfileImageResponseDto.md)
  - [DatabaseBackupConfig](doc//DatabaseBackupConfig.md)
+ - [DeduplicateAllDto](doc//DeduplicateAllDto.md)
  - [DownloadArchiveInfo](doc//DownloadArchiveInfo.md)
  - [DownloadInfoDto](doc//DownloadInfoDto.md)
  - [DownloadResponse](doc//DownloadResponse.md)

mobile/openapi/lib/api.dart

@@ -122,6 +122,7 @@ part 'model/create_album_dto.dart';
 part 'model/create_library_dto.dart';
 part 'model/create_profile_image_response_dto.dart';
 part 'model/database_backup_config.dart';
+part 'model/deduplicate_all_dto.dart';
 part 'model/download_archive_info.dart';
 part 'model/download_info_dto.dart';
 part 'model/download_response.dart';

mobile/openapi/lib/api/duplicates_api.dart

@@ -16,6 +16,45 @@ class DuplicatesApi {
 
   final ApiClient apiClient;
 
+  /// Performs an HTTP 'POST /duplicates/bulk/deduplicate' operation and returns the [Response].
+  /// Parameters:
+  ///
+  /// * [DeduplicateAllDto] deduplicateAllDto (required):
+  Future<Response> deduplicateAllWithHttpInfo(DeduplicateAllDto deduplicateAllDto,) async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/bulk/deduplicate';
+
+    // ignore: prefer_final_locals
+    Object? postBody = deduplicateAllDto;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>['application/json'];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'POST',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  /// Parameters:
+  ///
+  /// * [DeduplicateAllDto] deduplicateAllDto (required):
+  Future<void> deduplicateAll(DeduplicateAllDto deduplicateAllDto,) async {
+    final response = await deduplicateAllWithHttpInfo(deduplicateAllDto,);
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+  }
+
   /// Performs an HTTP 'GET /duplicates' operation and returns the [Response].
   Future<Response> getAssetDuplicatesWithHttpInfo() async {
     // ignore: prefer_const_declarations
@@ -59,4 +98,37 @@ class DuplicatesApi {
     }
     return null;
   }
+
+  /// Performs an HTTP 'POST /duplicates/bulk/keep' operation and returns the [Response].
+  Future<Response> keepAllWithHttpInfo() async {
+    // ignore: prefer_const_declarations
+    final apiPath = r'/duplicates/bulk/keep';
+
+    // ignore: prefer_final_locals
+    Object? postBody;
+
+    final queryParams = <QueryParam>[];
+    final headerParams = <String, String>{};
+    final formParams = <String, String>{};
+
+    const contentTypes = <String>[];
+
+
+    return apiClient.invokeAPI(
+      apiPath,
+      'POST',
+      queryParams,
+      postBody,
+      headerParams,
+      formParams,
+      contentTypes.isEmpty ? null : contentTypes.first,
+    );
+  }
+
+  Future<void> keepAll() async {
+    final response = await keepAllWithHttpInfo();
+    if (response.statusCode >= HttpStatus.badRequest) {
+      throw ApiException(response.statusCode, await _decodeBodyBytes(response));
+    }
+  }
 }

mobile/openapi/lib/api_client.dart

@@ -300,6 +300,8 @@ class ApiClient {
           return CreateProfileImageResponseDto.fromJson(value);
         case 'DatabaseBackupConfig':
           return DatabaseBackupConfig.fromJson(value);
+        case 'DeduplicateAllDto':
+          return DeduplicateAllDto.fromJson(value);
         case 'DownloadArchiveInfo':
           return DownloadArchiveInfo.fromJson(value);
         case 'DownloadInfoDto':

mobile/openapi/lib/model/all_job_status_response_dto.dart

@@ -18,7 +18,6 @@ class AllJobStatusResponseDto {
     required this.duplicateDetection,
     required this.faceDetection,
     required this.facialRecognition,
-    required this.integrityDatabaseCheck,
     required this.library_,
     required this.metadataExtraction,
     required this.migration,
@@ -41,8 +40,6 @@ class AllJobStatusResponseDto {
 
   JobStatusDto facialRecognition;
 
-  JobStatusDto integrityDatabaseCheck;
-
   JobStatusDto library_;
 
   JobStatusDto metadataExtraction;
@@ -70,7 +67,6 @@ class AllJobStatusResponseDto {
     other.duplicateDetection == duplicateDetection &&
     other.faceDetection == faceDetection &&
     other.facialRecognition == facialRecognition &&
-    other.integrityDatabaseCheck == integrityDatabaseCheck &&
     other.library_ == library_ &&
     other.metadataExtraction == metadataExtraction &&
     other.migration == migration &&
@@ -90,7 +86,6 @@ class AllJobStatusResponseDto {
     (duplicateDetection.hashCode) +
     (faceDetection.hashCode) +
     (facialRecognition.hashCode) +
-    (integrityDatabaseCheck.hashCode) +
     (library_.hashCode) +
     (metadataExtraction.hashCode) +
     (migration.hashCode) +
@@ -103,7 +98,7 @@ class AllJobStatusResponseDto {
     (videoConversion.hashCode);
 
   @override
-  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, integrityDatabaseCheck=$integrityDatabaseCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
+  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -112,7 +107,6 @@ class AllJobStatusResponseDto {
       json[r'duplicateDetection'] = this.duplicateDetection;
       json[r'faceDetection'] = this.faceDetection;
       json[r'facialRecognition'] = this.facialRecognition;
-      json[r'integrityDatabaseCheck'] = this.integrityDatabaseCheck;
       json[r'library'] = this.library_;
       json[r'metadataExtraction'] = this.metadataExtraction;
       json[r'migration'] = this.migration;
@@ -140,7 +134,6 @@ class AllJobStatusResponseDto {
         duplicateDetection: JobStatusDto.fromJson(json[r'duplicateDetection'])!,
         faceDetection: JobStatusDto.fromJson(json[r'faceDetection'])!,
         facialRecognition: JobStatusDto.fromJson(json[r'facialRecognition'])!,
-        integrityDatabaseCheck: JobStatusDto.fromJson(json[r'integrityDatabaseCheck'])!,
         library_: JobStatusDto.fromJson(json[r'library'])!,
         metadataExtraction: JobStatusDto.fromJson(json[r'metadataExtraction'])!,
         migration: JobStatusDto.fromJson(json[r'migration'])!,
@@ -203,7 +196,6 @@ class AllJobStatusResponseDto {
     'duplicateDetection',
     'faceDetection',
     'facialRecognition',
-    'integrityDatabaseCheck',
     'library',
     'metadataExtraction',
     'migration',

mobile/openapi/lib/model/deduplicate_all_dto.dart

@@ -0,0 +1,101 @@
+//
+// AUTO-GENERATED FILE, DO NOT MODIFY!
+//
+// @dart=2.18
+
+// ignore_for_file: unused_element, unused_import
+// ignore_for_file: always_put_required_named_parameters_first
+// ignore_for_file: constant_identifier_names
+// ignore_for_file: lines_longer_than_80_chars
+
+part of openapi.api;
+
+class DeduplicateAllDto {
+  /// Returns a new [DeduplicateAllDto] instance.
+  DeduplicateAllDto({
+    this.assetIdsToKeep = const [],
+  });
+
+  List<String> assetIdsToKeep;
+
+  @override
+  bool operator ==(Object other) => identical(this, other) || other is DeduplicateAllDto &&
+    _deepEquality.equals(other.assetIdsToKeep, assetIdsToKeep);
+
+  @override
+  int get hashCode =>
+    // ignore: unnecessary_parenthesis
+    (assetIdsToKeep.hashCode);
+
+  @override
+  String toString() => 'DeduplicateAllDto[assetIdsToKeep=$assetIdsToKeep]';
+
+  Map<String, dynamic> toJson() {
+    final json = <String, dynamic>{};
+      json[r'assetIdsToKeep'] = this.assetIdsToKeep;
+    return json;
+  }
+
+  /// Returns a new [DeduplicateAllDto] instance and imports its values from
+  /// [value] if it's a [Map], null otherwise.
+  // ignore: prefer_constructors_over_static_methods
+  static DeduplicateAllDto? fromJson(dynamic value) {
+    upgradeDto(value, "DeduplicateAllDto");
+    if (value is Map) {
+      final json = value.cast<String, dynamic>();
+
+      return DeduplicateAllDto(
+        assetIdsToKeep: json[r'assetIdsToKeep'] is Iterable
+            ? (json[r'assetIdsToKeep'] as Iterable).cast<String>().toList(growable: false)
+            : const [],
+      );
+    }
+    return null;
+  }
+
+  static List<DeduplicateAllDto> listFromJson(dynamic json, {bool growable = false,}) {
+    final result = <DeduplicateAllDto>[];
+    if (json is List && json.isNotEmpty) {
+      for (final row in json) {
+        final value = DeduplicateAllDto.fromJson(row);
+        if (value != null) {
+          result.add(value);
+        }
+      }
+    }
+    return result.toList(growable: growable);
+  }
+
+  static Map<String, DeduplicateAllDto> mapFromJson(dynamic json) {
+    final map = <String, DeduplicateAllDto>{};
+    if (json is Map && json.isNotEmpty) {
+      json = json.cast<String, dynamic>(); // ignore: parameter_assignments
+      for (final entry in json.entries) {
+        final value = DeduplicateAllDto.fromJson(entry.value);
+        if (value != null) {
+          map[entry.key] = value;
+        }
+      }
+    }
+    return map;
+  }
+
+  // maps a json object with a list of DeduplicateAllDto-objects as value to a dart map
+  static Map<String, List<DeduplicateAllDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
+    final map = <String, List<DeduplicateAllDto>>{};
+    if (json is Map && json.isNotEmpty) {
+      // ignore: parameter_assignments
+      json = json.cast<String, dynamic>();
+      for (final entry in json.entries) {
+        map[entry.key] = DeduplicateAllDto.listFromJson(entry.value, growable: growable,);
+      }
+    }
+    return map;
+  }
+
+  /// The list of required keys that must be present in a JSON.
+  static const requiredKeys = <String>{
+    'assetIdsToKeep',
+  };
+}
+

mobile/openapi/lib/model/job_name.dart

@@ -38,7 +38,6 @@ class JobName {
   static const library_ = JobName._(r'library');
   static const notifications = JobName._(r'notifications');
   static const backupDatabase = JobName._(r'backupDatabase');
-  static const integrityDatabaseCheck = JobName._(r'integrityDatabaseCheck');
 
   /// List of all possible values in this [enum][JobName].
   static const values = <JobName>[
@@ -57,7 +56,6 @@ class JobName {
     library_,
     notifications,
     backupDatabase,
-    integrityDatabaseCheck,
   ];
 
   static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value);
@@ -111,7 +109,6 @@ class JobNameTypeTransformer {
         case r'library': return JobName.library_;
         case r'notifications': return JobName.notifications;
         case r'backupDatabase': return JobName.backupDatabase;
-        case r'integrityDatabaseCheck': return JobName.integrityDatabaseCheck;
         default:
           if (!allowNull) {
             throw ArgumentError('Unknown enum value to decode: $data');

mobile/openapi/lib/model/manual_job_name.dart

@@ -29,7 +29,6 @@ class ManualJobName {
   static const memoryCleanup = ManualJobName._(r'memory-cleanup');
   static const memoryCreate = ManualJobName._(r'memory-create');
   static const backupDatabase = ManualJobName._(r'backup-database');
-  static const integrityDatabaseCheck = ManualJobName._(r'integrity-database-check');
 
   /// List of all possible values in this [enum][ManualJobName].
   static const values = <ManualJobName>[
@@ -39,7 +38,6 @@ class ManualJobName {
     memoryCleanup,
     memoryCreate,
     backupDatabase,
-    integrityDatabaseCheck,
   ];
 
   static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value);
@@ -84,7 +82,6 @@ class ManualJobNameTypeTransformer {
         case r'memory-cleanup': return ManualJobName.memoryCleanup;
         case r'memory-create': return ManualJobName.memoryCreate;
         case r'backup-database': return ManualJobName.backupDatabase;
-        case r'integrity-database-check': return ManualJobName.integrityDatabaseCheck;
         default:
           if (!allowNull) {
             throw ArgumentError('Unknown enum value to decode: $data');

open-api/immich-openapi-specs.json

@@ -2732,6 +2732,66 @@
         ]
       }
     },
+    "/duplicates/bulk/deduplicate": {
+      "post": {
+        "operationId": "deduplicateAll",
+        "parameters": [],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/DeduplicateAllDto"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "201": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ]
+      }
+    },
+    "/duplicates/bulk/keep": {
+      "post": {
+        "operationId": "keepAll",
+        "parameters": [],
+        "responses": {
+          "201": {
+            "description": ""
+          }
+        },
+        "security": [
+          {
+            "bearer": []
+          },
+          {
+            "cookie": []
+          },
+          {
+            "api_key": []
+          }
+        ],
+        "tags": [
+          "Duplicates"
+        ]
+      }
+    },
     "/faces": {
       "get": {
         "operationId": "getFaces",
@@ -8532,9 +8592,6 @@
           "facialRecognition": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
-          "integrityDatabaseCheck": {
-            "$ref": "#/components/schemas/JobStatusDto"
-          },
           "library": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
@@ -8572,7 +8629,6 @@
           "duplicateDetection",
           "faceDetection",
           "facialRecognition",
-          "integrityDatabaseCheck",
           "library",
           "metadataExtraction",
           "migration",
@@ -9659,6 +9715,21 @@
         ],
         "type": "object"
       },
+      "DeduplicateAllDto": {
+        "properties": {
+          "assetIdsToKeep": {
+            "items": {
+              "format": "uuid",
+              "type": "string"
+            },
+            "type": "array"
+          }
+        },
+        "required": [
+          "assetIdsToKeep"
+        ],
+        "type": "object"
+      },
       "DownloadArchiveInfo": {
         "properties": {
           "assetIds": {
@@ -10105,8 +10176,7 @@
           "sidecar",
           "library",
           "notifications",
-          "backupDatabase",
+          "backupDatabase"
-          "integrityDatabaseCheck"
         ],
         "type": "string"
       },
@@ -10341,8 +10411,7 @@
           "user-cleanup",
           "memory-cleanup",
           "memory-create",
-          "backup-database",
+          "backup-database"
-          "integrity-database-check"
         ],
         "type": "string"
       },

open-api/typescript-sdk/src/fetch-client.ts

@@ -560,6 +560,9 @@ export type DuplicateResponseDto = {
     assets: AssetResponseDto[];
     duplicateId: string;
 };
+export type DeduplicateAllDto = {
+    assetIdsToKeep: string[];
+};
 export type PersonResponseDto = {
     birthDate: string | null;
     /** This property was added in v1.126.0 */
@@ -622,7 +625,6 @@ export type AllJobStatusResponseDto = {
     duplicateDetection: JobStatusDto;
     faceDetection: JobStatusDto;
     facialRecognition: JobStatusDto;
-    integrityDatabaseCheck: JobStatusDto;
     library: JobStatusDto;
     metadataExtraction: JobStatusDto;
     migration: JobStatusDto;
@@ -2177,6 +2179,21 @@ export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
         ...opts
     }));
 }
+export function deduplicateAll({ deduplicateAllDto }: {
+    deduplicateAllDto: DeduplicateAllDto;
+}, opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/bulk/deduplicate", oazapfts.json({
+        ...opts,
+        method: "POST",
+        body: deduplicateAllDto
+    })));
+}
+export function keepAll(opts?: Oazapfts.RequestOpts) {
+    return oazapfts.ok(oazapfts.fetchText("/duplicates/bulk/keep", {
+        ...opts,
+        method: "POST"
+    }));
+}
 export function getFaces({ id }: {
     id: string;
 }, opts?: Oazapfts.RequestOpts) {
@@ -3790,8 +3807,7 @@ export enum ManualJobName {
     UserCleanup = "user-cleanup",
     MemoryCleanup = "memory-cleanup",
     MemoryCreate = "memory-create",
-    BackupDatabase = "backup-database",
+    BackupDatabase = "backup-database"
-    IntegrityDatabaseCheck = "integrity-database-check"
 }
 export enum JobName {
     ThumbnailGeneration = "thumbnailGeneration",
@@ -3808,8 +3824,7 @@ export enum JobName {
     Sidecar = "sidecar",
     Library = "library",
     Notifications = "notifications",
-    BackupDatabase = "backupDatabase",
+    BackupDatabase = "backupDatabase"
-    IntegrityDatabaseCheck = "integrityDatabaseCheck"
 }
 export enum JobCommand {
     Start = "start",

server/src/controllers/duplicate.controller.ts

@@ -1,7 +1,8 @@
-import { Controller, Get } from '@nestjs/common';
+import { Body, Controller, Get, Post } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import { AuthDto } from 'src/dtos/auth.dto';
-import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
+import { DeduplicateAllDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto';
+import { Permission } from 'src/enum';
 import { Auth, Authenticated } from 'src/middleware/auth.guard';
 import { DuplicateService } from 'src/services/duplicate.service';
 
@@ -15,4 +16,16 @@ export class DuplicateController {
   getAssetDuplicates(@Auth() auth: AuthDto): Promise<DuplicateResponseDto[]> {
     return this.service.getDuplicates(auth);
   }
+
+  @Post('/bulk/keep')
+  @Authenticated({ permission: Permission.ASSET_UPDATE })
+  async keepAll(@Auth() auth: AuthDto) {
+    await this.service.keepAll(auth);
+  }
+
+  @Post('/bulk/deduplicate')
+  @Authenticated({ permission: Permission.ASSET_DELETE })
+  async deduplicateAll(@Auth() auth: AuthDto, @Body() dto: DeduplicateAllDto) {
+    await this.service.deduplicateAll(auth, dto);
+  }
 }

server/src/dtos/duplicate.dto.ts

@@ -12,3 +12,9 @@ export class ResolveDuplicatesDto {
   @ValidateUUID({ each: true })
   assetIds!: string[];
 }
+
+export class DeduplicateAllDto {
+  @IsNotEmpty()
+  @ValidateUUID({ each: true })
+  assetIdsToKeep!: string[];
+}

server/src/dtos/job.dto.ts

@@ -99,7 +99,4 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
 
   @ApiProperty({ type: JobStatusDto })
   [QueueName.BACKUP_DATABASE]!: JobStatusDto;
-
-  @ApiProperty({ type: JobStatusDto })
-  [QueueName.DATABASE_INTEGRITY_CHECK]!: JobStatusDto;
 }

server/src/enum.ts

@@ -251,7 +251,6 @@ export enum ManualJobName {
   MEMORY_CLEANUP = 'memory-cleanup',
   MEMORY_CREATE = 'memory-create',
   BACKUP_DATABASE = 'backup-database',
-  INTEGRITY_DATABASE_CHECK = 'integrity-database-check',
 }
 
 export enum AssetPathType {
@@ -442,7 +441,6 @@ export enum QueueName {
   LIBRARY = 'library',
   NOTIFICATION = 'notifications',
   BACKUP_DATABASE = 'backupDatabase',
-  DATABASE_INTEGRITY_CHECK = 'integrityDatabaseCheck',
 }
 
 export enum JobName {
@@ -534,9 +532,6 @@ export enum JobName {
 
   // Version check
   VERSION_CHECK = 'version-check',
-
-  // Integrity
-  DATABASE_INTEGRITY_CHECK = 'database-integrity-check',
 }
 
 export enum JobCommand {

server/src/queries/asset.repository.sql

@@ -463,12 +463,3 @@ where
       and "libraryId" = $2::uuid
       and "isExternal" = $3
   )
-
--- AssetRepository.integrityCheckExif
-select
-  "id"
-from
-  "assets"
-  left join "exif" on "assets"."id" = "exif"."assetId"
-where
-  "exif"."assetId" is null

server/src/repositories/asset-job.repository.ts

@@ -146,10 +146,17 @@ export class AssetJobRepository {
 
   @GenerateSql({ params: [], stream: true })
   streamForSearchDuplicates(force?: boolean) {
-    return this.assetsWithPreviews()
+    return this.db
-      .where((eb) => eb.not((eb) => eb.exists(eb.selectFrom('smart_search').whereRef('assetId', '=', 'assets.id'))))
+      .selectFrom('assets')
-      .$if(!force, (qb) => qb.where('job_status.duplicatesDetectedAt', 'is', null))
       .select(['assets.id'])
+      .where('assets.visibility', '!=', AssetVisibility.HIDDEN)
+      .where('assets.deletedAt', 'is', null)
+      .innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
+      .$if(!force, (qb) =>
+        qb
+          .innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
+          .where('job_status.duplicatesDetectedAt', 'is', null),
+      )
       .stream();
   }
 

server/src/repositories/asset.repository.ts

@@ -632,57 +632,100 @@ export class AssetRepository {
 
   @GenerateSql({ params: [DummyValue.UUID] })
   getDuplicates(userId: string) {
-    return (
+    return this.db
-      this.db
+      .with('duplicates', (qb) =>
-        .with('duplicates', (qb) =>
+        qb
+          .selectFrom('assets')
+          .innerJoin('exif', 'assets.id', 'exif.assetId')
+          .leftJoinLateral(
+            (qb) =>
+              qb
+                .selectFrom(sql`(select 1)`.as('dummy'))
+                .selectAll('assets')
+                .select((eb) => eb.table('exif').as('exifInfo'))
+                .as('asset'),
+            (join) => join.onTrue(),
+          )
+          .select('assets.duplicateId')
+          .select((eb) => eb.fn.jsonAgg('asset').$castTo<MapAsset[]>().as('assets'))
+          .where('assets.ownerId', '=', asUuid(userId))
+          .where('assets.duplicateId', 'is not', null)
+          .$narrowType<{ duplicateId: NotNull }>()
+          .where('assets.deletedAt', 'is', null)
+          .where('assets.visibility', '!=', AssetVisibility.HIDDEN)
+          .where('assets.stackId', 'is', null)
+          .groupBy('assets.duplicateId'),
+      )
+      .with('unique', (qb) =>
+        qb
+          .selectFrom('duplicates')
+          .select('duplicateId')
+          .where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),
+      )
+      .with('removed_unique', (qb) =>
+        qb
+          .updateTable('assets')
+          .set({ duplicateId: null })
+          .from('unique')
+          .whereRef('assets.duplicateId', '=', 'unique.duplicateId'),
+      )
+      .selectFrom('duplicates')
+      .selectAll()
+      .where(({ not, exists }) =>
+        not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
+      )
+      .execute();
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID] })
+  streamDuplicates(userId: string) {
+    return this.db
+      .selectFrom('assets')
+      .innerJoin('exif', 'assets.id', 'exif.assetId')
+      .innerJoinLateral(
+        (qb) =>
           qb
-            .selectFrom('assets')
+            .selectFrom(sql`(select 1)`.as('dummy'))
-            .leftJoinLateral(
+            .selectAll('assets')
-              (qb) =>
+            .select((eb) => eb.table('exif').as('exifInfo'))
-                qb
+            .as('asset'),
-                  .selectFrom('exif')
+        (join) => join.onTrue(),
-                  .selectAll('assets')
+      )
-                  .select((eb) => eb.table('exif').as('exifInfo'))
+      .select('assets.duplicateId')
-                  .whereRef('exif.assetId', '=', 'assets.id')
+      .select((eb) => eb.fn.jsonAgg('asset').as('assets'))
-                  .as('asset'),
+      .where('assets.ownerId', '=', asUuid(userId))
-              (join) => join.onTrue(),
+      .where('assets.duplicateId', 'is not', null)
-            )
+      .$narrowType<{ duplicateId: NotNull }>()
-            .select('assets.duplicateId')
+      .where('assets.deletedAt', 'is', null)
-            .select((eb) =>
+      .where('assets.visibility', '!=', AssetVisibility.HIDDEN)
-              eb
+      .where('assets.stackId', 'is', null)
-                .fn('jsonb_agg', [eb.table('asset')])
+      .groupBy('assets.duplicateId')
-                .$castTo<MapAsset[]>()
+      .stream();
-                .as('assets'),
+  }
-            )
+
-            .where('assets.ownerId', '=', asUuid(userId))
+  @GenerateSql({ params: [DummyValue.UUID] })
-            .where('assets.duplicateId', 'is not', null)
+  keepAllDuplicates(userId: string) {
-            .$narrowType<{ duplicateId: NotNull }>()
+    return this.db
-            .where('assets.deletedAt', 'is', null)
+      .updateTable('assets')
-            .where('assets.visibility', '!=', AssetVisibility.HIDDEN)
+      .set({ duplicateId: null })
-            .where('assets.stackId', 'is', null)
+      .where('duplicateId', 'is not', null)
-            .groupBy('assets.duplicateId'),
+      .where('ownerId', '=', userId)
-        )
+      .execute();
-        .with('unique', (qb) =>
+  }
-          qb
+
-            .selectFrom('duplicates')
+  deduplicateAll(userId: string, keptAssetIds: string[], deduplicatedStatus: AssetStatus) {
-            .select('duplicateId')
+    return this.db
-            .where((eb) => eb(eb.fn('jsonb_array_length', ['assets']), '=', 1)),
+      .with('kept', (qb) =>
-        )
+        // anyUuid ensures the array is passed as a single parameter, so no need to chunk
-        .with('removed_unique', (qb) =>
+        qb.updateTable('assets').set({ duplicateId: null }).where('id', '=', anyUuid(keptAssetIds)).returning('id'),
-          qb
+      )
-            .updateTable('assets')
+      .updateTable('assets')
-            .set({ duplicateId: null })
+      .from('kept')
-            .from('unique')
+      .set({ duplicateId: null, status: deduplicatedStatus })
-            .whereRef('assets.duplicateId', '=', 'unique.duplicateId'),
+      .whereRef('id', '!=', 'kept.id')
-        )
+      .where('duplicateId', 'is not', null)
-        .selectFrom('duplicates')
+      .where('ownerId', '=', userId)
-        .selectAll()
+      .execute();
-        // TODO: compare with filtering by jsonb_array_length > 1
-        .where(({ not, exists }) =>
-          not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
-        )
-        .execute()
-    );
   }
 
   @GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
@@ -875,16 +918,4 @@ export class AssetRepository {
 
     return count;
   }
-
-  @GenerateSql()
-  async integrityCheckExif(): Promise<string[]> {
-    const result = await this.db
-      .selectFrom('assets')
-      .select('id')
-      .leftJoin('exif', 'assets.id', 'exif.assetId')
-      .where('exif.assetId', 'is', null)
-      .execute();
-
-    return result.map((row) => row.id);
-  }
 }

server/src/services/duplicate.service.ts

@@ -3,8 +3,8 @@ import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { OnJob } from 'src/decorators';
 import { mapAsset } from 'src/dtos/asset-response.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
-import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
+import { DeduplicateAllDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto';
-import { AssetFileType, AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
+import { AssetFileType, AssetStatus, AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
 import { AssetDuplicateResult } from 'src/repositories/search.repository';
 import { BaseService } from 'src/services/base.service';
 import { JobItem, JobOf } from 'src/types';
@@ -21,6 +21,20 @@ export class DuplicateService extends BaseService {
     }));
   }
 
+  keepAll(auth: AuthDto) {
+    return this.assetRepository.keepAllDuplicates(auth.user.id);
+  }
+
+  async deduplicateAll(auth: AuthDto, dto: DeduplicateAllDto) {
+    if (dto.assetIdsToKeep.length === 0) {
+      return;
+    }
+
+    const { trash } = await this.getConfig({ withCache: false });
+    const deduplicatedStatus = trash.enabled ? AssetStatus.TRASHED : AssetStatus.DELETED;
+    return this.assetRepository.deduplicateAll(auth.user.id, dto.assetIdsToKeep, deduplicatedStatus);
+  }
+
   @OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION })
   async handleQueueSearchDuplicates({ force }: JobOf<JobName.QUEUE_DUPLICATE_DETECTION>): Promise<JobStatus> {
     const { machineLearning } = await this.getConfig({ withCache: false });
@@ -29,20 +43,16 @@ export class DuplicateService extends BaseService {
     }
 
     let jobs: JobItem[] = [];
-    const queueAll = async () => {
-      await this.jobRepository.queueAll(jobs);
-      jobs = [];
-    };
-
     const assets = this.assetJobRepository.streamForSearchDuplicates(force);
     for await (const asset of assets) {
       jobs.push({ name: JobName.DUPLICATE_DETECTION, data: { id: asset.id } });
       if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
-        await queueAll();
+        await this.jobRepository.queueAll(jobs);
+        jobs = [];
       }
     }
 
-    await queueAll();
+    await this.jobRepository.queueAll(jobs);
 
     return JobStatus.SUCCESS;
   }

server/src/services/index.ts

@@ -11,7 +11,6 @@ import { CliService } from 'src/services/cli.service';
 import { DatabaseService } from 'src/services/database.service';
 import { DownloadService } from 'src/services/download.service';
 import { DuplicateService } from 'src/services/duplicate.service';
-import { IntegrityService } from 'src/services/integrity.service';
 import { JobService } from 'src/services/job.service';
 import { LibraryService } from 'src/services/library.service';
 import { MapService } from 'src/services/map.service';
@@ -55,7 +54,6 @@ export const services = [
   DatabaseService,
   DownloadService,
   DuplicateService,
-  IntegrityService,
   JobService,
   LibraryService,
   MapService,

server/src/services/integrity.service.ts

@@ -1,13 +0,0 @@
-import { Injectable } from '@nestjs/common';
-import { OnJob } from 'src/decorators';
-import { JobName, JobStatus, QueueName } from 'src/enum';
-import { BaseService } from 'src/services/base.service';
-
-@Injectable()
-export class IntegrityService extends BaseService {
-  @OnJob({ name: JobName.DATABASE_INTEGRITY_CHECK, queue: QueueName.DATABASE_INTEGRITY_CHECK })
-  async handleDatabaseIntegrityCheck(): Promise<JobStatus> {
-    console.log(JSON.stringify(await this.assetRepository.integrityCheckExif()));
-    return JobStatus.SUCCESS;
-  }
-}

server/src/services/job.service.ts

@@ -46,10 +46,6 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
       return { name: JobName.BACKUP_DATABASE };
     }
 
-    case ManualJobName.INTEGRITY_DATABASE_CHECK: {
-      return { name: JobName.DATABASE_INTEGRITY_CHECK };
-    }
-
     default: {
       throw new BadRequestException('Invalid job name');
     }
@@ -232,7 +228,6 @@ export class JobService extends BaseService {
       QueueName.STORAGE_TEMPLATE_MIGRATION,
       QueueName.DUPLICATE_DETECTION,
       QueueName.BACKUP_DATABASE,
-      QueueName.DATABASE_INTEGRITY_CHECK,
     ].includes(name);
   }
 

server/src/types.ts

@@ -164,7 +164,6 @@ export type ConcurrentQueueName = Exclude<
   | QueueName.FACIAL_RECOGNITION
   | QueueName.DUPLICATE_DETECTION
   | QueueName.BACKUP_DATABASE
-  | QueueName.DATABASE_INTEGRITY_CHECK
 >;
 
 export type Jobs = { [K in JobItem['name']]: (JobItem & { name: K })['data'] };
@@ -364,8 +363,9 @@ export type JobItem =
   // Version check
   | { name: JobName.VERSION_CHECK; data: IBaseJob }
 
-  // Integrity
+  // Memories
-  | { name: JobName.DATABASE_INTEGRITY_CHECK; data?: IBaseJob };
+  | { name: JobName.MEMORIES_CLEANUP; data?: IBaseJob }
+  | { name: JobName.MEMORIES_CREATE; data?: IBaseJob };
 
 export type VectorExtension = DatabaseExtension.VECTOR | DatabaseExtension.VECTORS;
 

web/src/lib/modals/JobCreateModal.svelte

@@ -20,7 +20,6 @@
     { title: $t('admin.memory_cleanup_job'), value: ManualJobName.MemoryCleanup },
     { title: $t('admin.memory_generate_job'), value: ManualJobName.MemoryCreate },
     { title: $t('admin.backup_database'), value: ManualJobName.BackupDatabase },
-    { title: 'integrity test', value: ManualJobName.IntegrityDatabaseCheck },
   ].map(({ value, title }) => ({ id: value, label: title, value }));
 
   let selectedJob: ComboBoxOption | undefined = $state(undefined);

web/src/routes/(user)/utilities/duplicates/[[photos=photos]]/[[assetId=id]]/+page.svelte

@@ -15,7 +15,7 @@
   import { suggestDuplicate } from '$lib/utils/duplicate-utils';
   import { handleError } from '$lib/utils/handle-error';
   import type { AssetResponseDto } from '@immich/sdk';
-  import { deleteAssets, updateAssets } from '@immich/sdk';
+  import { deduplicateAll, deleteAssets, keepAll, updateAssets } from '@immich/sdk';
   import { Button, HStack, IconButton, Text } from '@immich/ui';
   import { mdiCheckOutline, mdiInformationOutline, mdiKeyboard, mdiTrashCanOutline } from '@mdi/js';
   import { t } from 'svelte-i18n';
@@ -101,33 +101,30 @@
   };
 
   const handleDeduplicateAll = async () => {
-    const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
+    let assetCount = 0;
-    const idsToDelete = duplicates.flatMap((group, i) =>
+    const assetIdsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)!.id);
-      group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
+    for (const group of duplicates) {
-    );
+      assetCount += group.assets.length;
+      assetIdsToKeep.push(suggestDuplicate(group.assets)!.id);
+    }
+    const dedupedAssetCount = assetCount - assetIdsToKeep.length;
 
     let prompt, confirmText;
     if ($featureFlags.trash) {
-      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: idsToDelete.length } });
+      prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: dedupedAssetCount } });
       confirmText = $t('confirm');
     } else {
-      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: idsToDelete.length } });
+      prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: dedupedAssetCount } });
       confirmText = $t('permanently_delete');
     }
 
     return withConfirmation(
       async () => {
-        await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !$featureFlags.trash } });
+        await deduplicateAll({deduplicateAllDto: { assetIdsToKeep } });
-        await updateAssets({
-          assetBulkUpdateDto: {
-            ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)],
-            duplicateId: null,
-          },
-        });
 
         duplicates = [];
 
-        deletedNotification(idsToDelete.length);
+        deletedNotification(dedupedAssetCount);
       },
       prompt,
       confirmText,
@@ -135,10 +132,10 @@
   };
 
   const handleKeepAll = async () => {
-    const ids = duplicates.flatMap((group) => group.assets.map((asset) => asset.id));
+    const assetCount = duplicates.reduce((acc, cur) => acc + cur.assets.length, 0);
     return withConfirmation(
       async () => {
-        await updateAssets({ assetBulkUpdateDto: { ids, duplicateId: null } });
+        await keepAll();
 
         duplicates = [];
 
@@ -147,7 +144,7 @@
           type: NotificationType.Info,
         });
       },
-      $t('bulk_keep_duplicates_confirmation', { values: { count: ids.length } }),
+      $t('bulk_keep_duplicates_confirmation', { values: { count: assetCount } }),
       $t('confirm'),
     );
   };