feat(mobile): hash assets in isolate (#18924)

mobile/lib/domain/interfaces/local_album.interface.dart

@@ -29,6 +29,8 @@ abstract interface class ILocalAlbumRepository implements IDatabaseRepository {
     String albumId,
     Iterable<String> assetIdsToKeep,
   );
+
+  Future<List<LocalAsset>> getAssetsToHash(String albumId);
 }
 
 enum SortLocalAlbumsBy { id }

mobile/lib/domain/interfaces/local_asset.interface.dart

@@ -0,0 +1,6 @@
+import 'package:immich_mobile/domain/interfaces/db.interface.dart';
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
+
+abstract interface class ILocalAssetRepository implements IDatabaseRepository {
+  Future<void> updateHashes(Iterable<LocalAsset> hashes);
+}

mobile/lib/domain/interfaces/storage.interface.dart

@@ -0,0 +1,7 @@
+import 'dart:io';
+
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
+
+abstract interface class IStorageRepository {
+  Future<File?> getFileForAsset(LocalAsset asset);
+}

mobile/lib/domain/models/local_album.model.dart

@@ -1,13 +1,19 @@
 enum BackupSelection {
-  none,
-  selected,
-  excluded,
+  none._(1),
+  selected._(0),
+  excluded._(2);
+
+  // Used to sort albums based on the backupSelection
+  // selected -> none -> excluded
+  final int sortOrder;
+  const BackupSelection._(this.sortOrder);
 }
 
 class LocalAlbum {
   final String id;
   final String name;
   final DateTime updatedAt;
+  final bool isIosSharedAlbum;
 
   final int assetCount;
   final BackupSelection backupSelection;
@@ -18,6 +24,7 @@ class LocalAlbum {
     required this.updatedAt,
     this.assetCount = 0,
     this.backupSelection = BackupSelection.none,
+    this.isIosSharedAlbum = false,
   });
 
   LocalAlbum copyWith({
@@ -26,6 +33,7 @@ class LocalAlbum {
     DateTime? updatedAt,
     int? assetCount,
     BackupSelection? backupSelection,
+    bool? isIosSharedAlbum,
   }) {
     return LocalAlbum(
       id: id ?? this.id,
@@ -33,6 +41,7 @@ class LocalAlbum {
       updatedAt: updatedAt ?? this.updatedAt,
       assetCount: assetCount ?? this.assetCount,
       backupSelection: backupSelection ?? this.backupSelection,
+      isIosSharedAlbum: isIosSharedAlbum ?? this.isIosSharedAlbum,
     );
   }
 
@@ -45,7 +54,8 @@ class LocalAlbum {
         other.name == name &&
         other.updatedAt == updatedAt &&
         other.assetCount == assetCount &&
-        other.backupSelection == backupSelection;
+        other.backupSelection == backupSelection &&
+        other.isIosSharedAlbum == isIosSharedAlbum;
   }
 
   @override
@@ -54,7 +64,8 @@ class LocalAlbum {
         name.hashCode ^
         updatedAt.hashCode ^
         assetCount.hashCode ^
-        backupSelection.hashCode;
+        backupSelection.hashCode ^
+        isIosSharedAlbum.hashCode;
   }
 
   @override
@@ -65,6 +76,7 @@ name: $name,
 updatedAt: $updatedAt,
 assetCount: $assetCount,
 backupSelection: $backupSelection,
+isIosSharedAlbum: $isIosSharedAlbum
 }''';
   }
 }

mobile/lib/domain/services/hash.service.dart

@@ -0,0 +1,121 @@
+import 'dart:convert';
+
+import 'package:immich_mobile/constants/constants.dart';
+import 'package:immich_mobile/domain/interfaces/local_album.interface.dart';
+import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart';
+import 'package:immich_mobile/domain/interfaces/storage.interface.dart';
+import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
+import 'package:immich_mobile/platform/native_sync_api.g.dart';
+import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart';
+import 'package:logging/logging.dart';
+
+class HashService {
+  final int batchSizeLimit;
+  final int batchFileLimit;
+  final ILocalAlbumRepository _localAlbumRepository;
+  final ILocalAssetRepository _localAssetRepository;
+  final IStorageRepository _storageRepository;
+  final NativeSyncApi _nativeSyncApi;
+  final _log = Logger('HashService');
+
+  HashService({
+    required ILocalAlbumRepository localAlbumRepository,
+    required ILocalAssetRepository localAssetRepository,
+    required IStorageRepository storageRepository,
+    required NativeSyncApi nativeSyncApi,
+    this.batchSizeLimit = kBatchHashSizeLimit,
+    this.batchFileLimit = kBatchHashFileLimit,
+  })  : _localAlbumRepository = localAlbumRepository,
+        _localAssetRepository = localAssetRepository,
+        _storageRepository = storageRepository,
+        _nativeSyncApi = nativeSyncApi;
+
+  Future<void> hashAssets() async {
+    final Stopwatch stopwatch = Stopwatch()..start();
+    // Sorted by backupSelection followed by isCloud
+    final localAlbums = await _localAlbumRepository.getAll();
+    localAlbums.sort((a, b) {
+      final backupComparison =
+          a.backupSelection.sortOrder.compareTo(b.backupSelection.sortOrder);
+
+      if (backupComparison != 0) {
+        return backupComparison;
+      }
+
+      // Local albums come before iCloud albums
+      return (a.isIosSharedAlbum ? 1 : 0).compareTo(b.isIosSharedAlbum ? 1 : 0);
+    });
+
+    for (final album in localAlbums) {
+      final assetsToHash =
+          await _localAlbumRepository.getAssetsToHash(album.id);
+      if (assetsToHash.isNotEmpty) {
+        await _hashAssets(assetsToHash);
+      }
+    }
+
+    stopwatch.stop();
+    _log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
+    DLog.log("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
+  }
+
+  /// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
+  /// with hash for those that were successfully hashed. Hashes are looked up in a table
+  /// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.
+  Future<void> _hashAssets(List<LocalAsset> assetsToHash) async {
+    int bytesProcessed = 0;
+    final toHash = <_AssetToPath>[];
+
+    for (final asset in assetsToHash) {
+      final file = await _storageRepository.getFileForAsset(asset);
+      if (file == null) {
+        continue;
+      }
+
+      bytesProcessed += await file.length();
+      toHash.add(_AssetToPath(asset: asset, path: file.path));
+
+      if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) {
+        await _processBatch(toHash);
+        toHash.clear();
+        bytesProcessed = 0;
+      }
+    }
+
+    await _processBatch(toHash);
+  }
+
+  /// Processes a batch of assets.
+  Future<void> _processBatch(List<_AssetToPath> toHash) async {
+    if (toHash.isEmpty) {
+      return;
+    }
+
+    _log.fine("Hashing ${toHash.length} files");
+
+    final hashed = <LocalAsset>[];
+    final hashes =
+        await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList());
+
+    for (final (index, hash) in hashes.indexed) {
+      final asset = toHash[index].asset;
+      if (hash?.length == 20) {
+        hashed.add(asset.copyWith(checksum: base64.encode(hash!)));
+      } else {
+        _log.warning("Failed to hash file ${asset.id}");
+      }
+    }
+
+    _log.fine("Hashed ${hashed.length}/${toHash.length} assets");
+    DLog.log("Hashed ${hashed.length}/${toHash.length} assets");
+
+    await _localAssetRepository.updateHashes(hashed);
+  }
+}
+
+class _AssetToPath {
+  final LocalAsset asset;
+  final String path;
+
+  const _AssetToPath({required this.asset, required this.path});
+}

mobile/lib/domain/services/local_sync.service.dart

@@ -365,6 +365,7 @@ extension on Iterable<PlatformAsset> {
       (e) => LocalAsset(
         id: e.id,
         name: e.name,
+        checksum: null,
         type: AssetType.values.elementAtOrNull(e.type) ?? AssetType.other,
         createdAt: e.createdAt == null
             ? DateTime.now()

mobile/lib/domain/utils/background_sync.dart

@@ -7,6 +7,7 @@ import 'package:worker_manager/worker_manager.dart';
 class BackgroundSyncManager {
   Cancelable<void>? _syncTask;
   Cancelable<void>? _deviceAlbumSyncTask;
+  Cancelable<void>? _hashTask;
 
   BackgroundSyncManager();
 
@@ -45,6 +46,20 @@ class BackgroundSyncManager {
     });
   }
 
+// No need to cancel the task, as it can also be run when the user logs out
+  Future<void> hashAssets() {
+    if (_hashTask != null) {
+      return _hashTask!.future;
+    }
+
+    _hashTask = runInIsolateGentle(
+      computation: (ref) => ref.read(hashServiceProvider).hashAssets(),
+    );
+    return _hashTask!.whenComplete(() {
+      _hashTask = null;
+    });
+  }
+
   Future<void> syncRemote() {
     if (_syncTask != null) {
       return _syncTask!.future;