refactor: hashing service (#21997)

* download only backup selected assets * android impl * fix tests * limit concurrent hashing to 16 * extension cleanup * optimized hashing * hash only selected albums * remove concurrency limit * address review comments * log more info on failure * add native cancellation * small batch size on ios, large on android * fix: get correct resources * cleanup getResource * ios better hash cancellation * handle graceful cancellation android * do not trigger multiple hashing ops * ios: fix circular reference, improve cancellation * kotlin: more cancellation checks * no need to create result * cancel previous task * avoid race condition * ensure cancellation gets called * fix cancellation not happening --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Alex <alex.tran1502@gmail.com>
2025-09-18 10:12:37 +05:30
parent 2411bf8374
commit 532ec10d5f
18 changed files with 662 additions and 269 deletions
@@ -4,7 +4,6 @@
 *.moved-aside
 *.pbxuser
 *.perspectivev3
-**/*sync/
 .sconsign.dblite
 .tags*
 **/.vagrant/
@@ -5,7 +5,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {
  func enable() throws {
    BackgroundWorkerApiImpl.scheduleRefreshWorker()
    BackgroundWorkerApiImpl.scheduleProcessingWorker()
-    print("BackgroundUploadImpl:enbale Background worker scheduled")
+    print("BackgroundWorkerApiImpl:enable Background worker scheduled")
  }
  
  func configure(settings: BackgroundWorkerSettings) throws {
@@ -15,7 +15,7 @@ class BackgroundWorkerApiImpl: BackgroundWorkerFgHostApi {
  func disable() throws {
    BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.refreshTaskID);
    BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundWorkerApiImpl.processingTaskID);
-    print("BackgroundUploadImpl:disableUploadWorker Disabled background workers")
+    print("BackgroundWorkerApiImpl:disableUploadWorker Disabled background workers")
  }
  
  private static let refreshTaskID = "app.alextran.immich.background.refreshUpload"
@@ -267,6 +267,39 @@ struct SyncDelta: Hashable {
  }
 }

+/// Generated class from Pigeon that represents data sent in messages.
+struct HashResult: Hashable {
+  var assetId: String
+  var error: String? = nil
+  var hash: String? = nil
+
+
+  // swift-format-ignore: AlwaysUseLowerCamelCase
+  static func fromList(_ pigeonVar_list: [Any?]) -> HashResult? {
+    let assetId = pigeonVar_list[0] as! String
+    let error: String? = nilOrValue(pigeonVar_list[1])
+    let hash: String? = nilOrValue(pigeonVar_list[2])
+
+    return HashResult(
+      assetId: assetId,
+      error: error,
+      hash: hash
+    )
+  }
+  func toList() -> [Any?] {
+    return [
+      assetId,
+      error,
+      hash,
+    ]
+  }
+  static func == (lhs: HashResult, rhs: HashResult) -> Bool {
+    return deepEqualsMessages(lhs.toList(), rhs.toList())  }
+  func hash(into hasher: inout Hasher) {
+    deepHashMessages(value: toList(), hasher: &hasher)
+  }
+}
+
 private class MessagesPigeonCodecReader: FlutterStandardReader {
  override func readValue(ofType type: UInt8) -> Any? {
    switch type {
@@ -276,6 +309,8 @@ private class MessagesPigeonCodecReader: FlutterStandardReader {
      return PlatformAlbum.fromList(self.readValue() as! [Any?])
    case 131:
      return SyncDelta.fromList(self.readValue() as! [Any?])
+    case 132:
+      return HashResult.fromList(self.readValue() as! [Any?])
    default:
      return super.readValue(ofType: type)
    }
@@ -293,6 +328,9 @@ private class MessagesPigeonCodecWriter: FlutterStandardWriter {
    } else if let value = value as? SyncDelta {
      super.writeByte(131)
      super.writeValue(value.toList())
+    } else if let value = value as? HashResult {
+      super.writeByte(132)
+      super.writeValue(value.toList())
    } else {
      super.writeValue(value)
    }
@@ -313,6 +351,7 @@ class MessagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
  static let shared = MessagesPigeonCodec(readerWriter: MessagesPigeonCodecReaderWriter())
 }

+
 /// Generated protocol from Pigeon that represents a handler of messages from Flutter.
 protocol NativeSyncApi {
  func shouldFullSync() throws -> Bool
@@ -323,7 +362,8 @@ protocol NativeSyncApi {
  func getAlbums() throws -> [PlatformAlbum]
  func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64
  func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset]
-  func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?]
+  func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void)
+  func cancelHashing() throws
 }

 /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@@ -459,22 +499,38 @@ class NativeSyncApiSetup {
    } else {
      getAssetsForAlbumChannel.setMessageHandler(nil)
    }
-    let hashPathsChannel = taskQueue == nil
-      ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
-      : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)
+    let hashAssetsChannel = taskQueue == nil
+      ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
+      : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue)
    if let api = api {
-      hashPathsChannel.setMessageHandler { message, reply in
+      hashAssetsChannel.setMessageHandler { message, reply in
        let args = message as! [Any?]
-        let pathsArg = args[0] as! [String]
+        let assetIdsArg = args[0] as! [String]
+        let allowNetworkAccessArg = args[1] as! Bool
+        api.hashAssets(assetIds: assetIdsArg, allowNetworkAccess: allowNetworkAccessArg) { result in
+          switch result {
+          case .success(let res):
+            reply(wrapResult(res))
+          case .failure(let error):
+            reply(wrapError(error))
+          }
+        }
+      }
+    } else {
+      hashAssetsChannel.setMessageHandler(nil)
+    }
+    let cancelHashingChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
+    if let api = api {
+      cancelHashingChannel.setMessageHandler { _, reply in
        do {
-          let result = try api.hashPaths(paths: pathsArg)
-          reply(wrapResult(result))
+          try api.cancelHashing()
+          reply(wrapResult(nil))
        } catch {
          reply(wrapError(error))
        }
      }
    } else {
-      hashPathsChannel.setMessageHandler(nil)
+      cancelHashingChannel.setMessageHandler(nil)
    }
  }
 }
@@ -17,30 +17,16 @@ struct AssetWrapper: Hashable, Equatable {
  }
 }

-extension PHAsset {
-  func toPlatformAsset() -> PlatformAsset {
-    return PlatformAsset(
-      id: localIdentifier,
-      name: title(),
-      type: Int64(mediaType.rawValue),
-      createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },
-      updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },
-      width: Int64(pixelWidth),
-      height: Int64(pixelHeight),
-      durationInSeconds: Int64(duration),
-      orientation: 0,
-      isFavorite: isFavorite
-    )
-  }
-}
-
 class NativeSyncApiImpl: NativeSyncApi {
  private let defaults: UserDefaults
  private let changeTokenKey = "immich:changeToken"
  private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum]
  private let recoveredAlbumSubType = 1000000219
  
-  private let hashBufferSize = 2 * 1024 * 1024
+  private var hashTask: Task<Void, Error>?
+  private static let hashCancelledCode = "HASH_CANCELLED"
+  private static let hashCancelled = Result<[HashResult], Error>.failure(PigeonError(code: hashCancelledCode, message: "Hashing cancelled", details: nil))
+  
  
  init(with defaults: UserDefaults = .standard) {
    self.defaults = defaults
@@ -96,7 +82,7 @@ class NativeSyncApiImpl: NativeSyncApi {
      let collections = PHAssetCollection.fetchAssetCollections(with: type, subtype: .any, options: nil)
      for i in 0..<collections.count {
        let album = collections.object(at: i)
-      
+        
        // Ignore recovered album
        if(album.assetCollectionSubtype.rawValue == self.recoveredAlbumSubType) {
          continue;
@@ -254,7 +240,7 @@ class NativeSyncApiImpl: NativeSyncApi {
      let date = NSDate(timeIntervalSince1970: TimeInterval(updatedTimeCond!))
      options.predicate = NSPredicate(format: "creationDate > %@ OR modificationDate > %@", date, date)
    }
-
+    
    let result = PHAsset.fetchAssets(in: album, options: options)
    if(result.count == 0) {
      return []
@@ -267,23 +253,114 @@ class NativeSyncApiImpl: NativeSyncApi {
    return assets
  }
  
-  func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] {
-      return paths.map { path in
-          guard let file = FileHandle(forReadingAtPath: path) else {
-              print("Cannot open file: \(path)")
-              return nil
-          }
-          
-          var hasher = Insecure.SHA1()
-          while autoreleasepool(invoking: {
-              let chunk = file.readData(ofLength: hashBufferSize)
-              guard !chunk.isEmpty else { return false }
-              hasher.update(data: chunk)
-              return true
-          }) { }
-          
-          let digest = hasher.finalize()
-          return FlutterStandardTypedData(bytes: Data(digest))
+  func hashAssets(assetIds: [String], allowNetworkAccess: Bool, completion: @escaping (Result<[HashResult], Error>) -> Void) {
+    if let prevTask = hashTask {
+      prevTask.cancel()
+      hashTask = nil
+    }
+    hashTask = Task { [weak self] in
+      var missingAssetIds = Set(assetIds)
+      var assets = [PHAsset]()
+      assets.reserveCapacity(assetIds.count)
+      PHAsset.fetchAssets(withLocalIdentifiers: assetIds, options: nil).enumerateObjects { (asset, _, stop) in
+        if Task.isCancelled {
+          stop.pointee = true
+          return
+        }
+        missingAssetIds.remove(asset.localIdentifier)
+        assets.append(asset)
      }
+      
+      if Task.isCancelled {
+        return completion(Self.hashCancelled)
+      }
+      
+      await withTaskGroup(of: HashResult?.self) { taskGroup in
+        var results = [HashResult]()
+        results.reserveCapacity(assets.count)
+        for asset in assets {
+          if Task.isCancelled {
+            return completion(Self.hashCancelled)
+          }
+          taskGroup.addTask {
+            guard let self = self else { return nil }
+            return await self.hashAsset(asset, allowNetworkAccess: allowNetworkAccess)
+          }
+        }
+        
+        for await result in taskGroup {
+          guard let result = result else {
+            return completion(Self.hashCancelled)
+          }
+          results.append(result)
+        }
+        
+        for missing in missingAssetIds {
+          results.append(HashResult(assetId: missing, error: "Asset not found in library", hash: nil))
+        }
+        
+        completion(.success(results))
+      }
+    }
+  }
+  
+  func cancelHashing() {
+    hashTask?.cancel()
+    hashTask = nil
+  }
+  
+  private func hashAsset(_ asset: PHAsset, allowNetworkAccess: Bool) async -> HashResult? {
+    class RequestRef {
+      var id: PHAssetResourceDataRequestID?
+    }
+    let requestRef = RequestRef()
+    return await withTaskCancellationHandler(operation: {
+      if Task.isCancelled {
+        return nil
+      }
+      
+      guard let resource = asset.getResource() else {
+        return HashResult(assetId: asset.localIdentifier, error: "Cannot get asset resource", hash: nil)
+      }
+      
+      if Task.isCancelled {
+        return nil
+      }
+      
+      let options = PHAssetResourceRequestOptions()
+      options.isNetworkAccessAllowed = allowNetworkAccess
+      
+      return await withCheckedContinuation { continuation in
+        var hasher = Insecure.SHA1()
+        
+        requestRef.id = PHAssetResourceManager.default().requestData(
+          for: resource,
+          options: options,
+          dataReceivedHandler: { data in
+            hasher.update(data: data)
+          },
+          completionHandler: { error in
+            let result: HashResult? = switch (error) {
+            case let e as PHPhotosError where e.code == .userCancelled: nil
+            case let .some(e): HashResult(
+              assetId: asset.localIdentifier,
+              error: "Failed to hash asset: \(e.localizedDescription)",
+              hash: nil
+            )
+            case .none:
+              HashResult(
+                assetId: asset.localIdentifier,
+                error: nil,
+                hash: Data(hasher.finalize()).base64EncodedString()
+              )
+            }
+            continuation.resume(returning: result)
+          }
+        )
+      }
+    }, onCancel: {
+      guard let requestId = requestRef.id else { return }
+      PHAssetResourceManager.default().cancelDataRequest(requestId)
+    })
  }
 }
@@ -0,0 +1,77 @@
+import Photos
+
+extension PHAsset {
+  func toPlatformAsset() -> PlatformAsset {
+    return PlatformAsset(
+      id: localIdentifier,
+      name: title,
+      type: Int64(mediaType.rawValue),
+      createdAt: creationDate.map { Int64($0.timeIntervalSince1970) },
+      updatedAt: modificationDate.map { Int64($0.timeIntervalSince1970) },
+      width: Int64(pixelWidth),
+      height: Int64(pixelHeight),
+      durationInSeconds: Int64(duration),
+      orientation: 0,
+      isFavorite: isFavorite
+    )
+  }
+
+  var title: String {
+    return filename ?? originalFilename ?? "<unknown>"
+  }
+
+  var filename: String? {
+    return value(forKey: "filename") as? String
+  }
+
+  // This method is expected to be slow as it goes through the asset resources to fetch the originalFilename
+  var originalFilename: String? {
+    return getResource()?.originalFilename
+  }
+
+  func getResource() -> PHAssetResource? {
+    let resources = PHAssetResource.assetResources(for: self)
+
+    let filteredResources = resources.filter { $0.isMediaResource && isValidResourceType($0.type) }
+
+    guard !filteredResources.isEmpty else {
+      return nil
+    }
+
+    if filteredResources.count == 1 {
+      return filteredResources.first
+    }
+
+    if let currentResource = filteredResources.first(where: { $0.isCurrent }) {
+      return currentResource
+    }
+
+    if let fullSizeResource = filteredResources.first(where: { isFullSizeResourceType($0.type) }) {
+      return fullSizeResource
+    }
+
+    return nil
+  }
+
+  private func isValidResourceType(_ type: PHAssetResourceType) -> Bool {
+    switch mediaType {
+    case .image:
+      return [.photo, .alternatePhoto, .fullSizePhoto].contains(type)
+    case .video:
+      return [.video, .fullSizeVideo, .fullSizePairedVideo].contains(type)
+    default:
+      return false
+    }
+  }
+
+  private func isFullSizeResourceType(_ type: PHAssetResourceType) -> Bool {
+    switch mediaType {
+    case .image:
+      return type == .fullSizePhoto
+    case .video:
+      return type == .fullSizeVideo
+    default:
+      return false
+    }
+  }
+}
@@ -0,0 +1,16 @@
+
+import Photos
+
+extension PHAssetResource {
+  var isCurrent: Bool {
+    return value(forKey: "isCurrent") as? Bool ?? false
+  }
+  
+  var isMediaResource: Bool {
+    var isMedia = type != .adjustmentData
+    if #available(iOS 17, *) {
+      isMedia = isMedia && type != .photoProxy
+    }
+    return isMedia
+  }
+}