migrate hashes from remote asset table

This commit is contained in:
shenlong-tanwen
2025-09-05 04:15:16 +05:30
parent b11ea52704
commit 5236a72fb3
12 changed files with 78 additions and 62 deletions

View File

@@ -207,9 +207,10 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
}
Future<void> _syncAssets({Duration? hashTimeout}) async {
final futures = <Future<void>>[];
final localSyncFuture = _ref.read(backgroundSyncProvider).syncLocal().then((_) async {
await (
_ref.read(backgroundSyncProvider).syncLocal(),
_ref.read(backgroundSyncProvider).syncRemote(),
).wait.whenComplete(() async {
if (_isCleanedUp) {
return;
}
@@ -226,11 +227,6 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
return hashFuture;
});
futures.add(localSyncFuture);
futures.add(_ref.read(backgroundSyncProvider).syncRemote());
await Future.wait(futures);
}
}

View File

@@ -38,6 +38,10 @@ class HashService {
Future<void> hashAssets() async {
_log.info("Starting hashing of assets");
final Stopwatch stopwatch = Stopwatch()..start();
// Migrate hashes from cloud ID to local ID so we don't have to re-hash them
await _migrateHashes();
// Sorted by backupSelection followed by isCloud
final localAlbums = await _localAlbumRepository.getAll(
sortBy: {SortLocalAlbumsBy.backupSelection, SortLocalAlbumsBy.isIosSharedAlbum},
@@ -59,6 +63,15 @@ class HashService {
_log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
}
Future<void> _migrateHashes() async {
final hashMappings = await _localAssetRepository.getHashMappingFromCloudId();
if (hashMappings.isEmpty) {
return;
}
await _localAssetRepository.updateHashes(hashMappings);
}
/// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
/// with hash for those that were successfully hashed. Hashes are looked up in a table
/// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.
@@ -101,7 +114,7 @@ class HashService {
_log.fine("Hashing ${toHash.length} files");
final hashed = <LocalAsset>[];
final hashed = <LocalAssetHashMapping>[];
final hashes = await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList());
assert(
hashes.length == toHash.length,
@@ -117,7 +130,7 @@ class HashService {
final hash = hashes[i];
final asset = toHash[i].asset;
if (hash?.length == 20) {
hashed.add(asset.copyWith(checksum: base64.encode(hash!)));
hashed.add((assetId: asset.id, checksum: base64.encode(hash!)));
} else {
_log.warning(
"Failed to hash file for ${asset.id}: ${asset.name} created at ${asset.createdAt} from album: ${album.name}",

View File

@@ -1,6 +1,6 @@
import 'dart:async';
import 'package:immich_mobile/domain/utils/migrate_cloud_ids.dart';
import 'package:immich_mobile/domain/utils/migrate_cloud_ids.dart' as m;
import 'package:immich_mobile/domain/utils/sync_linked_album.dart';
import 'package:immich_mobile/providers/infrastructure/sync.provider.dart';
import 'package:immich_mobile/utils/isolate.dart';
@@ -195,7 +195,7 @@ class BackgroundSyncManager {
onCloudIdSyncStart?.call();
_cloudIdSyncTask = runInIsolateGentle(computation: migrateCloudIds);
_cloudIdSyncTask = runInIsolateGentle(computation: m.syncCloudIds);
return _cloudIdSyncTask!
.whenComplete(() {
onCloudIdSyncComplete?.call();

View File

@@ -12,7 +12,7 @@ import 'package:logging/logging.dart';
// ignore: import_rule_openapi
import 'package:openapi/api.dart';
Future<void> migrateCloudIds(ProviderContainer ref) async {
Future<void> syncCloudIds(ProviderContainer ref) async {
final db = ref.read(driftProvider);
// Populate cloud IDs for local assets that don't have one yet
await _populateCloudIds(db);

View File

@@ -7,8 +7,7 @@ import 'local_album_asset.entity.dart';
mergedAsset:
SELECT
rae.id as remote_id,
(SELECT lae.id FROM local_asset_entity lae
WHERE lae.checksum = rae.checksum OR lae.cloud_id = rae.cloud_id LIMIT 1) as local_id,
(SELECT lae.id FROM local_asset_entity lae WHERE lae.checksum = rae.checksum LIMIT 1) as local_id,
rae.name,
rae."type",
rae.created_at as created_at,
@@ -60,8 +59,7 @@ SELECT
FROM
local_asset_entity lae
WHERE NOT EXISTS (
SELECT 1 FROM remote_asset_entity rae WHERE
(rae.checksum = lae.checksum OR rae.cloud_id = lae.cloud_id) AND rae.owner_id IN :user_ids
SELECT 1 FROM remote_asset_entity rae WHERE rae.checksum = lae.checksum AND rae.owner_id IN :user_ids
)
AND EXISTS (
SELECT 1 FROM local_album_asset_entity laa
@@ -105,8 +103,7 @@ FROM
FROM
local_asset_entity lae
WHERE NOT EXISTS (
SELECT 1 FROM remote_asset_entity rae
WHERE (rae.checksum = lae.checksum OR rae.cloud_id = lae.cloud_id) AND rae.owner_id IN :user_ids
SELECT 1 FROM remote_asset_entity rae WHERE rae.checksum = lae.checksum AND rae.owner_id IN :user_ids
)
AND EXISTS (
SELECT 1 FROM local_album_asset_entity laa

View File

@@ -29,7 +29,7 @@ class MergedAssetDrift extends i1.ModularAccessor {
);
$arrayStartIndex += generatedlimit.amountOfVariables;
return customSelect(
'SELECT rae.id AS remote_id, (SELECT lae.id FROM local_asset_entity AS lae WHERE lae.checksum = rae.checksum OR lae.cloud_id = rae.cloud_id LIMIT 1) AS local_id, rae.name, rae.type, rae.created_at AS created_at, rae.updated_at, rae.width, rae.height, rae.duration_in_seconds, rae.is_favorite, rae.thumb_hash, rae.checksum, rae.owner_id, rae.live_photo_video_id, 0 AS orientation, rae.stack_id, rae.cloud_id FROM remote_asset_entity AS rae LEFT JOIN stack_entity AS se ON rae.stack_id = se.id WHERE rae.deleted_at IS NULL AND rae.visibility = 0 AND rae.owner_id IN ($expandeduserIds) AND(rae.stack_id IS NULL OR rae.id = se.primary_asset_id)UNION ALL SELECT NULL AS remote_id, lae.id AS local_id, lae.name, lae.type, lae.created_at AS created_at, lae.updated_at, lae.width, lae.height, lae.duration_in_seconds, lae.is_favorite, NULL AS thumb_hash, lae.checksum, NULL AS owner_id, NULL AS live_photo_video_id, lae.orientation, NULL AS stack_id, lae.cloud_id FROM local_asset_entity AS lae WHERE NOT EXISTS (SELECT 1 FROM remote_asset_entity AS rae WHERE(rae.checksum = lae.checksum OR rae.cloud_id = lae.cloud_id)AND rae.owner_id IN ($expandeduserIds)) AND EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 0) AND NOT EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 2) ORDER BY created_at DESC ${generatedlimit.sql}',
'SELECT rae.id AS remote_id, (SELECT lae.id FROM local_asset_entity AS lae WHERE lae.checksum = rae.checksum LIMIT 1) AS local_id, rae.name, rae.type, rae.created_at AS created_at, rae.updated_at, rae.width, rae.height, rae.duration_in_seconds, rae.is_favorite, rae.thumb_hash, rae.checksum, rae.owner_id, rae.live_photo_video_id, 0 AS orientation, rae.stack_id, rae.cloud_id FROM remote_asset_entity AS rae LEFT JOIN stack_entity AS se ON rae.stack_id = se.id WHERE rae.deleted_at IS NULL AND rae.visibility = 0 AND rae.owner_id IN ($expandeduserIds) AND(rae.stack_id IS NULL OR rae.id = se.primary_asset_id)UNION ALL SELECT NULL AS remote_id, lae.id AS local_id, lae.name, lae.type, lae.created_at AS created_at, lae.updated_at, lae.width, lae.height, lae.duration_in_seconds, lae.is_favorite, NULL AS thumb_hash, lae.checksum, NULL AS owner_id, NULL AS live_photo_video_id, lae.orientation, NULL AS stack_id, lae.cloud_id FROM local_asset_entity AS lae WHERE NOT EXISTS (SELECT 1 FROM remote_asset_entity AS rae WHERE rae.checksum = lae.checksum AND rae.owner_id IN ($expandeduserIds)) AND EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 0) AND NOT EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 2) ORDER BY created_at DESC ${generatedlimit.sql}',
variables: [
for (var $ in userIds) i0.Variable<String>($),
...generatedlimit.introducedVariables,
@@ -75,7 +75,7 @@ class MergedAssetDrift extends i1.ModularAccessor {
final expandeduserIds = $expandVar($arrayStartIndex, userIds.length);
$arrayStartIndex += userIds.length;
return customSelect(
'SELECT COUNT(*) AS asset_count, CASE WHEN ?1 = 0 THEN STRFTIME(\'%Y-%m-%d\', created_at, \'localtime\') WHEN ?1 = 1 THEN STRFTIME(\'%Y-%m\', created_at, \'localtime\') END AS bucket_date FROM (SELECT rae.created_at FROM remote_asset_entity AS rae LEFT JOIN stack_entity AS se ON rae.stack_id = se.id WHERE rae.deleted_at IS NULL AND rae.visibility = 0 AND rae.owner_id IN ($expandeduserIds) AND(rae.stack_id IS NULL OR rae.id = se.primary_asset_id)UNION ALL SELECT lae.created_at FROM local_asset_entity AS lae WHERE NOT EXISTS (SELECT 1 FROM remote_asset_entity AS rae WHERE(rae.checksum = lae.checksum OR rae.cloud_id = lae.cloud_id)AND rae.owner_id IN ($expandeduserIds)) AND EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 0) AND NOT EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 2)) GROUP BY bucket_date ORDER BY bucket_date DESC',
'SELECT COUNT(*) AS asset_count, CASE WHEN ?1 = 0 THEN STRFTIME(\'%Y-%m-%d\', created_at, \'localtime\') WHEN ?1 = 1 THEN STRFTIME(\'%Y-%m\', created_at, \'localtime\') END AS bucket_date FROM (SELECT rae.created_at FROM remote_asset_entity AS rae LEFT JOIN stack_entity AS se ON rae.stack_id = se.id WHERE rae.deleted_at IS NULL AND rae.visibility = 0 AND rae.owner_id IN ($expandeduserIds) AND(rae.stack_id IS NULL OR rae.id = se.primary_asset_id)UNION ALL SELECT lae.created_at FROM local_asset_entity AS lae WHERE NOT EXISTS (SELECT 1 FROM remote_asset_entity AS rae WHERE rae.checksum = lae.checksum AND rae.owner_id IN ($expandeduserIds)) AND EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 0) AND NOT EXISTS (SELECT 1 FROM local_album_asset_entity AS laa INNER JOIN local_album_entity AS la ON laa.album_id = la.id WHERE laa.asset_id = lae.id AND la.backup_selection = 2)) GROUP BY bucket_date ORDER BY bucket_date DESC',
variables: [
i0.Variable<int>(groupBy),
for (var $ in userIds) i0.Variable<String>($),

View File

@@ -64,8 +64,7 @@ class DriftBackupRepository extends DriftDatabaseRepository {
),
leftOuterJoin(
_db.remoteAssetEntity,
(_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId)) &
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) &
_db.remoteAssetEntity.ownerId.equals(userId),
useColumns: false,
),
@@ -95,8 +94,7 @@ class DriftBackupRepository extends DriftDatabaseRepository {
),
innerJoin(
_db.remoteAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum),
useColumns: false,
),
])
@@ -118,7 +116,7 @@ class DriftBackupRepository extends DriftDatabaseRepository {
final query = _db.localAssetEntity.select()
..where(
(lae) =>
(lae.checksum.isNotNull() | lae.cloudId.isNotNull()) &
lae.checksum.isNotNull() &
existsQuery(
_db.localAlbumAssetEntity.selectOnly()
..addColumns([_db.localAlbumAssetEntity.assetId])
@@ -131,9 +129,7 @@ class DriftBackupRepository extends DriftDatabaseRepository {
_db.remoteAssetEntity.selectOnly()
..addColumns([_db.remoteAssetEntity.checksum])
..where(
(_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId)) &
_db.remoteAssetEntity.ownerId.equals(userId),
_db.remoteAssetEntity.checksum.equalsExp(lae.checksum) & _db.remoteAssetEntity.ownerId.equals(userId),
),
) &
lae.id.isNotInQuery(_getExcludedSubquery()),

View File

@@ -5,6 +5,8 @@ import 'package:immich_mobile/infrastructure/entities/local_asset.entity.dart';
import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart';
import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
typedef LocalAssetHashMapping = ({String assetId, String checksum});
class DriftLocalAssetRepository extends DriftDatabaseRepository {
final Drift _db;
const DriftLocalAssetRepository(this._db) : super(_db);
@@ -13,8 +15,7 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {
final query = _db.localAssetEntity.select().addColumns([_db.remoteAssetEntity.id]).join([
leftOuterJoin(
_db.remoteAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum),
useColumns: false,
),
])..where(_db.localAssetEntity.id.equals(id));
@@ -29,17 +30,17 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {
Stream<LocalAsset?> watch(String id) => _assetSelectable(id).watchSingleOrNull();
Future<void> updateHashes(Iterable<LocalAsset> hashes) {
Future<void> updateHashes(Iterable<LocalAssetHashMapping> hashes) {
if (hashes.isEmpty) {
return Future.value();
}
return _db.batch((batch) async {
for (final asset in hashes) {
for (final mapping in hashes) {
batch.update(
_db.localAssetEntity,
LocalAssetEntityCompanion(checksum: Value(asset.checksum)),
where: (e) => e.id.equals(asset.id),
LocalAssetEntityCompanion(checksum: Value(mapping.checksum)),
where: (e) => e.id.equals(mapping.assetId),
);
}
});
@@ -70,4 +71,22 @@ class DriftLocalAssetRepository extends DriftDatabaseRepository {
Future<int> getHashedCount() {
return _db.managers.localAssetEntity.filter((e) => e.checksum.isNull().not()).count();
}
Future<List<LocalAssetHashMapping>> getHashMappingFromCloudId() async {
final query =
_db.localAssetEntity.selectOnly().join([
leftOuterJoin(
_db.remoteAssetEntity,
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
useColumns: false,
),
])
..addColumns([_db.localAssetEntity.id, _db.remoteAssetEntity.checksum])
..where(_db.remoteAssetEntity.cloudId.isNotNull() & _db.localAssetEntity.checksum.isNull());
return query
.map(
(row) => (assetId: row.read(_db.localAssetEntity.id)!, checksum: row.read(_db.remoteAssetEntity.checksum)!),
)
.get();
}
}

View File

@@ -34,8 +34,7 @@ class RemoteAssetRepository extends DriftDatabaseRepository {
_db.remoteAssetEntity.select().addColumns([_db.localAssetEntity.id]).join([
leftOuterJoin(
_db.localAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.remoteAssetEntity.checksum.equalsExp(_db.localAssetEntity.checksum),
useColumns: false,
),
])

View File

@@ -118,8 +118,7 @@ class DriftTimelineRepository extends DriftDatabaseRepository {
),
leftOuterJoin(
_db.remoteAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum),
useColumns: false,
),
])
@@ -145,8 +144,7 @@ class DriftTimelineRepository extends DriftDatabaseRepository {
),
leftOuterJoin(
_db.remoteAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum),
useColumns: false,
),
])
@@ -541,8 +539,7 @@ class DriftTimelineRepository extends DriftDatabaseRepository {
_db.remoteAssetEntity.select().join([
leftOuterJoin(
_db.localAssetEntity,
_db.localAssetEntity.checksum.equalsExp(_db.remoteAssetEntity.checksum) |
_db.localAssetEntity.cloudId.equalsExp(_db.remoteAssetEntity.cloudId),
_db.remoteAssetEntity.checksum.equalsExp(_db.localAssetEntity.checksum),
useColumns: false,
),
])

View File

@@ -31,7 +31,6 @@ import 'package:immich_mobile/providers/backup/backup.provider.dart';
import 'package:immich_mobile/services/app_settings.service.dart';
import 'package:immich_mobile/utils/diff.dart';
import 'package:isar/isar.dart';
import 'package:logging/logging.dart';
// ignore: import_rule_photo_manager
import 'package:photo_manager/photo_manager.dart';
@@ -268,21 +267,16 @@ class _DeviceAsset {
const _DeviceAsset({required this.assetId, this.hash, this.dateTime});
}
Future<List<void>> runNewSync(WidgetRef ref, {bool full = false}) {
Future<void> runNewSync(WidgetRef ref, {bool full = false}) {
ref.read(backupProvider.notifier).cancelBackup();
final backgroundManager = ref.read(backgroundSyncProvider);
final isAlbumLinkedSyncEnable = ref.read(appSettingsServiceProvider).getSetting(AppSettingsEnum.syncAlbums);
return Future.wait([
backgroundManager.syncLocal(full: full).then((_) {
Logger("runNewSync").fine("Hashing assets after syncLocal");
return backgroundManager.hashAssets();
}),
backgroundManager.syncRemote().then((_) {
if (isAlbumLinkedSyncEnable) {
return backgroundManager.syncLinkedAlbum();
}
}),
]);
return (backgroundManager.syncLocal(full: full), backgroundManager.syncRemote()).wait.whenComplete(() async {
await backgroundManager.hashAssets();
if (isAlbumLinkedSyncEnable) {
await backgroundManager.syncLinkedAlbum();
}
});
}

View File

@@ -3,9 +3,9 @@ import 'dart:io';
import 'dart:typed_data';
import 'package:flutter_test/flutter_test.dart';
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
import 'package:immich_mobile/domain/services/hash.service.dart';
import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
import 'package:mocktail/mocktail.dart';
import '../../fixtures/album.stub.dart';
@@ -39,6 +39,7 @@ void main() {
registerFallbackValue(LocalAlbumStub.recent);
registerFallbackValue(LocalAssetStub.image1);
when(() => mockAssetRepo.getHashMappingFromCloudId()).thenAnswer((_) async => []);
when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {});
when(() => mockStorageRepo.clearCache()).thenAnswer((_) async => {});
});
@@ -87,7 +88,8 @@ void main() {
await sut.hashAssets();
verify(() => mockNativeApi.hashPaths(['image-path'])).called(1);
final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
final captured =
verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAssetHashMapping>;
expect(captured.length, 1);
expect(captured[0].checksum, base64.encode(hash));
});
@@ -107,7 +109,8 @@ void main() {
await sut.hashAssets();
final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
final captured =
verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAssetHashMapping>;
expect(captured.length, 0);
});
@@ -128,7 +131,8 @@ void main() {
await sut.hashAssets();
final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
final captured =
verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAssetHashMapping>;
expect(captured.length, 0);
});
@@ -224,9 +228,10 @@ void main() {
await sut.hashAssets();
final captured = verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAsset>;
final captured =
verify(() => mockAssetRepo.updateHashes(captureAny())).captured.first as List<LocalAssetHashMapping>;
expect(captured.length, 1);
expect(captured.first.id, asset1.id);
expect(captured.first.assetId, asset1.id);
});
});
}