cjones/immich
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(mobile): assets + exif stream sync placeholder (#17677)

Browse Source 
* feat(mobile): assets + exif stream sync placeholder

* feat(mobile): assets + exif stream sync placeholder

* refactor

* fix: test

* fix:test

* refactor(mobile): sync stream service (#17687)

* refactor: sync stream to use callbacks

* pr feedback

* pr feedback

* pr feedback

* fix: test

---------

Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

---------

Co-authored-by: shenlong <139912620+shenlong-tanwen@users.noreply.github.com>
Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
This commit is contained in:
Alex
2025-04-18 14:01:16 -05:00
committed by GitHub
parent bd2deda50c
commit 0e6ac87645
11 changed files with 666 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobile/lib/domain/interfaces/sync_api.interface.dart
+6 -2
View File
@@ -1,8 +1,12 @@
import 'package:http/http.dart' as http;
import 'package:immich_mobile/domain/models/sync_event.model.dart';
import 'package:openapi/api.dart';
abstract interface class ISyncApiRepository {
Future<void> ack(List<String> data);
Stream<List<SyncEvent>> getSyncEvents(List<SyncRequestType> type);
Future<void> streamChanges(
Function(List<SyncEvent>, Function() abort) onData, {
int batchSize,
http.Client? httpClient,
});
}
mobile/lib/domain/interfaces/sync_stream.interface.dart
+12 -4
View File
@@ -2,9 +2,17 @@ import 'package:immich_mobile/domain/interfaces/db.interface.dart';
import 'package:openapi/api.dart';
abstract interface class ISyncStreamRepository implements IDatabaseRepository {
Future<bool> updateUsersV1(Iterable<SyncUserV1> data);
Future<bool> deleteUsersV1(Iterable<SyncUserDeleteV1> data);
Future<void> updateUsersV1(Iterable<SyncUserV1> data);
Future<void> deleteUsersV1(Iterable<SyncUserDeleteV1> data);
Future<bool> updatePartnerV1(Iterable<SyncPartnerV1> data);
Future<bool> deletePartnerV1(Iterable<SyncPartnerDeleteV1> data);
Future<void> updatePartnerV1(Iterable<SyncPartnerV1> data);
Future<void> deletePartnerV1(Iterable<SyncPartnerDeleteV1> data);
Future<void> updateAssetsV1(Iterable<SyncAssetV1> data);
Future<void> deleteAssetsV1(Iterable<SyncAssetDeleteV1> data);
Future<void> updateAssetsExifV1(Iterable<SyncAssetExifV1> data);
Future<void> updatePartnerAssetsV1(Iterable<SyncAssetV1> data);
Future<void> deletePartnerAssetsV1(Iterable<SyncAssetDeleteV1> data);
Future<void> updatePartnerAssetsExifV1(Iterable<SyncAssetExifV1> data);
}
mobile/lib/domain/services/sync_stream.service.dart
+60 -168
View File
@@ -2,25 +2,11 @@
import 'dart:async';
import 'package:collection/collection.dart';
import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart';
import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart';
import 'package:immich_mobile/domain/models/sync_event.model.dart';
import 'package:logging/logging.dart';
import 'package:openapi/api.dart';
import 'package:worker_manager/worker_manager.dart';
const _kSyncTypeOrder = [
SyncEntityType.userDeleteV1,
SyncEntityType.userV1,
SyncEntityType.partnerDeleteV1,
SyncEntityType.partnerV1,
SyncEntityType.assetDeleteV1,
SyncEntityType.assetV1,
SyncEntityType.assetExifV1,
SyncEntityType.partnerAssetDeleteV1,
SyncEntityType.partnerAssetV1,
SyncEntityType.partnerAssetExifV1,
];
class SyncStreamService {
final Logger _logger = Logger('SyncStreamService');
@@ -37,164 +23,70 @@ class SyncStreamService {
_syncStreamRepository = syncStreamRepository,
_cancelChecker = cancelChecker;
Future<bool> _handleSyncData(
bool get isCancelled => _cancelChecker?.call() ?? false;
Future<void> sync() => _syncApiRepository.streamChanges(_handleEvents);
Future<void> _handleEvents(List<SyncEvent> events, Function() abort) async {
List<SyncEvent> items = [];
for (final event in events) {
if (isCancelled) {
_logger.warning("Sync stream cancelled");
abort();
return;
}
if (event.type != items.firstOrNull?.type) {
await _processBatch(items);
}
items.add(event);
}
await _processBatch(items);
}
Future<void> _processBatch(List<SyncEvent> batch) async {
if (batch.isEmpty) {
return;
}
final type = batch.first.type;
await _handleSyncData(type, batch.map((e) => e.data));
await _syncApiRepository.ack([batch.last.ack]);
batch.clear();
}
Future<void> _handleSyncData(
SyncEntityType type,
// ignore: avoid-dynamic
Iterable<dynamic> data,
) async {
if (data.isEmpty) {
_logger.warning("Received empty sync data for $type");
return false;
}
_logger.fine("Processing sync data for $type of length ${data.length}");
try {
if (type == SyncEntityType.partnerV1) {
return await _syncStreamRepository.updatePartnerV1(data.cast());
}
if (type == SyncEntityType.partnerDeleteV1) {
return await _syncStreamRepository.deletePartnerV1(data.cast());
}
if (type == SyncEntityType.userV1) {
return await _syncStreamRepository.updateUsersV1(data.cast());
}
if (type == SyncEntityType.userDeleteV1) {
return await _syncStreamRepository.deleteUsersV1(data.cast());
}
} catch (error, stack) {
_logger.severe("Error processing sync data for $type", error, stack);
return false;
// ignore: prefer-switch-expression
switch (type) {
case SyncEntityType.userV1:
return _syncStreamRepository.updateUsersV1(data.cast());
case SyncEntityType.userDeleteV1:
return _syncStreamRepository.deleteUsersV1(data.cast());
case SyncEntityType.partnerV1:
return _syncStreamRepository.updatePartnerV1(data.cast());
case SyncEntityType.partnerDeleteV1:
return _syncStreamRepository.deletePartnerV1(data.cast());
case SyncEntityType.assetV1:
return _syncStreamRepository.updateAssetsV1(data.cast());
case SyncEntityType.assetDeleteV1:
return _syncStreamRepository.deleteAssetsV1(data.cast());
case SyncEntityType.assetExifV1:
return _syncStreamRepository.updateAssetsExifV1(data.cast());
case SyncEntityType.partnerAssetV1:
return _syncStreamRepository.updatePartnerAssetsV1(data.cast());
case SyncEntityType.partnerAssetDeleteV1:
return _syncStreamRepository.deletePartnerAssetsV1(data.cast());
case SyncEntityType.partnerAssetExifV1:
return _syncStreamRepository.updatePartnerAssetsExifV1(data.cast());
default:
_logger.warning("Unknown sync data type: $type");
}
_logger.warning("Unknown sync data type: $type");
return false;
}
Future<void> _syncEvent(List<SyncRequestType> types) {
_logger.info("Syncing Events: $types");
final streamCompleter = Completer();
bool shouldComplete = false;
// the onDone callback might fire before the events are processed
// the following flag ensures that the onDone callback is not called
// before the events are processed and also that events are processed sequentially
Completer? mutex;
StreamSubscription? subscription;
try {
subscription = _syncApiRepository.getSyncEvents(types).listen(
(events) async {
if (events.isEmpty) {
_logger.warning("Received empty sync events");
return;
}
// If previous events are still being processed, wait for them to finish
if (mutex != null) {
await mutex!.future;
}
if (_cancelChecker?.call() ?? false) {
_logger.info("Sync cancelled, stopping stream");
subscription?.cancel();
if (!streamCompleter.isCompleted) {
streamCompleter.completeError(
CanceledError(),
StackTrace.current,
);
}
return;
}
// Take control of the mutex and process the events
mutex = Completer();
try {
final eventsMap = events.groupListsBy((event) => event.type);
final Map<SyncEntityType, String> acks = {};
for (final type in _kSyncTypeOrder) {
final data = eventsMap[type];
if (data == null) {
continue;
}
if (_cancelChecker?.call() ?? false) {
_logger.info("Sync cancelled, stopping stream");
mutex?.complete();
mutex = null;
if (!streamCompleter.isCompleted) {
streamCompleter.completeError(
CanceledError(),
StackTrace.current,
);
}
return;
}
if (data.isEmpty) {
_logger.warning("Received empty sync events for $type");
continue;
}
if (await _handleSyncData(type, data.map((e) => e.data))) {
// ignore: avoid-unsafe-collection-methods
acks[type] = data.last.ack;
} else {
_logger.warning("Failed to handle sync events for $type");
}
}
if (acks.isNotEmpty) {
await _syncApiRepository.ack(acks.values.toList());
}
_logger.info("$types events processed");
} catch (error, stack) {
_logger.warning("Error handling sync events", error, stack);
} finally {
mutex?.complete();
mutex = null;
}
if (shouldComplete) {
_logger.info("Sync done, completing stream");
if (!streamCompleter.isCompleted) streamCompleter.complete();
}
},
onError: (error, stack) {
_logger.warning("Error in sync stream for $types", error, stack);
// Do not proceed if the stream errors
if (!streamCompleter.isCompleted) {
// ignore: avoid-missing-completer-stack-trace
streamCompleter.completeError(error, stack);
}
},
onDone: () {
_logger.info("$types stream done");
if (mutex == null && !streamCompleter.isCompleted) {
streamCompleter.complete();
} else {
// Marks the stream as done but does not complete the completer
// until the events are processed
shouldComplete = true;
}
},
);
} catch (error, stack) {
_logger.severe("Error starting sync stream", error, stack);
if (!streamCompleter.isCompleted) {
streamCompleter.completeError(error, stack);
}
}
return streamCompleter.future.whenComplete(() {
_logger.info("Sync stream completed");
return subscription?.cancel();
});
}
Future<void> syncUsers() =>
_syncEvent([SyncRequestType.usersV1, SyncRequestType.partnersV1]);
}
mobile/lib/domain/utils/background_sync.dart
+15 -13
View File
@@ -7,31 +7,33 @@ import 'package:immich_mobile/utils/isolate.dart';
import 'package:worker_manager/worker_manager.dart';
class BackgroundSyncManager {
Cancelable<void>? _userSyncTask;
Cancelable<void>? _syncTask;
BackgroundSyncManager();
Future<void> cancel() {
final futures = <Future>[];
if (_userSyncTask != null) {
futures.add(_userSyncTask!.future);
if (_syncTask != null) {
futures.add(_syncTask!.future);
}
_userSyncTask?.cancel();
_userSyncTask = null;
_syncTask?.cancel();
_syncTask = null;
return Future.wait(futures);
}
Future<void> syncUsers() {
if (_userSyncTask != null) {
return _userSyncTask!.future;
Future<void> sync() {
if (_syncTask != null) {
return _syncTask!.future;
}
_userSyncTask = runInIsolateGentle(
computation: (ref) => ref.read(syncStreamServiceProvider).syncUsers(),
_syncTask = runInIsolateGentle(
computation: (ref) => ref.read(syncStreamServiceProvider).sync(),
);
_userSyncTask!.whenComplete(() {
_userSyncTask = null;
_syncTask!.whenComplete(() {
_syncTask = null;
});
return _userSyncTask!.future;
return _syncTask!.future;
}
}
mobile/lib/infrastructure/repositories/sync_api.repository.dart
+55 -20
View File
@@ -12,22 +12,22 @@ import 'package:openapi/api.dart';
class SyncApiRepository implements ISyncApiRepository {
final Logger _logger = Logger('SyncApiRepository');
final ApiService _api;
final int _batchSize;
SyncApiRepository(this._api, {int batchSize = kSyncEventBatchSize})
: _batchSize = batchSize;
@override
Stream<List<SyncEvent>> getSyncEvents(List<SyncRequestType> type) {
return _getSyncStream(SyncStreamDto(types: type));
}
SyncApiRepository(this._api);
@override
Future<void> ack(List<String> data) {
return _api.syncApi.sendSyncAck(SyncAckSetDto(acks: data));
}
Stream<List<SyncEvent>> _getSyncStream(SyncStreamDto dto) async* {
final client = http.Client();
@override
Future<void> streamChanges(
Function(List<SyncEvent>, Function() abort) onData, {
int batchSize = kSyncEventBatchSize,
http.Client? httpClient,
}) async {
// ignore: avoid-unused-assignment
final stopwatch = Stopwatch()..start();
final client = httpClient ?? http.Client();
final endpoint = "${_api.apiClient.basePath}/sync/stream";
final headers = {
@@ -35,20 +35,38 @@ class SyncApiRepository implements ISyncApiRepository {
'Accept': 'application/jsonlines+json',
};
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
await _api.applyToParams(queryParams, headerParams);
await _api.applyToParams([], headerParams);
headers.addAll(headerParams);
final request = http.Request('POST', Uri.parse(endpoint));
request.headers.addAll(headers);
request.body = jsonEncode(dto.toJson());
request.body = jsonEncode(
SyncStreamDto(
types: [
SyncRequestType.usersV1,
SyncRequestType.partnersV1,
SyncRequestType.assetsV1,
SyncRequestType.partnerAssetsV1,
SyncRequestType.assetExifsV1,
SyncRequestType.partnerAssetExifsV1,
],
).toJson(),
);
String previousChunk = '';
List<String> lines = [];
bool shouldAbort = false;
void abort() {
_logger.warning("Abort requested, stopping sync stream");
shouldAbort = true;
}
try {
final response = await client.send(request);
final response =
await client.send(request).timeout(const Duration(seconds: 20));
if (response.statusCode != 200) {
final errorBody = await response.stream.bytesToString();
@@ -59,27 +77,38 @@ class SyncApiRepository implements ISyncApiRepository {
}
await for (final chunk in response.stream.transform(utf8.decoder)) {
if (shouldAbort) {
break;
}
previousChunk += chunk;
final parts = previousChunk.toString().split('\n');
previousChunk = parts.removeLast();
lines.addAll(parts);
if (lines.length < _batchSize) {
if (lines.length < batchSize) {
continue;
}
yield _parseSyncResponse(lines);
await onData(_parseLines(lines), abort);
lines.clear();
}
} finally {
if (lines.isNotEmpty) {
yield _parseSyncResponse(lines);
if (lines.isNotEmpty && !shouldAbort) {
await onData(_parseLines(lines), abort);
}
} catch (error, stack) {
_logger.severe("error processing stream", error, stack);
return Future.error(error, stack);
} finally {
client.close();
}
stopwatch.stop();
_logger
.info("Remote Sync completed in ${stopwatch.elapsed.inMilliseconds}ms");
}
List<SyncEvent> _parseSyncResponse(List<String> lines) {
List<SyncEvent> _parseLines(List<String> lines) {
final List<SyncEvent> data = [];
for (final line in lines) {
@@ -110,4 +139,10 @@ const _kResponseMap = <SyncEntityType, Function(dynamic)>{
SyncEntityType.userDeleteV1: SyncUserDeleteV1.fromJson,
SyncEntityType.partnerV1: SyncPartnerV1.fromJson,
SyncEntityType.partnerDeleteV1: SyncPartnerDeleteV1.fromJson,
SyncEntityType.assetV1: SyncAssetV1.fromJson,
SyncEntityType.assetDeleteV1: SyncAssetDeleteV1.fromJson,
SyncEntityType.assetExifV1: SyncAssetExifV1.fromJson,
SyncEntityType.partnerAssetV1: SyncAssetV1.fromJson,
SyncEntityType.partnerAssetDeleteV1: SyncAssetDeleteV1.fromJson,
SyncEntityType.partnerAssetExifV1: SyncAssetExifV1.fromJson,
};
mobile/lib/infrastructure/repositories/sync_stream.repository.dart
+46 -16
View File
@@ -1,4 +1,5 @@
import 'package:drift/drift.dart';
import 'package:flutter/foundation.dart';
import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart';
import 'package:immich_mobile/extensions/string_extensions.dart';
import 'package:immich_mobile/infrastructure/entities/partner.entity.drift.dart';
@@ -15,7 +16,7 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository
DriftSyncStreamRepository(super.db) : _db = db;
@override
Future<bool> deleteUsersV1(Iterable<SyncUserDeleteV1> data) async {
Future<void> deleteUsersV1(Iterable<SyncUserDeleteV1> data) async {
try {
await _db.batch((batch) {
for (final user in data) {
@@ -25,15 +26,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository
);
}
});
return true;
} catch (e, s) {
_logger.severe('Error while processing SyncUserDeleteV1', e, s);
return false;
} catch (error, stack) {
_logger.severe('Error while processing SyncUserDeleteV1', error, stack);
rethrow;
}
}
@override
Future<bool> updateUsersV1(Iterable<SyncUserV1> data) async {
Future<void> updateUsersV1(Iterable<SyncUserV1> data) async {
try {
await _db.batch((batch) {
for (final user in data) {
@@ -49,15 +49,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository
);
}
});
return true;
} catch (e, s) {
_logger.severe('Error while processing SyncUserV1', e, s);
return false;
} catch (error, stack) {
_logger.severe('Error while processing SyncUserV1', error, stack);
rethrow;
}
}
@override
Future<bool> deletePartnerV1(Iterable<SyncPartnerDeleteV1> data) async {
Future<void> deletePartnerV1(Iterable<SyncPartnerDeleteV1> data) async {
try {
await _db.batch((batch) {
for (final partner in data) {
@@ -70,15 +69,14 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository
);
}
});
return true;
} catch (e, s) {
_logger.severe('Error while processing SyncPartnerDeleteV1', e, s);
return false;
rethrow;
}
}
@override
Future<bool> updatePartnerV1(Iterable<SyncPartnerV1> data) async {
Future<void> updatePartnerV1(Iterable<SyncPartnerV1> data) async {
try {
await _db.batch((batch) {
for (final partner in data) {
@@ -95,10 +93,42 @@ class DriftSyncStreamRepository extends DriftDatabaseRepository
);
}
});
return true;
} catch (e, s) {
_logger.severe('Error while processing SyncPartnerV1', e, s);
return false;
rethrow;
}
}
// Assets
@override
Future<void> updateAssetsV1(Iterable<SyncAssetV1> data) async {
debugPrint("updateAssetsV1 - ${data.length}");
}
@override
Future<void> deleteAssetsV1(Iterable<SyncAssetDeleteV1> data) async {
debugPrint("deleteAssetsV1 - ${data.length}");
}
// Partner Assets
@override
Future<void> updatePartnerAssetsV1(Iterable<SyncAssetV1> data) async {
debugPrint("updatePartnerAssetsV1 - ${data.length}");
}
@override
Future<void> deletePartnerAssetsV1(Iterable<SyncAssetDeleteV1> data) async {
debugPrint("deletePartnerAssetsV1 - ${data.length}");
}
// EXIF
@override
Future<void> updateAssetsExifV1(Iterable<SyncAssetExifV1> data) async {
debugPrint("updateAssetsExifV1 - ${data.length}");
}
@override
Future<void> updatePartnerAssetsExifV1(Iterable<SyncAssetExifV1> data) async {
debugPrint("updatePartnerAssetsExifV1 - ${data.length}");
}
}
mobile/lib/widgets/common/immich_app_bar.dart
+7
View File
@@ -1,11 +1,13 @@
import 'package:auto_route/auto_route.dart';
import 'package:easy_localization/easy_localization.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/material.dart';
import 'package:flutter_svg/svg.dart';
import 'package:hooks_riverpod/hooks_riverpod.dart';
import 'package:immich_mobile/extensions/build_context_extensions.dart';
import 'package:immich_mobile/models/backup/backup_state.model.dart';
import 'package:immich_mobile/models/server_info/server_info.model.dart';
import 'package:immich_mobile/providers/background_sync.provider.dart';
import 'package:immich_mobile/providers/backup/backup.provider.dart';
import 'package:immich_mobile/providers/server_info.provider.dart';
import 'package:immich_mobile/providers/user.provider.dart';
@@ -178,6 +180,11 @@ class ImmichAppBar extends ConsumerWidget implements PreferredSizeWidget {
child: action,
),
),
if (kDebugMode)
IconButton(
onPressed: () => ref.read(backgroundSyncProvider).sync(),
icon: const Icon(Icons.sync),
),
if (showUploadButton)
Padding(
padding: const EdgeInsets.only(right: 20),