feat(mobile)!: batched full/initial sync (#4840)

* feat(mobile): batched full/initial sync

* use OptionalBetween

* skip/take as integer

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

mobile/lib/shared/services/asset.service.dart

@@ -62,20 +62,31 @@ class AssetService {
 
   /// Returns `null` if the server state did not change, else list of assets
   Future<List<Asset>?> _getRemoteAssets(User user) async {
+    const int chunkSize = 5000;
     try {
-      final List<AssetResponseDto>? assets =
-          await _apiService.assetApi.getAllAssets(
-        userId: user.id,
-      );
-      if (assets == null) {
-        return null;
-      } else if (assets.isNotEmpty && assets.first.ownerId != user.id) {
-        log.warning("Make sure that server and app versions match!"
-            " The server returned assets for user ${assets.first.ownerId}"
-            " while requesting assets of user ${user.id}");
-        return null;
+      final DateTime now = DateTime.now().toUtc();
+      final List<Asset> allAssets = [];
+      for (int i = 0;; i += chunkSize) {
+        final List<AssetResponseDto>? assets =
+            await _apiService.assetApi.getAllAssets(
+          userId: user.id,
+          // updatedBefore is important! without it we could
+          // a) get the same Asset multiple times in different versions (when
+          // the asset is modified while the chunks are loaded from the server)
+          // b) miss assets when new assets are inserted in between the calls
+          updatedBefore: now,
+          skip: i,
+          take: chunkSize,
+        );
+        if (assets == null) {
+          return null;
+        }
+        allAssets.addAll(assets.map(Asset.remote));
+        if (assets.length < chunkSize) {
+          break;
+        }
       }
-      return assets.map(Asset.remote).toList();
+      return allAssets;
     } catch (error, stack) {
       log.severe(
         'Error while getting remote assets: ${error.toString()}',

mobile/lib/shared/services/sync.service.dart

@@ -197,7 +197,7 @@ class SyncService {
     User user,
     FutureOr<List<Asset>?> Function(User user) loadAssets,
   ) async {
-    final DateTime now = DateTime.now();
+    final DateTime now = DateTime.now().toUtc();
     final List<Asset>? remote = await loadAssets(user);
     if (remote == null) {
       return false;
@@ -210,6 +210,10 @@ class SyncService {
     assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
 
     remote.sort(Asset.compareByChecksum);
+
+    // filter our duplicates that might be introduced by the chunked retrieval
+    remote.uniqueConsecutive(compare: Asset.compareByChecksum);
+
     final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
     if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
       await _updateUserAssetsETag(user, now);
@@ -759,6 +763,12 @@ class SyncService {
   final List<Asset> toAdd = [];
   final List<Asset> toUpdate = [];
   final List<Asset> toRemove = [];
+  if (assets.isEmpty || inDb.isEmpty) {
+    // fast path for trivial cases: halfes memory usage during initial sync
+    return assets.isEmpty
+        ? (toAdd, toUpdate, inDb) // remove all from DB
+        : (assets, toUpdate, toRemove); // add all assets
+  }
   diffSortedListsSync(
     inDb,
     assets,

mobile/openapi/doc/AssetApi.md

@@ -374,7 +374,7 @@ void (empty response body)
 [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
 
 # **getAllAssets**
-> List<AssetResponseDto> getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch)
+> List<AssetResponseDto> getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch)
 
 
 
@@ -399,15 +399,17 @@ import 'package:openapi/api.dart';
 //defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
 
 final api_instance = AssetApi();
+final skip = 56; // int | 
+final take = 56; // int | 
 final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String | 
 final isFavorite = true; // bool | 
 final isArchived = true; // bool | 
-final skip = 8.14; // num | 
 final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime | 
+final updatedBefore = 2013-10-20T19:20:30+01:00; // DateTime | 
 final ifNoneMatch = ifNoneMatch_example; // String | ETag of data already cached on the client
 
 try {
-    final result = api_instance.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch);
+    final result = api_instance.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch);
     print(result);
 } catch (e) {
     print('Exception when calling AssetApi->getAllAssets: $e\n');
@@ -418,11 +420,13 @@ try {
 
 Name | Type | Description  | Notes
 ------------- | ------------- | ------------- | -------------
+ **skip** | **int**|  | [optional] 
+ **take** | **int**|  | [optional] 
  **userId** | **String**|  | [optional] 
  **isFavorite** | **bool**|  | [optional] 
  **isArchived** | **bool**|  | [optional] 
- **skip** | **num**|  | [optional] 
  **updatedAfter** | **DateTime**|  | [optional] 
+ **updatedBefore** | **DateTime**|  | [optional] 
  **ifNoneMatch** | **String**| ETag of data already cached on the client | [optional] 
 
 ### Return type

mobile/openapi/lib/api/asset_api.dart

@@ -309,19 +309,23 @@ class AssetApi {
   ///
   /// Parameters:
   ///
+  /// * [int] skip:
+  ///
+  /// * [int] take:
+  ///
   /// * [String] userId:
   ///
   /// * [bool] isFavorite:
   ///
   /// * [bool] isArchived:
   ///
-  /// * [num] skip:
-  ///
   /// * [DateTime] updatedAfter:
   ///
+  /// * [DateTime] updatedBefore:
+  ///
   /// * [String] ifNoneMatch:
   ///   ETag of data already cached on the client
-  Future<Response> getAllAssetsWithHttpInfo({ String? userId, bool? isFavorite, bool? isArchived, num? skip, DateTime? updatedAfter, String? ifNoneMatch, }) async {
+  Future<Response> getAllAssetsWithHttpInfo({ int? skip, int? take, String? userId, bool? isFavorite, bool? isArchived, DateTime? updatedAfter, DateTime? updatedBefore, String? ifNoneMatch, }) async {
     // ignore: prefer_const_declarations
     final path = r'/asset';
 
@@ -332,6 +336,12 @@ class AssetApi {
     final headerParams = <String, String>{};
     final formParams = <String, String>{};
 
+    if (skip != null) {
+      queryParams.addAll(_queryParams('', 'skip', skip));
+    }
+    if (take != null) {
+      queryParams.addAll(_queryParams('', 'take', take));
+    }
     if (userId != null) {
       queryParams.addAll(_queryParams('', 'userId', userId));
     }
@@ -341,12 +351,12 @@ class AssetApi {
     if (isArchived != null) {
       queryParams.addAll(_queryParams('', 'isArchived', isArchived));
     }
-    if (skip != null) {
-      queryParams.addAll(_queryParams('', 'skip', skip));
-    }
     if (updatedAfter != null) {
       queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
     }
+    if (updatedBefore != null) {
+      queryParams.addAll(_queryParams('', 'updatedBefore', updatedBefore));
+    }
 
     if (ifNoneMatch != null) {
       headerParams[r'if-none-match'] = parameterToString(ifNoneMatch);
@@ -370,20 +380,24 @@ class AssetApi {
   ///
   /// Parameters:
   ///
+  /// * [int] skip:
+  ///
+  /// * [int] take:
+  ///
   /// * [String] userId:
   ///
   /// * [bool] isFavorite:
   ///
   /// * [bool] isArchived:
   ///
-  /// * [num] skip:
-  ///
   /// * [DateTime] updatedAfter:
   ///
+  /// * [DateTime] updatedBefore:
+  ///
   /// * [String] ifNoneMatch:
   ///   ETag of data already cached on the client
-  Future<List<AssetResponseDto>?> getAllAssets({ String? userId, bool? isFavorite, bool? isArchived, num? skip, DateTime? updatedAfter, String? ifNoneMatch, }) async {
-    final response = await getAllAssetsWithHttpInfo( userId: userId, isFavorite: isFavorite, isArchived: isArchived, skip: skip, updatedAfter: updatedAfter, ifNoneMatch: ifNoneMatch, );
+  Future<List<AssetResponseDto>?> getAllAssets({ int? skip, int? take, String? userId, bool? isFavorite, bool? isArchived, DateTime? updatedAfter, DateTime? updatedBefore, String? ifNoneMatch, }) async {
+    final response = await getAllAssetsWithHttpInfo( skip: skip, take: take, userId: userId, isFavorite: isFavorite, isArchived: isArchived, updatedAfter: updatedAfter, updatedBefore: updatedBefore, ifNoneMatch: ifNoneMatch, );
     if (response.statusCode >= HttpStatus.badRequest) {
       throw ApiException(response.statusCode, await _decodeBodyBytes(response));
     }

mobile/openapi/test/asset_api_test.dart

@@ -53,7 +53,7 @@ void main() {
 
     // Get all AssetEntity belong to the user
     //
-    //Future<List<AssetResponseDto>> getAllAssets({ String userId, bool isFavorite, bool isArchived, num skip, DateTime updatedAfter, String ifNoneMatch }) async
+    //Future<List<AssetResponseDto>> getAllAssets({ int skip, int take, String userId, bool isFavorite, bool isArchived, DateTime updatedAfter, DateTime updatedBefore, String ifNoneMatch }) async
     test('test getAllAssets', () async {
       // TODO
     });