refactor(server): library syncing (#12220)

* refactor: library scanning

fix tests

remove offline files step

cleanup library service

improve tests

cleanup tests

add db migration

fix e2e

cleanup openapi

fix tests

fix tests

update docs

update docs

update mobile code

fix formatting

don't remove assets from library with invalid import path

use trash for offline files

add migration

simplify scan endpoint

cleanup library panel

fix library tests

e2e lint

fix e2e

trash e2e

fix lint

add asset trash tests

add more tests

ensure thumbs are generated

cleanup svelte

cleanup queue names

fix tests

fix lint

add warning due to trash

fix trash tests

fix lint

fix tests

Admin message for offline asset

fix comments

Update web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

add permission to library scan endpoint

revert asset interface sort

add trash reason to shared link stub

improve path view in offline

update docs

improve trash performance

fix comments

remove stray comment

* refactor: add back isOffline and remove trashReason from asset, change sync job flow

* chore(server): drop coverage to 80% for functions

* chore: rebase and generated files

---------

Co-authored-by: Zack Pollard <zackpollard@ymail.com>

mobile/lib/entities/asset.entity.g.dart

@@ -57,69 +57,64 @@ const AssetSchema = CollectionSchema(
       name: r'isFavorite',
       type: IsarType.bool,
     ),
-    r'isOffline': PropertySchema(
-      id: 8,
-      name: r'isOffline',
-      type: IsarType.bool,
-    ),
     r'isTrashed': PropertySchema(
-      id: 9,
+      id: 8,
       name: r'isTrashed',
       type: IsarType.bool,
     ),
     r'livePhotoVideoId': PropertySchema(
-      id: 10,
+      id: 9,
       name: r'livePhotoVideoId',
       type: IsarType.string,
     ),
     r'localId': PropertySchema(
-      id: 11,
+      id: 10,
       name: r'localId',
       type: IsarType.string,
     ),
     r'ownerId': PropertySchema(
-      id: 12,
+      id: 11,
       name: r'ownerId',
       type: IsarType.long,
     ),
     r'remoteId': PropertySchema(
-      id: 13,
+      id: 12,
       name: r'remoteId',
       type: IsarType.string,
     ),
     r'stackCount': PropertySchema(
-      id: 14,
+      id: 13,
       name: r'stackCount',
       type: IsarType.long,
     ),
     r'stackId': PropertySchema(
-      id: 15,
+      id: 14,
       name: r'stackId',
       type: IsarType.string,
     ),
     r'stackPrimaryAssetId': PropertySchema(
-      id: 16,
+      id: 15,
       name: r'stackPrimaryAssetId',
       type: IsarType.string,
     ),
     r'thumbhash': PropertySchema(
-      id: 17,
+      id: 16,
       name: r'thumbhash',
       type: IsarType.string,
     ),
     r'type': PropertySchema(
-      id: 18,
+      id: 17,
       name: r'type',
       type: IsarType.byte,
       enumMap: _AssettypeEnumValueMap,
     ),
     r'updatedAt': PropertySchema(
-      id: 19,
+      id: 18,
       name: r'updatedAt',
       type: IsarType.dateTime,
     ),
     r'width': PropertySchema(
-      id: 20,
+      id: 19,
       name: r'width',
       type: IsarType.int,
     )
@@ -244,19 +239,18 @@ void _assetSerialize(
   writer.writeInt(offsets[5], object.height);
   writer.writeBool(offsets[6], object.isArchived);
   writer.writeBool(offsets[7], object.isFavorite);
-  writer.writeBool(offsets[8], object.isOffline);
-  writer.writeBool(offsets[9], object.isTrashed);
-  writer.writeString(offsets[10], object.livePhotoVideoId);
-  writer.writeString(offsets[11], object.localId);
-  writer.writeLong(offsets[12], object.ownerId);
-  writer.writeString(offsets[13], object.remoteId);
-  writer.writeLong(offsets[14], object.stackCount);
-  writer.writeString(offsets[15], object.stackId);
-  writer.writeString(offsets[16], object.stackPrimaryAssetId);
-  writer.writeString(offsets[17], object.thumbhash);
-  writer.writeByte(offsets[18], object.type.index);
-  writer.writeDateTime(offsets[19], object.updatedAt);
-  writer.writeInt(offsets[20], object.width);
+  writer.writeBool(offsets[8], object.isTrashed);
+  writer.writeString(offsets[9], object.livePhotoVideoId);
+  writer.writeString(offsets[10], object.localId);
+  writer.writeLong(offsets[11], object.ownerId);
+  writer.writeString(offsets[12], object.remoteId);
+  writer.writeLong(offsets[13], object.stackCount);
+  writer.writeString(offsets[14], object.stackId);
+  writer.writeString(offsets[15], object.stackPrimaryAssetId);
+  writer.writeString(offsets[16], object.thumbhash);
+  writer.writeByte(offsets[17], object.type.index);
+  writer.writeDateTime(offsets[18], object.updatedAt);
+  writer.writeInt(offsets[19], object.width);
 }
 
 Asset _assetDeserialize(
@@ -275,20 +269,19 @@ Asset _assetDeserialize(
     id: id,
     isArchived: reader.readBoolOrNull(offsets[6]) ?? false,
     isFavorite: reader.readBoolOrNull(offsets[7]) ?? false,
-    isOffline: reader.readBoolOrNull(offsets[8]) ?? false,
-    isTrashed: reader.readBoolOrNull(offsets[9]) ?? false,
-    livePhotoVideoId: reader.readStringOrNull(offsets[10]),
-    localId: reader.readStringOrNull(offsets[11]),
-    ownerId: reader.readLong(offsets[12]),
-    remoteId: reader.readStringOrNull(offsets[13]),
-    stackCount: reader.readLongOrNull(offsets[14]) ?? 0,
-    stackId: reader.readStringOrNull(offsets[15]),
-    stackPrimaryAssetId: reader.readStringOrNull(offsets[16]),
-    thumbhash: reader.readStringOrNull(offsets[17]),
-    type: _AssettypeValueEnumMap[reader.readByteOrNull(offsets[18])] ??
+    isTrashed: reader.readBoolOrNull(offsets[8]) ?? false,
+    livePhotoVideoId: reader.readStringOrNull(offsets[9]),
+    localId: reader.readStringOrNull(offsets[10]),
+    ownerId: reader.readLong(offsets[11]),
+    remoteId: reader.readStringOrNull(offsets[12]),
+    stackCount: reader.readLongOrNull(offsets[13]) ?? 0,
+    stackId: reader.readStringOrNull(offsets[14]),
+    stackPrimaryAssetId: reader.readStringOrNull(offsets[15]),
+    thumbhash: reader.readStringOrNull(offsets[16]),
+    type: _AssettypeValueEnumMap[reader.readByteOrNull(offsets[17])] ??
         AssetType.other,
-    updatedAt: reader.readDateTime(offsets[19]),
-    width: reader.readIntOrNull(offsets[20]),
+    updatedAt: reader.readDateTime(offsets[18]),
+    width: reader.readIntOrNull(offsets[19]),
   );
   return object;
 }
@@ -319,29 +312,27 @@ P _assetDeserializeProp<P>(
     case 8:
       return (reader.readBoolOrNull(offset) ?? false) as P;
     case 9:
-      return (reader.readBoolOrNull(offset) ?? false) as P;
+      return (reader.readStringOrNull(offset)) as P;
     case 10:
       return (reader.readStringOrNull(offset)) as P;
     case 11:
-      return (reader.readStringOrNull(offset)) as P;
-    case 12:
       return (reader.readLong(offset)) as P;
-    case 13:
+    case 12:
       return (reader.readStringOrNull(offset)) as P;
-    case 14:
+    case 13:
       return (reader.readLongOrNull(offset) ?? 0) as P;
+    case 14:
+      return (reader.readStringOrNull(offset)) as P;
     case 15:
       return (reader.readStringOrNull(offset)) as P;
     case 16:
       return (reader.readStringOrNull(offset)) as P;
     case 17:
-      return (reader.readStringOrNull(offset)) as P;
-    case 18:
       return (_AssettypeValueEnumMap[reader.readByteOrNull(offset)] ??
           AssetType.other) as P;
-    case 19:
+    case 18:
       return (reader.readDateTime(offset)) as P;
-    case 20:
+    case 19:
       return (reader.readIntOrNull(offset)) as P;
     default:
       throw IsarError('Unknown property with id $propertyId');
@@ -1362,16 +1353,6 @@ extension AssetQueryFilter on QueryBuilder<Asset, Asset, QFilterCondition> {
     });
   }
 
-  QueryBuilder<Asset, Asset, QAfterFilterCondition> isOfflineEqualTo(
-      bool value) {
-    return QueryBuilder.apply(this, (query) {
-      return query.addFilterCondition(FilterCondition.equalTo(
-        property: r'isOffline',
-        value: value,
-      ));
-    });
-  }
-
   QueryBuilder<Asset, Asset, QAfterFilterCondition> isTrashedEqualTo(
       bool value) {
     return QueryBuilder.apply(this, (query) {
@@ -2647,18 +2628,6 @@ extension AssetQuerySortBy on QueryBuilder<Asset, Asset, QSortBy> {
     });
   }
 
-  QueryBuilder<Asset, Asset, QAfterSortBy> sortByIsOffline() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addSortBy(r'isOffline', Sort.asc);
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterSortBy> sortByIsOfflineDesc() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addSortBy(r'isOffline', Sort.desc);
-    });
-  }
-
   QueryBuilder<Asset, Asset, QAfterSortBy> sortByIsTrashed() {
     return QueryBuilder.apply(this, (query) {
       return query.addSortBy(r'isTrashed', Sort.asc);
@@ -2913,18 +2882,6 @@ extension AssetQuerySortThenBy on QueryBuilder<Asset, Asset, QSortThenBy> {
     });
   }
 
-  QueryBuilder<Asset, Asset, QAfterSortBy> thenByIsOffline() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addSortBy(r'isOffline', Sort.asc);
-    });
-  }
-
-  QueryBuilder<Asset, Asset, QAfterSortBy> thenByIsOfflineDesc() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addSortBy(r'isOffline', Sort.desc);
-    });
-  }
-
   QueryBuilder<Asset, Asset, QAfterSortBy> thenByIsTrashed() {
     return QueryBuilder.apply(this, (query) {
       return query.addSortBy(r'isTrashed', Sort.asc);
@@ -3121,12 +3078,6 @@ extension AssetQueryWhereDistinct on QueryBuilder<Asset, Asset, QDistinct> {
     });
   }
 
-  QueryBuilder<Asset, Asset, QDistinct> distinctByIsOffline() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addDistinctBy(r'isOffline');
-    });
-  }
-
   QueryBuilder<Asset, Asset, QDistinct> distinctByIsTrashed() {
     return QueryBuilder.apply(this, (query) {
       return query.addDistinctBy(r'isTrashed');
@@ -3263,12 +3214,6 @@ extension AssetQueryProperty on QueryBuilder<Asset, Asset, QQueryProperty> {
     });
   }
 
-  QueryBuilder<Asset, bool, QQueryOperations> isOfflineProperty() {
-    return QueryBuilder.apply(this, (query) {
-      return query.addPropertyName(r'isOffline');
-    });
-  }
-
   QueryBuilder<Asset, bool, QQueryOperations> isTrashedProperty() {
     return QueryBuilder.apply(this, (query) {
       return query.addPropertyName(r'isTrashed');

mobile/lib/extensions/collection_extensions.dart

@@ -72,13 +72,14 @@ extension AssetListExtension on Iterable<Asset> {
   }
 
   /// Filters out offline assets and returns those that are still accessible by the Immich server
+  /// TODO: isOffline is removed from Immich, so this method is not useful anymore
   Iterable<Asset> nonOfflineOnly({
     void Function()? errorCallback,
   }) {
-    final bool onlyLive = every((e) => !e.isOffline);
+    final bool onlyLive = every((e) => false);
     if (!onlyLive) {
       if (errorCallback != null) errorCallback();
-      return where((a) => !a.isOffline);
+      return where((a) => false);
     }
     return this;
   }

mobile/lib/widgets/asset_viewer/bottom_gallery_bar.dart

@@ -172,29 +172,12 @@ class BottomGalleryBar extends ConsumerWidget {
     }
 
     shareAsset() {
-      if (asset.isOffline) {
-        ImmichToast.show(
-          durationInSecond: 1,
-          context: context,
-          msg: 'asset_action_share_err_offline'.tr(),
-          gravity: ToastGravity.BOTTOM,
-        );
-        return;
-      }
       ref.read(imageViewerStateProvider.notifier).shareAsset(asset, context);
     }
 
     void handleEdit() async {
       final image = Image(image: ImmichImage.imageProvider(asset: asset));
-      if (asset.isOffline) {
-        ImmichToast.show(
-          durationInSecond: 1,
-          context: context,
-          msg: 'asset_action_edit_err_offline'.tr(),
-          gravity: ToastGravity.BOTTOM,
-        );
-        return;
-      }
+
       Navigator.of(context).push(
         MaterialPageRoute(
           builder: (context) => EditImagePage(
@@ -219,16 +202,6 @@ class BottomGalleryBar extends ConsumerWidget {
       if (asset.isLocal) {
         return;
       }
-      if (asset.isOffline) {
-        ImmichToast.show(
-          durationInSecond: 1,
-          context: context,
-          msg: 'asset_action_share_err_offline'.tr(),
-          gravity: ToastGravity.BOTTOM,
-        );
-        return;
-      }
-
       ref.read(imageViewerStateProvider.notifier).downloadAsset(
             asset,
             context,

mobile/lib/widgets/asset_viewer/top_control_app_bar.dart

@@ -183,8 +183,7 @@ class TopControlAppBar extends HookConsumerWidget {
         if (asset.isRemote && isOwner) buildFavoriteButton(a),
         if (asset.livePhotoVideoId != null) buildLivePhotoButton(),
         if (asset.isLocal && !asset.isRemote) buildUploadButton(),
-        if (asset.isRemote && !asset.isLocal && !asset.isOffline && isOwner)
-          buildDownloadButton(),
+        if (asset.isRemote && !asset.isLocal && isOwner) buildDownloadButton(),
         if (asset.isRemote && (isOwner || isPartner) && !asset.isTrashed)
           buildAddToAlbumButton(),
         if (asset.isTrashed) buildRestoreButton(),