always use at-least 5 isolates

Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>

cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@immich/cli",
-  "version": "2.2.93",
+  "version": "2.2.94",
   "description": "Command Line Interface (CLI) for Immich",
   "type": "module",
   "exports": "./dist/index.js",

docs/static/archived-versions.json

@@ -1,4 +1,8 @@
 [
+  {
+    "label": "v1.144.1",
+    "url": "https://docs.v1.144.1.archive.immich.app"
+  },
   {
     "label": "v1.144.0",
     "url": "https://docs.v1.144.0.archive.immich.app"

e2e/package.json

@@ -1,6 +1,6 @@
 {
   "name": "immich-e2e",
-  "version": "1.144.0",
+  "version": "1.144.1",
   "description": "",
   "main": "index.js",
   "type": "module",

machine-learning/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "immich-ml"
-version = "minor"
+version = "1.144.1"
 description = ""
 authors = [{ name = "Hau Tran", email = "alex.tran1502@gmail.com" }]
 requires-python = ">=3.10,<4.0"

misc/release/pump-version.sh

@@ -80,7 +80,7 @@ if [ "$CURRENT_SERVER" != "$NEXT_SERVER" ]; then
 
   jq --arg version "$NEXT_SERVER" '.version = $version' e2e/package.json > e2e/package.json.tmp && mv e2e/package.json.tmp e2e/package.json
   pnpm install --frozen-lockfile --prefix e2e
-  uvx --from=toml-cli toml set --toml-path=machine-learning/pyproject.toml project.version "$SERVER_PUMP"
+  uvx --from=toml-cli toml set --toml-path=machine-learning/pyproject.toml project.version "$NEXT_SERVER"
 fi
 
 if [ "$CURRENT_MOBILE" != "$NEXT_MOBILE" ]; then

mobile/android/fastlane/Fastfile

@@ -35,8 +35,8 @@ platform :android do
       task: 'bundle', 
       build_type: 'Release',
       properties: {
-        "android.injected.version.code" => 3018,
-        "android.injected.version.name" => "1.144.0",
+        "android.injected.version.code" => 3019,
+        "android.injected.version.name" => "1.144.1",
       }
     )
     upload_to_play_store(skip_upload_apk: true, skip_upload_images: true, skip_upload_screenshots: true, aab: '../build/app/outputs/bundle/release/app-release.aab')

mobile/ios/Runner.xcodeproj/project.pbxproj

@@ -705,7 +705,7 @@
 				CODE_SIGN_ENTITLEMENTS = Runner/RunnerProfile.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
@@ -849,7 +849,7 @@
 				CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
@@ -879,7 +879,7 @@
 				CODE_SIGN_ENTITLEMENTS = Runner/Runner.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
@@ -913,7 +913,7 @@
 				CODE_SIGN_ENTITLEMENTS = WidgetExtension/WidgetExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu17;
@@ -956,7 +956,7 @@
 				CODE_SIGN_ENTITLEMENTS = WidgetExtension/WidgetExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu17;
@@ -996,7 +996,7 @@
 				CODE_SIGN_ENTITLEMENTS = WidgetExtension/WidgetExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
 				GCC_C_LANGUAGE_STANDARD = gnu17;
@@ -1035,7 +1035,7 @@
 				CODE_SIGN_ENTITLEMENTS = ShareExtension/ShareExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
@@ -1079,7 +1079,7 @@
 				CODE_SIGN_ENTITLEMENTS = ShareExtension/ShareExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;
@@ -1120,7 +1120,7 @@
 				CODE_SIGN_ENTITLEMENTS = ShareExtension/ShareExtension.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 226;
+				CURRENT_PROJECT_VERSION = 227;
 				CUSTOM_GROUP_ID = group.app.immich.share;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_USER_SCRIPT_SANDBOXING = YES;

mobile/ios/Runner/Info.plist

@@ -80,7 +80,7 @@
 	<key>CFBundlePackageType</key>
 	<string>APPL</string>
 	<key>CFBundleShortVersionString</key>
-	<string>1.143.1</string>
+	<string>1.144.0</string>
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleURLTypes</key>
@@ -107,7 +107,7 @@
 		</dict>
 	</array>
 	<key>CFBundleVersion</key>
-	<string>226</string>
+	<string>227</string>
 	<key>FLTEnableImpeller</key>
 	<true/>
 	<key>ITSAppUsesNonExemptEncryption</key>

mobile/ios/fastlane/Fastfile

@@ -22,7 +22,7 @@ platform :ios do
       path: "./Runner.xcodeproj",
     )
     increment_version_number(
-      version_number: "1.144.0"
+      version_number: "1.144.1"
     )
     increment_build_number(
       build_number: latest_testflight_build_number + 1,

mobile/lib/domain/services/background_worker.service.dart

@@ -32,9 +32,9 @@ import 'package:immich_mobile/services/upload.service.dart';
 import 'package:immich_mobile/utils/bootstrap.dart';
 import 'package:immich_mobile/utils/debug_print.dart';
 import 'package:immich_mobile/utils/http_ssl_options.dart';
+import 'package:immich_mobile/wm_executor.dart';
 import 'package:isar/isar.dart';
 import 'package:logging/logging.dart';
-import 'package:worker_manager/worker_manager.dart';
 
 class BackgroundWorkerFgService {
   final BackgroundWorkerFgHostApi _foregroundHostApi;
@@ -93,7 +93,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
       await Future.wait(
         [
           loadTranslations(),
-          workerManager.init(dynamicSpawning: true),
+          workerManagerPatch.init(dynamicSpawning: true),
           _ref?.read(authServiceProvider).setOpenApiServiceEndpoint(),
           // Initialize the file downloader
           FileDownloader().configure(
@@ -198,7 +198,7 @@ class BackgroundWorkerBgService extends BackgroundWorkerFlutterApi {
       _cancellationToken.cancel();
       _logger.info("Cleaning up background worker");
       final cleanupFutures = [
-        workerManager.dispose().catchError((_) async {
+        workerManagerPatch.dispose().catchError((_) async {
           // Discard any errors on the dispose call
           return;
         }),

mobile/lib/infrastructure/repositories/backup.repository.dart

@@ -81,7 +81,7 @@ class DriftBackupRepository extends DriftDatabaseRepository {
     );
   }
 
-  Future<List<LocalAsset>> getCandidates(String userId) async {
+  Future<List<LocalAsset>> getCandidates(String userId, {bool onlyHashed = true}) async {
     final selectedAlbumIds = _db.localAlbumEntity.selectOnly(distinct: true)
       ..addColumns([_db.localAlbumEntity.id])
       ..where(_db.localAlbumEntity.backupSelection.equalsValue(BackupSelection.selected));
@@ -89,7 +89,6 @@ class DriftBackupRepository extends DriftDatabaseRepository {
     final query = _db.localAssetEntity.select()
       ..where(
         (lae) =>
-            lae.checksum.isNotNull() &
             existsQuery(
               _db.localAlbumAssetEntity.selectOnly()
                 ..addColumns([_db.localAlbumAssetEntity.assetId])
@@ -109,6 +108,10 @@ class DriftBackupRepository extends DriftDatabaseRepository {
       )
       ..orderBy([(localAsset) => OrderingTerm.desc(localAsset.createdAt)]);
 
+    if (onlyHashed) {
+      query.where((lae) => lae.checksum.isNotNull());
+    }
+
     return query.map((localAsset) => localAsset.toDto()).get();
   }
 }

mobile/lib/main.dart

@@ -1,5 +1,6 @@
 import 'dart:async';
 import 'dart:io';
+import 'dart:math';
 
 import 'package:auto_route/auto_route.dart';
 import 'package:background_downloader/background_downloader.dart';
@@ -38,10 +39,10 @@ import 'package:immich_mobile/utils/debug_print.dart';
 import 'package:immich_mobile/utils/http_ssl_options.dart';
 import 'package:immich_mobile/utils/licenses.dart';
 import 'package:immich_mobile/utils/migration.dart';
+import 'package:immich_mobile/wm_executor.dart';
 import 'package:intl/date_symbol_data_local.dart';
 import 'package:logging/logging.dart';
 import 'package:timezone/data/latest.dart';
-import 'package:worker_manager/worker_manager.dart';
 
 void main() async {
   ImmichWidgetsBinding();
@@ -50,7 +51,7 @@ void main() async {
   await Bootstrap.initDomain(isar, drift, logDb);
   await initApp();
   // Warm-up isolate pool for worker manager
-  await workerManager.init(dynamicSpawning: true);
+  await workerManagerPatch.init(dynamicSpawning: true, isolatesCount: max(Platform.numberOfProcessors - 1, 5));
   await migrateDatabaseIfNeeded(isar, drift);
   HttpSSLOptions.apply();
 

mobile/lib/providers/backup/drift_backup.provider.dart

@@ -405,7 +405,7 @@ final driftBackupCandidateProvider = FutureProvider.autoDispose<List<LocalAsset>
     return [];
   }
 
-  return ref.read(backupRepositoryProvider).getCandidates(user.id);
+  return ref.read(backupRepositoryProvider).getCandidates(user.id, onlyHashed: false);
 });
 
 final driftCandidateBackupAlbumInfoProvider = FutureProvider.autoDispose.family<List<LocalAlbum>, String>((

mobile/lib/utils/isolate.dart

@@ -11,6 +11,7 @@ import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
 import 'package:immich_mobile/utils/bootstrap.dart';
 import 'package:immich_mobile/utils/debug_print.dart';
 import 'package:immich_mobile/utils/http_ssl_options.dart';
+import 'package:immich_mobile/wm_executor.dart';
 import 'package:logging/logging.dart';
 import 'package:worker_manager/worker_manager.dart';
 
@@ -31,7 +32,7 @@ Cancelable<T?> runInIsolateGentle<T>({
     throw const InvalidIsolateUsageException();
   }
 
-  return workerManager.executeGentle((cancelledChecker) async {
+  return workerManagerPatch.executeGentle((cancelledChecker) async {
     T? result;
     await runZonedGuarded(
       () async {

mobile/lib/wm_executor.dart

@@ -0,0 +1,251 @@
+// part of 'package:worker_manager/worker_manager.dart';
+// ignore_for_file: implementation_imports, avoid_print
+
+import 'dart:async';
+import 'dart:math';
+
+import 'package:collection/collection.dart';
+import 'package:flutter/foundation.dart';
+import 'package:worker_manager/src/number_of_processors/processors_io.dart';
+import 'package:worker_manager/src/worker/worker.dart';
+import 'package:worker_manager/worker_manager.dart';
+
+final workerManagerPatch = _Executor();
+
+// [-2^54; 2^53] is compatible with dart2js, see core.int doc
+const _minId = -9007199254740992;
+const _maxId = 9007199254740992;
+
+class Mixinable<T> {
+  late final itSelf = this as T;
+}
+
+mixin _ExecutorLogger on Mixinable<_Executor> {
+  var log = false;
+
+  @mustCallSuper
+  void init() {
+    logMessage("${itSelf._isolatesCount} workers have been spawned and initialized");
+  }
+
+  void logTaskAdded<R>(String uid) {
+    logMessage("added task with number $uid");
+  }
+
+  @mustCallSuper
+  void dispose() {
+    logMessage("worker_manager have been disposed");
+  }
+
+  @mustCallSuper
+  void _cancel(Task task) {
+    logMessage("Task ${task.id} have been canceled");
+  }
+
+  void logMessage(String message) {
+    if (log) print(message);
+  }
+}
+
+class _Executor extends Mixinable<_Executor> with _ExecutorLogger {
+  final _queue = PriorityQueue<Task>();
+  final _pool = <Worker>[];
+  var _nextTaskId = _minId;
+  var _dynamicSpawning = false;
+  var _isolatesCount = numberOfProcessors;
+
+  @override
+  Future<void> init({int? isolatesCount, bool? dynamicSpawning}) async {
+    if (_pool.isNotEmpty) {
+      print("worker_manager already warmed up, init is ignored. Dispose before init");
+      return;
+    }
+    if (isolatesCount != null) {
+      if (isolatesCount < 0) {
+        throw Exception("isolatesCount must be greater than 0");
+      }
+
+      _isolatesCount = isolatesCount;
+    }
+    _dynamicSpawning = dynamicSpawning ?? false;
+    await _ensureWorkersInitialized();
+    super.init();
+  }
+
+  @override
+  Future<void> dispose() async {
+    _queue.clear();
+    for (final worker in _pool) {
+      worker.kill();
+    }
+    _pool.clear();
+    super.dispose();
+  }
+
+  Cancelable<R> execute<R>(Execute<R> execution, {WorkPriority priority = WorkPriority.immediately}) {
+    return _createCancelable<R>(execution: execution, priority: priority);
+  }
+
+  Cancelable<R> executeNow<R>(ExecuteGentle<R> execution) {
+    final task = TaskGentle<R>(
+      id: "",
+      workPriority: WorkPriority.immediately,
+      execution: execution,
+      completer: Completer<R>(),
+    );
+
+    Future<void> run() async {
+      try {
+        final result = await execution(() => task.canceled);
+        task.complete(result, null, null);
+      } catch (error, st) {
+        task.complete(null, error, st);
+      }
+    }
+
+    run();
+    return Cancelable(completer: task.completer, onCancel: () => _cancel(task));
+  }
+
+  Cancelable<R> executeWithPort<R, T>(
+    ExecuteWithPort<R> execution, {
+    WorkPriority priority = WorkPriority.immediately,
+    required void Function(T value) onMessage,
+  }) {
+    return _createCancelable<R>(
+      execution: execution,
+      priority: priority,
+      onMessage: (message) => onMessage(message as T),
+    );
+  }
+
+  Cancelable<R> executeGentle<R>(ExecuteGentle<R> execution, {WorkPriority priority = WorkPriority.immediately}) {
+    return _createCancelable<R>(execution: execution, priority: priority);
+  }
+
+  Cancelable<R> executeGentleWithPort<R, T>(
+    ExecuteGentleWithPort<R> execution, {
+    WorkPriority priority = WorkPriority.immediately,
+    required void Function(T value) onMessage,
+  }) {
+    return _createCancelable<R>(
+      execution: execution,
+      priority: priority,
+      onMessage: (message) => onMessage(message as T),
+    );
+  }
+
+  void _createWorkers() {
+    for (var i = 0; i < _isolatesCount; i++) {
+      _pool.add(Worker());
+    }
+  }
+
+  Future<void> _initializeWorkers() async {
+    await Future.wait(_pool.map((e) => e.initialize()));
+  }
+
+  Cancelable<R> _createCancelable<R>({
+    required Function execution,
+    WorkPriority priority = WorkPriority.immediately,
+    void Function(Object value)? onMessage,
+  }) {
+    if (_nextTaskId + 1 == _maxId) {
+      _nextTaskId = _minId;
+    }
+    final id = _nextTaskId.toString();
+    _nextTaskId++;
+    late final Task<R> task;
+    final completer = Completer<R>();
+    if (execution is Execute<R>) {
+      task = TaskRegular<R>(id: id, workPriority: priority, execution: execution, completer: completer);
+    } else if (execution is ExecuteWithPort<R>) {
+      task = TaskWithPort<R>(
+        id: id,
+        workPriority: priority,
+        execution: execution,
+        completer: completer,
+        onMessage: onMessage!,
+      );
+    } else if (execution is ExecuteGentle<R>) {
+      task = TaskGentle<R>(id: id, workPriority: priority, execution: execution, completer: completer);
+    } else if (execution is ExecuteGentleWithPort<R>) {
+      task = TaskGentleWithPort<R>(
+        id: id,
+        workPriority: priority,
+        execution: execution,
+        completer: completer,
+        onMessage: onMessage!,
+      );
+    }
+    _queue.add(task);
+    _schedule();
+    logTaskAdded(task.id);
+    return Cancelable(completer: task.completer, onCancel: () => _cancel(task));
+  }
+
+  Future<void> _ensureWorkersInitialized() async {
+    if (_pool.isEmpty) {
+      _createWorkers();
+      if (!_dynamicSpawning) {
+        await _initializeWorkers();
+        final poolSize = _pool.length;
+        final queueSize = _queue.length;
+        for (int i = 0; i <= min(poolSize, queueSize); i++) {
+          _schedule();
+        }
+      }
+    }
+    if (_pool.every((worker) => worker.taskId != null)) {
+      return;
+    }
+    if (_dynamicSpawning) {
+      final freeWorker = _pool.firstWhereOrNull(
+        (worker) => worker.taskId == null && !worker.initialized && !worker.initializing,
+      );
+      await freeWorker?.initialize();
+      _schedule();
+    }
+  }
+
+  void _schedule() {
+    final availableWorker = _pool.firstWhereOrNull((worker) => worker.taskId == null && worker.initialized);
+    if (availableWorker == null) {
+      _ensureWorkersInitialized();
+      return;
+    }
+    if (_queue.isEmpty) return;
+    final task = _queue.removeFirst();
+
+    availableWorker
+        .work(task)
+        .then(
+          (value) {
+            //could be completed already by cancel and it is normal.
+            //Assuming that worker finished with error and cleaned gracefully
+            task.complete(value, null, null);
+          },
+          onError: (error, st) {
+            task.complete(null, error, st);
+          },
+        )
+        .whenComplete(() {
+          if (_dynamicSpawning && _queue.isEmpty) availableWorker.kill();
+          _schedule();
+        });
+  }
+
+  @override
+  void _cancel(Task task) {
+    task.cancel();
+    _queue.remove(task);
+    final targetWorker = _pool.firstWhereOrNull((worker) => worker.taskId == task.id);
+    if (task is Gentle) {
+      targetWorker?.cancelGentle();
+    } else {
+      targetWorker?.kill();
+      if (!_dynamicSpawning) targetWorker?.initialize();
+    }
+    super._cancel(task);
+  }
+}

mobile/openapi/README.md

@@ -3,7 +3,7 @@ Immich API
 
 This Dart package is automatically generated by the [OpenAPI Generator](https://openapi-generator.tech) project:
 
-- API version: 1.144.0
+- API version: 1.144.1
 - Generator version: 7.8.0
 - Build package: org.openapitools.codegen.languages.DartClientCodegen
 

mobile/pubspec.yaml

@@ -2,7 +2,7 @@ name: immich_mobile
 description: Immich - selfhosted backup media file on mobile phone
 
 publish_to: 'none'
-version: 1.144.0+3018
+version: 1.144.1+3019
 
 environment:
   sdk: '>=3.8.0 <4.0.0'

open-api/immich-openapi-specs.json

@@ -9858,7 +9858,7 @@
   "info": {
     "title": "Immich",
     "description": "Immich API",
-    "version": "1.144.0",
+    "version": "1.144.1",
     "contact": {}
   },
   "tags": [],

open-api/typescript-sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@immich/sdk",
-  "version": "1.144.0",
+  "version": "1.144.1",
   "description": "Auto-generated TypeScript SDK for the Immich API",
   "type": "module",
   "main": "./build/index.js",

open-api/typescript-sdk/src/fetch-client.ts

@@ -1,6 +1,6 @@
 /**
  * Immich
- * 1.144.0
+ * 1.144.1
  * DO NOT MODIFY - This file has been generated using oazapfts.
  * See https://www.npmjs.com/package/oazapfts
  */

server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "immich",
-  "version": "1.144.0",
+  "version": "1.144.1",
   "description": "",
   "author": "",
   "private": true,

web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "immich-web",
-  "version": "1.144.0",
+  "version": "1.144.1",
   "license": "GNU Affero General Public License version 3",
   "type": "module",
   "scripts": {