kotlin impl, avoid message passing overhead

mobile/android/app/src/main/cpp/native_buffer.c

@@ -0,0 +1,13 @@
+#include <jni.h>
+
+JNIEXPORT jobject JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapPointer(
+    JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+    return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
+}
+
+JNIEXPORT jobject JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_wrapPointer(
+    JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+    return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
+}

mobile/android/app/src/main/kotlin/app/alextran/immich/MainActivity.kt

@@ -2,7 +2,8 @@ package app.alextran.immich
 
 import android.os.Build
 import android.os.ext.SdkExtensions
-import androidx.annotation.NonNull
+import app.alextran.immich.images.ThumbnailApi
+import app.alextran.immich.images.ThumbnailsImpl
 import app.alextran.immich.sync.NativeSyncApi
 import app.alextran.immich.sync.NativeSyncApiImpl26
 import app.alextran.immich.sync.NativeSyncApiImpl30
@@ -10,7 +11,7 @@ import io.flutter.embedding.android.FlutterFragmentActivity
 import io.flutter.embedding.engine.FlutterEngine
 
 class MainActivity : FlutterFragmentActivity() {
-  override fun configureFlutterEngine(@NonNull flutterEngine: FlutterEngine) {
+  override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
     super.configureFlutterEngine(flutterEngine)
     flutterEngine.plugins.add(BackgroundServicePlugin())
     flutterEngine.plugins.add(HttpSSLOptionsPlugin())
@@ -23,5 +24,6 @@ class MainActivity : FlutterFragmentActivity() {
         NativeSyncApiImpl30(this)
       }
     NativeSyncApi.setUp(flutterEngine.dartExecutor.binaryMessenger, nativeSyncApiImpl)
+    ThumbnailApi.setUp(flutterEngine.dartExecutor.binaryMessenger, ThumbnailsImpl(this))
   }
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt

@@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface ThumbnailApi {
-  fun getThumbnail(assetId: String, width: Long, height: Long, callback: (Result<ByteArray>) -> Unit)
+  fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result<Unit>) -> Unit)
 
   companion object {
     /** The codec used by ThumbnailApi. */
@@ -71,20 +71,20 @@ interface ThumbnailApi {
     fun setUp(binaryMessenger: BinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") {
       val separatedMessageChannelSuffix = if (messageChannelSuffix.isNotEmpty()) ".$messageChannelSuffix" else ""
       run {
-        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbnail$separatedMessageChannelSuffix", codec)
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.ThumbnailApi.setThumbnailToBuffer$separatedMessageChannelSuffix", codec)
         if (api != null) {
           channel.setMessageHandler { message, reply ->
             val args = message as List<Any?>
-            val assetIdArg = args[0] as String
-            val widthArg = args[1] as Long
-            val heightArg = args[2] as Long
-            api.getThumbnail(assetIdArg, widthArg, heightArg) { result: Result<ByteArray> ->
+            val pointerArg = args[0] as Long
+            val assetIdArg = args[1] as String
+            val widthArg = args[2] as Long
+            val heightArg = args[3] as Long
+            api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result<Unit> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(ThumbnailsPigeonUtils.wrapError(error))
               } else {
-                val data = result.getOrNull()
-                reply.reply(ThumbnailsPigeonUtils.wrapResult(data))
+                reply.reply(ThumbnailsPigeonUtils.wrapResult(null))
               }
             }
           }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt

@@ -0,0 +1,148 @@
+package app.alextran.immich.images
+
+import android.content.ContentResolver
+import android.content.ContentUris
+import android.content.Context
+import android.graphics.*
+import android.media.MediaMetadataRetriever
+import android.media.ThumbnailUtils
+import android.net.Uri
+import android.os.Build
+import android.provider.MediaStore
+import android.util.Size
+import java.nio.ByteBuffer
+import kotlin.math.max
+import java.util.concurrent.Executors
+
+class ThumbnailsImpl(context: Context) : ThumbnailApi {
+  private val ctx: Context = context.applicationContext
+  private val contentResolver: ContentResolver = ctx.contentResolver
+  private val threadPool =
+    Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors()))
+
+  companion object {
+    val PROJECTION = arrayOf(
+      MediaStore.MediaColumns.DATE_MODIFIED,
+      MediaStore.Files.FileColumns.MEDIA_TYPE,
+    )
+    const val SELECTION = "${MediaStore.MediaColumns._ID} = ?"
+    val URI: Uri = MediaStore.Files.getContentUri("external")
+
+    const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE
+    const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO
+
+    init {
+      System.loadLibrary("native_buffer")
+    }
+
+    @JvmStatic
+    external fun wrapPointer(address: Long, capacity: Int): ByteBuffer
+  }
+
+  override fun setThumbnailToBuffer(
+    pointer: Long,
+    assetId: String,
+    width: Long,
+    height: Long,
+    callback: (Result<Unit>) -> Unit
+  ) {
+    threadPool.execute {
+      try {
+        val targetWidth = width.toInt()
+        val targetHeight = height.toInt()
+
+        val cursor = contentResolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null)
+          ?: return@execute callback(Result.failure(RuntimeException("Asset not found")))
+
+        cursor.use { c ->
+          if (!c.moveToNext()) {
+            return@execute callback(Result.failure(RuntimeException("Asset not found")))
+          }
+
+          val mediaType = c.getInt(1)
+          val bitmap = when (mediaType) {
+            MEDIA_TYPE_IMAGE -> decodeImageThumbnail(assetId, targetWidth, targetHeight)
+            MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight)
+            else -> return@execute callback(Result.failure(RuntimeException("Unsupported media type")))
+          }
+
+          val croppedBitmap = ThumbnailUtils.extractThumbnail(
+            bitmap,
+            targetWidth,
+            targetHeight,
+            ThumbnailUtils.OPTIONS_RECYCLE_INPUT
+          )
+          val buffer = wrapPointer(pointer, (width * height * 4).toInt())
+          croppedBitmap.copyPixelsToBuffer(buffer)
+          croppedBitmap.recycle()
+          callback(Result.success(Unit))
+        }
+      } catch (e: Exception) {
+        callback(Result.failure(e))
+      }
+    }
+  }
+
+  private fun decodeImageThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
+    val uri =
+      ContentUris.withAppendedId(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
+    return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+      contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
+    } else {
+      decodeSampledBitmap(uri, targetWidth, targetHeight)
+    }
+  }
+
+  private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
+    val uri =
+      ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
+    return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+      contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
+    } else {
+      val retriever = MediaMetadataRetriever()
+      try {
+        retriever.setDataSource(ctx, uri)
+        retriever.getFrameAtTime(0L) ?: throw RuntimeException("Failed to extract video frame")
+      } finally {
+        retriever.release()
+      }
+    }
+  }
+
+  private fun decodeSampledBitmap(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap {
+    val options = BitmapFactory.Options().apply {
+      inJustDecodeBounds = true
+    }
+
+    contentResolver.openInputStream(uri)?.use { stream ->
+      BitmapFactory.decodeStream(stream, null, options)
+    }
+
+    options.apply {
+      inSampleSize = getSampleSize(this, targetWidth, targetHeight)
+      inJustDecodeBounds = false
+      inPreferredConfig = Bitmap.Config.ARGB_8888
+    }
+
+    return contentResolver.openInputStream(uri)?.use { stream ->
+      BitmapFactory.decodeStream(stream, null, options)
+    } ?: throw RuntimeException("Failed to decode bitmap")
+  }
+
+  private fun getSampleSize(options: BitmapFactory.Options, reqWidth: Int, reqHeight: Int): Int {
+    val height = options.outHeight
+    val width = options.outWidth
+    var inSampleSize = 1
+
+    if (height > reqHeight || width > reqWidth) {
+      val halfHeight = height / 2
+      val halfWidth = width / 2
+
+      while ((halfHeight / inSampleSize) >= reqHeight && (halfWidth / inSampleSize) >= reqWidth) {
+        inSampleSize *= 2
+      }
+    }
+
+    return inSampleSize
+  }
+}