light at the end of the tunnel

mobile/android/app/src/main/cpp/native_buffer.c

@@ -1,13 +1,52 @@
 #include <jni.h>
+#include <stdlib.h>
+
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative(
+    JNIEnv *env, jclass clazz, jint size)
+{
+    void *ptr = malloc(size);
+    return (jlong)ptr;
+}
+
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative(
+    JNIEnv *env, jclass clazz, jint size)
+{
+    void *ptr = malloc(size);
+    return (jlong)ptr;
+}
+
+JNIEXPORT void JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative(
+    JNIEnv *env, jclass clazz, jlong address)
+{
+    if (address != 0)
+    {
+        free((void *)address);
+    }
+}
+
+JNIEXPORT void JNICALL
+Java_app_alextran_immich_images_ThumbnailsImpl_freeNative(
+    JNIEnv *env, jclass clazz, jlong address)
+{
+    if (address != 0)
+    {
+        free((void *)address);
+    }
+}
 
 JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapPointer(
-    JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer(
+    JNIEnv *env, jclass clazz, jlong address, jint capacity)
+{
     return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
 }
 
 JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_wrapPointer(
-    JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer(
+    JNIEnv *env, jclass clazz, jlong address, jint capacity)
+{
     return (*env)->NewDirectByteBuffer(env, (void*)address, capacity);
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/AppGlideModule.kt

@@ -1,7 +1,21 @@
 package app.alextran.immich
 
+import android.content.Context
+import com.bumptech.glide.GlideBuilder
 import com.bumptech.glide.annotation.GlideModule
+import com.bumptech.glide.load.engine.bitmap_recycle.BitmapPoolAdapter
+import com.bumptech.glide.load.engine.cache.DiskCacheAdapter
+import com.bumptech.glide.load.engine.cache.MemoryCache
+import com.bumptech.glide.load.engine.cache.MemoryCacheAdapter
 import com.bumptech.glide.module.AppGlideModule
 
 @GlideModule
-class AppGlideModule : AppGlideModule()
+class AppGlideModule : AppGlideModule() {
+  override fun applyOptions(context: Context, builder: GlideBuilder) {
+    super.applyOptions(context, builder)
+    // disable caching as this is already done on the Flutter side
+    builder.setMemoryCache(MemoryCacheAdapter())
+    builder.setDiskCache(DiskCacheAdapter.Factory())
+    builder.setBitmapPool(BitmapPoolAdapter())
+  }
+}

mobile/android/app/src/main/kotlin/app/alextran/immich/images/Thumbnails.g.kt

@@ -59,7 +59,7 @@ private open class ThumbnailsPigeonCodec : StandardMessageCodec() {
 
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface ThumbnailApi {
-  fun setThumbnailToBuffer(pointer: Long, assetId: String, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit)
+  fun setThumbnailToBuffer(assetId: String, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit)
 
   companion object {
     /** The codec used by ThumbnailApi. */
@@ -75,11 +75,10 @@ interface ThumbnailApi {
         if (api != null) {
           channel.setMessageHandler { message, reply ->
             val args = message as List<Any?>
-            val pointerArg = args[0] as Long
-            val assetIdArg = args[1] as String
-            val widthArg = args[2] as Long
-            val heightArg = args[3] as Long
-            api.setThumbnailToBuffer(pointerArg, assetIdArg, widthArg, heightArg) { result: Result<Map<String, Long>> ->
+            val assetIdArg = args[0] as String
+            val widthArg = args[1] as Long
+            val heightArg = args[2] as Long
+            api.setThumbnailToBuffer(assetIdArg, widthArg, heightArg) { result: Result<Map<String, Long>> ->
               val error = result.exceptionOrNull()
               if (error != null) {
                 reply.reply(ThumbnailsPigeonUtils.wrapError(error))

mobile/android/app/src/main/kotlin/app/alextran/immich/images/ThumbnailsImpl.kt

@@ -4,154 +4,176 @@ import android.content.ContentResolver
 import android.content.ContentUris
 import android.content.Context
 import android.graphics.*
-import android.media.MediaMetadataRetriever
-import android.media.ThumbnailUtils
 import android.net.Uri
 import android.os.Build
 import android.provider.MediaStore
+import android.provider.MediaStore.Images
+import android.provider.MediaStore.Video
 import android.util.Size
 import java.nio.ByteBuffer
-import kotlin.math.max
+import kotlin.math.*
 import java.util.concurrent.Executors
+import com.bumptech.glide.Glide
+import com.bumptech.glide.Priority
+import com.bumptech.glide.load.DecodeFormat
+import kotlin.time.TimeSource
 
 class ThumbnailsImpl(context: Context) : ThumbnailApi {
-  private val ctx: Context = context.applicationContext
-  private val contentResolver: ContentResolver = ctx.contentResolver
-  private val threadPool =
-    Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors()))
+    private val ctx: Context = context.applicationContext
+    private val contentResolver: ContentResolver = ctx.contentResolver
+    private val threadPool =
+        Executors.newFixedThreadPool(max(4, Runtime.getRuntime().availableProcessors()))
 
-  companion object {
-    val PROJECTION = arrayOf(
-      MediaStore.MediaColumns.DATE_MODIFIED,
-      MediaStore.Files.FileColumns.MEDIA_TYPE,
-    )
-    const val SELECTION = "${MediaStore.MediaColumns._ID} = ?"
-    val URI: Uri = MediaStore.Files.getContentUri("external")
+    companion object {
+        val PROJECTION = arrayOf(
+            MediaStore.MediaColumns.DATE_MODIFIED,
+            MediaStore.Files.FileColumns.MEDIA_TYPE,
+        )
+        const val SELECTION = "${MediaStore.MediaColumns._ID} = ?"
+        val URI: Uri = MediaStore.Files.getContentUri("external")
 
-    const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE
-    const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO
+        const val MEDIA_TYPE_IMAGE = MediaStore.Files.FileColumns.MEDIA_TYPE_IMAGE
+        const val MEDIA_TYPE_VIDEO = MediaStore.Files.FileColumns.MEDIA_TYPE_VIDEO
 
-    init {
-      System.loadLibrary("native_buffer")
+        init {
+            System.loadLibrary("native_buffer")
+        }
+
+        @JvmStatic
+        external fun allocateNative(size: Int): Long
+
+        @JvmStatic
+        external fun freeNative(pointer: Long)
+
+        @JvmStatic
+        external fun wrapAsBuffer(address: Long, capacity: Int): ByteBuffer
     }
 
-    @JvmStatic
-    external fun wrapPointer(address: Long, capacity: Int): ByteBuffer
-  }
+    override fun setThumbnailToBuffer(
+        assetId: String, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit
+    ) {
+        threadPool.execute {
+            try {
+                setThumbnailToBufferInternal(assetId, width, height, callback)
+            } catch (e: Exception) {
+                callback(Result.failure(e))
+            }
+        }
+    }
 
-  override fun setThumbnailToBuffer(
-    pointer: Long,
-    assetId: String,
-    width: Long,
-    height: Long,
-    callback: (Result<Unit>) -> Unit
-  ) {
-    threadPool.execute {
-      try {
+    private fun setThumbnailToBufferInternal(
+        assetId: String, width: Long, height: Long, callback: (Result<Map<String, Long>>) -> Unit
+    ) {
         val targetWidth = width.toInt()
         val targetHeight = height.toInt()
 
         val cursor = contentResolver.query(URI, PROJECTION, SELECTION, arrayOf(assetId), null)
-          ?: return@execute callback(Result.failure(RuntimeException("Asset not found")))
+            ?: return callback(Result.failure(RuntimeException("Asset not found")))
 
         cursor.use { c ->
-          if (!c.moveToNext()) {
-            return@execute callback(Result.failure(RuntimeException("Asset not found")))
-          }
+            if (!c.moveToNext()) {
+                return callback(Result.failure(RuntimeException("Asset not found")))
+            }
 
-          val mediaType = c.getInt(1)
-          val bitmap = when (mediaType) {
-            MEDIA_TYPE_IMAGE -> decodeImageThumbnail(assetId, targetWidth, targetHeight)
-            MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight)
-            else -> return@execute callback(Result.failure(RuntimeException("Unsupported media type")))
-          }
+            val mediaType = c.getInt(1)
+            val bitmap = when (mediaType) {
+                MEDIA_TYPE_IMAGE -> decodeImage(assetId, targetWidth, targetHeight)
+                MEDIA_TYPE_VIDEO -> decodeVideoThumbnail(assetId, targetWidth, targetHeight)
+                else -> return callback(Result.failure(RuntimeException("Unsupported media type")))
+            }
 
-          val croppedBitmap = ThumbnailUtils.extractThumbnail(
-            bitmap,
-            targetWidth,
-            targetHeight,
-            ThumbnailUtils.OPTIONS_RECYCLE_INPUT
-          )
-          val buffer = wrapPointer(pointer, (width * height * 4).toInt())
-          croppedBitmap.copyPixelsToBuffer(buffer)
-          croppedBitmap.recycle()
-          callback(Result.success(Unit))
+            val actualWidth = bitmap.width
+            val actualHeight = bitmap.height
+
+            val size = actualWidth * actualHeight * 4
+            val pointer = allocateNative(size)
+            try {
+                val buffer = wrapAsBuffer(pointer, size)
+                bitmap.copyPixelsToBuffer(buffer)
+                bitmap.recycle()
+                callback(
+                    Result.success(
+                        mapOf(
+                            "pointer" to pointer,
+                            "width" to actualWidth.toLong(),
+                            "height" to actualHeight.toLong()
+                        )
+                    )
+                )
+            } catch (e: Exception) {
+                freeNative(pointer)
+                callback(Result.failure(e))
+            }
         }
-      } catch (e: Exception) {
-        callback(Result.failure(e))
-      }
-    }
-  }
-
-  private fun decodeImageThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
-    val uri =
-      ContentUris.withAppendedId(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
-    return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-      contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
-    } else {
-      decodeSampledBitmap(uri, targetWidth, targetHeight)
-    }
-  }
-
-  private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
-    val uri =
-      ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
-    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
-      return contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
     }
 
-    val retriever = MediaMetadataRetriever()
-    try {
-      retriever.setDataSource(ctx, uri)
-      return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
-        retriever.getScaledFrameAtTime(
-          0L,
-          MediaMetadataRetriever.OPTION_NEXT_SYNC,
-          targetWidth,
-          targetHeight
+    private fun decodeImage(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
+        val uri = ContentUris.withAppendedId(Images.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
+
+        if (targetHeight <= 768 && targetWidth <= 768) {
+            return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
+            } else {
+                Images.Thumbnails.getThumbnail(
+                    contentResolver,
+                    assetId.toLong(),
+                    Images.Thumbnails.MINI_KIND,
+                    BitmapFactory.Options().apply {
+                        inPreferredConfig = Bitmap.Config.ARGB_8888
+                    }
+                )
+            }
+        }
+
+        return decodeSource(uri, targetWidth, targetHeight)
+    }
+
+    private fun decodeVideoThumbnail(assetId: String, targetWidth: Int, targetHeight: Int): Bitmap {
+        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            val uri = ContentUris.withAppendedId(Video.Media.EXTERNAL_CONTENT_URI, assetId.toLong())
+            contentResolver.loadThumbnail(uri, Size(targetWidth, targetHeight), null)
+        } else {
+            Video.Thumbnails.getThumbnail(
+                contentResolver,
+                assetId.toLong(),
+                Video.Thumbnails.MINI_KIND,
+                BitmapFactory.Options().apply {
+                    inPreferredConfig = Bitmap.Config.ARGB_8888
+                }
+            )
+        }
+    }
+
+    private fun decodeSource(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap {
+        return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            val source = ImageDecoder.createSource(contentResolver, uri)
+
+            ImageDecoder.decodeBitmap(source) { decoder, info, _ ->
+                val sampleSize =
+                    getSampleSize(info.size.width, info.size.height, targetWidth, targetHeight)
+                decoder.setTargetSampleSize(sampleSize)
+                decoder.allocator = ImageDecoder.ALLOCATOR_SOFTWARE
+                decoder.setTargetColorSpace(ColorSpace.get(ColorSpace.Named.SRGB))
+            }
+        } else {
+            Glide.with(ctx)
+                .asBitmap()
+                .priority(Priority.IMMEDIATE)
+                .load(uri)
+                .disallowHardwareConfig()
+                .format(DecodeFormat.PREFER_ARGB_8888)
+                .submit(targetWidth, targetHeight).get()
+        }
+    }
+
+    private fun getSampleSize(fullWidth: Int, fullHeight: Int, reqWidth: Int, reqHeight: Int): Int {
+        return 1 shl max(
+            0, floor(
+                min(
+                    log2(fullWidth / (2.0 * reqWidth)),
+                    log2(fullHeight / (2.0 * reqHeight)),
+                )
+            ).toInt()
         )
-      } else {
-        retriever.getFrameAtTime(0L)
-      } ?: throw RuntimeException("Failed to extract video frame")
-    } finally {
-      retriever.release()
     }
-  }
-
-  private fun decodeSampledBitmap(uri: Uri, targetWidth: Int, targetHeight: Int): Bitmap {
-    val options = BitmapFactory.Options().apply {
-      inJustDecodeBounds = true
-    }
-
-    contentResolver.openInputStream(uri)?.use { stream ->
-      BitmapFactory.decodeStream(stream, null, options)
-    }
-
-    options.apply {
-      inSampleSize = getSampleSize(this, targetWidth, targetHeight)
-      inJustDecodeBounds = false
-      inPreferredConfig = Bitmap.Config.ARGB_8888
-    }
-
-    return contentResolver.openInputStream(uri)?.use { stream ->
-      BitmapFactory.decodeStream(stream, null, options)
-    } ?: throw RuntimeException("Failed to decode bitmap")
-  }
-
-  private fun getSampleSize(options: BitmapFactory.Options, reqWidth: Int, reqHeight: Int): Int {
-    val height = options.outHeight
-    val width = options.outWidth
-    var inSampleSize = 1
-
-    if (height > reqHeight || width > reqWidth) {
-      val halfHeight = height / 2
-      val halfWidth = width / 2
-
-      while ((halfHeight / inSampleSize) >= reqHeight && (halfWidth / inSampleSize) >= reqWidth) {
-        inSampleSize *= 2
-      }
-    }
-
-    return inSampleSize
-  }
 }