Experimental: Render inline images / custom emotes in the timeline

Change-Id: I50111b5a36045067d326fd057a536ff5098dfde3

FEATURES.md

@@ -37,7 +37,7 @@ Note that following list of changes compared to Element X is likely incomplete,
 
 - Allow sending freeform reactions
 - Don't waste horizontal space in message bubbles with forced line-breaks that do not make full use of the available width
-- Show alt text for inline images / custom emotes, instead of not showing them at all
+- Render inline images such as custom emotes in text messages
 
 - Disable Element's pinned message overlay on top of the conversation screen †
 - Access pinned messages via toolbar action when the pinned message overlay is disabled †

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/ScTimelineItemTextViewExtensions.kt

@@ -1,19 +1,47 @@
 package io.element.android.features.messages.impl.timeline.components.event
 
+import android.text.Spannable
 import android.text.SpannableString
+import android.text.Spanned
+import android.text.style.ImageSpan
 import androidx.compose.foundation.ExperimentalFoundationApi
 import androidx.compose.foundation.combinedClickable
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.MutableState
+import androidx.compose.runtime.SideEffect
 import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.remember
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalDensity
+import androidx.compose.ui.unit.dp
+import androidx.core.graphics.drawable.toBitmapOrNull
+import androidx.core.text.getSpans
 import chat.schildi.lib.compose.thenIf
+import chat.schildi.lib.preferences.ScPrefs
+import chat.schildi.lib.preferences.value
+import coil.compose.AsyncImagePainter
+import coil.compose.rememberAsyncImagePainter
+import coil.request.ImageRequest
+import coil.size.Scale
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemTextBasedContent
+import io.element.android.libraries.matrix.api.media.MediaSource
+import io.element.android.libraries.matrix.ui.media.MediaRequestData
 import io.element.android.libraries.matrix.ui.messages.LocalRoomMemberProfilesCache
 import io.element.android.libraries.textcomposer.mentions.LocalMentionSpanTheme
 import io.element.android.libraries.textcomposer.mentions.updateMentionStyles
+import io.element.android.wysiwyg.view.spans.InlineImageSpan
+import timber.log.Timber
+
+private const val CUSTOM_EMOTE_SIZE = 32
+private const val DEFAULT_IMAGE_WIDTH = CUSTOM_EMOTE_SIZE
+private const val DEFAULT_IMAGE_HEIGHT = CUSTOM_EMOTE_SIZE
+private const val MAX_IMAGE_WIDTH = 128
+private const val MAX_IMAGE_HEIGHT = 128
+private const val MIN_IMAGE_WIDTH = 8
+private const val MIN_IMAGE_HEIGHT = 8
 
 @Composable
 internal fun scGetTextWithResolvedMentions(
@@ -40,7 +68,7 @@ internal fun Modifier.scCollapseClick(
     )
 }
 
-@Composable // SC: Copy from upstream code in the non-extension file, then added formattedContent override
+@Composable // SC: Copy from upstream code in the non-extension file, then added formattedContent override + inline image resolution
 private fun getTextWithResolvedMentions(toFormat: CharSequence?, content: TimelineItemTextBasedContent): CharSequence {
     val userProfileCache = LocalRoomMemberProfilesCache.current
     val lastCacheUpdate by userProfileCache.lastCacheUpdate.collectAsState()
@@ -50,6 +78,52 @@ private fun getTextWithResolvedMentions(toFormat: CharSequence?, content: Timeli
         updateMentionSpans(formattedBody, userProfileCache)
         mentionSpanTheme.updateMentionStyles(formattedBody)
         formattedBody
-    }
+    }.resolveInlineImageSpans()
     return SpannableString(textWithMentions)
 }
+
+@Composable
+fun CharSequence.resolveInlineImageSpans(): CharSequence {
+    if (!ScPrefs.RENDER_INLINE_IMAGES.value()) {
+        return this
+    }
+    val context = LocalContext.current
+    val density = LocalDensity.current
+    val inlineImageSpans = (this as? Spanned)?.getSpans<InlineImageSpan>() ?: return this
+    val spansToReplace = inlineImageSpans.mapNotNull { inSpan ->
+        val src = inSpan.src.takeIf { it.startsWith("mxc://") } ?: return@mapNotNull null
+        val originWidth = if (inSpan.isEmoticon) CUSTOM_EMOTE_SIZE else inSpan.width ?: inSpan.height ?: DEFAULT_IMAGE_WIDTH
+        val originHeight = if (inSpan.isEmoticon) CUSTOM_EMOTE_SIZE else inSpan.height ?: inSpan.width ?: DEFAULT_IMAGE_HEIGHT
+        val width = density.run { originWidth.coerceIn(MIN_IMAGE_WIDTH, MAX_IMAGE_WIDTH).dp.roundToPx() }
+        val height = density.run { originHeight.coerceIn(MIN_IMAGE_HEIGHT, MAX_IMAGE_HEIGHT).dp.roundToPx() }
+        val painter = rememberAsyncImagePainter(
+            model = ImageRequest.Builder(context)
+                .data(MediaRequestData(MediaSource(url = src), MediaRequestData.Kind.Content))
+                .scale(Scale.FILL)
+                .size(width, height)
+                .build()
+        )
+        LaunchedEffect(painter.state) {
+            val state = painter.state
+            if (state is AsyncImagePainter.State.Error) {
+                Timber.tag("InlineImage").e(state.result.throwable, "Inline image failed to query \"$src\"")
+            } else {
+                Timber.tag("InlineImage").v("Inline image \"$src\" state is $state")
+            }
+        }
+        val bitmap = (painter.state as? AsyncImagePainter.State.Success)?.result?.drawable?.toBitmapOrNull() ?: return@mapNotNull null
+        Pair(inSpan, bitmap)
+    }
+    if (spansToReplace.isEmpty()) {
+        return this
+    }
+    return remember(this, spansToReplace) {
+        SpannableString(this).apply {
+            spansToReplace.forEach { (inSpan, bitmap) ->
+                val start = getSpanStart(inSpan).takeIf { it != -1 } ?: return@forEach
+                val end = getSpanEnd(inSpan).takeIf { it != -1 } ?: return@forEach
+                setSpan(ImageSpan(context, bitmap), start, end, Spannable.SPAN_INCLUSIVE_EXCLUSIVE)
+            }
+        }
+    }
+}

features/messages/impl/src/main/kotlin/io/element/android/features/messages/impl/timeline/components/event/TimelineItemTextView.kt

@@ -22,16 +22,13 @@ import androidx.compose.ui.Modifier
 import androidx.compose.ui.semantics.contentDescription
 import androidx.compose.ui.semantics.semantics
 import androidx.compose.ui.tooling.preview.PreviewParameter
-import chat.schildi.lib.preferences.ScPrefs.SC_TIMELINE_LAYOUT
-import chat.schildi.lib.preferences.value
 import chat.schildi.matrixsdk.containsOnlyEmojis
 import chat.schildi.theme.scBubbleFont
 import io.element.android.compound.theme.ElementTheme
 import io.element.android.features.messages.impl.timeline.components.layout.ContentAvoidingLayout
 import io.element.android.features.messages.impl.timeline.components.layout.ContentAvoidingLayoutData
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemTextBasedContent
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemTextBasedContentProvider
-import io.element.android.features.messages.impl.utils.containsOnlyEmojis
 import io.element.android.libraries.designsystem.preview.ElementPreview
 import io.element.android.libraries.designsystem.preview.PreviewsDayNight
 import io.element.android.libraries.matrix.api.core.UserId

schildi/lib/src/main/kotlin/chat/schildi/lib/preferences/ScPrefs.kt

@@ -80,6 +80,7 @@ object ScPrefs {
     val PREFER_FREEFORM_REACTIONS = ScBoolPref("PREFER_FREEFORM_REACTIONS", false, R.string.sc_pref_prefer_freeform_reactions_title, R.string.sc_pref_prefer_freeform_reactions_summary, authorsChoice = false)
     val PREFER_FULLSCREEN_REACTION_SHEET = ScBoolPref("PREFER_FULLSCREEN_REACTION_SHEET", false, R.string.sc_pref_prefer_fullscreen_reaction_sheet_title, R.string.sc_pref_prefer_fullscreen_reaction_sheet_summary, authorsChoice = false, upstreamChoice = false)
     val JUMP_TO_UNREAD = ScBoolPref("JUMP_TO_UNREAD", false, R.string.sc_pref_jump_to_unread_title, R.string.sc_pref_jump_to_unread_option_summary, authorsChoice = true, upstreamChoice = false)
+    val RENDER_INLINE_IMAGES = ScBoolPref("RENDER_INLINE_IMAGES", false, R.string.sc_pref_render_inline_images_title, R.string.sc_pref_render_inline_images_summary, authorsChoice = true, upstreamChoice = false)
 
     // Advanced theming options - Light theme
     val BUBBLE_BG_LIGHT_OUTGOING = ScColorPref("BUBBLE_BG_LIGHT_OUTGOING", R.string.sc_pref_bubble_color_outgoing_title)
@@ -180,6 +181,7 @@ object ScPrefs {
                     SPACE_MANAGEMENT,
                 )),
                 ScPrefCategory(R.string.sc_pref_category_timeline, null, listOf(
+                    RENDER_INLINE_IMAGES,
                     PL_DISPLAY_NAME,
                     JUMP_TO_UNREAD,
                     SYNC_READ_RECEIPT_AND_MARKER,
@@ -246,6 +248,7 @@ object ScPrefs {
         SC_TIMELINE_LAYOUT.copy(titleRes = R.string.sc_pref_sc_layout_title),
         PINNED_MESSAGE_OVERLAY.copy(titleRes = R.string.sc_pref_pinned_message_overlay_title_short),
         PINNED_MESSAGE_TOOLBAR_ACTION.copy(titleRes = R.string.sc_pref_pinned_message_toolbar_title_short),
+        RENDER_INLINE_IMAGES,
         OPEN_LINKS_IN_CUSTOM_TAB,
         ScPrefCategory(R.string.sc_pref_screen_experimental_title, null, listOf(
             PL_DISPLAY_NAME,

schildi/lib/src/main/res/values/strings.xml

@@ -143,4 +143,7 @@
     <string name="sc_space_list_loading">Spaces loading…</string>
     <string name="sc_space_list_empty">You don\'t have any spaces!</string>
     <string name="sc_space_list_permissions_missing_for_following">Spaces managed by others</string>
+
+    <string name="sc_pref_render_inline_images_title">Inline images</string>
+    <string name="sc_pref_render_inline_images_summary">Render images such as custom emotes in text messages</string>
 </resources>