Merge pull request #4031 from element-hq/feature/bma/fileListAudioPlayer

Render audio file in the files list and improve media viewer for audio/voice files
element-hq · Dec 13, 2024 · 2a3fd99 · 2a3fd99
2 parents 29eca97 + b236389
commit 2a3fd99
Show file tree

Hide file tree

Showing 58 changed files with 1,034 additions and 135 deletions.
diff --git a/...ssages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt b/...ssages/impl/src/main/kotlin/io/element/android/features/messages/impl/MessagesFlowNode.kt
@@ -47,6 +47,7 @@ import io.element.android.features.messages.impl.timeline.model.event.TimelineIt
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemLocationContent
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemStickerContent
 import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVideoContent
+import io.element.android.features.messages.impl.timeline.model.event.TimelineItemVoiceContent
 import io.element.android.features.poll.api.create.CreatePollEntryPoint
 import io.element.android.features.poll.api.create.CreatePollMode
 import io.element.android.libraries.architecture.BackstackWithOverlayBox
@@ -447,6 +448,7 @@ class MessagesFlowNode @AssistedInject constructor(
                     timestamp = event.sentTimeMillis,
                     mode = DateFormatterMode.Full,
                 ),
+                waveform = (content as? TimelineItemVoiceContent)?.waveform,
             ),
             mediaSource = mediaSource,
             thumbnailSource = thumbnailSource,

diff --git a/...kotlin/io/element/android/libraries/designsystem/components/media/WaveformPlaybackView.kt b/...kotlin/io/element/android/libraries/designsystem/components/media/WaveformPlaybackView.kt
@@ -189,7 +189,7 @@ internal fun WaveformPlaybackViewPreview() = ElementPreview {
             showCursor = false,
             playbackProgress = 0.5f,
             onSeek = {},
-            waveform = persistentListOf(0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 8f, 7f, 6f, 5f, 4f, 3f, 2f, 1f, 0f),
+            waveform = aWaveForm().toPersistentList(),
         )
         WaveformPlaybackView(
             modifier = Modifier.height(34.dp),
@@ -219,3 +219,45 @@ private fun ImmutableList<Float>.normalisedData(maxSamplesCount: Int): Immutable
 
     return result.toPersistentList()
 }
+
+fun aWaveForm(): List<Float> {
+    return listOf(
+        0.000f,
+        0.000f,
+        0.000f,
+        0.003f,
+        0.354f,
+        0.353f,
+        0.365f,
+        0.790f,
+        0.787f,
+        0.167f,
+        0.333f,
+        0.975f,
+        0.000f,
+        0.102f,
+        0.003f,
+        0.531f,
+        0.584f,
+        0.317f,
+        0.140f,
+        0.475f,
+        0.496f,
+        0.561f,
+        0.042f,
+        0.263f,
+        0.169f,
+        0.829f,
+        0.349f,
+        0.010f,
+        0.000f,
+        0.000f,
+        1.000f,
+        0.334f,
+        0.321f,
+        0.011f,
+        0.000f,
+        0.000f,
+        0.003f,
+    )
+}
diff --git a/...mediaviewer/api/src/main/kotlin/io/element/android/libraries/mediaviewer/api/MediaInfo.kt b/...mediaviewer/api/src/main/kotlin/io/element/android/libraries/mediaviewer/api/MediaInfo.kt
@@ -24,6 +24,7 @@ data class MediaInfo(
     val senderAvatar: String?,
     val dateSent: String?,
     val dateSentFull: String?,
+    val waveform: List<Float>?,
 ) : Parcelable
 
 fun anImageMediaInfo(
@@ -43,6 +44,7 @@ fun anImageMediaInfo(
     senderAvatar = null,
     dateSent = dateSent,
     dateSentFull = dateSentFull,
+    waveform = null,
 )
 
 fun aVideoMediaInfo(
@@ -61,6 +63,7 @@ fun aVideoMediaInfo(
     senderAvatar = null,
     dateSent = dateSent,
     dateSentFull = dateSentFull,
+    waveform = null,
 )
 
 fun aPdfMediaInfo(
@@ -80,6 +83,7 @@ fun aPdfMediaInfo(
     senderAvatar = null,
     dateSent = dateSent,
     dateSentFull = dateSentFull,
+    waveform = null,
 )
 
 fun anApkMediaInfo(
@@ -98,15 +102,19 @@ fun anApkMediaInfo(
     senderAvatar = null,
     dateSent = dateSent,
     dateSentFull = dateSentFull,
+    waveform = null,
 )
 
 fun anAudioMediaInfo(
+    filename: String = "an audio file.mp3",
+    caption: String? = null,
     senderName: String? = null,
     dateSent: String? = null,
     dateSentFull: String? = null,
+    waveForm: List<Float>? = null,
 ): MediaInfo = MediaInfo(
-    filename = "an audio file.mp3",
-    caption = null,
+    filename = filename,
+    caption = caption,
     mimeType = MimeTypes.Mp3,
     formattedFileSize = "7MB",
     fileExtension = "mp3",
@@ -115,4 +123,5 @@ fun anAudioMediaInfo(
     senderAvatar = null,
     dateSent = dateSent,
     dateSentFull = dateSentFull,
+    waveform = waveForm,
 )
diff --git a/...main/kotlin/io/element/android/libraries/mediaviewer/impl/DefaultMediaViewerEntryPoint.kt b/...main/kotlin/io/element/android/libraries/mediaviewer/impl/DefaultMediaViewerEntryPoint.kt
@@ -54,6 +54,7 @@ class DefaultMediaViewerEntryPoint @Inject constructor() : MediaViewerEntryPoint
                             senderAvatar = null,
                             dateSent = null,
                             dateSentFull = null,
+                            waveform = null,
                         ),
                         mediaSource = MediaSource(url = avatarUrl),
                         thumbnailSource = null,

diff --git a/...src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/EventItemFactory.kt b/...src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/EventItemFactory.kt
@@ -86,7 +86,7 @@ class EventItemFactory @Inject constructor(
                         Timber.w("Should not happen: ${content.type}")
                         null
                     }
-                    is AudioMessageType -> MediaItem.File(
+                    is AudioMessageType -> MediaItem.Audio(
                         id = currentTimelineItem.uniqueId,
                         eventId = currentTimelineItem.eventId,
                         mediaInfo = MediaInfo(
@@ -100,8 +100,11 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = null,
                         ),
                         mediaSource = type.source,
+                        duration = type.info?.duration?.inWholeMilliseconds?.toHumanReadableDuration(),
+                        waveform = null,
                     )
                     is FileMessageType -> MediaItem.File(
                         id = currentTimelineItem.uniqueId,
@@ -117,6 +120,7 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = null,
                         ),
                         mediaSource = type.source,
                     )
@@ -134,6 +138,7 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = null,
                         ),
                         mediaSource = type.source,
                         thumbnailSource = null,
@@ -152,6 +157,7 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = null,
                         ),
                         mediaSource = type.source,
                         thumbnailSource = null,
@@ -170,12 +176,13 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = null,
                         ),
                         mediaSource = type.source,
                         thumbnailSource = type.info?.thumbnailSource,
                         duration = type.info?.duration?.inWholeMilliseconds?.toHumanReadableDuration(),
                     )
-                    is VoiceMessageType -> MediaItem.File(
+                    is VoiceMessageType -> MediaItem.Audio(
                         id = currentTimelineItem.uniqueId,
                         eventId = currentTimelineItem.eventId,
                         mediaInfo = MediaInfo(
@@ -189,8 +196,11 @@ class EventItemFactory @Inject constructor(
                             senderAvatar = event.senderProfile.getAvatarUrl(),
                             dateSent = dateSent,
                             dateSentFull = dateSentFull,
+                            waveform = type.details?.waveform.orEmpty(),
                         ),
                         mediaSource = type.source,
+                        duration = type.info?.duration?.inWholeMilliseconds?.toHumanReadableDuration(),
+                        waveform = type.details?.waveform,
                     )
                 }
             }

diff --git a/...ain/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryPresenter.kt b/...ain/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryPresenter.kt
@@ -135,6 +135,7 @@ class MediaGalleryPresenter @AssistedInject constructor(
                         thumbnailSource = when (event.mediaItem) {
                             is MediaItem.Image -> event.mediaItem.thumbnailSource ?: event.mediaItem.mediaSource
                             is MediaItem.Video -> event.mediaItem.thumbnailSource ?: event.mediaItem.mediaSource
+                            is MediaItem.Audio -> null
                             is MediaItem.File -> null
                         },
                     )

diff --git a/...kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryStateProvider.kt b/...kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryStateProvider.kt
@@ -9,9 +9,11 @@ package io.element.android.libraries.mediaviewer.impl.gallery
 
 import androidx.compose.ui.tooling.preview.PreviewParameterProvider
 import io.element.android.libraries.architecture.AsyncData
+import io.element.android.libraries.designsystem.components.media.aWaveForm
 import io.element.android.libraries.matrix.api.core.UniqueId
 import io.element.android.libraries.mediaviewer.impl.details.MediaBottomSheetState
 import io.element.android.libraries.mediaviewer.impl.details.aMediaDetailsBottomSheetState
+import io.element.android.libraries.mediaviewer.impl.gallery.ui.aMediaItemAudio
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.aMediaItemDateSeparator
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.aMediaItemFile
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.aMediaItemImage
@@ -62,7 +64,11 @@ open class MediaGalleryStateProvider : PreviewParameterProvider<MediaGalleryStat
                                 formattedDate = "September 2004",
                             ),
                             aMediaItemFile(id = UniqueId("3")),
-                            aMediaItemFile(id = UniqueId("4")),
+                            aMediaItemAudio(id = UniqueId("4")),
+                            aMediaItemAudio(
+                                id = UniqueId("5"),
+                                waveform = aWaveForm(),
+                            ),
                             aMediaItemLoadingIndicator(),
                         ).toImmutableList()
                     )

diff --git a/...src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryView.kt b/...src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaGalleryView.kt
@@ -59,6 +59,7 @@ import io.element.android.libraries.mediaviewer.impl.R
 import io.element.android.libraries.mediaviewer.impl.details.MediaBottomSheetState
 import io.element.android.libraries.mediaviewer.impl.details.MediaDeleteConfirmationBottomSheet
 import io.element.android.libraries.mediaviewer.impl.details.MediaDetailsBottomSheet
+import io.element.android.libraries.mediaviewer.impl.gallery.ui.AudioItemView
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.DateItemView
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.FileItemView
 import io.element.android.libraries.mediaviewer.impl.gallery.ui.ImageItemView
@@ -257,6 +258,13 @@ private fun MediaGalleryFilesList(
                     onDownloadClick = { eventSink(MediaGalleryEvents.SaveOnDisk(item)) },
                     onInfoClick = { eventSink(MediaGalleryEvents.OpenInfo(item)) },
                 )
+                is MediaItem.Audio -> AudioItemView(
+                    item,
+                    onClick = { onItemClick(item) },
+                    onShareClick = { eventSink(MediaGalleryEvents.Share(item)) },
+                    onDownloadClick = { eventSink(MediaGalleryEvents.SaveOnDisk(item)) },
+                    onInfoClick = { eventSink(MediaGalleryEvents.OpenInfo(item)) },
+                )
                 is MediaItem.DateSeparator -> DateItemView(item)
                 is MediaItem.Image,
                 is MediaItem.Video -> {
@@ -312,6 +320,9 @@ private fun MediaGalleryImageGrid(
                 is MediaItem.DateSeparator -> {
                     DateItemView(item)
                 }
+                is MediaItem.Audio -> {
+                    // Should not happen
+                }
                 is MediaItem.File -> {
                     // Should not happen
                 }

diff --git a/...r/impl/src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaItem.kt b/...r/impl/src/main/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaItem.kt
@@ -13,6 +13,7 @@ import io.element.android.libraries.matrix.api.media.MediaSource
 import io.element.android.libraries.matrix.api.timeline.Timeline
 import io.element.android.libraries.matrix.ui.media.MediaRequestData
 import io.element.android.libraries.mediaviewer.api.MediaInfo
+import kotlinx.collections.immutable.ImmutableList
 
 sealed interface MediaItem {
     data class DateSeparator(
@@ -51,6 +52,15 @@ sealed interface MediaItem {
             get() = MediaRequestData(thumbnailSource ?: mediaSource, MediaRequestData.Kind.Thumbnail(100))
     }
 
+    data class Audio(
+        val id: UniqueId,
+        val eventId: EventId?,
+        val mediaInfo: MediaInfo,
+        val mediaSource: MediaSource,
+        val duration: String?,
+        val waveform: ImmutableList<Float>?,
+    ) : Event
+
     data class File(
         val id: UniqueId,
         val eventId: EventId?,
@@ -66,6 +76,7 @@ fun MediaItem.id(): UniqueId {
         is MediaItem.Image -> id
         is MediaItem.Video -> id
         is MediaItem.File -> id
+        is MediaItem.Audio -> id
     }
 }
 
@@ -74,6 +85,7 @@ fun MediaItem.Event.eventId(): EventId? {
         is MediaItem.Image -> eventId
         is MediaItem.Video -> eventId
         is MediaItem.File -> eventId
+        is MediaItem.Audio -> eventId
     }
 }
 
@@ -82,6 +94,7 @@ fun MediaItem.Event.mediaInfo(): MediaInfo {
         is MediaItem.Image -> mediaInfo
         is MediaItem.Video -> mediaInfo
         is MediaItem.File -> mediaInfo
+        is MediaItem.Audio -> mediaInfo
     }
 }
 
@@ -90,6 +103,7 @@ fun MediaItem.Event.mediaSource(): MediaSource {
         is MediaItem.Image -> mediaSource
         is MediaItem.Video -> mediaSource
         is MediaItem.File -> mediaSource
+        is MediaItem.Audio -> mediaSource
     }
 }
 
@@ -98,5 +112,6 @@ fun MediaItem.Event.thumbnailSource(): MediaSource? {
         is MediaItem.Image -> thumbnailSource
         is MediaItem.Video -> thumbnailSource
         is MediaItem.File -> null
+        is MediaItem.Audio -> null
     }
 }
diff --git a/...n/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaItemsPostProcessor.kt b/...n/kotlin/io/element/android/libraries/mediaviewer/impl/gallery/MediaItemsPostProcessor.kt
@@ -56,6 +56,7 @@ class MediaItemsPostProcessor @Inject constructor() {
                         is MediaItem.Video -> {
                             imageAndVideoItemsSubList.add(0, item)
                         }
+                        is MediaItem.Audio,
                         is MediaItem.File -> {
                             fileItemsSublist.add(0, item)
                         }