From e8340baee0557fa99aabec22ba2e2e47a3aebbd5 Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Wed, 3 Nov 2021 15:42:35 +0000 Subject: [PATCH 1/6] supporting images in the room notifications 15:40:32 - downloads and exports any images whilst resolving the notification event --- .../app/core/extensions/BasicExtensions.kt | 4 +++ .../notifications/NotifiableEventResolver.kt | 32 ++++++++++++++++--- .../notifications/NotifiableMessageEvent.kt | 2 ++ .../NotificationBroadcastReceiver.kt | 1 + .../notifications/PushRuleTriggerListener.kt | 29 +++++++++++------ .../notifications/RoomGroupMessageCreator.kt | 11 ++++++- 6 files changed, 64 insertions(+), 15 deletions(-) diff --git a/vector/src/main/java/im/vector/app/core/extensions/BasicExtensions.kt b/vector/src/main/java/im/vector/app/core/extensions/BasicExtensions.kt index ee3d79d8461..dbe90dfdc1b 100644 --- a/vector/src/main/java/im/vector/app/core/extensions/BasicExtensions.kt +++ b/vector/src/main/java/im/vector/app/core/extensions/BasicExtensions.kt @@ -66,3 +66,7 @@ fun String?.insertBeforeLast(insert: String, delimiter: String = "."): String { replaceRange(idx, idx, insert) } } + +inline fun Any?.takeAs(): R? { + return takeIf { it is R } as R? +} diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt index d2db73af3d1..951b5933586 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt @@ -15,8 +15,10 @@ */ package im.vector.app.features.notifications +import android.net.Uri import im.vector.app.BuildConfig import im.vector.app.R +import im.vector.app.core.extensions.takeAs import im.vector.app.core.resources.StringProvider import im.vector.app.features.displayname.getBestName import im.vector.app.features.home.room.detail.timeline.format.DisplayableEventFormatter @@ -31,9 +33,11 @@ import org.matrix.android.sdk.api.session.events.model.isEdition import org.matrix.android.sdk.api.session.events.model.toModel import org.matrix.android.sdk.api.session.room.model.Membership import org.matrix.android.sdk.api.session.room.model.RoomMemberContent +import org.matrix.android.sdk.api.session.room.model.message.MessageWithAttachmentContent import org.matrix.android.sdk.api.session.room.sender.SenderInfo import org.matrix.android.sdk.api.session.room.timeline.TimelineEvent import org.matrix.android.sdk.api.session.room.timeline.getEditedEventId +import org.matrix.android.sdk.api.session.room.timeline.getLastMessageContent import org.matrix.android.sdk.api.util.toMatrixItem import org.matrix.android.sdk.internal.crypto.algorithms.olm.OlmDecryptionResult import timber.log.Timber @@ -49,11 +53,12 @@ import javax.inject.Inject class NotifiableEventResolver @Inject constructor( private val stringProvider: StringProvider, private val noticeEventFormatter: NoticeEventFormatter, - private val displayableEventFormatter: DisplayableEventFormatter) { + private val displayableEventFormatter: DisplayableEventFormatter +) { // private val eventDisplay = RiotEventDisplay(context) - fun resolveEvent(event: Event/*, roomState: RoomState?, bingRule: PushRule?*/, session: Session, isNoisy: Boolean): NotifiableEvent? { + suspend fun resolveEvent(event: Event/*, roomState: RoomState?, bingRule: PushRule?*/, session: Session, isNoisy: Boolean): NotifiableEvent? { val roomID = event.roomId ?: return null val eventId = event.eventId ?: return null if (event.getClearType() == EventType.STATE_ROOM_MEMBER) { @@ -89,7 +94,7 @@ class NotifiableEventResolver @Inject constructor( } } - fun resolveInMemoryEvent(session: Session, event: Event, canBeReplaced: Boolean): NotifiableEvent? { + suspend fun resolveInMemoryEvent(session: Session, event: Event, canBeReplaced: Boolean): NotifiableEvent? { if (event.getClearType() != EventType.MESSAGE) return null // Ignore message edition @@ -120,7 +125,7 @@ class NotifiableEventResolver @Inject constructor( } } - private fun resolveMessageEvent(event: TimelineEvent, session: Session, canBeReplaced: Boolean, isNoisy: Boolean): NotifiableEvent { + private suspend fun resolveMessageEvent(event: TimelineEvent, session: Session, canBeReplaced: Boolean, isNoisy: Boolean): NotifiableEvent { // The event only contains an eventId, and roomId (type is m.room.*) , we need to get the displayable content (names, avatar, text, etc...) val room = session.getRoom(event.root.roomId!! /*roomID cannot be null*/) @@ -140,6 +145,7 @@ class NotifiableEventResolver @Inject constructor( senderName = senderDisplayName, senderId = event.root.senderId, body = body.toString(), + imageUri = event.fetchImageIfPresent(session), roomId = event.root.roomId!!, roomName = roomName, matrixID = session.myUserId @@ -173,6 +179,7 @@ class NotifiableEventResolver @Inject constructor( senderName = senderDisplayName, senderId = event.root.senderId, body = body, + imageUri = event.fetchImageIfPresent(session), roomId = event.root.roomId!!, roomName = roomName, roomIsDirect = room.roomSummary()?.isDirect ?: false, @@ -192,6 +199,22 @@ class NotifiableEventResolver @Inject constructor( } } + private suspend fun TimelineEvent.fetchImageIfPresent(session: Session): Uri? { + return when { + root.isEncrypted() && root.mxDecryptionResult == null -> null + root.getClearType() == EventType.MESSAGE -> downloadAndExportImage(session) + else -> null + } + } + + private suspend fun TimelineEvent.downloadAndExportImage(session: Session): Uri? { + return getLastMessageContent()?.takeAs()?.let { imageMessage -> + val fileService = session.fileService() + fileService.downloadFile(imageMessage) + fileService.getTemporarySharableURI(imageMessage) + } + } + private fun resolveStateRoomEvent(event: Event, session: Session, canBeReplaced: Boolean, isNoisy: Boolean): NotifiableEvent? { val content = event.content?.toModel() ?: return null val roomId = event.roomId ?: return null @@ -224,3 +247,4 @@ class NotifiableEventResolver @Inject constructor( return null } } + diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotifiableMessageEvent.kt b/vector/src/main/java/im/vector/app/features/notifications/NotifiableMessageEvent.kt index 161c9f74a69..35718666b0f 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotifiableMessageEvent.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotifiableMessageEvent.kt @@ -15,6 +15,7 @@ */ package im.vector.app.features.notifications +import android.net.Uri import org.matrix.android.sdk.api.session.events.model.EventType data class NotifiableMessageEvent( @@ -26,6 +27,7 @@ data class NotifiableMessageEvent( val senderName: String?, val senderId: String?, val body: String?, + val imageUri: Uri?, val roomId: String, val roomName: String?, val roomIsDirect: Boolean = false, diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotificationBroadcastReceiver.kt b/vector/src/main/java/im/vector/app/features/notifications/NotificationBroadcastReceiver.kt index 06ef3f4aebe..b1905059a1f 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotificationBroadcastReceiver.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotificationBroadcastReceiver.kt @@ -138,6 +138,7 @@ class NotificationBroadcastReceiver : BroadcastReceiver() { ?: context?.getString(R.string.notification_sender_me), senderId = session.myUserId, body = message, + imageUri = null, roomId = room.roomId, roomName = room.roomSummary()?.displayName ?: room.roomId, roomIsDirect = room.roomSummary()?.isDirect == true, diff --git a/vector/src/main/java/im/vector/app/features/notifications/PushRuleTriggerListener.kt b/vector/src/main/java/im/vector/app/features/notifications/PushRuleTriggerListener.kt index f0633b24de5..ff817520db5 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/PushRuleTriggerListener.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/PushRuleTriggerListener.kt @@ -16,6 +16,11 @@ package im.vector.app.features.notifications +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.launch import org.matrix.android.sdk.api.pushrules.PushEvents import org.matrix.android.sdk.api.pushrules.PushRuleService import org.matrix.android.sdk.api.pushrules.getActions @@ -31,21 +36,24 @@ class PushRuleTriggerListener @Inject constructor( ) : PushRuleService.PushRuleListener { private var session: Session? = null + private val scope: CoroutineScope = CoroutineScope(SupervisorJob()) override fun onEvents(pushEvents: PushEvents) { - session?.let { session -> - val notifiableEvents = createNotifiableEvents(pushEvents, session) - notificationDrawerManager.updateEvents { queuedEvents -> - notifiableEvents.forEach { notifiableEvent -> - queuedEvents.onNotifiableEventReceived(notifiableEvent) + scope.launch { + session?.let { session -> + val notifiableEvents = createNotifiableEvents(pushEvents, session) + notificationDrawerManager.updateEvents { queuedEvents -> + notifiableEvents.forEach { notifiableEvent -> + queuedEvents.onNotifiableEventReceived(notifiableEvent) + } + queuedEvents.syncRoomEvents(roomsLeft = pushEvents.roomsLeft, roomsJoined = pushEvents.roomsJoined) + queuedEvents.markRedacted(pushEvents.redactedEventIds) } - queuedEvents.syncRoomEvents(roomsLeft = pushEvents.roomsLeft, roomsJoined = pushEvents.roomsJoined) - queuedEvents.markRedacted(pushEvents.redactedEventIds) - } - } ?: Timber.e("Called without active session") + } ?: Timber.e("Called without active session") + } } - private fun createNotifiableEvents(pushEvents: PushEvents, session: Session): List { + private suspend fun createNotifiableEvents(pushEvents: PushEvents, session: Session): List { return pushEvents.matchedEvents.mapNotNull { (event, pushRule) -> Timber.v("Push rule match for event ${event.eventId}") val action = pushRule.getActions().toNotificationAction() @@ -67,6 +75,7 @@ class PushRuleTriggerListener @Inject constructor( } fun stop() { + scope.coroutineContext.cancelChildren(CancellationException("PushRuleTriggerListener stopping")) session?.removePushRuleListener(this) session = null notificationDrawerManager.clearAllEvents() diff --git a/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt b/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt index bdd7d026f97..2178e200e57 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt @@ -19,6 +19,7 @@ package im.vector.app.features.notifications import android.content.Context import android.graphics.Bitmap import android.os.Build +import android.util.Log import androidx.core.app.NotificationCompat import androidx.core.app.Person import androidx.core.content.pm.ShortcutInfoCompat @@ -103,6 +104,7 @@ class RoomGroupMessageCreator @Inject constructor( private fun NotificationCompat.MessagingStyle.addMessagesFromEvents(events: List) { events.forEach { event -> + Log.e("!!!", "event: $event") val senderPerson = if (event.outGoingMessage) { null } else { @@ -114,7 +116,14 @@ class RoomGroupMessageCreator @Inject constructor( } when { event.isSmartReplyError() -> addMessage(stringProvider.getString(R.string.notification_inline_reply_failed), event.timestamp, senderPerson) - else -> addMessage(event.body, event.timestamp, senderPerson) + else -> { + val message = NotificationCompat.MessagingStyle.Message(event.body, event.timestamp, senderPerson).also { message -> + event.imageUri?.let { + message.setData("image/", it) + } + } + addMessage(message) + } } } } From 40a5c254f1d76082bd15b93279415120882521fe Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Wed, 3 Nov 2021 15:44:58 +0000 Subject: [PATCH 2/6] catching any potential errors whilst download/exporting the notification image, will allow us to continue to show the notifications --- .../notifications/NotifiableEventResolver.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt index 951b5933586..23f9fc307bb 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt @@ -208,11 +208,15 @@ class NotifiableEventResolver @Inject constructor( } private suspend fun TimelineEvent.downloadAndExportImage(session: Session): Uri? { - return getLastMessageContent()?.takeAs()?.let { imageMessage -> - val fileService = session.fileService() - fileService.downloadFile(imageMessage) - fileService.getTemporarySharableURI(imageMessage) - } + return kotlin.runCatching { + getLastMessageContent()?.takeAs()?.let { imageMessage -> + val fileService = session.fileService() + fileService.downloadFile(imageMessage) + fileService.getTemporarySharableURI(imageMessage) + } + }.onFailure { + Timber.e(it, "Failed to download and export image for notification") + }.getOrNull() } private fun resolveStateRoomEvent(event: Event, session: Session, canBeReplaced: Boolean, isNoisy: Boolean): NotifiableEvent? { From f97595465751a18a4bc5e3ef4ae213a0267b2368 Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Wed, 3 Nov 2021 15:51:52 +0000 Subject: [PATCH 3/6] removing debug log --- .../app/features/notifications/RoomGroupMessageCreator.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt b/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt index 2178e200e57..c1df6fe641a 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/RoomGroupMessageCreator.kt @@ -19,7 +19,6 @@ package im.vector.app.features.notifications import android.content.Context import android.graphics.Bitmap import android.os.Build -import android.util.Log import androidx.core.app.NotificationCompat import androidx.core.app.Person import androidx.core.content.pm.ShortcutInfoCompat @@ -104,7 +103,6 @@ class RoomGroupMessageCreator @Inject constructor( private fun NotificationCompat.MessagingStyle.addMessagesFromEvents(events: List) { events.forEach { event -> - Log.e("!!!", "event: $event") val senderPerson = if (event.outGoingMessage) { null } else { From 76d1eb01acc304cb05154c5f3e05a63b7632e100 Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Thu, 4 Nov 2021 09:55:05 +0000 Subject: [PATCH 4/6] add changelog entry --- changelog.d/4402.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4402.feature diff --git a/changelog.d/4402.feature b/changelog.d/4402.feature new file mode 100644 index 00000000000..29b9f9a3377 --- /dev/null +++ b/changelog.d/4402.feature @@ -0,0 +1 @@ +Adds support for images inside message notifications \ No newline at end of file From 1928aa325a5660df052a16660c1e73d28174c7e9 Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Thu, 4 Nov 2021 10:10:36 +0000 Subject: [PATCH 5/6] checking if the event is an image message to avoid attepting to render non image based attachments --- .../app/features/notifications/NotifiableEventResolver.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt index 23f9fc307bb..fa669244971 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt @@ -30,6 +30,7 @@ import org.matrix.android.sdk.api.session.crypto.MXCryptoError import org.matrix.android.sdk.api.session.events.model.Event import org.matrix.android.sdk.api.session.events.model.EventType import org.matrix.android.sdk.api.session.events.model.isEdition +import org.matrix.android.sdk.api.session.events.model.isImageMessage import org.matrix.android.sdk.api.session.events.model.toModel import org.matrix.android.sdk.api.session.room.model.Membership import org.matrix.android.sdk.api.session.room.model.RoomMemberContent @@ -202,7 +203,7 @@ class NotifiableEventResolver @Inject constructor( private suspend fun TimelineEvent.fetchImageIfPresent(session: Session): Uri? { return when { root.isEncrypted() && root.mxDecryptionResult == null -> null - root.getClearType() == EventType.MESSAGE -> downloadAndExportImage(session) + root.isImageMessage() -> downloadAndExportImage(session) else -> null } } From 1eb544efdbc38bb64f6dbed215f841fb6709719f Mon Sep 17 00:00:00 2001 From: Adam Brown Date: Thu, 4 Nov 2021 10:43:51 +0000 Subject: [PATCH 6/6] fixing auto merge issues --- .../app/features/notifications/NotifiableEventResolver.kt | 1 - .../java/im/vector/app/test/fixtures/NotifiableEventFixture.kt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt index fa669244971..87b31fa92a9 100644 --- a/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt +++ b/vector/src/main/java/im/vector/app/features/notifications/NotifiableEventResolver.kt @@ -252,4 +252,3 @@ class NotifiableEventResolver @Inject constructor( return null } } - diff --git a/vector/src/test/java/im/vector/app/test/fixtures/NotifiableEventFixture.kt b/vector/src/test/java/im/vector/app/test/fixtures/NotifiableEventFixture.kt index a3dab7e0699..53d38aa2284 100644 --- a/vector/src/test/java/im/vector/app/test/fixtures/NotifiableEventFixture.kt +++ b/vector/src/test/java/im/vector/app/test/fixtures/NotifiableEventFixture.kt @@ -76,5 +76,6 @@ fun aNotifiableMessageEvent( roomName = "room-name", roomIsDirect = false, canBeReplaced = false, - isRedacted = isRedacted + isRedacted = isRedacted, + imageUri = null )