diff --git a/changelog.d/1-api-changes/FS-1467 b/changelog.d/1-api-changes/FS-1467 new file mode 100644 index 0000000000..d7eff3eeb9 --- /dev/null +++ b/changelog.d/1-api-changes/FS-1467 @@ -0,0 +1 @@ +Updating conversation meta-data APIs to be fault tolerant of unavailable federation servers. \ No newline at end of file diff --git a/services/galley/src/Galley/API/Action.hs b/services/galley/src/Galley/API/Action.hs index dfe49a0595..95cae36a04 100644 --- a/services/galley/src/Galley/API/Action.hs +++ b/services/galley/src/Galley/API/Action.hs @@ -37,6 +37,7 @@ module Galley.API.Action notifyConversationAction, notifyRemoteConversationAction, ConversationUpdate, + FederationFailEarly (..), ) where @@ -632,10 +633,16 @@ updateLocalConversationUnchecked lconv qusr con action = do (extraTargets, action') <- performAction tag qusr lconv action notifyConversationAction - -- Removing members should be fault tolerant. ( case tag of - SConversationRemoveMembersTag -> False - _ -> True + -- Removing members should be fault tolerant. + SConversationRemoveMembersTag -> FaultTolerant + -- Conversation metadata updates should be fault tolerant. + SConversationRenameTag -> FaultTolerant + SConversationMessageTimerUpdateTag -> FaultTolerant + SConversationReceiptModeUpdateTag -> FaultTolerant + SConversationAccessDataTag -> FaultTolerant + SConversationMemberUpdateTag -> FaultTolerant + _ -> FailEarly ) (sing @tag) qusr @@ -689,6 +696,11 @@ addMembersToLocalConversation lcnv users role = do let action = ConversationJoin neUsers role pure (bmFromMembers lmems rmems, action) +data FederationFailEarly + = FailEarly + | FaultTolerant + deriving (Eq, Show) + notifyConversationAction :: forall tag r. ( Member FederatorAccess r, @@ -697,7 +709,7 @@ notifyConversationAction :: Member (Input UTCTime) r, Member (Logger (Log.Msg -> Log.Msg)) r ) => - Bool -> + FederationFailEarly -> Sing tag -> Qualified UserId -> Bool -> @@ -769,7 +781,9 @@ notifyConversationAction failEarly tag quid notifyOrigDomain con lconv targets a "An error occurred while communicating with federated server: " pure update - update <- if failEarly then errorIntolerant else errorTolerant + update <- case failEarly of + FailEarly -> errorIntolerant + FaultTolerant -> errorTolerant -- notify local participants and bots pushConversationEvent con e (qualifyAs lcnv (bmLocals targets)) (bmBots targets) @@ -856,7 +870,7 @@ kickMember qusr lconv targets victim = void . runError @NoChanges $ do lconv () notifyConversationAction - False + FaultTolerant (sing @'ConversationRemoveMembersTag) qusr True diff --git a/services/galley/src/Galley/API/Federation.hs b/services/galley/src/Galley/API/Federation.hs index 11f24556ef..2824835dd5 100644 --- a/services/galley/src/Galley/API/Federation.hs +++ b/services/galley/src/Galley/API/Federation.hs @@ -372,7 +372,7 @@ leaveConversation requestingDomain lc = do let botsAndMembers = BotsAndMembers mempty (Set.fromList remotes) mempty _ <- notifyConversationAction - False + FaultTolerant SConversationLeaveTag (tUntagged leaver) False @@ -500,7 +500,7 @@ onUserDeleted origDomain udcn = do removeUser (qualifyAs lc conv) (tUntagged deletedUser) void $ notifyConversationAction - False + FaultTolerant (sing @'ConversationLeaveTag) untaggedDeletedUser False diff --git a/services/galley/src/Galley/API/Update.hs b/services/galley/src/Galley/API/Update.hs index 2460533ccb..c09a74bba7 100644 --- a/services/galley/src/Galley/API/Update.hs +++ b/services/galley/src/Galley/API/Update.hs @@ -766,7 +766,7 @@ joinConversation lusr zcon conv access = do addMembersToLocalConversation lcnv (UserList users []) roleNameWireMember lcuEvent <$> notifyConversationAction - False + FaultTolerant (sing @'ConversationJoinTag) (tUntagged lusr) False diff --git a/services/galley/test/integration/API.hs b/services/galley/test/integration/API.hs index 5689797051..4d2d6242ac 100644 --- a/services/galley/test/integration/API.hs +++ b/services/galley/test/integration/API.hs @@ -203,6 +203,7 @@ tests s = test s "rename conversation" putConvRenameOk, test s "rename qualified conversation" putQualifiedConvRenameOk, test s "rename qualified conversation with remote members" putQualifiedConvRenameWithRemotesOk, + test s "rename qualified conversation with unavailable remote" putQualifiedConvRenameWithRemotesUnavailable, test s "rename qualified conversation failure" putQualifiedConvRenameFailure, test s "other member update role" putOtherMemberOk, test s "qualified other member update role" putQualifiedOtherMemberOk, @@ -216,6 +217,7 @@ tests s = test s "remote conversation member update (everything)" putRemoteConvMemberAllOk, test s "conversation receipt mode update" putReceiptModeOk, test s "conversation receipt mode update with remote members" putReceiptModeWithRemotesOk, + test s "conversation receipt mode update with unavailable remote members" putReceiptModeWithRemotesUnavailable, test s "remote conversation receipt mode update" putRemoteReceiptModeOk, test s "leave connect conversation" leaveConnectConversation, test s "post conversations/:cnv/otr/message: message delivery and missing clients" postCryptoMessageVerifyMsgSentAndRejectIfMissingClient, @@ -238,6 +240,7 @@ tests s = test s "convert invite to code-access conversation" postConvertCodeConv, test s "convert code to team-access conversation" postConvertTeamConv, test s "local and remote guests are removed when access changes" testAccessUpdateGuestRemoved, + test s "local and remote guests are removed when access changes remotes unavailable" testAccessUpdateGuestRemovedRemotesUnavailable, test s "team member can't join via guest link if access role removed" testTeamMemberCantJoinViaGuestLinkIfAccessRoleRemoved, test s "cannot join private conversation" postJoinConvFail, test s "revoke guest links for team conversation" testJoinTeamConvGuestLinksDisabled, @@ -1846,6 +1849,90 @@ testAccessUpdateGuestRemoved = do -- @END +testAccessUpdateGuestRemovedRemotesUnavailable :: TestM () +testAccessUpdateGuestRemovedRemotesUnavailable = do + -- alice, bob are in a team + (tid, alice, [bob]) <- createBindingTeamWithQualifiedMembers 2 + + -- charlie is a local guest + charlie <- randomQualifiedUser + connectUsers (qUnqualified alice) (pure (qUnqualified charlie)) + + -- dee is a remote guest + let remoteDomain = Domain "far-away.example.com" + dee <- Qualified <$> randomId <*> pure remoteDomain + + connectWithRemoteUser (qUnqualified alice) dee + + -- they are all in a local conversation + conv <- + responseJsonError + =<< postConvWithRemoteUsers + (qUnqualified alice) + Nothing + defNewProteusConv + { newConvQualifiedUsers = [bob, charlie, dee], + newConvTeam = Just (ConvTeamInfo tid) + } + do + -- conversation access role changes to team only + (_, reqs) <- withTempMockFederator' (throw $ MockErrorResponse HTTP.status503 "Down for maintenance") $ do + -- This request should still succeed even with an unresponsive federation member. + putQualifiedAccessUpdate + (qUnqualified alice) + (cnvQualifiedId conv) + (ConversationAccessData mempty (Set.fromList [TeamMemberAccessRole])) + !!! const 200 === statusCode + -- charlie and dee are kicked out + -- + -- note that removing users happens asynchronously, so this check should + -- happen while the mock federator is still available + WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $ + wsAssertMembersLeave (cnvQualifiedId conv) alice [charlie] + WS.assertMatchN_ (5 # Second) [wsA, wsB, wsC] $ + wsAssertMembersLeave (cnvQualifiedId conv) alice [dee] + + let compareLists [] ys = [] @?= ys + compareLists (x : xs) ys = case break (== x) ys of + (ys1, _ : ys2) -> compareLists xs (ys1 <> ys2) + _ -> assertFailure $ "Could not find " <> show x <> " in " <> show ys + liftIO $ + compareLists + ( map + ( \fr -> do + cu <- eitherDecode (frBody fr) + pure (F.cuOrigUserId cu, F.cuAction cu) + ) + ( filter + ( \fr -> + frComponent fr == Galley + && frRPC fr == "on-conversation-updated" + ) + reqs + ) + ) + [ Right (alice, SomeConversationAction (sing @'ConversationRemoveMembersTag) (pure charlie)), + Right (alice, SomeConversationAction (sing @'ConversationRemoveMembersTag) (pure dee)), + Right + ( alice, + SomeConversationAction + (sing @'ConversationAccessDataTag) + ConversationAccessData + { cupAccess = mempty, + cupAccessRoles = Set.fromList [TeamMemberAccessRole] + } + ) + ] + -- only alice and bob remain + conv2 <- + responseJsonError + =<< getConvQualified (qUnqualified alice) (cnvQualifiedId conv) + randomId <*> pure remoteDomain + qbob <- randomQualifiedUser + let bob = qUnqualified qbob + + connectWithRemoteUser bob qalice + + resp <- + postConvWithRemoteUsers + bob + Nothing + defNewProteusConv {newConvQualifiedUsers = [qalice]} + do + (_, requests) <- + withTempMockFederator' (throw $ MockErrorResponse HTTP.status503 "Down for maintenance") $ + putQualifiedConversationName bob qconv "gossip++" !!! const 200 === statusCode + + req <- assertOne requests + liftIO $ do + frTargetDomain req @?= remoteDomain + frComponent req @?= Galley + frRPC req @?= "on-conversation-updated" + Right cu <- pure . eitherDecode . frBody $ req + F.cuConvId cu @?= qUnqualified qconv + F.cuAction cu @?= SomeConversationAction (sing @'ConversationRenameTag) (ConversationRename "gossip++") + + void . liftIO . WS.assertMatch (5 # Second) wsB $ \n -> do + let e = List1.head (WS.unpackPayload n) + ntfTransient n @?= False + evtConv e @?= qconv + evtType e @?= ConvRename + evtFrom e @?= qbob + evtData e @?= EdConvRename (ConversationRename "gossip++") + putConvDeprecatedRenameOk :: TestM () putConvDeprecatedRenameOk = do c <- view tsCannon @@ -4025,6 +4152,48 @@ putReceiptModeWithRemotesOk = do @?= EdConvReceiptModeUpdate (ConversationReceiptModeUpdate (ReceiptMode 43)) +putReceiptModeWithRemotesUnavailable :: TestM () +putReceiptModeWithRemotesUnavailable = do + c <- view tsCannon + let remoteDomain = Domain "alice.example.com" + qalice <- Qualified <$> randomId <*> pure remoteDomain + qbob <- randomQualifiedUser + let bob = qUnqualified qbob + + connectWithRemoteUser bob qalice + + resp <- + postConvWithRemoteUsers + bob + Nothing + defNewProteusConv {newConvQualifiedUsers = [qalice]} + let qconv = decodeQualifiedConvId resp + + WS.bracketR c bob $ \wsB -> do + (_, requests) <- + withTempMockFederator' (throw $ MockErrorResponse HTTP.status503 "Down for maintenance") $ + putQualifiedReceiptMode bob qconv (ReceiptMode 43) !!! const 200 === statusCode + + req <- assertOne requests + liftIO $ do + frTargetDomain req @?= remoteDomain + frComponent req @?= Galley + frRPC req @?= "on-conversation-updated" + Right cu <- pure . eitherDecode . frBody $ req + F.cuConvId cu @?= qUnqualified qconv + F.cuAction cu + @?= SomeConversationAction (sing @'ConversationReceiptModeUpdateTag) (ConversationReceiptModeUpdate (ReceiptMode 43)) + + void . liftIO . WS.assertMatch (5 # Second) wsB $ \n -> do + let e = List1.head (WS.unpackPayload n) + ntfTransient n @?= False + evtConv e @?= qconv + evtType e @?= ConvReceiptModeUpdate + evtFrom e @?= qbob + evtData e + @?= EdConvReceiptModeUpdate + (ConversationReceiptModeUpdate (ReceiptMode 43)) + postTypingIndicatorsV2 :: TestM () postTypingIndicatorsV2 = do c <- view tsCannon diff --git a/services/galley/test/integration/API/Federation.hs b/services/galley/test/integration/API/Federation.hs index f05cfe537e..d0e6fe37a9 100644 --- a/services/galley/test/integration/API/Federation.hs +++ b/services/galley/test/integration/API/Federation.hs @@ -21,6 +21,7 @@ module API.Federation where import API.Util import Bilge hiding (head) import Bilge.Assert +import Control.Exception import Control.Lens hiding ((#)) import qualified Data.Aeson as A import Data.ByteString.Conversion (toByteString') @@ -42,6 +43,7 @@ import Data.Timeout (TimeoutUnit (..), (#)) import Data.UUID.V4 (nextRandom) import Federator.MockServer import Imports +import qualified Network.HTTP.Types as Http import Test.QuickCheck (arbitrary, generate) import Test.Tasty import qualified Test.Tasty.Cannon as WS @@ -86,7 +88,11 @@ tests s = test s "POST /federation/on-message-sent : Receive a message from another backend" onMessageSent, test s "POST /federation/send-message : Post a message sent from another backend" sendMessage, test s "POST /federation/on-user-deleted-conversations : Remove deleted remote user from local conversations" onUserDeleted, - test s "POST /federation/update-conversation : Update local conversation by a remote admin " updateConversationByRemoteAdmin + test s "POST /federation/update-conversation : Update local conversation by a remote admin " updateConversationByRemoteAdmin, + test s "POST /federation/on-conversation-updated : Notify local user about conversation rename with an unavailable federator" notifyConvRenameUnavailable, + test s "POST /federation/on-conversation-updated : Notify local user about message timer update with an unavailable federator" notifyMessageTimerUnavailable, + test s "POST /federation/on-conversation-updated : Notify local user about receipt mode update with an unavailable federator" notifyReceiptModeUnavailable, + test s "POST /federation/on-conversation-updated : Notify local user about access update with an unavailable federator" notifyAccessUnavailable ] getConversationsAllFound :: TestM () @@ -473,6 +479,50 @@ notifyUpdate extras action etype edata = do evtData e @?= edata WS.assertNoEvent (1 # Second) [wsC] +notifyUpdateUnavailable :: [Qualified UserId] -> SomeConversationAction -> EventType -> EventData -> TestM () +notifyUpdateUnavailable extras action etype edata = do + c <- view tsCannon + qalice <- randomQualifiedUser + let alice = qUnqualified qalice + bob <- randomId + charlie <- randomUser + conv <- randomId + let bdom = Domain "bob.example.com" + qbob = Qualified bob bdom + qconv = Qualified conv bdom + mkMember quid = OtherMember quid Nothing roleNameWireMember + fedGalleyClient <- view tsFedGalleyClient + + connectWithRemoteUser alice qbob + registerRemoteConv + qconv + bob + (Just "gossip") + (Set.fromList (map mkMember (qalice : extras))) + + now <- liftIO getCurrentTime + let cu = + FedGalley.ConversationUpdate + { FedGalley.cuTime = now, + FedGalley.cuOrigUserId = qbob, + FedGalley.cuConvId = conv, + FedGalley.cuAlreadyPresentUsers = [alice, charlie], + FedGalley.cuAction = action + } + WS.bracketR2 c alice charlie $ \(wsA, wsC) -> do + ((), _fedRequests) <- + withTempMockFederator' (throw $ MockErrorResponse Http.status500 "Down for maintenance") $ + runFedClient @"on-conversation-updated" fedGalleyClient bdom cu + liftIO $ do + WS.assertMatch_ (5 # Second) wsA $ \n -> do + let e = List1.head (WS.unpackPayload n) + ntfTransient n @?= False + evtConv e @?= qconv + evtType e @?= etype + evtFrom e @?= qbob + evtData e @?= edata + WS.assertNoEvent (1 # Second) [wsC] + notifyConvRename :: TestM () notifyConvRename = do let d = ConversationRename "gossip++" @@ -505,6 +555,38 @@ notifyAccess = do ConvAccessUpdate (EdConvAccessUpdate d) +notifyConvRenameUnavailable :: TestM () +notifyConvRenameUnavailable = do + let d = ConversationRename "gossip++" + notifyUpdateUnavailable [] (SomeConversationAction (sing @'ConversationRenameTag) d) ConvRename (EdConvRename d) + +notifyMessageTimerUnavailable :: TestM () +notifyMessageTimerUnavailable = do + let d = ConversationMessageTimerUpdate (Just 5000) + notifyUpdateUnavailable + [] + (SomeConversationAction (sing @'ConversationMessageTimerUpdateTag) d) + ConvMessageTimerUpdate + (EdConvMessageTimerUpdate d) + +notifyReceiptModeUnavailable :: TestM () +notifyReceiptModeUnavailable = do + let d = ConversationReceiptModeUpdate (ReceiptMode 42) + notifyUpdateUnavailable + [] + (SomeConversationAction (sing @'ConversationReceiptModeUpdateTag) d) + ConvReceiptModeUpdate + (EdConvReceiptModeUpdate d) + +notifyAccessUnavailable :: TestM () +notifyAccessUnavailable = do + let d = ConversationAccessData (Set.fromList [InviteAccess, LinkAccess]) (Set.fromList [TeamMemberAccessRole]) + notifyUpdateUnavailable + [] + (SomeConversationAction (sing @'ConversationAccessDataTag) d) + ConvAccessUpdate + (EdConvAccessUpdate d) + notifyMemberUpdate :: TestM () notifyMemberUpdate = do qdee <- randomQualifiedUser diff --git a/services/galley/test/integration/API/MessageTimer.hs b/services/galley/test/integration/API/MessageTimer.hs index d1354b1e0a..696fcb7d5f 100644 --- a/services/galley/test/integration/API/MessageTimer.hs +++ b/services/galley/test/integration/API/MessageTimer.hs @@ -23,6 +23,7 @@ where import API.Util import Bilge hiding (timeout) import Bilge.Assert +import Control.Exception import Control.Lens (view) import Data.Aeson (eitherDecode) import Data.Domain @@ -34,6 +35,7 @@ import Data.Qualified import Data.Singletons import Federator.MockServer import Imports hiding (head) +import qualified Network.HTTP.Types as Http import Network.Wai.Utilities.Error import Test.Tasty import Test.Tasty.Cannon (TimeoutUnit (..), (#)) @@ -61,6 +63,7 @@ tests s = test s "timer can be changed" messageTimerChange, test s "timer can be changed with the qualified endpoint" messageTimerChangeQualified, test s "timer changes are propagated to remote users" messageTimerChangeWithRemotes, + test s "timer changes unavailable remotes" messageTimerUnavailableRemotes, test s "timer can't be set by conv member without allowed action" messageTimerChangeWithoutAllowedAction, test s "timer can't be set in 1:1 conversations" messageTimerChangeO2O, test s "setting the timer generates an event" messageTimerEvent @@ -179,6 +182,46 @@ messageTimerChangeWithRemotes = do evtFrom e @?= qbob evtData e @?= EdConvMessageTimerUpdate (ConversationMessageTimerUpdate timer1sec) +messageTimerUnavailableRemotes :: TestM () +messageTimerUnavailableRemotes = do + c <- view tsCannon + let remoteDomain = Domain "alice.example.com" + qalice <- Qualified <$> randomId <*> pure remoteDomain + qbob <- randomQualifiedUser + let bob = qUnqualified qbob + connectWithRemoteUser bob qalice + + resp <- + postConvWithRemoteUsers + bob + Nothing + defNewProteusConv {newConvQualifiedUsers = [qalice]} + let qconv = decodeQualifiedConvId resp + + WS.bracketR c bob $ \wsB -> do + (_, requests) <- + withTempMockFederator' (throw $ MockErrorResponse Http.status503 "Down for maintenance") $ + putMessageTimerUpdateQualified bob qconv (ConversationMessageTimerUpdate timer1sec) + !!! const 200 === statusCode + + req <- assertOne requests + liftIO $ do + frTargetDomain req @?= remoteDomain + frComponent req @?= Galley + frRPC req @?= "on-conversation-updated" + Right cu <- pure . eitherDecode . frBody $ req + F.cuConvId cu @?= qUnqualified qconv + F.cuAction cu + @?= SomeConversationAction (sing @'ConversationMessageTimerUpdateTag) (ConversationMessageTimerUpdate timer1sec) + + void . liftIO . WS.assertMatch (5 # Second) wsB $ \n -> do + let e = List1.head (WS.unpackPayload n) + ntfTransient n @?= False + evtConv e @?= qconv + evtType e @?= ConvMessageTimerUpdate + evtFrom e @?= qbob + evtData e @?= EdConvMessageTimerUpdate (ConversationMessageTimerUpdate timer1sec) + messageTimerChangeWithoutAllowedAction :: TestM () messageTimerChangeWithoutAllowedAction = do -- Create a team and a guest user diff --git a/services/galley/test/integration/API/Roles.hs b/services/galley/test/integration/API/Roles.hs index 5b9847cefc..2eabd8deb9 100644 --- a/services/galley/test/integration/API/Roles.hs +++ b/services/galley/test/integration/API/Roles.hs @@ -20,6 +20,7 @@ module API.Roles where import API.Util import Bilge hiding (timeout) import Bilge.Assert +import Control.Exception import Control.Lens (view) import Data.Aeson hiding (json) import Data.ByteString.Conversion (toByteString') @@ -32,6 +33,7 @@ import qualified Data.Set as Set import Data.Singletons import Federator.MockServer import Imports +import qualified Network.HTTP.Types as Http import Network.Wai.Utilities.Error import Test.Tasty import Test.Tasty.Cannon (TimeoutUnit (..), (#)) @@ -54,6 +56,7 @@ tests s = [ test s "conversation roles admin (and downgrade)" handleConversationRoleAdmin, test s "conversation roles member (and upgrade)" handleConversationRoleMember, test s "conversation role update with remote users present" roleUpdateWithRemotes, + test s "conversation role update with remote users present remotes unavailable" roleUpdateWithRemotesUnavailable, test s "conversation access update with remote users present" accessUpdateWithRemotes, test s "conversation role update of remote member" roleUpdateRemoteMember, test s "get all conversation roles" testAllConversationRoles, @@ -284,6 +287,65 @@ roleUpdateWithRemotes = do evtFrom e @?= qbob evtData e @?= EdMemberUpdate mu +roleUpdateWithRemotesUnavailable :: TestM () +roleUpdateWithRemotesUnavailable = do + c <- view tsCannon + let remoteDomain = Domain "alice.example.com" + qalice <- Qualified <$> randomId <*> pure remoteDomain + qbob <- randomQualifiedUser + qcharlie <- randomQualifiedUser + let bob = qUnqualified qbob + charlie = qUnqualified qcharlie + + connectUsers bob (singleton charlie) + connectWithRemoteUser bob qalice + resp <- + postConvWithRemoteUsers + bob + Nothing + defNewProteusConv {newConvQualifiedUsers = [qalice, qcharlie]} + let qconv = decodeQualifiedConvId resp + + WS.bracketR2 c bob charlie $ \(wsB, wsC) -> do + (_, requests) <- + withTempMockFederator' (throw $ MockErrorResponse Http.status503 "Down for maintenance") $ + putOtherMemberQualified + bob + qcharlie + (OtherMemberUpdate (Just roleNameWireAdmin)) + qconv + !!! const 200 === statusCode + + req <- assertOne requests + let mu = + MemberUpdateData + { misTarget = qcharlie, + misOtrMutedStatus = Nothing, + misOtrMutedRef = Nothing, + misOtrArchived = Nothing, + misOtrArchivedRef = Nothing, + misHidden = Nothing, + misHiddenRef = Nothing, + misConvRoleName = Just roleNameWireAdmin + } + liftIO $ do + frTargetDomain req @?= remoteDomain + frComponent req @?= Galley + frRPC req @?= "on-conversation-updated" + Right cu <- pure . eitherDecode . frBody $ req + F.cuConvId cu @?= qUnqualified qconv + F.cuAction cu + @?= SomeConversationAction (sing @'ConversationMemberUpdateTag) (ConversationMemberUpdate qcharlie (OtherMemberUpdate (Just roleNameWireAdmin))) + F.cuAlreadyPresentUsers cu @?= [qUnqualified qalice] + + liftIO . WS.assertMatchN_ (5 # Second) [wsB, wsC] $ \n -> do + let e = List1.head (WS.unpackPayload n) + ntfTransient n @?= False + evtConv e @?= qconv + evtType e @?= MemberStateUpdate + evtFrom e @?= qbob + evtData e @?= EdMemberUpdate mu + accessUpdateWithRemotes :: TestM () accessUpdateWithRemotes = do c <- view tsCannon