[FIXED] Deadlock when removing a peer that happened to be the leader.

server/raft.go

@@ -2915,7 +2915,7 @@ func (n *raft) applyCommit(index uint64) error {
 
 			// If this is us and we are the leader we should attempt to stepdown.
 			if peer == n.id && n.State() == Leader {
-				n.stepdown(n.selectNextLeader())
+				n.stepdownLocked(n.selectNextLeader())
 			}
 
 			// Remove from string intern map.

server/raft_test.go

@@ -14,6 +14,7 @@
 package server
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"math/rand"
@@ -917,3 +918,26 @@ func TestNRGCandidateDontStepdownDueToLeaderOfPreviousTerm(t *testing.T) {
 	// Check that the candidate's term is still ahead of the leader's term
 	require_True(t, candidate.term > ae.term)
 }
+
+func TestNRGRemoveLeaderPeerDeadlockBug(t *testing.T) {
+	c := createJetStreamClusterExplicit(t, "R3S", 3)
+	defer c.shutdown()
+
+	rg := c.createMemRaftGroup("TEST", 3, newStateAdder)
+	rg.waitOnLeader()
+
+	n := rg.leader().node().(*raft)
+	leader := n.ID()
+
+	// Propose to remove the leader as a peer. Will lead to a deadlock with bug.
+	require_NoError(t, n.ProposeRemovePeer(leader))
+	rg.waitOnLeader()
+
+	checkFor(t, 10*time.Second, 200*time.Millisecond, func() error {
+		nl := n.GroupLeader()
+		if nl != leader {
+			return nil
+		}
+		return errors.New("Leader has not moved")
+	})
+}