Chain stalls while scaling out from 1 to 4 nodes, changing quorum size fixes things

consensus/istanbul/backend/engine.go

@@ -27,8 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/istanbul"
-	istanbulCore "github.com/ethereum/go-ethereum/consensus/istanbul/core"
 	"github.com/ethereum/go-ethereum/consensus/istanbul/validator"
+	istanbulCore "github.com/ethereum/go-ethereum/consensus/istanbul/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
@@ -289,8 +289,8 @@ func (sb *backend) verifyCommittedSeals(chain consensus.ChainReader, header *typ
 		}
 	}
 
-	// The length of validSeal should be larger than number of faulty node + 1
-	if validSeal <= 2*snap.ValSet.F() {
+	// The validSeal number should be greater than equal to ceil(2N/3) i.e., quorum size   
+	if validSeal < snap.ValSet.QuorumSize() {
 		return errInvalidCommittedSeals
 	}
 

consensus/istanbul/core/commit.go

@@ -72,7 +72,7 @@ func (c *core) handleCommit(msg *message, src istanbul.Validator) error {
 	//
 	// If we already have a proposal, we may have chance to speed up the consensus process
 	// by committing the proposal without PREPARE messages.
-	if c.current.Commits.Size() > 2*c.valSet.F() && c.state.Cmp(StateCommitted) < 0 {
+	if c.current.Commits.Size() >= c.valSet.QuorumSize() && c.state.Cmp(StateCommitted) < 0 {
 		// Still need to call LockHash here since state can skip Prepared state and jump directly to the Committed state.
 		c.current.LockHash()
 		c.commit()

consensus/istanbul/core/commit_test.go

@@ -191,18 +191,18 @@ OUTER:
 			if r0.state != StatePrepared {
 				t.Errorf("state mismatch: have %v, want %v", r0.state, StatePrepared)
 			}
-			if r0.current.Commits.Size() > 2*r0.valSet.F() {
-				t.Errorf("the size of commit messages should be less than %v", 2*r0.valSet.F()+1)
+			if r0.current.Commits.Size() >= r0.valSet.QuorumSize() {
+				t.Errorf("the size of commit messages should be less than %v", r0.valSet.QuorumSize())
 			}
 			if r0.current.IsHashLocked() {
 				t.Errorf("block should not be locked")
 			}
 			continue
 		}
 
-		// core should have 2F+1 prepare messages
-		if r0.current.Commits.Size() <= 2*r0.valSet.F() {
-			t.Errorf("the size of commit messages should be larger than 2F+1: size %v", r0.current.Commits.Size())
+		// core should have ceil(2N/3) prepare messages for N validators
+		if r0.current.Commits.Size() < r0.valSet.QuorumSize() {
+			t.Errorf("the size of commit messages should be atleast ceil(2N/3): size %v", r0.valSet.QuorumSize())
 		}
 
 		// check signatures large than 2F+1

consensus/istanbul/core/prepare.go

@@ -59,7 +59,7 @@ func (c *core) handlePrepare(msg *message, src istanbul.Validator) error {
 
 	// Change to Prepared state if we've received enough PREPARE messages or it is locked
 	// and we are in earlier state before Prepared state.
-	if ((c.current.IsHashLocked() && prepare.Digest == c.current.GetLockedHash()) || c.current.GetPrepareOrCommitSize() > 2*c.valSet.F()) &&
+	if ((c.current.IsHashLocked() && prepare.Digest == c.current.GetLockedHash()) || c.current.GetPrepareOrCommitSize() >= c.valSet.QuorumSize()) &&
 		c.state.Cmp(StatePrepared) < 0 {
 		c.current.LockHash()
 		c.setState(StatePrepared)

consensus/istanbul/core/prepare_test.go

@@ -214,8 +214,8 @@ OUTER:
 			if r0.state != StatePreprepared {
 				t.Errorf("state mismatch: have %v, want %v", r0.state, StatePreprepared)
 			}
-			if r0.current.Prepares.Size() > 2*r0.valSet.F() {
-				t.Errorf("the size of PREPARE messages should be less than %v", 2*r0.valSet.F()+1)
+			if r0.current.Prepares.Size() >= r0.valSet.QuorumSize() {
+				t.Errorf("the size of PREPARE messages should be less than %v", r0.valSet.QuorumSize())
 			}
 			if r0.current.IsHashLocked() {
 				t.Errorf("block should not be locked")
@@ -224,12 +224,12 @@ OUTER:
 			continue
 		}
 
-		// core should have 2F+1 PREPARE messages
-		if r0.current.Prepares.Size() <= 2*r0.valSet.F() {
-			t.Errorf("the size of PREPARE messages should be larger than 2F+1: size %v", r0.current.Commits.Size())
+		// core should have ceil(2N/3) PREPARE messages
+		if r0.current.Prepares.Size() < r0.valSet.QuorumSize() {
+			t.Errorf("the size of PREPARE messages should be atleast ceil(2N/3) size %v", r0.current.Commits.Size())
 		}
 
-		// a message will be delivered to backend if 2F+1
+		// a message will be delivered to backend if ceil(2N/3)
 		if int64(len(v0.sentMsgs)) != 1 {
 			t.Errorf("the Send() should be called once: times %v", len(test.system.backends[0].sentMsgs))
 		}

consensus/istanbul/core/roundchange.go

@@ -98,8 +98,8 @@ func (c *core) handleRoundChange(msg *message, src istanbul.Validator) error {
 			c.sendRoundChange(roundView.Round)
 		}
 		return nil
-	} else if num == int(2*c.valSet.F()+1) && (c.waitingForRoundChange || cv.Round.Cmp(roundView.Round) < 0) {
-		// We've received 2f+1 ROUND CHANGE messages, start a new round immediately.
+	} else if num == c.valSet.QuorumSize() && (c.waitingForRoundChange || cv.Round.Cmp(roundView.Round) < 0) {
+		// We've received ceil(2N/3) ROUND CHANGE messages, start a new round immediately.
 		c.startNewRound(roundView.Round)
 		return nil
 	} else if cv.Round.Cmp(roundView.Round) < 0 {

consensus/istanbul/validator.go

@@ -73,6 +73,8 @@ type ValidatorSet interface {
 	F() int
 	// Get proposer policy
 	Policy() ProposerPolicy
+	// Get quorum size based on quorum formula type
+	QuorumSize() int
 }
 
 // ----------------------------------------------------------------------------

consensus/istanbul/validator/default.go

@@ -199,3 +199,5 @@ func (valSet *defaultSet) Copy() istanbul.ValidatorSet {
 func (valSet *defaultSet) F() int { return int(math.Ceil(float64(valSet.Size())/3)) - 1 }
 
 func (valSet *defaultSet) Policy() istanbul.ProposerPolicy { return valSet.policy }
+
+func (valSet *defaultSet) QuorumSize() int { return int(math.Ceil(float64(2*valSet.Size())/3)) }

consensus/istanbul/validator/default_test.go

@@ -37,6 +37,7 @@ func TestValidatorSet(t *testing.T) {
 	testEmptyValSet(t)
 	testStickyProposer(t)
 	testAddAndRemoveValidator(t)
+	testQuorumSize(t)
 }
 
 func testNewValidatorSet(t *testing.T) {
@@ -206,3 +207,13 @@ func testStickyProposer(t *testing.T) {
 		t.Errorf("proposer mismatch: have %v, want %v", val, val2)
 	}
 }
+
+func testQuorumSize(t *testing.T) {
+	valSet := newDefaultSet([]common.Address{}, istanbul.RoundRobin)
+	for i:=1; i<=1000; i++ {
+		valSet.AddValidator(common.StringToAddress(string(i)));
+		if valSet.QuorumSize() <= (valSet.Size() - valSet.QuorumSize() + valSet.F()) {
+			t.Errorf("quorumSize constraint failed, expected value (QuorumSize > Size-QuorumSize+F) to be:%v, got: %v, for size: %v", true, false, valSet.Size());
+		}
+	}
+}