state proofs: update state proof totals calculation

config/consensus.go

@@ -391,6 +391,10 @@ type ConsensusParams struct {
 	// release resources allocated for creating state proofs.
 	StateProofMaxRecoveryIntervals uint64
 
+	// StateProofExcludeTotalWeightWithRewards specifies whether to subtract rewards from excluded online accounts along with
+	// their account balances.
+	StateProofExcludeTotalWeightWithRewards bool
+
 	// EnableAssetCloseAmount adds an extra field to the ApplyData. The field contains the amount of the remaining
 	// asset that were sent to the close-to address.
 	EnableAssetCloseAmount bool
@@ -1191,14 +1195,22 @@ func initConsensusProtocols() {
 
 	Consensus[protocol.ConsensusV34] = v34
 
-	// v33 can be upgraded to v34, with an update delay of 12h:
+	v35 := v34
+	v35.StateProofExcludeTotalWeightWithRewards = true
+
+	v35.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
+
+	Consensus[protocol.ConsensusV35] = v35
+
+	// v33 and v34 can be upgraded to v35, with an update delay of 12h:
 	// 10046 = (12 * 60 * 60 / 4.3)
 	// for the sake of future manual calculations, we'll round that down a bit :
-	v33.ApprovedUpgrades[protocol.ConsensusV34] = 10000
+	v33.ApprovedUpgrades[protocol.ConsensusV35] = 10000
+	v34.ApprovedUpgrades[protocol.ConsensusV35] = 10000
 
 	// ConsensusFuture is used to test features that are implemented
 	// but not yet released in a production protocol version.
-	vFuture := v34
+	vFuture := v35
 	vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
 	vFuture.LogicSigVersion = 8 // When moving this to a release, put a new higher LogicSigVersion here

ledger/acctonline.go

@@ -747,7 +747,7 @@ func (ao *onlineAccounts) lookupOnlineAccountData(rnd basics.Round, addr basics.
 // not participate in round == voteRnd.
 // See the normalization description in AccountData.NormalizedOnlineBalance().
 // The return value of totalOnlineStake represents the total stake that is online for voteRnd: it is an approximation since voteRnd did not yet occur.
-func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64) (topOnlineAccounts []*ledgercore.OnlineAccount, totalOnlineStake basics.MicroAlgos, err error) {
+func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64, params *config.ConsensusParams, rewardsLevel uint64) (topOnlineAccounts []*ledgercore.OnlineAccount, totalOnlineStake basics.MicroAlgos, err error) {
 	genesisProto := ao.ledger.GenesisProto()
 	ao.accountsMu.RLock()
 	for {
@@ -902,6 +902,13 @@ func (ao *onlineAccounts) TopOnlineAccounts(rnd basics.Round, voteRnd basics.Rou
 			if ot.Overflowed {
 				return nil, basics.MicroAlgos{}, fmt.Errorf("TopOnlineAccounts: overflow in stakeOfflineInVoteRound")
 			}
+			if params.StateProofExcludeTotalWeightWithRewards {
+				rewards := basics.PendingRewards(&ot, *params, oa.MicroAlgos, oa.RewardsBase, rewardsLevel)
+				totalOnlineStake = ot.SubA(totalOnlineStake, rewards)
+				if ot.Overflowed {
+					return nil, basics.MicroAlgos{}, fmt.Errorf("TopOnlineAccounts: overflow in stakeOfflineInVoteRound rewards")
+				}
+			}
 		}
 
 		return topOnlineAccounts, totalOnlineStake, nil

ledger/acctonline_test.go

@@ -1392,7 +1392,8 @@ func TestAcctOnlineTopInBatches(t *testing.T) {
 	_, oa := newAcctUpdates(t, ml, conf)
 	defer oa.close()
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 2048)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 2048, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 }
@@ -1437,7 +1438,8 @@ func TestAcctOnlineTopBetweenCommitAndPostCommit(t *testing.T) {
 	defer oa.close()
 	ml.trackers.trackers = append([]ledgerTracker{stallingTracker}, ml.trackers.trackers...)
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 5)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 5, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 
@@ -1475,7 +1477,8 @@ func TestAcctOnlineTopBetweenCommitAndPostCommit(t *testing.T) {
 			time.Sleep(2 * time.Second)
 			stallingTracker.postCommitReleaseLock <- struct{}{}
 		}()
-		top, _, err = oa.TopOnlineAccounts(2, 2, 5)
+
+		top, _, err = oa.TopOnlineAccounts(2, 2, 5, &proto, 0)
 		a.NoError(err)
 
 		accountToBeUpdated := allAccts[numAccts-1]
@@ -1528,7 +1531,8 @@ func TestAcctOnlineTopDBBehindMemRound(t *testing.T) {
 	defer oa.close()
 	ml.trackers.trackers = append([]ledgerTracker{stallingTracker}, ml.trackers.trackers...)
 
-	top, _, err := oa.TopOnlineAccounts(0, 0, 5)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, _, err := oa.TopOnlineAccounts(0, 0, 5, &proto, 0)
 	a.NoError(err)
 	compareTopAccounts(a, top, allAccts)
 
@@ -1571,7 +1575,8 @@ func TestAcctOnlineTopDBBehindMemRound(t *testing.T) {
 			})
 			stallingTracker.postCommitReleaseLock <- struct{}{}
 		}()
-		_, _, err = oa.TopOnlineAccounts(2, 2, 5)
+
+		_, _, err = oa.TopOnlineAccounts(2, 2, 5, &proto, 0)
 		a.Error(err)
 		a.Contains(err.Error(), "is behind in-memory round")
 
@@ -1680,7 +1685,8 @@ func (m *MicroAlgoOperations) Add(x, y basics.MicroAlgos) basics.MicroAlgos {
 }
 
 func compareOnlineTotals(a *require.Assertions, oa *onlineAccounts, rnd, voteRnd basics.Round, n uint64, expectedForRnd, expectedForVoteRnd basics.MicroAlgos) []*ledgercore.OnlineAccount {
-	top, onlineTotalVoteRnd, err := oa.TopOnlineAccounts(rnd, voteRnd, n)
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
+	top, onlineTotalVoteRnd, err := oa.TopOnlineAccounts(rnd, voteRnd, n, &proto, 0)
 	a.NoError(err)
 	a.Equal(expectedForVoteRnd, onlineTotalVoteRnd)
 	onlineTotalsRnd, err := oa.onlineTotals(rnd)

ledger/internal/eval_blackbox_test.go

@@ -910,6 +910,7 @@ var consensusByNumber = []protocol.ConsensusVersion{
 	protocol.ConsensusV32, // unlimited assets and apps
 	protocol.ConsensusV33, // 320 rounds
 	protocol.ConsensusV34, // AVM v7, stateproofs
+	protocol.ConsensusV35, // stateproofs stake fix
 	protocol.ConsensusFuture,
 }
 

ledger/ledgercore/votersForRound.go

@@ -18,14 +18,14 @@ package ledgercore
 
 import (
 	"fmt"
-	"github.com/algorand/go-algorand/crypto/merklesignature"
 	"sync"
 
 	"github.com/algorand/go-deadlock"
 
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/crypto/merklearray"
+	"github.com/algorand/go-algorand/crypto/merklesignature"
 	"github.com/algorand/go-algorand/crypto/stateproof"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
@@ -36,7 +36,7 @@ type OnlineAccountsFetcher interface {
 	// TopOnlineAccounts returns the top n online accounts, sorted by their normalized
 	// balance and address, whose voting keys are valid in voteRnd.  See the
 	// normalization description in AccountData.NormalizedOnlineBalance().
-	TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64) (topOnlineAccounts []*OnlineAccount, totalOnlineStake basics.MicroAlgos, err error)
+	TopOnlineAccounts(rnd basics.Round, voteRnd basics.Round, n uint64, params *config.ConsensusParams, rewardsLevel uint64) (topOnlineAccounts []*OnlineAccount, totalOnlineStake basics.MicroAlgos, err error)
 }
 
 // VotersForRound tracks the top online voting accounts as of a particular
@@ -113,7 +113,7 @@ func (tr *VotersForRound) LoadTree(onlineAccountsFetcher OnlineAccountsFetcher,
 	// using the balances from round r.
 	stateProofRound := r + basics.Round(tr.Proto.StateProofVotersLookback+tr.Proto.StateProofInterval)
 
-	top, totalOnlineWeight, err := onlineAccountsFetcher.TopOnlineAccounts(r, stateProofRound, tr.Proto.StateProofTopVoters)
+	top, totalOnlineWeight, err := onlineAccountsFetcher.TopOnlineAccounts(r, stateProofRound, tr.Proto.StateProofTopVoters, &tr.Proto, hdr.RewardsLevel)
 	if err != nil {
 		return err
 	}

protocol/consensus.go

@@ -187,6 +187,11 @@ const ConsensusV34 = ConsensusVersion(
 	"https://github.com/algorandfoundation/specs/tree/2dd5435993f6f6d65691140f592ebca5ef19ffbd",
 )
 
+// ConsensusV35 updates the calculation of total stake in state proofs.
+const ConsensusV35 = ConsensusVersion(
+	"https://github.com/algorandfoundation/specs/tree/433d8e9a7274b6fca703d91213e05c7e6a589e69",
+)
+
 // ConsensusFuture is a protocol that should not appear in any production
 // network, but is used to test features before they are released.
 const ConsensusFuture = ConsensusVersion(
@@ -199,7 +204,7 @@ const ConsensusFuture = ConsensusVersion(
 
 // ConsensusCurrentVersion is the latest version and should be used
 // when a specific version is not provided.
-const ConsensusCurrentVersion = ConsensusV34
+const ConsensusCurrentVersion = ConsensusV35
 
 // Error is used to indicate that an unsupported protocol has been detected.
 type Error ConsensusVersion

test/e2e-go/features/stateproofs/stateproofs_test.go

@@ -17,10 +17,12 @@
 package stateproofs
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -34,6 +36,8 @@ import (
 	"github.com/algorand/go-algorand/crypto/merklearray"
 	sp "github.com/algorand/go-algorand/crypto/stateproof"
 	"github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated"
+	generatedV2 "github.com/algorand/go-algorand/daemon/algod/api/server/v2/generated"
+	v1 "github.com/algorand/go-algorand/daemon/algod/api/spec/v1"
 	"github.com/algorand/go-algorand/data/account"
 	"github.com/algorand/go-algorand/data/basics"
 	"github.com/algorand/go-algorand/data/bookkeeping"
@@ -122,8 +126,12 @@ func TestStateProofs(t *testing.T) {
 }
 
 func TestStateProofsMultiWallets(t *testing.T) {
-	t.Skip("this test is heavy and should be run manually")
 	partitiontest.PartitionTest(t)
+
+	if strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" {
+		t.Skip()
+	}
+
 	defer fixtures.ShutdownSynchronizedTest(t)
 
 	configurableConsensus := make(config.ConsensusProtocols)
@@ -1155,3 +1163,140 @@ func getWellformedSPTransaction(round uint64, genesisHash crypto.Digest, consens
 
 	return stxn
 }
+
+func TestStateProofCheckTotalStake(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	if strings.ToUpper(os.Getenv("CIRCLECI")) == "TRUE" {
+		t.Skip()
+	}
+
+	defer fixtures.ShutdownSynchronizedTest(t)
+
+	r := require.New(fixtures.SynchronizedTest(t))
+
+	configurableConsensus := make(config.ConsensusProtocols)
+	consensusVersion := protocol.ConsensusVersion("test-fast-stateproofs")
+	consensusParams := getDefaultStateProofConsensusParams()
+	consensusParams.StateProofWeightThreshold = (1 << 32) * 90 / 100
+	consensusParams.StateProofStrengthTarget = 3
+	consensusParams.AgreementFilterTimeout = 1000 * time.Millisecond
+	consensusParams.AgreementFilterTimeoutPeriod0 = 1000 * time.Millisecond
+	consensusParams.SeedLookback = 2
+	consensusParams.SeedRefreshInterval = 8
+	consensusParams.MaxBalLookback = 2 * consensusParams.SeedLookback * consensusParams.SeedRefreshInterval // 32
+	configurableConsensus[consensusVersion] = consensusParams
+
+	var fixture fixtures.RestClientFixture
+	pNodes := 5
+	expectedNumberOfStateProofs := uint64(4)
+	fixture.SetConsensus(configurableConsensus)
+	fixture.Setup(t, filepath.Join("nettemplates", "StateProof.json"))
+
+	defer fixture.Shutdown()
+
+	// Get node libgoal clients in order to update their participation keys
+	libgoalNodeClients := make([]libgoal.Client, pNodes, pNodes)
+	accountsAddresses := make([]string, pNodes, pNodes)
+	for i := 0; i < pNodes; i++ {
+		nodeName := fmt.Sprintf("Node%d", i)
+		libgoalNodeClients[i] = fixture.GetLibGoalClientForNamedNode(nodeName)
+		parts, err := libgoalNodeClients[i].GetParticipationKeys()
+		r.NoError(err)
+		accountsAddresses[i] = parts[0].Address
+	}
+
+	participations := make([]account.Participation, pNodes, pNodes)
+	var lastStateProofBlock bookkeeping.Block
+	libgoalClient := fixture.LibGoalClient
+
+	var totalSupplyAtRound [100]v1.Supply
+	var accountSnapshotAtRound [100][]generatedV2.Account
+
+	for rnd := uint64(1); rnd <= consensusParams.StateProofInterval*(expectedNumberOfStateProofs+1); rnd++ {
+		if rnd == consensusParams.StateProofInterval+consensusParams.StateProofVotersLookback { // here we register the keys of address 0 so it won't be able the sign a state proof (its stake would be removed for the total)
+			_, part, err := installParticipationKey(t, libgoalNodeClients[0], accountsAddresses[0], 0, consensusParams.StateProofInterval*2-1)
+			r.NoError(err)
+			participations[0] = part
+			registerParticipationAndWait(t, libgoalNodeClients[0], participations[0])
+		}
+
+		//send a dummy payment transaction.
+		paymentSender{
+			from:   accountFetcher{nodeName: "Node3", accountNumber: 0},
+			to:     accountFetcher{nodeName: "Node4", accountNumber: 0},
+			amount: 1,
+		}.sendPayment(r, &fixture, rnd)
+
+		err := fixture.WaitForRound(rnd, timeoutUntilNextRound)
+		r.NoError(err)
+
+		// this is the round in we take a snapshot of the account balances.
+		// We would use this snapshot later on to compare the weights on the state proof, and to make sure that
+		// the totalWeight commitment is correct
+		if ((rnd + 2) % consensusParams.StateProofInterval) == 0 {
+			totalSupply, err := libgoalClient.LedgerSupply()
+			r.NoError(err)
+
+			r.Equal(rnd, totalSupply.Round, "could not capture total stake at the target round. The machine might be too slow for this test")
+			totalSupplyAtRound[rnd] = totalSupply
+
+			accountSnapshotAtRound[rnd] = make([]generatedV2.Account, pNodes, pNodes)
+			for i := 0; i < pNodes; i++ {
+				accountSnapshotAtRound[rnd][i], err = libgoalClient.AccountInformationV2(accountsAddresses[i], false)
+				r.NoError(err)
+				r.NotEqual(accountSnapshotAtRound[rnd][i].Amount, uint64(0))
+				r.Equal(rnd, accountSnapshotAtRound[rnd][i].Round, "could not capture the account at the target round. The machine might be too slow for this test")
+			}
+		}
+
+		blk, err := libgoalClient.BookkeepingBlock(rnd)
+		r.NoErrorf(err, "failed to retrieve block from algod on round %d", rnd)
+
+		if (rnd % consensusParams.StateProofInterval) == 0 {
+			if rnd >= consensusParams.StateProofInterval*2 {
+				// since account 0 would no longer be able to sign the state proof, its stake should
+				// be removed from the total stake in the commitment
+				total := totalSupplyAtRound[rnd-consensusParams.StateProofVotersLookback].OnlineMoney
+				total = total - accountSnapshotAtRound[rnd-consensusParams.StateProofVotersLookback][0].Amount
+				r.Equal(total, blk.StateProofTracking[protocol.StateProofBasic].StateProofOnlineTotalWeight.Raw)
+			} else {
+				r.Equal(totalSupplyAtRound[rnd-consensusParams.StateProofVotersLookback].OnlineMoney, blk.StateProofTracking[protocol.StateProofBasic].StateProofOnlineTotalWeight.Raw)
+			}
+
+			// Special case: bootstrap validation with the first block
+			// that has a merkle root.
+			if lastStateProofBlock.Round() == 0 {
+				lastStateProofBlock = blk
+			}
+		}
+
+		for lastStateProofBlock.Round() != 0 && lastStateProofBlock.Round()+basics.Round(consensusParams.StateProofInterval) < blk.StateProofTracking[protocol.StateProofBasic].StateProofNextRound {
+			nextStateProofRound := uint64(lastStateProofBlock.Round()) + consensusParams.StateProofInterval
+
+			t.Logf("found a state proof for round %d at round %d", nextStateProofRound, blk.Round())
+
+			stateProof, stateProofMsg := getStateProofByLastRound(r, &fixture, nextStateProofRound, expectedNumberOfStateProofs)
+
+			accountSnapshot := accountSnapshotAtRound[stateProofMsg.LastAttestedRound-consensusParams.StateProofInterval-consensusParams.StateProofVotersLookback]
+
+			// once the state proof is accepted we want to make sure that the weight
+			for _, v := range stateProof.Reveals {
+				found := false
+				for i := 0; i < len(accountSnapshot); i++ {
+					if bytes.Compare(v.Part.PK.Commitment[:], *accountSnapshot[i].Participation.StateProofKey) == 0 {
+						r.Equal(v.Part.Weight, accountSnapshot[i].Amount)
+						found = true
+						break
+					}
+				}
+				r.True(found)
+			}
+			nextStateProofBlock, err := fixture.LibGoalClient.BookkeepingBlock(nextStateProofRound)
+			r.NoError(err)
+			lastStateProofBlock = nextStateProofBlock
+		}
+	}
+
+	r.Equalf(int(consensusParams.StateProofInterval*expectedNumberOfStateProofs), int(lastStateProofBlock.Round()), "the expected last state proof block wasn't the one that was observed")
+}