incentives: cache top online accounts and use when building AbsentParticipationAccounts

Makefile

@@ -285,6 +285,9 @@ $(GOPATH1)/bin/%:
 test: build
 	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
+testc:
+	echo $(UNIT_TEST_SOURCES) | xargs -P8 -n1 go test -c
+
 benchcheck: build
 	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -run ^NOTHING -bench Benchmark -benchtime 1x -timeout 1h
 

cmd/tealdbg/localLedger.go

@@ -359,6 +359,10 @@ func (l *localLedger) LookupAgreement(rnd basics.Round, addr basics.Address) (ba
 	}, nil
 }
 
+func (l *localLedger) GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return nil, nil
+}
+
 func (l *localLedger) OnlineCirculation(rnd basics.Round, voteRound basics.Round) (basics.MicroAlgos, error) {
 	// A constant is fine for tealdbg
 	return basics.Algos(1_000_000_000), nil // 1B

daemon/algod/api/server/v2/dryrun.go

@@ -329,6 +329,10 @@ func (dl *dryrunLedger) LookupAgreement(rnd basics.Round, addr basics.Address) (
 	}, nil
 }
 
+func (dl *dryrunLedger) GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return nil, nil
+}
+
 func (dl *dryrunLedger) OnlineCirculation(rnd basics.Round, voteRnd basics.Round) (basics.MicroAlgos, error) {
 	// dryrun doesn't support setting the global online stake, so we'll just return a constant
 	return basics.Algos(1_000_000_000), nil // 1B

data/basics/userBalance.go

@@ -111,7 +111,10 @@ type VotingData struct {
 type OnlineAccountData struct {
 	MicroAlgosWithRewards MicroAlgos
 	VotingData
+
 	IncentiveEligible bool
+	LastProposed      Round
+	LastHeartbeat     Round
 }
 
 // AccountData contains the data associated with a given address.

ledger/acctdeltas.go

@@ -690,7 +690,11 @@ func accountDataToOnline(address basics.Address, ad *ledgercore.AccountData, pro
 		NormalizedOnlineBalance: ad.NormalizedOnlineBalance(proto),
 		VoteFirstValid:          ad.VoteFirstValid,
 		VoteLastValid:           ad.VoteLastValid,
+		VoteID:                  ad.VoteID,
 		StateProofID:            ad.StateProofID,
+		LastProposed:            ad.LastProposed,
+		LastHeartbeat:           ad.LastHeartbeat,
+		IncentiveEligible:       ad.IncentiveEligible,
 	}
 }
 

ledger/acctonline.go

@@ -622,11 +622,6 @@ func (ao *onlineAccounts) onlineTotals(rnd basics.Round) (basics.MicroAlgos, pro
 	return basics.MicroAlgos{Raw: onlineRoundParams.OnlineSupply}, onlineRoundParams.CurrentProtocol, nil
 }
 
-// LookupOnlineAccountData returns the online account data for a given address at a given round.
-func (ao *onlineAccounts) LookupOnlineAccountData(rnd basics.Round, addr basics.Address) (data basics.OnlineAccountData, err error) {
-	return ao.lookupOnlineAccountData(rnd, addr)
-}
-
 // roundOffset calculates the offset of the given round compared to the current dbRound. Requires that the lock would be taken.
 func (ao *onlineAccounts) roundOffset(rnd basics.Round) (offset uint64, err error) {
 	if rnd < ao.cachedDBRoundOnline {

ledger/eval/appcow_test.go

@@ -56,6 +56,10 @@ func (ml *emptyLedger) onlineStake() (basics.MicroAlgos, error) {
 	return basics.MicroAlgos{}, nil
 }
 
+func (ml *emptyLedger) incentiveCandidates(uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return nil, nil
+}
+
 func (ml *emptyLedger) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	return ledgercore.AppParamsDelta{}, true, nil
 }

ledger/eval/cow.go

@@ -47,6 +47,7 @@ type roundCowParent interface {
 	// lookup retrieves agreement data about an address, querying the ledger if necessary.
 	lookupAgreement(basics.Address) (basics.OnlineAccountData, error)
 	onlineStake() (basics.MicroAlgos, error)
+	incentiveCandidates(rewardsLevel uint64) (data map[basics.Address]basics.OnlineAccountData, err error)
 
 	// lookupAppParams, lookupAssetParams, lookupAppLocalState, and lookupAssetHolding retrieve data for a given address and ID.
 	// If cacheOnly is set, the ledger DB will not be queried, and only the cache will be consulted.
@@ -192,6 +193,10 @@ func (cb *roundCowState) lookupAgreement(addr basics.Address) (data basics.Onlin
 	return cb.lookupParent.lookupAgreement(addr)
 }
 
+func (cb *roundCowState) incentiveCandidates(rewardsLevel uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return cb.lookupParent.incentiveCandidates(rewardsLevel)
+}
+
 func (cb *roundCowState) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	params, ok := cb.mods.Accts.GetAppParams(addr, aidx)
 	if ok {

ledger/eval/cow_test.go

@@ -73,6 +73,10 @@ func (ml *mockLedger) onlineStake() (basics.MicroAlgos, error) {
 	return basics.Algos(55_555), nil
 }
 
+func (ml *mockLedger) incentiveCandidates(uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return nil, nil
+}
+
 func (ml *mockLedger) lookupAppParams(addr basics.Address, aidx basics.AppIndex, cacheOnly bool) (ledgercore.AppParamsDelta, bool, error) {
 	params, ok := ml.balanceMap[addr].AppParams[aidx]
 	return ledgercore.AppParamsDelta{Params: &params}, ok, nil // XXX make a copy?

ledger/eval/eval.go

@@ -48,6 +48,7 @@ type LedgerForCowBase interface {
 	CheckDup(config.ConsensusParams, basics.Round, basics.Round, basics.Round, transactions.Txid, ledgercore.Txlease) error
 	LookupWithoutRewards(basics.Round, basics.Address) (ledgercore.AccountData, basics.Round, error)
 	LookupAgreement(basics.Round, basics.Address) (basics.OnlineAccountData, error)
+	GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (data map[basics.Address]basics.OnlineAccountData, err error)
 	LookupAsset(basics.Round, basics.Address, basics.AssetIndex) (ledgercore.AssetResource, error)
 	LookupApplication(basics.Round, basics.Address, basics.AppIndex) (ledgercore.AppResource, error)
 	LookupKv(basics.Round, string) ([]byte, error)
@@ -237,6 +238,10 @@ func (x *roundCowBase) lookupAgreement(addr basics.Address) (basics.OnlineAccoun
 	return ad, err
 }
 
+func (x *roundCowBase) incentiveCandidates(rewardsLevel uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return x.l.GetIncentiveKickoffCandidates(x.rnd, x.proto, rewardsLevel)
+}
+
 // onlineStake returns the total online stake as of the start of the round. It
 // caches the result to prevent repeated calls to the ledger.
 func (x *roundCowBase) onlineStake() (basics.MicroAlgos, error) {
@@ -1620,12 +1625,61 @@ func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
 
 	ch := activeChallenge(&eval.proto, uint64(eval.Round()), eval.state)
 
+	// Make a set of candidate addresses to check for expired or absentee status.
+	type candidateData struct {
+		VoteLastValid         basics.Round
+		VoteID                crypto.OneTimeSignatureVerifier
+		Status                basics.Status
+		LastProposed          basics.Round
+		LastHeartbeat         basics.Round
+		MicroAlgosWithRewards basics.MicroAlgos
+		IncentiveEligible     bool // currently unused below, but may be needed in the future
+	}
+	candidates := make(map[basics.Address]candidateData)
+
+	// First, ask the ledger for the top N online accounts, with their latest
+	// online account data, current up to the previous round.
+	incentiveCandidates, err := eval.state.incentiveCandidates(eval.state.rewardsLevel())
+	if err != nil {
+		// Log an error and keep going; generating lists of absent and expired
+		// accounts is not required by block validation rules.
+		logging.Base().Warnf("error fetching incentiveCandidates: %v", err)
+		incentiveCandidates = nil
+	}
+	for accountAddr, acctData := range incentiveCandidates {
+		// acctData is from previous block: doesn't include any updates in mods
+		candidates[accountAddr] = candidateData{
+			VoteLastValid:         acctData.VoteLastValid,
+			VoteID:                acctData.VoteID,
+			Status:                basics.Online, // from lookupOnlineAccountData, which only returns online accounts
+			LastProposed:          acctData.LastProposed,
+			LastHeartbeat:         acctData.LastHeartbeat,
+			MicroAlgosWithRewards: acctData.MicroAlgosWithRewards,
+			IncentiveEligible:     acctData.IncentiveEligible,
+		}
+	}
+
+	// Then add any accounts modified in this block, with their state at the
+	// end of the round.
 	for _, accountAddr := range eval.state.modifiedAccounts() {
 		acctData, found := eval.state.mods.Accts.GetData(accountAddr)
 		if !found {
 			continue
 		}
+		// This will overwrite data from the incentiveCandidates() list, if they were modified in the current block.
+		candidates[accountAddr] = candidateData{
+			VoteLastValid:         acctData.VoteLastValid,
+			VoteID:                acctData.VoteID,
+			Status:                acctData.Status,
+			LastProposed:          acctData.LastProposed,
+			LastHeartbeat:         acctData.LastHeartbeat,
+			MicroAlgosWithRewards: acctData.RewardedMicroAlgos,
+			IncentiveEligible:     acctData.IncentiveEligible,
+		}
+	}
 
+	// Now, check these candidate accounts to see if they are expired or absent.
+	for accountAddr, acctData := range candidates {
 		// Regardless of being online or suspended, if voting data exists, the
 		// account can be expired to remove it.  This means an offline account
 		// can be expired (because it was already suspended).
@@ -1647,7 +1701,7 @@ func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
 
 		if acctData.Status == basics.Online {
 			lastSeen := max(acctData.LastProposed, acctData.LastHeartbeat)
-			if isAbsent(eval.state.prevTotals.Online.Money, acctData.MicroAlgos, lastSeen, current) ||
+			if isAbsent(eval.state.prevTotals.Online.Money, acctData.MicroAlgosWithRewards, lastSeen, current) ||
 				failsChallenge(ch, accountAddr, lastSeen) {
 				updates.AbsentParticipationAccounts = append(
 					updates.AbsentParticipationAccounts,
@@ -1658,14 +1712,6 @@ func (eval *BlockEvaluator) generateKnockOfflineAccountsList() {
 	}
 }
 
-// delete me in Go 1.21
-func max(a, b basics.Round) basics.Round {
-	if a > b {
-		return a
-	}
-	return b
-}
-
 // bitsMatch checks if the first n bits of two byte slices match. Written to
 // work on arbitrary slices, but we expect that n is small. Only user today
 // calls with n=5.

ledger/eval/eval_test.go

@@ -793,6 +793,10 @@ func (ledger *evalTestLedger) LookupAgreement(rnd basics.Round, addr basics.Addr
 	return convertToOnline(ad), err
 }
 
+func (ledger *evalTestLedger) GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (data map[basics.Address]basics.OnlineAccountData, err error) {
+	return nil, nil
+}
+
 // OnlineCirculation just returns a deterministic value for a given round.
 func (ledger *evalTestLedger) OnlineCirculation(rnd, voteRound basics.Round) (basics.MicroAlgos, error) {
 	return basics.MicroAlgos{Raw: uint64(rnd) * 1_000_000}, nil
@@ -1025,6 +1029,10 @@ func (l *testCowBaseLedger) LookupAgreement(rnd basics.Round, addr basics.Addres
 	return basics.OnlineAccountData{}, errors.New("not implemented")
 }
 
+func (l *testCowBaseLedger) GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, errors.New("not implemented")
+}
+
 func (l *testCowBaseLedger) OnlineCirculation(rnd, voteRnd basics.Round) (basics.MicroAlgos, error) {
 	return basics.MicroAlgos{}, errors.New("not implemented")
 }

ledger/eval/prefetcher/prefetcher_alignment_test.go

@@ -119,16 +119,23 @@ func (l *prefetcherAlignmentTestLedger) LookupWithoutRewards(_ basics.Round, add
 	}
 	return ledgercore.AccountData{}, 0, nil
 }
+
 func (l *prefetcherAlignmentTestLedger) LookupAgreement(_ basics.Round, addr basics.Address) (basics.OnlineAccountData, error) {
 	// prefetch alignment tests do not check for prefetching of online account data
 	// because it's quite different and can only occur in AVM opcodes, which
 	// aren't handled anyway (just as we don't know if a holding or app local
 	// will be accessed in AVM.)
 	return basics.OnlineAccountData{}, errors.New("not implemented")
 }
+
 func (l *prefetcherAlignmentTestLedger) OnlineCirculation(rnd, voteRnd basics.Round) (basics.MicroAlgos, error) {
-	return basics.MicroAlgos{}, errors.New("not implemented")
+	panic("not implemented")
+}
+
+func (l *prefetcherAlignmentTestLedger) GetIncentiveKickoffCandidates(basics.Round, config.ConsensusParams, uint64) (map[basics.Address]basics.OnlineAccountData, error) {
+	return nil, errors.New("not implemented")
 }
+
 func (l *prefetcherAlignmentTestLedger) LookupApplication(rnd basics.Round, addr basics.Address, aidx basics.AppIndex) (ledgercore.AppResource, error) {
 	l.mu.Lock()
 	if l.requestedApps == nil {
@@ -144,6 +151,7 @@ func (l *prefetcherAlignmentTestLedger) LookupApplication(rnd basics.Round, addr
 
 	return l.apps[addr][aidx], nil
 }
+
 func (l *prefetcherAlignmentTestLedger) LookupAsset(rnd basics.Round, addr basics.Address, aidx basics.AssetIndex) (ledgercore.AssetResource, error) {
 	l.mu.Lock()
 	if l.requestedAssets == nil {
@@ -159,9 +167,11 @@ func (l *prefetcherAlignmentTestLedger) LookupAsset(rnd basics.Round, addr basic
 
 	return l.assets[addr][aidx], nil
 }
+
 func (l *prefetcherAlignmentTestLedger) LookupKv(rnd basics.Round, key string) ([]byte, error) {
 	panic("not implemented")
 }
+
 func (l *prefetcherAlignmentTestLedger) GetCreatorForRound(_ basics.Round, cidx basics.CreatableIndex, ctype basics.CreatableType) (basics.Address, bool, error) {
 	l.mu.Lock()
 	if l.requestedCreators == nil {
@@ -175,6 +185,7 @@ func (l *prefetcherAlignmentTestLedger) GetCreatorForRound(_ basics.Round, cidx
 	}
 	return basics.Address{}, false, nil
 }
+
 func (l *prefetcherAlignmentTestLedger) GenesisHash() crypto.Digest {
 	return crypto.Digest{}
 }

ledger/ledger.go

@@ -89,6 +89,7 @@ type Ledger struct {
 	notifier       blockNotifier
 	metrics        metricsTracker
 	spVerification spVerificationTracker
+	topOnlineCache topOnlineCache
 
 	trackers  trackerRegistry
 	trackerMu deadlock.RWMutex
@@ -635,10 +636,35 @@ func (l *Ledger) LookupAgreement(rnd basics.Round, addr basics.Address) (basics.
 	defer l.trackerMu.RUnlock()
 
 	// Intentionally apply (pending) rewards up to rnd.
-	data, err := l.acctsOnline.LookupOnlineAccountData(rnd, addr)
+	data, err := l.acctsOnline.lookupOnlineAccountData(rnd, addr)
 	return data, err
 }
 
+// GetIncentiveKickoffCandidates retrieves a list of online accounts who may not have
+// proposed or sent a heartbeat recently.
+func (l *Ledger) GetIncentiveKickoffCandidates(rnd basics.Round, proto config.ConsensusParams, rewardsLevel uint64) (map[basics.Address]basics.OnlineAccountData, error) {
+	l.trackerMu.RLock()
+	defer l.trackerMu.RUnlock()
+
+	// get cached list of top N addresses
+	addrs, err := l.topOnlineCache.topN(&l.acctsOnline, rnd, proto, rewardsLevel)
+	if err != nil {
+		return nil, err
+	}
+
+	// fetch data for this round from online account cache. These accounts should all
+	// be in cache, as long as topOnlineCacheSize < onlineAccountsCacheMaxSize.
+	ret := make(map[basics.Address]basics.OnlineAccountData)
+	for _, addr := range addrs {
+		data, err := l.acctsOnline.lookupOnlineAccountData(rnd, addr)
+		if err != nil {
+			continue // skip missing / not online accounts
+		}
+		ret[addr] = data
+	}
+	return ret, nil
+}
+
 // LookupWithoutRewards is like Lookup but does not apply pending rewards up
 // to the requested round rnd.
 func (l *Ledger) LookupWithoutRewards(rnd basics.Round, addr basics.Address) (ledgercore.AccountData, basics.Round, error) {

ledger/ledgercore/onlineacct.go

@@ -17,14 +17,15 @@
 package ledgercore
 
 import (
+	"github.com/algorand/go-algorand/crypto"
 	"github.com/algorand/go-algorand/crypto/merklesignature"
 	"github.com/algorand/go-algorand/data/basics"
 )
 
 // An OnlineAccount corresponds to an account whose AccountData.Status
-// is Online.  This is used for a Merkle tree commitment of online
+// is Online. This is used for a Merkle tree commitment of online
 // accounts, which is subsequently used to validate participants for
-// a state proof.
+// a state proof. It is also used to track incentives participants.
 type OnlineAccount struct {
 	// These are a subset of the fields from the corresponding AccountData.
 	Address                 basics.Address
@@ -33,5 +34,9 @@ type OnlineAccount struct {
 	NormalizedOnlineBalance uint64
 	VoteFirstValid          basics.Round
 	VoteLastValid           basics.Round
+	VoteID                  crypto.OneTimeSignatureVerifier
 	StateProofID            merklesignature.Commitment
+	LastProposed            basics.Round
+	LastHeartbeat           basics.Round
+	IncentiveEligible       bool
 }

ledger/store/trackerdb/data.go

@@ -152,6 +152,8 @@ type BaseOnlineAccountData struct {
 
 	BaseVotingData
 
+	LastProposed      basics.Round      `codec:"V"`
+	LastHeartbeat     basics.Round      `codec:"W"`
 	IncentiveEligible bool              `codec:"X"`
 	MicroAlgos        basics.MicroAlgos `codec:"Y"`
 	RewardsBase       uint64            `codec:"Z"`
@@ -469,7 +471,11 @@ func (bo *BaseOnlineAccountData) GetOnlineAccount(addr basics.Address, normBalan
 		NormalizedOnlineBalance: normBalance,
 		VoteFirstValid:          bo.VoteFirstValid,
 		VoteLastValid:           bo.VoteLastValid,
+		VoteID:                  bo.VoteID,
 		StateProofID:            bo.StateProofID,
+		LastHeartbeat:           bo.LastHeartbeat,
+		LastProposed:            bo.LastProposed,
+		IncentiveEligible:       bo.IncentiveEligible,
 	}
 }