metrics: measure block assembly re-evaluation failures

data/pools/errors.go

@@ -21,6 +21,11 @@ import (
 	"fmt"
 
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/go-algorand/data/transactions"
+	"github.com/algorand/go-algorand/data/transactions/logic"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/util/metrics"
 )
 
 // ErrStaleBlockAssemblyRequest returned by AssembleBlock when requested block number is older than the current transaction pool round
@@ -45,3 +50,155 @@ func (e *ErrTxPoolFeeError) Error() string {
 	return fmt.Sprintf("fee %d below threshold %d (%d per byte * %d bytes)",
 		e.fee, e.feeThreshold, e.feePerByte, e.encodedLength)
 }
+
+// TxPoolErrorTag constants for categorizing transaction pool errors.
+// These are used by both the txHandler (initial remember) and the
+// transaction pool (re-evaluation) to classify errors consistently.
+const (
+	TxPoolErrTagCap          = "cap"
+	TxPoolErrTagPendingEval  = "pending_eval"
+	TxPoolErrTagNoSpace      = "no_space"
+	TxPoolErrTagFee          = "fee"
+	TxPoolErrTagTxnDead      = "txn_dead"
+	TxPoolErrTagTxnEarly     = "txn_early"
+	TxPoolErrTagTooLarge     = "too_large"
+	TxPoolErrTagGroupID      = "groupid"
+	TxPoolErrTagTxID         = "txid"
+	TxPoolErrTagLease        = "lease"
+	TxPoolErrTagTxIDEval     = "txid_eval"
+	TxPoolErrTagLeaseEval    = "lease_eval"
+	TxPoolErrTagNotWell      = "not_well"    // TxnNotWellFormedError - malformed transaction
+	TxPoolErrTagTealErr      = "teal_err"    // TEAL runtime error (logic.EvalError)
+	TxPoolErrTagTealReject   = "teal_reject" // TEAL returned false ("rejected by ApprovalProgram")
+	TxPoolErrTagMinBalance   = "min_balance" // Account balance below minimum
+	TxPoolErrTagOverspend    = "overspend"   // Insufficient Algo funds
+	TxPoolErrTagAssetBalance = "asset_bal"   // Insufficient asset balance
+	TxPoolErrTagEvalGeneric  = "eval"        // Other evaluation errors not matching known patterns
+)
+
+// TxPoolErrTags is the list of all error tags for use with TagCounter.
+var TxPoolErrTags = []string{
+	TxPoolErrTagCap, TxPoolErrTagPendingEval, TxPoolErrTagNoSpace, TxPoolErrTagFee,
+	TxPoolErrTagTxnDead, TxPoolErrTagTxnEarly, TxPoolErrTagTooLarge, TxPoolErrTagGroupID,
+	TxPoolErrTagTxID, TxPoolErrTagLease, TxPoolErrTagTxIDEval, TxPoolErrTagLeaseEval,
+	TxPoolErrTagNotWell, TxPoolErrTagTealErr, TxPoolErrTagTealReject,
+	TxPoolErrTagMinBalance, TxPoolErrTagOverspend, TxPoolErrTagAssetBalance, TxPoolErrTagEvalGeneric,
+}
+
+// txPoolReevalCounter tracks transaction groups that failed during block assembly
+// re-evaluation. These are transactions that were accepted into the pool but
+// failed when re-evaluated against the latest confirmed block.
+var txPoolReevalCounter = metrics.NewTagCounter(
+	"algod_tx_pool_reeval_{TAG}",
+	"Number of transaction groups removed from pool during re-evaluation due to {TAG}",
+	TxPoolErrTags...,
+)
+
+// txPoolReevalSuccess tracks transaction groups successfully re-evaluated
+// after a new block commits.
+var txPoolReevalSuccess = metrics.MakeCounter(metrics.MetricName{
+	Name:        "algod_tx_pool_reeval_success",
+	Description: "Number of transaction groups successfully re-evaluated after new block",
+})
+
+// txPoolReevalCommitted tracks transaction groups removed during re-evaluation
+// because they were already committed in the latest block.
+var txPoolReevalCommitted = metrics.MakeCounter(metrics.MetricName{
+	Name:        "algod_tx_pool_reeval_committed",
+	Description: "Number of transaction groups removed because already committed in latest block",
+})
+
+// ClassifyTxPoolError examines an error from BlockEvaluator.TransactionGroup
+// and returns the appropriate tag for metrics. Both errors.Is (for sentinel
+// errors) and errors.As (for typed errors) traverse wrapped error chains.
+func ClassifyTxPoolError(err error) string {
+	if err == nil {
+		return ""
+	}
+
+	// Sentinel errors (specific values)
+	if errors.Is(err, ErrPendingQueueReachedMaxCap) {
+		return TxPoolErrTagCap
+	}
+	if errors.Is(err, ErrNoPendingBlockEvaluator) {
+		return TxPoolErrTagPendingEval
+	}
+	if errors.Is(err, ledgercore.ErrNoSpace) {
+		return TxPoolErrTagNoSpace
+	}
+
+	// Typed errors
+	var feeErr *ErrTxPoolFeeError
+	if errors.As(err, &feeErr) {
+		return TxPoolErrTagFee
+	}
+
+	var minFeeErr *transactions.MinFeeError
+	if errors.As(err, &minFeeErr) {
+		return TxPoolErrTagFee
+	}
+
+	var deadErr *bookkeeping.TxnDeadError
+	if errors.As(err, &deadErr) {
+		if deadErr.Early {
+			return TxPoolErrTagTxnEarly
+		}
+		return TxPoolErrTagTxnDead
+	}
+
+	var txInLedgerErr *ledgercore.TransactionInLedgerError
+	if errors.As(err, &txInLedgerErr) {
+		if txInLedgerErr.InBlockEvaluator {
+			return TxPoolErrTagTxIDEval
+		}
+		return TxPoolErrTagTxID
+	}
+
+	var leaseErr *ledgercore.LeaseInLedgerError
+	if errors.As(err, &leaseErr) {
+		if leaseErr.InBlockEvaluator {
+			return TxPoolErrTagLeaseEval
+		}
+		return TxPoolErrTagLease
+	}
+
+	var groupErr *ledgercore.TxGroupMalformedError
+	if errors.As(err, &groupErr) {
+		if groupErr.Reason == ledgercore.TxGroupMalformedErrorReasonExceedMaxSize {
+			return TxPoolErrTagTooLarge
+		}
+		return TxPoolErrTagGroupID
+	}
+
+	var notWellErr *ledgercore.TxnNotWellFormedError
+	if errors.As(err, &notWellErr) {
+		return TxPoolErrTagNotWell
+	}
+
+	var overspendErr *ledgercore.OverspendError
+	if errors.As(err, &overspendErr) {
+		return TxPoolErrTagOverspend
+	}
+
+	var minBalErr *ledgercore.MinBalanceError
+	if errors.As(err, &minBalErr) {
+		return TxPoolErrTagMinBalance
+	}
+
+	var assetBalErr *ledgercore.AssetBalanceError
+	if errors.As(err, &assetBalErr) {
+		return TxPoolErrTagAssetBalance
+	}
+
+	var approvalErr *ledgercore.ApprovalProgramRejectedError
+	if errors.As(err, &approvalErr) {
+		return TxPoolErrTagTealReject
+	}
+
+	var evalErr logic.EvalError
+	if errors.As(err, &evalErr) {
+		return TxPoolErrTagTealErr
+	}
+
+	return TxPoolErrTagEvalGeneric
+}

data/pools/errors_test.go

@@ -0,0 +1,147 @@
+// Copyright (C) 2019-2026 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package pools
+
+import (
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
+	"github.com/algorand/go-algorand/data/transactions"
+	"github.com/algorand/go-algorand/data/transactions/logic"
+	"github.com/algorand/go-algorand/ledger/ledgercore"
+	"github.com/algorand/go-algorand/test/partitiontest"
+	"github.com/algorand/go-algorand/util/metrics"
+)
+
+func TestClassifyTxPoolErrorGeneralCoverage(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// Table mirrors the distinct branches inside ClassifyTxPoolError/classifyUnwrappedError.
+	tcases := []struct {
+		name string
+		err  error
+		tag  string
+		wrap bool
+	}{
+		{name: "cap", err: ErrPendingQueueReachedMaxCap, tag: TxPoolErrTagCap},
+		{name: "cap_wrapped", err: ErrPendingQueueReachedMaxCap, tag: TxPoolErrTagCap, wrap: true},
+		{name: "pending_eval", err: ErrNoPendingBlockEvaluator, tag: TxPoolErrTagPendingEval},
+		{name: "pending_eval_wrapped", err: ErrNoPendingBlockEvaluator, tag: TxPoolErrTagPendingEval, wrap: true},
+		{name: "no_space", err: ledgercore.ErrNoSpace, tag: TxPoolErrTagNoSpace},
+		{name: "no_space_wrapped", err: ledgercore.ErrNoSpace, tag: TxPoolErrTagNoSpace, wrap: true},
+		{name: "fee_escalation", err: &ErrTxPoolFeeError{}, tag: TxPoolErrTagFee},
+		{name: "fee_escalation_wrapped", err: &ErrTxPoolFeeError{}, tag: TxPoolErrTagFee, wrap: true},
+		{name: "fee_min_fee", err: func() error { e := transactions.MinFeeError("min"); return &e }(), tag: TxPoolErrTagFee},
+		{name: "fee_min_fee_wrapped", err: func() error { e := transactions.MinFeeError("min"); return &e }(), tag: TxPoolErrTagFee, wrap: true},
+		{name: "txn_dead", err: &bookkeeping.TxnDeadError{}, tag: TxPoolErrTagTxnDead},
+		{name: "txn_dead_wrapped", err: &bookkeeping.TxnDeadError{}, tag: TxPoolErrTagTxnDead, wrap: true},
+		{name: "txn_early", err: &bookkeeping.TxnDeadError{Early: true}, tag: TxPoolErrTagTxnEarly},
+		{name: "txn_early_wrapped", err: &bookkeeping.TxnDeadError{Early: true}, tag: TxPoolErrTagTxnEarly, wrap: true},
+		{name: "txid_ledger", err: &ledgercore.TransactionInLedgerError{InBlockEvaluator: false}, tag: TxPoolErrTagTxID},
+		{name: "txid_ledger_wrapped", err: &ledgercore.TransactionInLedgerError{InBlockEvaluator: false}, tag: TxPoolErrTagTxID, wrap: true},
+		{name: "txid_eval", err: &ledgercore.TransactionInLedgerError{InBlockEvaluator: true}, tag: TxPoolErrTagTxIDEval},
+		{name: "txid_eval_wrapped", err: &ledgercore.TransactionInLedgerError{InBlockEvaluator: true}, tag: TxPoolErrTagTxIDEval, wrap: true},
+		{name: "lease_ledger", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Lease: [32]byte{1}}, false), tag: TxPoolErrTagLease},
+		{name: "lease_ledger_wrapped", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Lease: [32]byte{1}}, false), tag: TxPoolErrTagLease, wrap: true},
+		{name: "lease_eval", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Lease: [32]byte{2}}, true), tag: TxPoolErrTagLeaseEval},
+		{name: "lease_eval_wrapped", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Lease: [32]byte{2}}, true), tag: TxPoolErrTagLeaseEval, wrap: true},
+		{name: "group_too_large", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonExceedMaxSize}, tag: TxPoolErrTagTooLarge},
+		{name: "group_too_large_wrapped", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonExceedMaxSize}, tag: TxPoolErrTagTooLarge, wrap: true},
+		{name: "group_other", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonGeneric}, tag: TxPoolErrTagGroupID},
+		{name: "group_other_wrapped", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonGeneric}, tag: TxPoolErrTagGroupID, wrap: true},
+		{name: "not_well", err: func() error { e := ledgercore.TxnNotWellFormedError("bad txn"); return &e }(), tag: TxPoolErrTagNotWell},
+		{name: "not_well_wrapped", err: func() error { e := ledgercore.TxnNotWellFormedError("bad txn"); return &e }(), tag: TxPoolErrTagNotWell, wrap: true},
+		{name: "overspend", err: &ledgercore.OverspendError{Account: basics.Address{}, Data: ledgercore.AccountData{}, Tried: basics.MicroAlgos{Raw: 1}}, tag: TxPoolErrTagOverspend},
+		{name: "overspend_wrapped", err: &ledgercore.OverspendError{Account: basics.Address{}, Data: ledgercore.AccountData{}, Tried: basics.MicroAlgos{Raw: 1}}, tag: TxPoolErrTagOverspend, wrap: true},
+		{name: "min_balance", err: &ledgercore.MinBalanceError{Account: basics.Address{}, Balance: 1, MinBalance: 2, TotalAssets: 3}, tag: TxPoolErrTagMinBalance},
+		{name: "min_balance_wrapped", err: &ledgercore.MinBalanceError{Account: basics.Address{}, Balance: 1, MinBalance: 2, TotalAssets: 3}, tag: TxPoolErrTagMinBalance, wrap: true},
+		{name: "asset_balance", err: &ledgercore.AssetBalanceError{Amount: 10, SenderAmount: 5}, tag: TxPoolErrTagAssetBalance},
+		{name: "asset_balance_wrapped", err: &ledgercore.AssetBalanceError{Amount: 10, SenderAmount: 5}, tag: TxPoolErrTagAssetBalance, wrap: true},
+		{name: "approval_reject", err: &ledgercore.ApprovalProgramRejectedError{}, tag: TxPoolErrTagTealReject},
+		{name: "approval_reject_wrapped", err: &ledgercore.ApprovalProgramRejectedError{}, tag: TxPoolErrTagTealReject, wrap: true},
+		{name: "logic_eval", err: logic.EvalError{Err: errors.New("logic")}, tag: TxPoolErrTagTealErr},
+		{name: "logic_eval_wrapped", err: logic.EvalError{Err: errors.New("logic")}, tag: TxPoolErrTagTealErr, wrap: true},
+		{name: "unknown_error", err: errors.New("unknown"), tag: TxPoolErrTagEvalGeneric},
+		{name: "unknown_error_wrapped", err: errors.New("unknown"), tag: TxPoolErrTagEvalGeneric, wrap: true},
+	}
+
+	for _, tc := range tcases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.err
+			if tc.wrap {
+				err = fmt.Errorf("wrap: %w", err)
+			}
+			require.Equal(t, tc.tag, ClassifyTxPoolError(err))
+		})
+	}
+}
+
+func TestTxPoolReevalCounterCoversAllTags(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	// Re-eval counter uses TxPoolErrTags to ensure all possible classification results are predeclared.
+	reevalCases := []struct {
+		name string
+		err  error
+		tag  string
+	}{
+		{name: "fee", err: &ErrTxPoolFeeError{}, tag: TxPoolErrTagFee},
+		{name: "txn_dead", err: &bookkeeping.TxnDeadError{}, tag: TxPoolErrTagTxnDead},
+		{name: "txn_early", err: &bookkeeping.TxnDeadError{Early: true}, tag: TxPoolErrTagTxnEarly},
+		{name: "too_large", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonExceedMaxSize}, tag: TxPoolErrTagTooLarge},
+		{name: "groupid", err: &ledgercore.TxGroupMalformedError{Reason: ledgercore.TxGroupMalformedErrorReasonInconsistentGroupID}, tag: TxPoolErrTagGroupID},
+		{name: "txid", err: &ledgercore.TransactionInLedgerError{Txid: transactions.Txid{}, InBlockEvaluator: false}, tag: TxPoolErrTagTxID},
+		{name: "txid_eval", err: &ledgercore.TransactionInLedgerError{Txid: transactions.Txid{}, InBlockEvaluator: true}, tag: TxPoolErrTagTxIDEval},
+		{name: "lease", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Sender: basics.Address{}, Lease: [32]byte{3}}, false), tag: TxPoolErrTagLease},
+		{name: "lease_eval", err: ledgercore.MakeLeaseInLedgerError(transactions.Txid{}, ledgercore.Txlease{Sender: basics.Address{}, Lease: [32]byte{4}}, true), tag: TxPoolErrTagLeaseEval},
+		{name: "no_space", err: ledgercore.ErrNoSpace, tag: TxPoolErrTagNoSpace},
+		{name: "not_well", err: func() error { e := ledgercore.TxnNotWellFormedError("bad"); return &e }(), tag: TxPoolErrTagNotWell},
+		{name: "teal_err", err: logic.EvalError{Err: errors.New("logic")}, tag: TxPoolErrTagTealErr},
+		{name: "teal_reject", err: &ledgercore.ApprovalProgramRejectedError{}, tag: TxPoolErrTagTealReject},
+		{name: "min_balance", err: &ledgercore.MinBalanceError{Account: basics.Address{}, Balance: 1, MinBalance: 2, TotalAssets: 3}, tag: TxPoolErrTagMinBalance},
+		{name: "overspend", err: &ledgercore.OverspendError{Account: basics.Address{}, Data: ledgercore.AccountData{}, Tried: basics.MicroAlgos{Raw: 1}}, tag: TxPoolErrTagOverspend},
+		{name: "asset_balance", err: &ledgercore.AssetBalanceError{Amount: 10, SenderAmount: 5}, tag: TxPoolErrTagAssetBalance},
+	}
+
+	orig := txPoolReevalCounter
+	txPoolReevalCounter = metrics.NewTagCounter(
+		"algod_tx_pool_reeval_{TAG}",
+		"Number of transaction groups removed from pool during re-evaluation due to {TAG}",
+		TxPoolErrTags...,
+	)
+	t.Cleanup(func() { txPoolReevalCounter = orig })
+
+	for _, tc := range reevalCases {
+		t.Run(tc.name, func(t *testing.T) {
+			tag := ClassifyTxPoolError(tc.err)
+			require.Equal(t, tc.tag, tag)
+			require.Contains(t, TxPoolErrTags, tag)
+			txPoolReevalCounter.Add(tag, 1)
+		})
+	}
+
+	metricsMap := map[string]float64{}
+	txPoolReevalCounter.AddMetric(metricsMap)
+	for _, tc := range reevalCases {
+		require.Equal(t, float64(1), metricsMap["algod_tx_pool_reeval_"+tc.tag])
+	}
+}

data/pools/transactionPool.go

@@ -749,13 +749,17 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIDs map[transact
 		}
 		if _, alreadyCommitted := committedTxIDs[txgroup[0].ID()]; alreadyCommitted {
 			asmStats.EarlyCommittedCount++
+			txPoolReevalCommitted.Inc(nil)
 			continue
 		}
 		err := pool.add(txgroup, &asmStats)
 		if err != nil {
 			for _, tx := range txgroup {
 				pool.statusCache.put(tx, err.Error())
 			}
+
+			txPoolReevalCounter.Add(ClassifyTxPoolError(err), 1)
+
 			// metrics here are duplicated for historic reasons. stats is hardly used and should be removed in favor of asmstats
 			switch terr := err.(type) {
 			case *ledgercore.TransactionInLedgerError:
@@ -785,6 +789,8 @@ func (pool *TransactionPool) recomputeBlockEvaluator(committedTxIDs map[transact
 				stats.RemovedInvalidCount++
 				pool.log.Infof("Pending transaction in pool no longer valid: %v", err)
 			}
+		} else {
+			txPoolReevalSuccess.Inc(nil)
 		}
 	}