From ea1df48e2ca0cf6973eaa4ed1d1590f6912de257 Mon Sep 17 00:00:00 2001 From: Ankan Date: Mon, 6 Apr 2026 10:36:07 +0200 Subject: [PATCH 1/8] collect errors not spam --- substrate/frame/nomination-pools/src/lib.rs | 57 +++++++--- .../frame/staking-async/src/pallet/impls.rs | 102 +++++++++++++----- 2 files changed, 116 insertions(+), 43 deletions(-) diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index 1cb1a2bcb3d56..ee2b2f150297e 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -4008,7 +4008,13 @@ impl Pallet { })?; let mut expected_tvl: BalanceOf = Default::default(); + let mut depositor_undermin: Vec<(PoolId, T::AccountId)> = Vec::new(); + let mut depositor_undermin_total: u32 = 0; + let mut total_pools: u32 = 0; + const MAX_EXAMPLES: usize = 10; + BondedPools::::iter().try_for_each(|(id, inner)| -> Result<(), TryRuntimeError> { + total_pools += 1; let bonded_pool = BondedPool { id, inner }; ensure!( pools_members.get(&id).copied().unwrap_or_default() == @@ -4026,14 +4032,10 @@ impl Pallet { .is_destroying_and_only_depositor(depositor.active_points()) || depositor.active_points() >= MinCreateBond::::get(); if !depositor_has_enough_stake { - log!( - warn, - "pool {:?} has depositor {:?} with insufficient stake {:?}, minimum required is {:?}", - id, - bonded_pool.roles.depositor, - depositor.active_points(), - MinCreateBond::::get() - ); + depositor_undermin_total += 1; + if depositor_undermin.len() < MAX_EXAMPLES { + depositor_undermin.push((id, bonded_pool.roles.depositor.clone())); + } } ensure!( @@ -4046,6 +4048,17 @@ impl Pallet { Ok(()) })?; + if depositor_undermin_total > 0 { + log!( + warn, + "{}/{} pools have depositor with insufficient stake, minimum required is {:?}. Examples: {:?}", + depositor_undermin_total, + total_pools, + MinCreateBond::::get(), + depositor_undermin, + ); + } + ensure!( MaxPoolMembers::::get().map_or(true, |max| all_members <= max), Error::::MaxPoolMembers @@ -4108,8 +4121,13 @@ impl Pallet { debug_assertions ))] pub fn check_ed_imbalance() -> Result { - let mut needs_adjust = 0; + let mut needs_adjust: u32 = 0; + let mut total_pools: u32 = 0; + let mut ed_examples: Vec = Vec::new(); + const MAX_EXAMPLES: usize = 10; + BondedPools::::iter_keys().for_each(|id| { + total_pools += 1; let reward_acc = Self::generate_reward_account(id); let frozen_balance = T::Currency::balance_frozen(&FreezeReason::PoolMinBalance.into(), &reward_acc); @@ -4117,16 +4135,23 @@ impl Pallet { let expected_frozen_balance = T::Currency::minimum_balance(); if frozen_balance != expected_frozen_balance { needs_adjust += 1; - log!( - warn, - "pool {:?} has incorrect ED frozen that can result from change in ED. Expected = {:?}, Actual = {:?}. Use `adjust_pool_deposit` to fix it", - id, - expected_frozen_balance, - frozen_balance, - ); + if ed_examples.len() < MAX_EXAMPLES { + ed_examples.push(id); + } } }); + if needs_adjust > 0 { + log!( + warn, + "{}/{} pools have incorrect ED frozen (expected {:?}). Use `adjust_pool_deposit` to fix. Examples: {:?}", + needs_adjust, + total_pools, + T::Currency::minimum_balance(), + ed_examples, + ); + } + Ok(needs_adjust) } /// Fully unbond the shares of `member`, when executed from `origin`. diff --git a/substrate/frame/staking-async/src/pallet/impls.rs b/substrate/frame/staking-async/src/pallet/impls.rs index 07314b0087565..b3f9b37493585 100644 --- a/substrate/frame/staking-async/src/pallet/impls.rs +++ b/substrate/frame/staking-async/src/pallet/impls.rs @@ -1945,8 +1945,17 @@ impl Pallet { /// * For virtual stakers, locked funds should be zero and payee should be non-stash account. /// * Staking ledger and bond are not corrupted. fn check_ledgers() -> Result<(), TryRuntimeError> { + let mut chilled_undermin: Vec = Vec::new(); + let mut chilled_total: u32 = 0; + let mut nominator_undermin: Vec = Vec::new(); + let mut nominator_total: u32 = 0; + let mut validator_undermin: Vec = Vec::new(); + let mut validator_total: u32 = 0; + let mut total_ledgers: u32 = 0; + Bonded::::iter() .map(|(stash, ctrl)| { + total_ledgers += 1; // ensure locks consistency. if VirtualStakers::::contains_key(stash.clone()) { ensure!( @@ -1990,10 +1999,50 @@ impl Pallet { } Self::ensure_ledger_consistent(&ctrl)?; - Self::ensure_ledger_role_and_min_bond(&ctrl)?; + Self::collect_min_bond_violations( + &ctrl, + &mut chilled_undermin, + &mut chilled_total, + &mut nominator_undermin, + &mut nominator_total, + &mut validator_undermin, + &mut validator_total, + )?; Ok(()) }) .collect::, _>>()?; + + if chilled_total > 0 { + log!( + warn, + "{}/{} chilled stashes have less stake than minimum role bond ({:?}). Examples: {:?}", + chilled_total, + total_ledgers, + Self::min_chilled_bond(), + chilled_undermin, + ); + } + if nominator_total > 0 { + log!( + warn, + "{}/{} nominators have less stake than minimum role bond ({:?}). Examples: {:?}", + nominator_total, + total_ledgers, + Self::min_nominator_bond(), + nominator_undermin, + ); + } + if validator_total > 0 { + log!( + warn, + "{}/{} validators have less stake than minimum role bond ({:?}). Examples: {:?}", + validator_total, + total_ledgers, + Self::min_validator_bond(), + validator_undermin, + ); + } + Ok(()) } @@ -2133,7 +2182,18 @@ impl Pallet { Ok(()) } - fn ensure_ledger_role_and_min_bond(ctrl: &T::AccountId) -> Result<(), TryRuntimeError> { + /// Checks if a ledger's stash has less stake than the minimum for its role and collects + /// violations into the provided accumulators (up to `MAX_EXAMPLES` examples per role). + fn collect_min_bond_violations( + ctrl: &T::AccountId, + chilled_undermin: &mut Vec, + chilled_total: &mut u32, + nominator_undermin: &mut Vec, + nominator_total: &mut u32, + validator_undermin: &mut Vec, + validator_total: &mut u32, + ) -> Result<(), TryRuntimeError> { + const MAX_EXAMPLES: usize = 10; let ledger = Self::ledger(StakingAccount::Controller(ctrl.clone()))?; let stash = ledger.stash; @@ -2142,40 +2202,28 @@ impl Pallet { match (is_nominator, is_validator) { (false, false) => { + // chilled accounts allow to go to zero and fully unbond if ledger.active < Self::min_chilled_bond() && !ledger.active.is_zero() { - // chilled accounts allow to go to zero and fully unbond ^^^^^^^^^ - log!( - warn, - "Chilled stash {:?} has less stake ({:?}) than minimum role bond ({:?})", - stash, - ledger.active, - Self::min_chilled_bond() - ); + *chilled_total += 1; + if chilled_undermin.len() < MAX_EXAMPLES { + chilled_undermin.push(stash); + } } - // is chilled }, (true, false) => { - // Nominators must have a minimum bond. if ledger.active < Self::min_nominator_bond() { - log!( - warn, - "Nominator {:?} has less stake ({:?}) than minimum role bond ({:?})", - stash, - ledger.active, - Self::min_nominator_bond() - ); + *nominator_total += 1; + if nominator_undermin.len() < MAX_EXAMPLES { + nominator_undermin.push(stash); + } } }, (false, true) => { - // Validators must have a minimum bond. if ledger.active < Self::min_validator_bond() { - log!( - warn, - "Validator {:?} has less stake ({:?}) than minimum role bond ({:?})", - stash, - ledger.active, - Self::min_validator_bond() - ); + *validator_total += 1; + if validator_undermin.len() < MAX_EXAMPLES { + validator_undermin.push(stash); + } } }, (true, true) => { From 82af640cec6c68990a5cc95409ecfc8637c30c50 Mon Sep 17 00:00:00 2001 From: Ankan Date: Mon, 6 Apr 2026 13:09:50 +0200 Subject: [PATCH 2/8] prdoc --- prdoc/11649.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 prdoc/11649.md diff --git a/prdoc/11649.md b/prdoc/11649.md new file mode 100644 index 0000000000000..37fd888329404 --- /dev/null +++ b/prdoc/11649.md @@ -0,0 +1,15 @@ +title: 'Consolidate try-state warnings into summary counts' +doc: +- audience: Runtime Dev + description: |- + Aggregates repetitive try-state warnings (min bond violations, pool ED imbalance, depositor + insufficient stake) into single summary lines with counts and up to 10 example accounts. + Reduces log noise from hundreds of expected per-item warnings on production runtimes. + + Closes #11646. + +crates: +- name: pallet-staking-async + bump: patch +- name: pallet-nomination-pools + bump: patch From 23bb0269b5aa824c0ff5cf7997db97ba19db5f95 Mon Sep 17 00:00:00 2001 From: Ankan Date: Mon, 6 Apr 2026 13:11:18 +0200 Subject: [PATCH 3/8] fix path --- prdoc/{11649.md => pr_11649.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename prdoc/{11649.md => pr_11649.md} (100%) diff --git a/prdoc/11649.md b/prdoc/pr_11649.md similarity index 100% rename from prdoc/11649.md rename to prdoc/pr_11649.md From 33a1e37df03de00e9f6817c164ce2d13932c44aa Mon Sep 17 00:00:00 2001 From: Ankan Date: Mon, 6 Apr 2026 18:44:20 +0200 Subject: [PATCH 4/8] comeon --- prdoc/pr_11649.md | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 prdoc/pr_11649.md diff --git a/prdoc/pr_11649.md b/prdoc/pr_11649.md deleted file mode 100644 index 37fd888329404..0000000000000 --- a/prdoc/pr_11649.md +++ /dev/null @@ -1,15 +0,0 @@ -title: 'Consolidate try-state warnings into summary counts' -doc: -- audience: Runtime Dev - description: |- - Aggregates repetitive try-state warnings (min bond violations, pool ED imbalance, depositor - insufficient stake) into single summary lines with counts and up to 10 example accounts. - Reduces log noise from hundreds of expected per-item warnings on production runtimes. - - Closes #11646. - -crates: -- name: pallet-staking-async - bump: patch -- name: pallet-nomination-pools - bump: patch From cb6c8eeb2c2b3feff670bf71b2d9e5c8e86dabbb Mon Sep 17 00:00:00 2001 From: Ankan Date: Mon, 6 Apr 2026 18:50:57 +0200 Subject: [PATCH 5/8] really --- prdoc/pr_11649.prdoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 prdoc/pr_11649.prdoc diff --git a/prdoc/pr_11649.prdoc b/prdoc/pr_11649.prdoc new file mode 100644 index 0000000000000..37fd888329404 --- /dev/null +++ b/prdoc/pr_11649.prdoc @@ -0,0 +1,15 @@ +title: 'Consolidate try-state warnings into summary counts' +doc: +- audience: Runtime Dev + description: |- + Aggregates repetitive try-state warnings (min bond violations, pool ED imbalance, depositor + insufficient stake) into single summary lines with counts and up to 10 example accounts. + Reduces log noise from hundreds of expected per-item warnings on production runtimes. + + Closes #11646. + +crates: +- name: pallet-staking-async + bump: patch +- name: pallet-nomination-pools + bump: patch From 8f058d55d706f4264033063d627b05fb230b10c0 Mon Sep 17 00:00:00 2001 From: Ankan Date: Tue, 7 Apr 2026 20:24:37 +0200 Subject: [PATCH 6/8] preserve trace logs --- substrate/frame/nomination-pools/src/lib.rs | 15 +++++++++ .../frame/staking-async/src/pallet/impls.rs | 33 +++++++++++++++---- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index ee2b2f150297e..fb59dd56956a5 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -4036,6 +4036,14 @@ impl Pallet { if depositor_undermin.len() < MAX_EXAMPLES { depositor_undermin.push((id, bonded_pool.roles.depositor.clone())); } + log!( + trace, + "pool {:?} has depositor {:?} with insufficient stake {:?}, minimum required is {:?}", + id, + bonded_pool.roles.depositor, + depositor.active_points(), + MinCreateBond::::get() + ); } ensure!( @@ -4138,6 +4146,13 @@ impl Pallet { if ed_examples.len() < MAX_EXAMPLES { ed_examples.push(id); } + log!( + trace, + "pool {:?} has incorrect ED frozen. Expected = {:?}, Actual = {:?}. Use `adjust_pool_deposit` to fix.", + id, + expected_frozen_balance, + frozen_balance, + ); } }); diff --git a/substrate/frame/staking-async/src/pallet/impls.rs b/substrate/frame/staking-async/src/pallet/impls.rs index b3f9b37493585..1026394c6789b 100644 --- a/substrate/frame/staking-async/src/pallet/impls.rs +++ b/substrate/frame/staking-async/src/pallet/impls.rs @@ -2015,7 +2015,7 @@ impl Pallet { if chilled_total > 0 { log!( warn, - "{}/{} chilled stashes have less stake than minimum role bond ({:?}). Examples: {:?}", + "{} chilled stashes (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", chilled_total, total_ledgers, Self::min_chilled_bond(), @@ -2025,7 +2025,7 @@ impl Pallet { if nominator_total > 0 { log!( warn, - "{}/{} nominators have less stake than minimum role bond ({:?}). Examples: {:?}", + "{} nominators (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", nominator_total, total_ledgers, Self::min_nominator_bond(), @@ -2035,7 +2035,7 @@ impl Pallet { if validator_total > 0 { log!( warn, - "{}/{} validators have less stake than minimum role bond ({:?}). Examples: {:?}", + "{} validators (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", validator_total, total_ledgers, Self::min_validator_bond(), @@ -2206,24 +2206,45 @@ impl Pallet { if ledger.active < Self::min_chilled_bond() && !ledger.active.is_zero() { *chilled_total += 1; if chilled_undermin.len() < MAX_EXAMPLES { - chilled_undermin.push(stash); + chilled_undermin.push(stash.clone()); } + log!( + trace, + "Chilled stash {:?} has less stake ({:?}) than minimum bond ({:?})", + stash, + ledger.active, + Self::min_chilled_bond() + ); } }, (true, false) => { if ledger.active < Self::min_nominator_bond() { *nominator_total += 1; if nominator_undermin.len() < MAX_EXAMPLES { - nominator_undermin.push(stash); + nominator_undermin.push(stash.clone()); } + log!( + trace, + "Nominator {:?} has less stake ({:?}) than minimum bond ({:?})", + stash, + ledger.active, + Self::min_nominator_bond() + ); } }, (false, true) => { if ledger.active < Self::min_validator_bond() { *validator_total += 1; if validator_undermin.len() < MAX_EXAMPLES { - validator_undermin.push(stash); + validator_undermin.push(stash.clone()); } + log!( + trace, + "Validator {:?} has less stake ({:?}) than minimum bond ({:?})", + stash, + ledger.active, + Self::min_validator_bond() + ); } }, (true, true) => { From 14d51a7352fcb376c16d06b38578b0c38709c16d Mon Sep 17 00:00:00 2001 From: Ankan Date: Tue, 7 Apr 2026 20:31:24 +0200 Subject: [PATCH 7/8] fix prdoc --- prdoc/pr_11649.prdoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prdoc/pr_11649.prdoc b/prdoc/pr_11649.prdoc index 37fd888329404..674f71f1adc96 100644 --- a/prdoc/pr_11649.prdoc +++ b/prdoc/pr_11649.prdoc @@ -2,11 +2,11 @@ title: 'Consolidate try-state warnings into summary counts' doc: - audience: Runtime Dev description: |- - Aggregates repetitive try-state warnings (min bond violations, pool ED imbalance, depositor - insufficient stake) into single summary lines with counts and up to 10 example accounts. - Reduces log noise from hundreds of expected per-item warnings on production runtimes. + Aggregates repetitive try-state warnings (min bond violations, pool ED imbalance, depositor + insufficient stake) into single summary lines with counts and up to 10 example accounts. + Reduces log noise from hundreds of expected per-item warnings on production runtimes. - Closes #11646. + Closes #11646. crates: - name: pallet-staking-async From 63e3b7f873af867d63dfdc04663afd6bff0b2125 Mon Sep 17 00:00:00 2001 From: Ankan Date: Tue, 7 Apr 2026 20:41:54 +0200 Subject: [PATCH 8/8] minimise diff --- substrate/frame/nomination-pools/src/lib.rs | 2 +- .../frame/staking-async/src/pallet/impls.rs | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index fb59dd56956a5..61557c28adbdc 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -4148,7 +4148,7 @@ impl Pallet { } log!( trace, - "pool {:?} has incorrect ED frozen. Expected = {:?}, Actual = {:?}. Use `adjust_pool_deposit` to fix.", + "pool {:?} has incorrect ED frozen that can result from change in ED. Expected = {:?}, Actual = {:?}. Use `adjust_pool_deposit` to fix it", id, expected_frozen_balance, frozen_balance, diff --git a/substrate/frame/staking-async/src/pallet/impls.rs b/substrate/frame/staking-async/src/pallet/impls.rs index e075618bf761e..19eb519341f55 100644 --- a/substrate/frame/staking-async/src/pallet/impls.rs +++ b/substrate/frame/staking-async/src/pallet/impls.rs @@ -2015,7 +2015,7 @@ impl Pallet { if chilled_total > 0 { log!( warn, - "{} chilled stashes (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", + "{} chilled stashes (out of {} total ledgers) have less stake than minimum role bond ({:?}). Examples: {:?}", chilled_total, total_ledgers, Self::min_chilled_bond(), @@ -2025,7 +2025,7 @@ impl Pallet { if nominator_total > 0 { log!( warn, - "{} nominators (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", + "{} nominators (out of {} total ledgers) have less stake than minimum role bond ({:?}). Examples: {:?}", nominator_total, total_ledgers, Self::min_nominator_bond(), @@ -2035,7 +2035,7 @@ impl Pallet { if validator_total > 0 { log!( warn, - "{} validators (out of {} total ledgers) have less stake than minimum bond ({:?}). Examples: {:?}", + "{} validators (out of {} total ledgers) have less stake than minimum role bond ({:?}). Examples: {:?}", validator_total, total_ledgers, Self::min_validator_bond(), @@ -2204,22 +2204,24 @@ impl Pallet { match (is_nominator, is_validator) { (false, false) => { - // chilled accounts allow to go to zero and fully unbond if ledger.active < Self::min_chilled_bond() && !ledger.active.is_zero() { + // chilled accounts allow to go to zero and fully unbond ^^^^^^^^^ *chilled_total += 1; if chilled_undermin.len() < MAX_EXAMPLES { chilled_undermin.push(stash.clone()); } log!( trace, - "Chilled stash {:?} has less stake ({:?}) than minimum bond ({:?})", + "Chilled stash {:?} has less stake ({:?}) than minimum role bond ({:?})", stash, ledger.active, Self::min_chilled_bond() ); } + // is chilled }, (true, false) => { + // Nominators must have a minimum bond. if ledger.active < Self::min_nominator_bond() { *nominator_total += 1; if nominator_undermin.len() < MAX_EXAMPLES { @@ -2227,7 +2229,7 @@ impl Pallet { } log!( trace, - "Nominator {:?} has less stake ({:?}) than minimum bond ({:?})", + "Nominator {:?} has less stake ({:?}) than minimum role bond ({:?})", stash, ledger.active, Self::min_nominator_bond() @@ -2235,6 +2237,7 @@ impl Pallet { } }, (false, true) => { + // Validators must have a minimum bond. if ledger.active < Self::min_validator_bond() { *validator_total += 1; if validator_undermin.len() < MAX_EXAMPLES { @@ -2242,7 +2245,7 @@ impl Pallet { } log!( trace, - "Validator {:?} has less stake ({:?}) than minimum bond ({:?})", + "Validator {:?} has less stake ({:?}) than minimum role bond ({:?})", stash, ledger.active, Self::min_validator_bond()