From e4bc3e1a66e084d40783d2ebb76e54719d5fbe14 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 6 Oct 2020 18:24:31 +0900 Subject: [PATCH 1/6] Detect vote hash mismatch even at run-sanity.sh --- ci/run-sanity.sh | 3 ++- programs/vote/src/vote_state/mod.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 36f423c53e4b3c..6cb3ac433fd37e 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -31,4 +31,5 @@ wait $pid $solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" config/snapshot-ledger cp config/ledger/genesis.tar.bz2 config/snapshot-ledger -$solana_ledger_tool verify --ledger config/snapshot-ledger +$solana_ledger_tool verify --ledger config/snapshot-ledger | tee "ledger-verify.log" +! grep "failed to match hash" "ledger-verify.log" diff --git a/programs/vote/src/vote_state/mod.rs b/programs/vote/src/vote_state/mod.rs index 9b61d847279c52..6ebc970f719ecd 100644 --- a/programs/vote/src/vote_state/mod.rs +++ b/programs/vote/src/vote_state/mod.rs @@ -323,8 +323,9 @@ impl VoteState { return Err(VoteError::SlotsMismatch); } if slot_hashes[j].1 != vote.hash { + // UPDATE ci/run-sanity.sh AS WELL WHEN CHANGING THIS LOG FORMAT! warn!( - "{} dropped vote {:?} failed to match hash {} {}", + "{} dropped vote {:?} failed to match hash: {} (vorter) != {} (local)", self.node_pubkey, vote, vote.hash, slot_hashes[j].1 ); inc_new_counter_info!("dropped-vote-hash", 1); From df34e27135a66def47054a5ffa2ff64ab7bdb328 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 6 Oct 2020 19:43:33 +0900 Subject: [PATCH 2/6] Redirect stderr... --- ci/run-sanity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 6cb3ac433fd37e..6a395bead6f6d7 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -31,5 +31,5 @@ wait $pid $solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" config/snapshot-ledger cp config/ledger/genesis.tar.bz2 config/snapshot-ledger -$solana_ledger_tool verify --ledger config/snapshot-ledger | tee "ledger-verify.log" +$solana_ledger_tool verify --ledger config/snapshot-ledger 2>&1 | tee "ledger-verify.log" ! grep "failed to match hash" "ledger-verify.log" From ae3bfef88860919913d98e600e2040dec49a672f Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 6 Oct 2020 19:44:21 +0900 Subject: [PATCH 3/6] Improve message clarity a bit --- programs/vote/src/vote_state/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/vote/src/vote_state/mod.rs b/programs/vote/src/vote_state/mod.rs index 6ebc970f719ecd..84f35d81f2b6ce 100644 --- a/programs/vote/src/vote_state/mod.rs +++ b/programs/vote/src/vote_state/mod.rs @@ -325,7 +325,7 @@ impl VoteState { if slot_hashes[j].1 != vote.hash { // UPDATE ci/run-sanity.sh AS WELL WHEN CHANGING THIS LOG FORMAT! warn!( - "{} dropped vote {:?} failed to match hash: {} (vorter) != {} (local)", + "{} dropped vote {:?} failed to match hash: {} (vorter's) != {} (local's)", self.node_pubkey, vote, vote.hash, slot_hashes[j].1 ); inc_new_counter_info!("dropped-vote-hash", 1); From eb374df4462bee2ee44d00e5ce922caf91803e76 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 6 Oct 2020 19:47:56 +0900 Subject: [PATCH 4/6] Fix type and tweak message even more --- programs/vote/src/vote_state/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/vote/src/vote_state/mod.rs b/programs/vote/src/vote_state/mod.rs index 84f35d81f2b6ce..9a2dcbc183eeac 100644 --- a/programs/vote/src/vote_state/mod.rs +++ b/programs/vote/src/vote_state/mod.rs @@ -325,7 +325,7 @@ impl VoteState { if slot_hashes[j].1 != vote.hash { // UPDATE ci/run-sanity.sh AS WELL WHEN CHANGING THIS LOG FORMAT! warn!( - "{} dropped vote {:?} failed to match hash: {} (vorter's) != {} (local's)", + "{} dropped vote {:?} failed to match hash: {} (vote's) != {} (local's)", self.node_pubkey, vote, vote.hash, slot_hashes[j].1 ); inc_new_counter_info!("dropped-vote-hash", 1); From b62f4f262ab77edfa3eaaf7cdab33eb2eb456a55 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Tue, 6 Oct 2020 23:25:48 +0900 Subject: [PATCH 5/6] Really fix... --- ci/run-sanity.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index 6a395bead6f6d7..cd8b8eb645ba1a 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -29,7 +29,8 @@ curl -X POST -H 'Content-Type: application/json' -d '{"jsonrpc":"2.0","id":1, "m wait $pid -$solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" config/snapshot-ledger +$solana_ledger_tool create-snapshot --ledger config/ledger "$snapshot_slot" config/snapshot-ledger 2>&1 \ + | tee -a "ledger-tool.log" cp config/ledger/genesis.tar.bz2 config/snapshot-ledger -$solana_ledger_tool verify --ledger config/snapshot-ledger 2>&1 | tee "ledger-verify.log" -! grep "failed to match hash" "ledger-verify.log" +$solana_ledger_tool verify --ledger config/snapshot-ledger 2>&1 | tee -a "ledger-tool.log" +! grep "failed to match hash" "ledger-tool.log" From 21d30594d8103e3c98b010a789356aa9cd22f63f Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Wed, 7 Oct 2020 00:36:46 +0900 Subject: [PATCH 6/6] Well, accept compromise; prefer debug over clarity --- ci/run-sanity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run-sanity.sh b/ci/run-sanity.sh index cd8b8eb645ba1a..50d73dd5e9155c 100755 --- a/ci/run-sanity.sh +++ b/ci/run-sanity.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -ex cd "$(dirname "$0")/.." # shellcheck source=multinode-demo/common.sh