[wip] Improve frozen abi coverage, docs and DX

ci/test-checks.sh

@@ -27,6 +27,8 @@ echo --- build environment
 )
 
 export RUST_BACKTRACE=1
+# to suppress noisy complation warnings because of enabled specialization for
+# frozen abi thing...
 export RUSTFLAGS="-D warnings -A incomplete_features"
 
 # Only force up-to-date lock files on edge

ci/test-coverage.sh

@@ -11,6 +11,10 @@ annotate() {
 source ci/upload-ci-artifact.sh
 source scripts/ulimit-n.sh
 
+# to suppress noisy complation warnings because of enabled specialization for
+# frozen abi thing...
+export RUSTFLAGS="-D warnings -A incomplete_features"
+
 scripts/coverage.sh "$@"
 
 if [[ -z $CI ]]; then
@@ -21,9 +25,6 @@ report=coverage-"${CI_COMMIT:0:9}".tar.gz
 mv target/cov/report.tar.gz "$report"
 upload-ci-artifact "$report"
 
-gzip -f target/cov/coverage-stderr.log
-upload-ci-artifact target/cov/coverage-stderr.log.gz
-
 annotate --style success --context lcov-report \
   "lcov report: <a href=\"artifact://$report\">$report</a>"
 

core/src/serve_repair.rs

@@ -33,7 +33,10 @@ use std::{
 pub const MAX_ORPHAN_REPAIR_RESPONSES: usize = 10;
 pub const DEFAULT_NONCE: u32 = 42;
 
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq)]
+#[frozen_abi(digest = "BZQSnWkoizpajDnMmz6axZE1NnTgThq1jcRzxAE2E4EC")]
+#[derive(
+    Serialize, Deserialize, Debug, Clone, Copy, Hash, PartialEq, Eq, AbiExample, AbiEnumVisitor,
+)]
 pub enum RepairType {
     Orphan(Slot),
     HighestShred(Slot, u64),
@@ -62,7 +65,8 @@ pub struct ServeRepairStats {
 }
 
 /// Window protocol messages
-#[derive(Serialize, Deserialize, Debug)]
+#[frozen_abi(digest = "8RYRhKQwGABY5hipqQvZMRrnzFxe3aE1sEFx1sgimFG2")]
+#[derive(Serialize, Deserialize, Debug, AbiExample, AbiEnumVisitor)]
 pub enum RepairProtocol {
     WindowIndex(ContactInfo, u64, u64),
     HighestWindowIndex(ContactInfo, u64, u64),

docs/src/implemented-proposals/abi-management.md

@@ -69,14 +69,51 @@ fields.
 
 # Developer's workflow
 
+## Addition of the digest
+
 To know the digest for new ABI items, developers can add `frozen_abi` with a
 random digest value and run the unit tests and replace it with the correct
 digest from the assertion test error message.
 
+and derive AbiExample, AbiEnumVisitor
+
 In general, once we add `frozen_abi` and its change is published in the stable
 release channel, its digest should never change. If such a change is needed, we
-should opt for defining a new `struct` like `FooV1`. And special release flow
-like hard forks should be approached.
+should carefully do so like defining a new `struct` like `FooV1` and using special
+release flow like hard forks.
+
+
+
+## Update of the digest
+
+### High-level work flow
+
+The mismatched digest is currently disigned to be very conservative. It just says something changed, which might be necesallily incompatible or compatible.
+So, developer should manually confirm the reason of difference of abi digest before updating the digest.
+
+
+for example, depending on context adding another variat to enum might be ok or not ok, so frozen abi alters enginner by failing test and then enginner expected to confirm this case-by-case basis.
+
+So, we must first create digest files pre-/post- suspected abi changes, from which the digest is derived.
+Also, the actual digest is just a base58-encoded sha256
+
+### Actual steps
+
+1. (If the frozen abi test failure is found on CI), prepare to use nightly rust to run tests locally because `frozen_abi` requires rust's unstable feature (specialization).
+2. run the abi test locally without the developer's changes (usually the base commit of the topic branch)
+    ./script/dump-abi-digest-files.sh
+    all tests should generate different digest files for each.
+3. run the abi test locally with the developer's changes
+    ./script/dump-abi-digest-files.sh
+4. diff /tmp/ /tmp/
+
+5. Update the digest
+
+After that, the failing test should start to pass. And if there should be another change, it should start to fail again.
+
+
+# Advanced topics
+IgnoreAsHelper
 
 # Implementation remarks
 

ledger/Cargo.toml

@@ -39,6 +39,7 @@ solana-perf = { path = "../perf", version = "1.4.0" }
 solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "1.4.0" }
 solana-runtime = { path = "../runtime", version = "1.4.0" }
 solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.4.0" }
 solana-stake-program = { path = "../programs/stake", version = "1.4.0" }
 solana-vote-program = { path = "../programs/vote", version = "1.4.0" }
 tempfile = "3.1.0"
@@ -57,6 +58,9 @@ assert_matches = "1.3.0"
 matches = "0.1.6"
 solana-budget-program = { path = "../programs/budget", version = "1.4.0" }
 
+[build-dependencies]
+rustc_version = "0.2"
+
 [lib]
 crate-type = ["lib"]
 name = "solana_ledger"

ledger/build.rs

@@ -0,0 +1 @@
+../sdk/build.rs

ledger/src/blockstore_db.rs

@@ -75,7 +75,7 @@ pub enum IteratorMode<Index> {
     End,
     From(Index, IteratorDirection),
 }
-
+// abi!
 pub mod columns {
     #[derive(Debug)]
     /// The slot metadata column

ledger/src/blockstore_meta.rs

@@ -3,7 +3,10 @@ use serde::{Deserialize, Serialize};
 use solana_sdk::clock::Slot;
 use std::{collections::BTreeSet, ops::RangeBounds};
 
-#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq)]
+// abi
+
+#[frozen_abi(digest = "DBRxERf9gS7cmKDc3sbtQVp7HUdsgAHfSRH5vw8JAEB1")]
+#[derive(Clone, Debug, Default, Deserialize, Serialize, Eq, PartialEq, AbiExample)]
 // The Meta column family
 pub struct SlotMeta {
     // The number of slots above the root (the genesis block). The first

ledger/src/lib.rs

@@ -1,3 +1,5 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
+
 pub mod bank_forks_utils;
 pub mod block_error;
 #[macro_use]
@@ -27,3 +29,6 @@ extern crate log;
 
 #[macro_use]
 extern crate lazy_static;
+
+#[macro_use]
+extern crate solana_sdk_macro_frozen_abi;

ledger/src/shred.rs

@@ -90,16 +90,16 @@ pub enum ShredError {
 
 pub type Result<T> = std::result::Result<T, ShredError>;
 
-#[derive(Serialize, Clone, Deserialize, PartialEq, Debug)]
+#[derive(Serialize, Clone, Deserialize, PartialEq, Debug, AbiExample)]
 pub struct ShredType(pub u8);
 impl Default for ShredType {
     fn default() -> Self {
         ShredType(DATA_SHRED)
     }
 }
-
 /// A common header that is present in data and code shred headers
-#[derive(Serialize, Clone, Deserialize, Default, PartialEq, Debug)]
+#[frozen_abi(digest = "6dscpJYWaLHEpBGEUZR2UUnft1mT3LkvmdUSxG47sjCs")]
+#[derive(Serialize, Clone, Deserialize, Default, PartialEq, Debug, AbiExample)]
 pub struct ShredCommonHeader {
     pub signature: Signature,
     pub shred_type: ShredType,
@@ -110,22 +110,24 @@ pub struct ShredCommonHeader {
 }
 
 /// The data shred header has parent offset and flags
-#[derive(Serialize, Clone, Default, Deserialize, PartialEq, Debug)]
+#[frozen_abi(digest = "8een3rwRLK2wtf7L2Jj6Spp1xv8Ze7LQqnWtmaD2nTnd")]
+#[derive(Serialize, Clone, Default, Deserialize, PartialEq, Debug, AbiExample)]
 pub struct DataShredHeader {
     pub parent_offset: u16,
     pub flags: u8,
     pub size: u16,
 }
 
 /// The coding shred header has FEC information
-#[derive(Serialize, Clone, Default, Deserialize, PartialEq, Debug)]
+#[frozen_abi(digest = "prkkDnjtmAdp644hDa2CiXKWsAA5D9spyiTo2ZPxeto")]
+#[derive(Serialize, Clone, Default, Deserialize, PartialEq, Debug, AbiExample)]
 pub struct CodingShredHeader {
     pub num_data_shreds: u16,
     pub num_coding_shreds: u16,
     pub position: u16,
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, AbiExample)]
 pub struct Shred {
     pub common_header: ShredCommonHeader,
     pub data_header: DataShredHeader,

net-utils/Cargo.toml

@@ -21,10 +21,15 @@ socket2 = "0.3.12"
 solana-clap-utils = { path = "../clap-utils", version = "1.4.0" }
 solana-logger = { path = "../logger", version = "1.4.0" }
 solana-version = { path = "../version", version = "1.4.0" }
+solana-sdk = { path = "../sdk", version = "1.4.0" }
+solana-sdk-macro-frozen-abi = { path = "../sdk/macro-frozen-abi", version = "1.4.0" }
 tokio = "0.1"
 tokio-codec = "0.1"
 url = "2.1.1"
 
+[build-dependencies]
+rustc_version = "0.2"
+
 [lib]
 name = "solana_net_utils"
 

net-utils/build.rs

@@ -0,0 +1 @@
+../sdk/build.rs

net-utils/src/ip_echo_server.rs

@@ -9,7 +9,8 @@ pub type IpEchoServer = Runtime;
 
 pub const MAX_PORT_COUNT_PER_MESSAGE: usize = 4;
 
-#[derive(Serialize, Deserialize, Default)]
+#[frozen_abi(digest = "7NQSrBcS4djwS5ZCdzfDj9dvsPp2Zzx9E4NzZzN52QhG")]
+#[derive(Serialize, Deserialize, Default, AbiExample)]
 pub(crate) struct IpEchoServerMessage {
     tcp_ports: [u16; MAX_PORT_COUNT_PER_MESSAGE], // Fixed size list of ports to avoid vec serde
     udp_ports: [u16; MAX_PORT_COUNT_PER_MESSAGE], // Fixed size list of ports to avoid vec serde

net-utils/src/lib.rs

@@ -1,3 +1,5 @@
+#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(specialization))]
+
 //! The `net_utils` module assists with networking
 use log::*;
 use rand::{thread_rng, Rng};
@@ -13,6 +15,9 @@ mod ip_echo_server;
 use ip_echo_server::IpEchoServerMessage;
 pub use ip_echo_server::{ip_echo_server, IpEchoServer, MAX_PORT_COUNT_PER_MESSAGE};
 
+#[macro_use]
+extern crate solana_sdk_macro_frozen_abi;
+
 /// A data type representing a public Udp socket
 pub struct UdpSocketPair {
     pub addr: SocketAddr,    // Public address of the socket

programs/stake/src/stake_instruction.rs

@@ -48,7 +48,8 @@ impl<E> DecodeError<E> for StakeError {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
+#[frozen_abi(digest = "CQqMwt8fA1LD5FhwLDNCUxn7NKZ1RCX1TnBRf3xU1KhK")]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, AbiExample, AbiEnumVisitor)]
 pub enum StakeInstruction {
     /// Initialize a stake with lockup and authorization information
     ///
@@ -133,7 +134,7 @@ pub enum StakeInstruction {
     Merge,
 }
 
-#[derive(Default, Debug, Serialize, Deserialize, PartialEq, Clone, Copy)]
+#[derive(Default, Debug, Serialize, Deserialize, PartialEq, Clone, Copy, AbiExample)]
 pub struct LockupArgs {
     pub unix_timestamp: Option<UnixTimestamp>,
     pub epoch: Option<Epoch>,

programs/stake/src/stake_state.rs

@@ -91,7 +91,7 @@ impl StakeState {
     }
 }
 
-#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Copy, AbiExample)]
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Copy, AbiExample, AbiEnumVisitor)]
 pub enum StakeAuthorize {
     Staker,
     Withdrawer,

programs/vote/src/vote_instruction.rs

@@ -50,7 +50,8 @@ impl<E> DecodeError<E> for VoteError {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
+#[frozen_abi(digest = "GyBYC41AUZHvF1QTfYupgL3tgFj57NHVYWFwStuxxjPm")]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, AbiExample, AbiEnumVisitor)]
 pub enum VoteInstruction {
     /// Initialize a vote account
     ///

programs/vote/src/vote_state/mod.rs

@@ -90,15 +90,15 @@ impl Lockout {
     }
 }
 
-#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy)]
+#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, AbiExample)]
 pub struct VoteInit {
     pub node_pubkey: Pubkey,
     pub authorized_voter: Pubkey,
     pub authorized_withdrawer: Pubkey,
     pub commission: u8,
 }
 
-#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy)]
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone, Copy, AbiExample, AbiEnumVisitor)]
 pub enum VoteAuthorize {
     Voter,
     Withdrawer,

scripts/coverage.sh

@@ -51,14 +51,23 @@ if [[ -n $CI || -z $1 ]]; then
     $(git grep -l "proc-macro.*true" :**/Cargo.toml | sed 's|Cargo.toml|src/lib.rs|')
 fi
 
+log_file=target/cov/coverage-stderr.log
+
 RUST_LOG=solana=trace _ cargo +$rust_nightly test --target-dir target/cov --no-run "${packages[@]}"
-if RUST_LOG=solana=trace _ cargo +$rust_nightly test --target-dir target/cov "${packages[@]}" 2> target/cov/coverage-stderr.log; then
+if RUST_LOG=solana=trace _ cargo +$rust_nightly test --target-dir target/cov "${packages[@]}" 2> "${log_file}"; then
   test_status=0
 else
   test_status=$?
   echo "Failed: $test_status"
   echo "^^^ +++"
   if [[ -n $CI ]]; then
+    # This should reside in ci/test-coverage.sh but placing here is less complicated
+    # than the ideal coding..
+    tail -n 500 "${log_file}"
+    gzip -f  "${log_file}"
+    source ci/upload-ci-artifact.sh
+    upload-ci-artifact "${log_file}.gz"
+
     exit $test_status
   fi
 fi