Skip to content

Commit a9ee3e8

Browse files
committed
Auto merge of #14031 - epage:snap, r=weihanglo
tests: Migrate alt_registry to snapbox ### What does this PR try to resolve? The overall goal is to enable the use of snapshot testing on as many cargo tests as possible to reduce the burden when having to touch a lot of tests. Towards that aim, this PR - Adds snapshot testing to `cargo_test_support::Execs` - Migrates `alt_registry` tests over as an example (and to vet it) I've taken the approach of deprecating all other output assertions on `Execs` with `#[allow(deprecated)]` in every test file. This let's us easily identity what files haven't been migrated, what in a file needs migration, and helps prevent a file from regressing. This should make it easier to do a gradual migration that (as many people as they want) can chip in. It comes at the cost of losing visibility into deprecated items we use. Hopefully we won't be in this intermediate state for too long. To reduce manual touch ups of snapshots, I've added some regex redactions. My main concern with the `FILE_SIZE` redaction was when we test for specific sizes. That shouldn't be a problem because we don't use `Execs::with_stderr` to test those but we capture the output and have a custom asserter for it. ### How should we test and review this PR? Yes, this puts us in an intermediate state which isn't ideal but much better than one person trying to do all of this in a single branch / PR. The main risk is that we'll hit a snag with snapbox being able to support our needs. We got away with a lot because everything was custom, down to the diffing algorithm. This is why I at least started with `alt_registry` to get a feel for what problems we might have. There will likely be some we uncover. I'm fairly confident that we can resolve them in some way, ### Additional information This is a continuation of the work done in #13980.
2 parents cc34b84 + e589ed7 commit a9ee3e8

File tree

149 files changed

+1235
-565
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

149 files changed

+1235
-565
lines changed

Cargo.lock

+2-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ sha1 = "0.10.6"
9191
sha2 = "0.10.8"
9292
shell-escape = "0.1.5"
9393
supports-hyperlinks = "3.0.0"
94-
snapbox = { version = "0.6.5", features = ["diff", "dir", "term-svg", "regex"] }
94+
snapbox = { version = "0.6.9", features = ["diff", "dir", "term-svg", "regex", "json"] }
9595
tar = { version = "0.4.40", default-features = false }
9696
tempfile = "3.10.1"
9797
thiserror = "1.0.59"

crates/cargo-test-support/Cargo.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "cargo-test-support"
3-
version = "0.2.1"
3+
version = "0.2.2"
44
edition.workspace = true
55
rust-version = "1.78" # MSRV:1
66
license.workspace = true

crates/cargo-test-support/src/compare.rs

+20
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,16 @@ pub fn assert_ui() -> snapbox::Assert {
9292
regex::Regex::new("Finished.*in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
9393
)
9494
.unwrap();
95+
subs.insert(
96+
"[FILE_SIZE]",
97+
regex::Regex::new("(?<redacted>[0-9]+(\\.[0-9]+)([a-zA-Z]i)?)B").unwrap(),
98+
)
99+
.unwrap();
100+
subs.insert(
101+
"[HASH]",
102+
regex::Regex::new("home/\\.cargo/registry/src/-(?<redacted>[a-z0-9]+)").unwrap(),
103+
)
104+
.unwrap();
95105
snapbox::Assert::new()
96106
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
97107
.redact_with(subs)
@@ -146,6 +156,16 @@ pub fn assert_e2e() -> snapbox::Assert {
146156
regex::Regex::new("[FINISHED].*in (?<redacted>[0-9]+(\\.[0-9]+))s").unwrap(),
147157
)
148158
.unwrap();
159+
subs.insert(
160+
"[FILE_SIZE]",
161+
regex::Regex::new("(?<redacted>[0-9]+(\\.[0-9]+)([a-zA-Z]i)?)B").unwrap(),
162+
)
163+
.unwrap();
164+
subs.insert(
165+
"[HASH]",
166+
regex::Regex::new("home/\\.cargo/registry/src/-(?<redacted>[a-z0-9]+)").unwrap(),
167+
)
168+
.unwrap();
149169
snapbox::Assert::new()
150170
.action_env(snapbox::assert::DEFAULT_ACTION_ENV)
151171
.redact_with(subs)

crates/cargo-test-support/src/lib.rs

+58
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ use std::time::{self, Duration};
2525

2626
use anyhow::{bail, Result};
2727
use cargo_util::{is_ci, ProcessBuilder, ProcessError};
28+
use snapbox::IntoData as _;
2829
use url::Url;
2930

3031
use self::paths::CargoPathExt;
@@ -534,6 +535,8 @@ pub struct Execs {
534535
expect_stdin: Option<String>,
535536
expect_stderr: Option<String>,
536537
expect_exit_code: Option<i32>,
538+
expect_stdout_data: Option<snapbox::Data>,
539+
expect_stderr_data: Option<snapbox::Data>,
537540
expect_stdout_contains: Vec<String>,
538541
expect_stderr_contains: Vec<String>,
539542
expect_stdout_contains_n: Vec<(String, usize)>,
@@ -545,6 +548,7 @@ pub struct Execs {
545548
expect_json: Option<String>,
546549
expect_json_contains_unordered: Option<String>,
547550
stream_output: bool,
551+
assert: snapbox::Assert,
548552
}
549553

550554
impl Execs {
@@ -555,18 +559,36 @@ impl Execs {
555559

556560
/// Verifies that stdout is equal to the given lines.
557561
/// See [`compare`] for supported patterns.
562+
#[deprecated(note = "replaced with `Execs::with_stdout_data(expected)`")]
558563
pub fn with_stdout<S: ToString>(&mut self, expected: S) -> &mut Self {
559564
self.expect_stdout = Some(expected.to_string());
560565
self
561566
}
562567

563568
/// Verifies that stderr is equal to the given lines.
564569
/// See [`compare`] for supported patterns.
570+
#[deprecated(note = "replaced with `Execs::with_stderr_data(expected)`")]
565571
pub fn with_stderr<S: ToString>(&mut self, expected: S) -> &mut Self {
566572
self.expect_stderr = Some(expected.to_string());
567573
self
568574
}
569575

576+
/// Verifies that stdout is equal to the given lines.
577+
///
578+
/// See [`compare::assert_e2e`] for assertion details.
579+
pub fn with_stdout_data(&mut self, expected: impl snapbox::IntoData) -> &mut Self {
580+
self.expect_stdout_data = Some(expected.into_data());
581+
self
582+
}
583+
584+
/// Verifies that stderr is equal to the given lines.
585+
///
586+
/// See [`compare::assert_e2e`] for assertion details.
587+
pub fn with_stderr_data(&mut self, expected: impl snapbox::IntoData) -> &mut Self {
588+
self.expect_stderr_data = Some(expected.into_data());
589+
self
590+
}
591+
570592
/// Writes the given lines to stdin.
571593
pub fn with_stdin<S: ToString>(&mut self, expected: S) -> &mut Self {
572594
self.expect_stdin = Some(expected.to_string());
@@ -593,6 +615,7 @@ impl Execs {
593615
/// its output.
594616
///
595617
/// See [`compare`] for supported patterns.
618+
#[deprecated(note = "replaced with `Execs::with_stdout_data(expected)`")]
596619
pub fn with_stdout_contains<S: ToString>(&mut self, expected: S) -> &mut Self {
597620
self.expect_stdout_contains.push(expected.to_string());
598621
self
@@ -602,6 +625,7 @@ impl Execs {
602625
/// its output.
603626
///
604627
/// See [`compare`] for supported patterns.
628+
#[deprecated(note = "replaced with `Execs::with_stderr_data(expected)`")]
605629
pub fn with_stderr_contains<S: ToString>(&mut self, expected: S) -> &mut Self {
606630
self.expect_stderr_contains.push(expected.to_string());
607631
self
@@ -611,6 +635,7 @@ impl Execs {
611635
/// its output, and should be repeated `number` times.
612636
///
613637
/// See [`compare`] for supported patterns.
638+
#[deprecated(note = "replaced with `Execs::with_stdout_data(expected)`")]
614639
pub fn with_stdout_contains_n<S: ToString>(&mut self, expected: S, number: usize) -> &mut Self {
615640
self.expect_stdout_contains_n
616641
.push((expected.to_string(), number));
@@ -622,6 +647,7 @@ impl Execs {
622647
/// See [`compare`] for supported patterns.
623648
///
624649
/// See note on [`Self::with_stderr_does_not_contain`].
650+
#[deprecated]
625651
pub fn with_stdout_does_not_contain<S: ToString>(&mut self, expected: S) -> &mut Self {
626652
self.expect_stdout_not_contains.push(expected.to_string());
627653
self
@@ -636,6 +662,7 @@ impl Execs {
636662
/// your test will pass without verifying the correct behavior. If
637663
/// possible, write the test first so that it fails, and then implement
638664
/// your fix/feature to make it pass.
665+
#[deprecated]
639666
pub fn with_stderr_does_not_contain<S: ToString>(&mut self, expected: S) -> &mut Self {
640667
self.expect_stderr_not_contains.push(expected.to_string());
641668
self
@@ -645,6 +672,7 @@ impl Execs {
645672
/// ignoring the order of the lines.
646673
///
647674
/// See [`Execs::with_stderr_unordered`] for more details.
675+
#[deprecated(note = "replaced with `Execs::with_stdout_data(expected.unordered())`")]
648676
pub fn with_stdout_unordered<S: ToString>(&mut self, expected: S) -> &mut Self {
649677
self.expect_stdout_unordered.push(expected.to_string());
650678
self
@@ -671,6 +699,7 @@ impl Execs {
671699
///
672700
/// This will randomly fail if the other crate name is `bar`, and the
673701
/// order changes.
702+
#[deprecated(note = "replaced with `Execs::with_stderr_data(expected.unordered())`")]
674703
pub fn with_stderr_unordered<S: ToString>(&mut self, expected: S) -> &mut Self {
675704
self.expect_stderr_unordered.push(expected.to_string());
676705
self
@@ -698,6 +727,7 @@ impl Execs {
698727
///
699728
/// Be careful writing the `without` fragments, see note in
700729
/// `with_stderr_does_not_contain`.
730+
#[deprecated]
701731
pub fn with_stderr_line_without<S: ToString>(
702732
&mut self,
703733
with: &[S],
@@ -730,6 +760,7 @@ impl Execs {
730760
/// - The order of arrays is ignored.
731761
/// - Strings support patterns described in [`compare`].
732762
/// - Use `"{...}"` to match any object.
763+
#[deprecated(note = "replaced with `Execs::with_stdout_data(expected.json_lines())`")]
733764
pub fn with_json(&mut self, expected: &str) -> &mut Self {
734765
self.expect_json = Some(expected.to_string());
735766
self
@@ -744,6 +775,7 @@ impl Execs {
744775
/// what you are doing.
745776
///
746777
/// See `with_json` for more detail.
778+
#[deprecated]
747779
pub fn with_json_contains_unordered(&mut self, expected: &str) -> &mut Self {
748780
match &mut self.expect_json_contains_unordered {
749781
None => self.expect_json_contains_unordered = Some(expected.to_string()),
@@ -908,11 +940,14 @@ impl Execs {
908940
}
909941
}
910942

943+
#[track_caller]
911944
fn verify_checks_output(&self, stdout: &[u8], stderr: &[u8]) {
912945
if self.expect_exit_code.unwrap_or(0) != 0
913946
&& self.expect_stdout.is_none()
914947
&& self.expect_stdin.is_none()
915948
&& self.expect_stderr.is_none()
949+
&& self.expect_stdout_data.is_none()
950+
&& self.expect_stderr_data.is_none()
916951
&& self.expect_stdout_contains.is_empty()
917952
&& self.expect_stderr_contains.is_empty()
918953
&& self.expect_stdout_contains_n.is_empty()
@@ -934,6 +969,7 @@ impl Execs {
934969
}
935970
}
936971

972+
#[track_caller]
937973
fn match_process(&self, process: &ProcessBuilder) -> Result<RawOutput> {
938974
println!("running {}", process);
939975
let res = if self.stream_output {
@@ -984,6 +1020,7 @@ impl Execs {
9841020
}
9851021
}
9861022

1023+
#[track_caller]
9871024
fn match_output(&self, code: Option<i32>, stdout: &[u8], stderr: &[u8]) -> Result<()> {
9881025
self.verify_checks_output(stdout, stderr);
9891026
let stdout = std::str::from_utf8(stdout).expect("stdout is not utf8");
@@ -1008,6 +1045,24 @@ impl Execs {
10081045
if let Some(expect_stderr) = &self.expect_stderr {
10091046
compare::match_exact(expect_stderr, stderr, "stderr", stdout, cwd)?;
10101047
}
1048+
if let Some(expect_stdout_data) = &self.expect_stdout_data {
1049+
if let Err(err) = self.assert.try_eq(
1050+
Some(&"stdout"),
1051+
stdout.into_data(),
1052+
expect_stdout_data.clone(),
1053+
) {
1054+
panic!("{err}")
1055+
}
1056+
}
1057+
if let Some(expect_stderr_data) = &self.expect_stderr_data {
1058+
if let Err(err) = self.assert.try_eq(
1059+
Some(&"stderr"),
1060+
stderr.into_data(),
1061+
expect_stderr_data.clone(),
1062+
) {
1063+
panic!("{err}")
1064+
}
1065+
}
10111066
for expect in self.expect_stdout_contains.iter() {
10121067
compare::match_contains(expect, stdout, cwd)?;
10131068
}
@@ -1060,6 +1115,8 @@ pub fn execs() -> Execs {
10601115
expect_stderr: None,
10611116
expect_stdin: None,
10621117
expect_exit_code: Some(0),
1118+
expect_stdout_data: None,
1119+
expect_stderr_data: None,
10631120
expect_stdout_contains: Vec::new(),
10641121
expect_stderr_contains: Vec::new(),
10651122
expect_stdout_contains_n: Vec::new(),
@@ -1071,6 +1128,7 @@ pub fn execs() -> Execs {
10711128
expect_json: None,
10721129
expect_json_contains_unordered: None,
10731130
stream_output: false,
1131+
assert: compare::assert_e2e(),
10741132
}
10751133
}
10761134

tests/build-std/main.rs

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
//! Otherwise the tests are skipped.
2020
2121
#![allow(clippy::disallowed_methods)]
22+
#![allow(deprecated)]
2223

2324
use cargo_test_support::*;
2425
use std::env;

0 commit comments

Comments
 (0)