Skip to content

compiletest: Overhaul the code for running an incremental test revision #155768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rust-bors merged 1 commit into from
Apr 28, 2026
+89 −91
Merged

compiletest: Overhaul the code for running an incremental test revision #155768

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions src/tools/compiletest/src/runtest.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,20 +319,6 @@ impl<'test> TestCx<'test> {
TestMode::RustdocJs => true,
TestMode::Ui => pm.is_some() || self.props.fail_mode > Some(FailMode::Build),
TestMode::Crashes => false,
TestMode::Incremental => {
let revision =
self.revision.expect("incremental tests require a list of revisions");
if revision.starts_with("cpass")
|| revision.starts_with("bpass")
|| revision.starts_with("rpass")
{
true
} else if revision.starts_with("bfail") {
false
} else {
panic!("revision name must begin with `cpass`, `bfail`, `bpass`, or `rpass`");
}
}
mode => panic!("unimplemented for mode {:?}", mode),
}
}
Expand Down
166 changes: 89 additions & 77 deletions src/tools/compiletest/src/runtest/incremental.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,43 @@
use super::{Emit, FailMode, PassMode, ProcRes, TestCx, WillExecute};
use std::sync::LazyLock;

use crate::runtest::{Emit, TestCx, WillExecute};
use crate::util::string_enum;

string_enum!(
/// How far an incremental test revision should proceed through the compile/run
/// sequence, and whether the last step should succeed or fail, as determined
/// from the start of the revision name.
#[derive(Clone, Copy, PartialEq, Eq)]
enum IncrRevKind {
CheckPass => "cpass",
BuildFail => "bfail",
BuildPass => "bpass",
RunPass => "rpass",
}
);

impl IncrRevKind {
fn for_revision_name(rev_name: &str) -> Result<Self, &'static str> {
static MESSAGE: LazyLock<String> = LazyLock::new(|| {
let values = IncrRevKind::STR_VARIANTS
.iter()
.map(|s| format!("`{s}`"))
.collect::<Vec<_>>()
.join(", ");
format!("incremental revision name must begin with one of: {values}")
});

IncrRevKind::VARIANTS
.iter()
.copied()
.find(|kind| rev_name.starts_with(kind.to_str()))
.ok_or_else(|| MESSAGE.as_str())
}
}

impl TestCx<'_> {
/// Runs a single revision of an incremental test.
pub(super) fn run_incremental_test(&self) {
// Basic plan for a test incremental/foo/bar.rs:
// - load list of revisions rpass1, bfail2, rpass3
// - each should begin with `bfail`, `bpass`, or `rpass`
// - if `bfail`, expect compilation to fail
// - if `bpass`, expect compilation to succeed, don't execute
// - if `rpass`, expect compilation and execution to succeed
// - create a directory build/foo/bar.incremental
// - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C rpass1
// - because name of revision starts with "rpass", expect success
// - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C bfail2
// - because name of revision starts with "bfail", expect an error
// - load expected errors as usual, but filter for those with `[bfail2]`
// - compile foo/bar.rs with -C incremental=.../foo/bar.incremental and -C rpass3
// - because name of revision starts with "rpass", expect success
// - execute build/foo/bar.exe and save output
//
// FIXME -- use non-incremental mode as an oracle? That doesn't apply
// to #[rustc_clean] tests I guess

let revision = self.revision.expect("incremental tests require a list of revisions");

// Incremental workproduct directory should have already been created.
Expand All @@ -31,68 +48,63 @@ impl TestCx<'_> {
write!(self.stdout, "revision={:?} props={:#?}", revision, self.props);
}

if revision.starts_with("cpass") {
self.run_cpass_test();
} else if revision.starts_with("bpass") {
self.run_bpass_test();
} else if revision.starts_with("rpass") {
self.run_rpass_test();
} else if revision.starts_with("bfail") {
self.run_bfail_test();
} else {
self.fatal("revision name must begin with `bfail`, `bpass`, or `rpass`");
// Determine the revision kind from the revision name.
// The revision kind should be matched exhaustively to ensure that no cases are missed.
let rev_kind = IncrRevKind::for_revision_name(revision).unwrap_or_else(|e| self.fatal(e));

// Compile the test for this revision.
let emit = match rev_kind {
IncrRevKind::CheckPass => Emit::Metadata, // Do a check build.
IncrRevKind::BuildFail | IncrRevKind::BuildPass | IncrRevKind::RunPass => Emit::None,
};
let will_execute = match rev_kind {
IncrRevKind::CheckPass | IncrRevKind::BuildFail | IncrRevKind::BuildPass => {
WillExecute::No
}
IncrRevKind::RunPass => {
// Yes, unless running test binaries is disabled.
self.run_if_enabled()
}
};
let proc_res = &self.compile_test(will_execute, emit);

// Check the compiler's exit status.
match rev_kind {
IncrRevKind::CheckPass | IncrRevKind::BuildPass | IncrRevKind::RunPass => {
// Compilation should have succeeded.
if !proc_res.status.success() {
self.fatal_proc_rec("test compilation failed although it shouldn't!", proc_res);
}
}

IncrRevKind::BuildFail => {
// Compilation should have failed, with the expected status code.
if proc_res.status.success() {
self.fatal_proc_rec("incremental test did not emit an error", proc_res);
}
if !self.props.dont_check_failure_status {
self.check_correct_failure_status(proc_res);
}
}
}
}

fn run_cpass_test(&self) {
let proc_res = self.compile_test(WillExecute::No, Emit::Metadata);
self.check_if_test_should_compile(None, Some(PassMode::Check), &proc_res);
self.check_compiler_output_for_incr(&proc_res);
}

fn run_bpass_test(&self) {
let emit_metadata = self.should_emit_metadata(self.pass_mode());
let proc_res = self.compile_test(WillExecute::No, emit_metadata);

if !proc_res.status.success() {
self.fatal_proc_rec("compilation failed!", &proc_res);
}

self.check_compiler_output_for_incr(&proc_res);
}

fn run_rpass_test(&self) {
let emit_metadata = self.should_emit_metadata(self.pass_mode());
let should_run = self.run_if_enabled();
let proc_res = self.compile_test(should_run, emit_metadata);

if !proc_res.status.success() {
self.fatal_proc_rec("compilation failed!", &proc_res);
}

self.check_compiler_output_for_incr(&proc_res);

if let WillExecute::Disabled = should_run {
return;
}

let proc_res = self.exec_compiled_test();
if !proc_res.status.success() {
self.fatal_proc_rec("test run failed!", &proc_res);
}
}

fn run_bfail_test(&self) {
let pm = self.pass_mode();
let proc_res = self.compile_test(WillExecute::No, self.should_emit_metadata(pm));
self.check_if_test_should_compile(Some(FailMode::Build), pm, &proc_res);
self.check_compiler_output_for_incr(&proc_res);
}

fn check_compiler_output_for_incr(&self, proc_res: &ProcRes) {
// Check compilation output.
let output_to_check = self.get_output(proc_res);
self.check_expected_errors(&proc_res);
self.check_all_error_patterns(&output_to_check, proc_res);
self.check_forbid_output(&output_to_check, proc_res);

// Run the binary and check its exit status, if appropriate.
match rev_kind {
IncrRevKind::CheckPass | IncrRevKind::BuildFail | IncrRevKind::BuildPass => {}
IncrRevKind::RunPass => {
if self.config.run_enabled() {
let run_proc_res = self.exec_compiled_test();
if !run_proc_res.status.success() {
self.fatal_proc_rec("test run failed!", &run_proc_res);
}
}
}
}
}
}
Loading