[red-knot] Add GitHub PR annotations when mdtests fail in CI

.github/workflows/ci.yaml

@@ -36,6 +36,8 @@ jobs:
       code: ${{ steps.check_code.outputs.changed }}
       # Flag that is raised when any code that affects the fuzzer is changed
       fuzz: ${{ steps.check_fuzzer.outputs.changed }}
+      # Flag that is set to "true" when code related to red-knot changes.
+      red_knot: ${{ steps.check_red_knot.outputs.changed }}
 
       # Flag that is set to "true" when code related to the playground changes.
       playground: ${{ steps.check_playground.outputs.changed }}
@@ -166,6 +168,29 @@ jobs:
               echo "changed=true" >> "$GITHUB_OUTPUT"
           fi
 
+      - name: Check if the red-knot code changed
+        id: check_red_knot
+        env:
+          MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
+        run: |
+          if git diff --quiet "${MERGE_BASE}...HEAD" -- \
+            ':Cargo.toml' \
+            ':Cargo.lock' \
+            ':crates/red_knot*/**' \
+            ':crates/ruff_db/**' \
+            ':crates/ruff_annotate_snippets/**' \
+            ':crates/ruff_python_ast/**' \
+            ':crates/ruff_python_parser/**' \
+            ':crates/ruff_python_trivia/**' \
+            ':crates/ruff_source_file/**' \
+            ':crates/ruff_text_size/**' \
+            ':.github/workflows/ci.yaml' \
+          ; then
+              echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+              echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
   cargo-fmt:
     name: "cargo fmt"
     runs-on: ubuntu-latest
@@ -221,6 +246,14 @@ jobs:
         uses: taiki-e/install-action@6aca1cfa12ef3a6b98ee8c70e0171bfa067604f5 # v2
         with:
           tool: cargo-insta
+      - name: Red-knot mdtests (GitHub annotations)
+        if: ${{ needs.determine_changes.outputs.red_knot == 'true' }}
+        env:
+          NO_COLOR: 1
+          MDTEST_GITHUB_ANNOTATIONS_FORMAT: 1
+        # Ignore errors if this step fails; we want to continue to later steps in the workflow anyway.
+        # This step is just to get nice GitHub annotations on the PR diff in the files-changed tab.
+        run: cargo test -p red_knot_python_semantic --test mdtest || true
       - name: "Run tests"
         shell: bash
         env:

crates/red_knot_python_semantic/tests/mdtest.rs

@@ -1,5 +1,6 @@
 use camino::Utf8Path;
 use dir_test::{dir_test, Fixture};
+use red_knot_test::OutputFormat;
 
 /// See `crates/red_knot_test/README.md` for documentation on these tests.
 #[dir_test(
@@ -18,12 +19,19 @@ fn mdtest(fixture: Fixture<&str>) {
 
     let test_name = test_name("mdtest", absolute_fixture_path);
 
+    let output_format = if std::env::var("MDTEST_GITHUB_ANNOTATIONS_FORMAT").is_ok() {
+        OutputFormat::GitHub
+    } else {
+        OutputFormat::Cli
+    };
+
     red_knot_test::run(
         absolute_fixture_path,
         relative_fixture_path,
         &snapshot_path,
         short_title,
         &test_name,
+        output_format,
     );
 }
 

crates/red_knot_test/src/lib.rs

@@ -34,6 +34,7 @@ pub fn run(
     snapshot_path: &Utf8Path,
     short_title: &str,
     test_name: &str,
+    output_format: OutputFormat,
 ) {
     let source = std::fs::read_to_string(absolute_fixture_path).unwrap();
     let suite = match test_parser::parse(short_title, &source) {
@@ -59,7 +60,10 @@ pub fn run(
 
         if let Err(failures) = run_test(&mut db, relative_fixture_path, snapshot_path, &test) {
             any_failures = true;
-            println!("\n{}\n", test.name().bold().underline());
+
+            if output_format.is_cli() {
+                println!("\n{}\n", test.name().bold().underline());
+            }
 
             let md_index = LineIndex::from_source_text(&source);
 
@@ -72,21 +76,31 @@ pub fn run(
                         source_map.to_absolute_line_number(relative_line_number);
 
                     for failure in failures {
-                        let line_info =
-                            format!("{relative_fixture_path}:{absolute_line_number}").cyan();
-                        println!("  {line_info} {failure}");
+                        match output_format {
+                            OutputFormat::Cli => {
+                                let line_info =
+                                    format!("{relative_fixture_path}:{absolute_line_number}")
+                                        .cyan();
+                                println!("  {line_info} {failure}");
+                            }
+                            OutputFormat::GitHub => println!(
+                                "::error file={absolute_fixture_path},line={absolute_line_number}::{failure}"
+                            ),
+                        }
                     }
                 }
             }
 
             let escaped_test_name = test.name().replace('\'', "\\'");
 
-            println!(
-                "\nTo rerun this specific test, set the environment variable: {MDTEST_TEST_FILTER}='{escaped_test_name}'",
-            );
-            println!(
-                "{MDTEST_TEST_FILTER}='{escaped_test_name}' cargo test -p red_knot_python_semantic --test mdtest -- {test_name}",
-            );
+            if output_format.is_cli() {
+                println!(
+                    "\nTo rerun this specific test, set the environment variable: {MDTEST_TEST_FILTER}='{escaped_test_name}'",
+                );
+                println!(
+                    "{MDTEST_TEST_FILTER}='{escaped_test_name}' cargo test -p red_knot_python_semantic --test mdtest -- {test_name}",
+                );
+            }
         }
     }
 
@@ -95,6 +109,23 @@ pub fn run(
     assert!(!any_failures, "Some tests failed.");
 }
 
+/// Defines the format in which mdtest should print an error to the terminal
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum OutputFormat {
+    /// The format `cargo test` should use by default.
+    Cli,
+    /// A format that will provide annotations from GitHub Actions
+    /// if mdtest fails on a PR.
+    /// See <https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-error-message>
+    GitHub,
+}
+
+impl OutputFormat {
+    const fn is_cli(self) -> bool {
+        matches!(self, OutputFormat::Cli)
+    }
+}
+
 fn run_test(
     db: &mut db::Db,
     relative_fixture_path: &Utf8Path,