diff --git a/Cargo.lock b/Cargo.lock index 07fee7477725..941bc6bace9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6438,6 +6438,8 @@ dependencies = [ "biome_js_parser", "biome_js_semantic", "biome_js_syntax", + "biome_markdown_parser", + "biome_markdown_syntax", "biome_parser", "biome_rowan", "biome_string_case", diff --git a/crates/biome_markdown_parser/tests/commonmark_spec.rs b/crates/biome_markdown_parser/tests/commonmark_spec.rs deleted file mode 100644 index df715286cf8b..000000000000 --- a/crates/biome_markdown_parser/tests/commonmark_spec.rs +++ /dev/null @@ -1,371 +0,0 @@ -//! CommonMark specification compliance test harness. -//! -//! This test runs all 652 CommonMark spec examples against Biome's markdown parser -//! and reports the compliance percentage. -//! -//! Run with: `cargo test -p biome_markdown_parser --test commonmark_spec -- --nocapture` - -use biome_markdown_parser::{document_to_html, parse_markdown}; -use biome_markdown_syntax::MdDocument; -use biome_rowan::AstNode; -use serde::Deserialize; - -/// Embedded CommonMark spec test cases. -const SPEC_JSON: &str = include_str!("spec.json"); - -/// A single test case from the CommonMark spec. -#[derive(Debug, Deserialize)] -struct SpecTest { - /// The markdown input - markdown: String, - /// The expected HTML output - html: String, - /// The example number in the spec - example: u32, - /// The section name - section: String, -} - -/// Information about a failed test. -#[derive(Debug)] -struct FailedTest { - example: u32, - section: String, - markdown: String, - expected: String, - actual: String, -} - -/// Normalize HTML for comparison. -/// -/// CommonMark spec tests are strict about HTML output, but there are some -/// acceptable variations in whitespace that we handle here. -/// -/// IMPORTANT: We preserve whitespace inside `
` blocks since trailing
-/// spaces are significant in code blocks per CommonMark spec.
-fn normalize_html(html: &str) -> String {
-    let mut result = Vec::new();
-    let mut in_pre = false;
-
-    for line in html.lines() {
-        // Track 
 block state
-        // Note: CommonMark output has 
 on same line, so check for 
 blocks
-        if in_pre {
-            result.push(line.to_string());
-        } else {
-            result.push(line.trim_end().to_string());
-        }
-
-        // Check for 
after processing the line - if line.contains("
") { - in_pre = false; - } - } - - result.join("\n").trim().to_string() + "\n" -} - -/// Show a unified diff between expected and actual HTML. -fn diff(expected: &str, actual: &str) -> String { - let mut result = String::new(); - let expected_lines: Vec<&str> = expected.lines().collect(); - let actual_lines: Vec<&str> = actual.lines().collect(); - - let max_lines = expected_lines.len().max(actual_lines.len()); - - for i in 0..max_lines { - let exp = expected_lines.get(i).unwrap_or(&""); - let act = actual_lines.get(i).unwrap_or(&""); - - if exp != act { - result.push_str(&format!("- {}\n", exp)); - result.push_str(&format!("+ {}\n", act)); - } else { - result.push_str(&format!(" {}\n", exp)); - } - } - - result -} - -#[test] -fn commonmark_spec_compliance() { - let tests: Vec = serde_json::from_str(SPEC_JSON).expect("Failed to parse spec.json"); - let total = tests.len(); - - let mut passed = 0; - let mut failed: Vec = Vec::new(); - let mut section_stats: std::collections::HashMap = - std::collections::HashMap::new(); - - let log_progress = std::env::var("CMARK_PROGRESS").is_ok(); - for (index, test) in tests.iter().enumerate() { - if log_progress { - println!( - "progress {}/{} example {} {}", - index + 1, - total, - test.example, - test.section - ); - } - let parsed = parse_markdown(&test.markdown); - - // Handle bogus nodes gracefully - count as failure instead of panicking - let Some(document) = MdDocument::cast(parsed.syntax()) else { - let section_entry = section_stats.entry(test.section.clone()).or_insert((0, 0)); - section_entry.1 += 1; - failed.push(FailedTest { - example: test.example, - section: test.section.clone(), - markdown: test.markdown.clone(), - expected: test.html.clone(), - actual: format!("", parsed.syntax().kind()), - }); - continue; - }; - - let actual = document_to_html( - &document, - parsed.list_tightness(), - parsed.list_item_indents(), - parsed.quote_indents(), - ); - - let expected_normalized = normalize_html(&test.html); - let actual_normalized = normalize_html(&actual); - - let section_entry = section_stats.entry(test.section.clone()).or_insert((0, 0)); - section_entry.1 += 1; // total for section - - if expected_normalized == actual_normalized { - passed += 1; - section_entry.0 += 1; // passed for section - } else { - failed.push(FailedTest { - example: test.example, - section: test.section.clone(), - markdown: test.markdown.clone(), - expected: test.html.clone(), - actual, - }); - } - } - - // Print summary - println!("\n"); - println!("═══════════════════════════════════════════════════════════════════════════════"); - println!(" CommonMark Spec Compliance Report"); - println!("═══════════════════════════════════════════════════════════════════════════════"); - println!(); - println!( - "Overall: {}/{} ({:.1}%)", - passed, - total, - (passed as f64 / total as f64) * 100.0 - ); - println!(); - - // Print section breakdown - println!("Section Breakdown:"); - println!("─────────────────────────────────────────────────────────────────────────────────"); - - let mut sections: Vec<_> = section_stats.iter().collect(); - sections.sort_by_key(|(name, _)| *name); - - for (section, (section_passed, section_total)) in sections { - let pct = (*section_passed as f64 / *section_total as f64) * 100.0; - let status = if pct == 100.0 { - "✓" - } else if pct >= 80.0 { - "○" - } else { - "✗" - }; - println!( - " {} {:40} {:3}/{:3} ({:5.1}%)", - status, section, section_passed, section_total, pct - ); - } - println!(); - - // Print failures (limited to first 50) - if !failed.is_empty() { - println!("Failed Examples (showing first 50):"); - println!( - "─────────────────────────────────────────────────────────────────────────────────" - ); - - for (i, failure) in failed.iter().take(50).enumerate() { - println!(); - println!( - "{}. Example {} [{}]", - i + 1, - failure.example, - failure.section - ); - println!(" Input:"); - for line in failure.markdown.lines() { - println!(" │ {:?}", line); - } - println!(" Expected:"); - for line in failure.expected.lines() { - println!(" │ {}", line); - } - println!(" Actual:"); - for line in failure.actual.lines() { - println!(" │ {}", line); - } - println!(" Diff:"); - for line in diff(&failure.expected, &failure.actual).lines() { - println!(" │ {}", line); - } - } - - if failed.len() > 50 { - println!(); - println!("... and {} more failures", failed.len() - 50); - } - } - - println!(); - println!("═══════════════════════════════════════════════════════════════════════════════"); - - // For now, we don't fail the test - we're just measuring compliance - // Once we reach high compliance, we can enable this assertion - // assert!(passed == total, "Not all CommonMark spec tests pass"); - - // Report the overall result - let compliance_pct = (passed as f64 / total as f64) * 100.0; - if compliance_pct < 50.0 { - println!( - "WARNING: Compliance is below 50% ({:.1}%). Parser may need significant work.", - compliance_pct - ); - } -} - -/// Run a single example for debugging. -#[test] -#[ignore] -fn debug_single_example() { - let tests: Vec = serde_json::from_str(SPEC_JSON).expect("Failed to parse spec.json"); - - // Change this to debug a specific example - let example_num = 228; - - if let Some(test) = tests.iter().find(|t| t.example == example_num) { - println!("Example {}: {}", test.example, test.section); - println!("Markdown: {:?}", test.markdown); - println!(); - - let parsed = parse_markdown(&test.markdown); - - println!("CST (raw syntax):"); - println!("{:#?}", parsed.syntax()); - println!(); - - println!("AST:"); - if parsed.syntax().kind() == biome_markdown_syntax::MarkdownSyntaxKind::MD_DOCUMENT { - println!("{:#?}", parsed.tree()); - } else { - println!( - "Cannot cast to MdDocument - root is {:?}", - parsed.syntax().kind() - ); - } - println!(); - println!(); - - if parsed.has_errors() { - println!("Parse errors:"); - for diag in parsed.diagnostics() { - println!(" - {:?}", diag); - } - println!(); - } - - println!("List tightness: {:?}", parsed.list_tightness()); - println!(); - - let actual = document_to_html( - &parsed.tree(), - parsed.list_tightness(), - parsed.list_item_indents(), - parsed.quote_indents(), - ); - - println!("Expected HTML:"); - println!("{}", test.html); - println!(); - - println!("Actual HTML:"); - println!("{}", actual); - println!(); - - let expected_normalized = normalize_html(&test.html); - let actual_normalized = normalize_html(&actual); - - if expected_normalized == actual_normalized { - println!("✓ PASS"); - } else { - println!("✗ FAIL"); - println!("Diff:"); - println!("{}", diff(&test.html, &actual)); - } - } else { - println!("Example {} not found", example_num); - } -} - -/// Test specific sections for focused debugging. -#[test] -#[ignore] -fn debug_section() { - let tests: Vec = serde_json::from_str(SPEC_JSON).expect("Failed to parse spec.json"); - - // Change this to debug a specific section - let section = "List items"; - - let section_tests: Vec<_> = tests.iter().filter(|t| t.section == section).collect(); - - println!("Section: {} ({} tests)", section, section_tests.len()); - println!(); - - let mut passed = 0; - for test in §ion_tests { - let parsed = parse_markdown(&test.markdown); - let actual = document_to_html( - &parsed.tree(), - parsed.list_tightness(), - parsed.list_item_indents(), - parsed.quote_indents(), - ); - - let expected_normalized = normalize_html(&test.html); - let actual_normalized = normalize_html(&actual); - - if expected_normalized == actual_normalized { - passed += 1; - println!(" ✓ Example {}", test.example); - } else { - println!(" ✗ Example {}", test.example); - println!(" Input: {:?}", test.markdown); - println!(" Expected: {:?}", test.html); - println!(" Actual: {:?}", actual); - } - } - - println!(); - println!( - "Result: {}/{} ({:.1}%)", - passed, - section_tests.len(), - (passed as f64 / section_tests.len() as f64) * 100.0 - ); -} diff --git a/justfile b/justfile index c88e0e0ade6e..45bf1c163914 100644 --- a/justfile +++ b/justfile @@ -232,7 +232,11 @@ test-doc: # Run CommonMark conformance tests for the markdown parser test-markdown-conformance: - cargo test -p biome_markdown_parser --test commonmark_spec -- --nocapture + cargo run -p xtask_coverage -- --suites=markdown/commonmark + +# Update the CommonMark spec.json to a specific version +update-commonmark-spec version: + ./scripts/update-commonmark-spec.sh {{version}} # Tests a lint rule. The name of the rule needs to be camel case test-lintrule name: diff --git a/scripts/update-commonmark-spec.sh b/scripts/update-commonmark-spec.sh new file mode 100755 index 000000000000..21c64f1d1d75 --- /dev/null +++ b/scripts/update-commonmark-spec.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Script: update-commonmark-spec.sh +# Purpose: +# Download a specific version of the CommonMark spec.json and update +# both the spec file and the provenance comment in commonmark.rs. +# +# Usage: +# ./scripts/update-commonmark-spec.sh +# +# Example: +# ./scripts/update-commonmark-spec.sh 0.31.2 +# +# After running, verify with: +# just test-markdown-conformance + +SPEC_PATH="xtask/coverage/src/markdown/spec.json" +RS_PATH="xtask/coverage/src/markdown/commonmark.rs" + +print_help() { + cat <<'EOF' +Download a specific version of the CommonMark spec.json and update provenance. + +Usage: + update-commonmark-spec.sh + +Example: + update-commonmark-spec.sh 0.31.2 + +This will: + 1. Download https://spec.commonmark.org//spec.json + 2. Replace xtask/coverage/src/markdown/spec.json + 3. Update the provenance comment in commonmark.rs + +After updating, verify with: + just test-markdown-conformance +EOF +} + +if [[ $# -lt 1 ]]; then + print_help + exit 1 +fi + +if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + print_help + exit 0 +fi + +VERSION="$1" +URL="https://spec.commonmark.org/${VERSION}/spec.json" +TODAY=$(date +%Y-%m-%d) + +echo "Downloading CommonMark spec version ${VERSION}..." +echo "URL: ${URL}" + +if ! curl -fsSL "${URL}" -o "${SPEC_PATH}.tmp"; then + echo "Error: Failed to download spec from ${URL}" >&2 + echo "Check that the version exists at https://spec.commonmark.org/" >&2 + rm -f "${SPEC_PATH}.tmp" + exit 1 +fi + +mv "${SPEC_PATH}.tmp" "${SPEC_PATH}" + +# Count examples +if command -v jq >/dev/null 2>&1; then + COUNT=$(jq 'length' "${SPEC_PATH}") +else + echo "Warning: jq not found, cannot count examples" >&2 + COUNT="unknown" +fi + +# Update provenance comment in commonmark.rs +echo "Updating provenance in ${RS_PATH}..." + +sed -i.bak \ + -e "s|// Version: .*|// Version: ${VERSION}|" \ + -e "s|// URL: .*|// URL: ${URL}|" \ + -e "s|// Downloaded: .*|// Downloaded: ${TODAY}|" \ + -e "s|// Examples: .*|// Examples: ${COUNT}|" \ + "${RS_PATH}" + +rm -f "${RS_PATH}.bak" + +# Print summary +echo +echo "Updated:" +echo " - ${SPEC_PATH}" +echo " - ${RS_PATH}" +echo +echo "Provenance:" +echo " Version: ${VERSION}" +echo " URL: ${URL}" +echo " Downloaded: ${TODAY}" +echo " Examples: ${COUNT}" +echo +echo "Next step:" +echo " just test-markdown-conformance" diff --git a/xtask/coverage/Cargo.toml b/xtask/coverage/Cargo.toml index d5acb91d40d9..20a6b151606d 100644 --- a/xtask/coverage/Cargo.toml +++ b/xtask/coverage/Cargo.toml @@ -5,29 +5,31 @@ edition = "2024" publish = false [dependencies] -ascii_table = "4.0.8" -backtrace = "0.3.76" -biome_console = { workspace = true } -biome_diagnostics = { workspace = true } -biome_js_parser = { workspace = true } -biome_js_semantic = { workspace = true } -biome_js_syntax = { workspace = true } -biome_parser = { workspace = true } -biome_rowan = { workspace = true } -biome_string_case = { workspace = true } -camino = { workspace = true } -colored = "3.0.0" -indicatif = { version = "0.18.3", features = ["improved_unicode"] } -pico-args = { version = "0.5.0", features = ["eq-separator"] } -regex = { workspace = true } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } -serde_yaml = "0.9.34" -tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["env-filter", "std"] } -walkdir = { workspace = true } -xtask_glue = { workspace = true } -yastl = "0.1.2" +ascii_table = "4.0.8" +backtrace = "0.3.76" +biome_console = { workspace = true } +biome_diagnostics = { workspace = true } +biome_js_parser = { workspace = true } +biome_js_semantic = { workspace = true } +biome_js_syntax = { workspace = true } +biome_markdown_parser = { workspace = true, features = ["test_utils"] } +biome_markdown_syntax = { workspace = true } +biome_parser = { workspace = true } +biome_rowan = { workspace = true } +biome_string_case = { workspace = true } +camino = { workspace = true } +colored = "3.0.0" +indicatif = { version = "0.18.3", features = ["improved_unicode"] } +pico-args = { version = "0.5.0", features = ["eq-separator"] } +regex = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_yaml = "0.9.34" +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "std"] } +walkdir = { workspace = true } +xtask_glue = { workspace = true } +yastl = "0.1.2" [lints] workspace = true diff --git a/xtask/coverage/src/lib.rs b/xtask/coverage/src/lib.rs index e7eb89ad53ea..f9c73ecd2991 100644 --- a/xtask/coverage/src/lib.rs +++ b/xtask/coverage/src/lib.rs @@ -1,6 +1,7 @@ pub mod compare; pub mod js; pub mod jsx; +pub mod markdown; mod reporters; pub mod results; mod runner; @@ -19,6 +20,7 @@ use crate::runner::{TestRunContext, TestSuite, run_test_suite}; use biome_parser::diagnostic::ParseDiagnostic; use biome_string_case::StrOnlyExtension; use jsx::jsx_babel::BabelJsxTestSuite; +use markdown::commonmark::CommonMarkTestSuite; use serde::{Deserialize, Serialize}; use std::any::Any; use symbols::msts::SymbolsMicrosoftTestSuite; @@ -164,6 +166,7 @@ const ALL_JS_SUITES: &str = "js"; const ALL_TS_SUITES: &str = "ts"; const ALL_JSX_SUITES: &str = "jsx"; const ALL_SYMBOLS_SUITES: &str = "symbols"; +const ALL_MARKDOWN_SUITES: &str = "markdown"; fn get_test_suites(suites: Option<&str>) -> Vec> { let suites = suites.unwrap_or("*").to_lowercase_cow(); @@ -177,13 +180,15 @@ fn get_test_suites(suites: Option<&str>) -> Vec> { ALL_TS_SUITES | "typescript" => ids.extend(["ts/microsoft", "ts/babel"]), ALL_JSX_SUITES => ids.extend(["jsx/babel"]), ALL_SYMBOLS_SUITES => ids.extend(["symbols/microsoft"]), - ALL_SUITES => ids.extend(["js", "ts", "jsx", "symbols"]), + ALL_MARKDOWN_SUITES => ids.extend(["markdown/commonmark"]), + ALL_SUITES => ids.extend(["js", "ts", "jsx", "symbols", "markdown"]), "js/262" => suites.push(Box::new(Test262TestSuite)), "ts/microsoft" => suites.push(Box::new(MicrosoftTypescriptTestSuite)), "ts/babel" => suites.push(Box::new(BabelTypescriptTestSuite)), "jsx/babel" => suites.push(Box::new(BabelJsxTestSuite)), "symbols/microsoft" => suites.push(Box::new(SymbolsMicrosoftTestSuite)), + "markdown/commonmark" => suites.push(Box::new(CommonMarkTestSuite)), _ => {} } diff --git a/xtask/coverage/src/main.rs b/xtask/coverage/src/main.rs index 58c8d62a4b7b..4e07297a81cc 100644 --- a/xtask/coverage/src/main.rs +++ b/xtask/coverage/src/main.rs @@ -48,10 +48,12 @@ OPTIONS js: will run all javascript suites; Same as \"js/262\"; ts: will run all typescript suites; Same as \"ts/microsoft,ts/babel\"; jsx: will run all jsx suites; Same as \"jsx/babel\"; + markdown: will run all markdown suites; Same as \"markdown/commonmark\"; js/262: will run https://github.com/tc39/test262/tree/main/test; ts/microsoft: will run https://github.com/microsoft/Typescript/tree/main/tests/cases ts/babel: will run https://github.com/babel/babel/tree/main/packages/babel-parser/test/fixtures/typescript jsx/babel: will run https://github.com/babel/babel/tree/main/packages/babel-parser/test/fixtures/jsx/basic + markdown/commonmark: will run CommonMark spec tests (https://spec.commonmark.org/) Default is \"*\". --filter= Filters out tests that don't match the query. --help Prints this help. diff --git a/xtask/coverage/src/markdown/commonmark.rs b/xtask/coverage/src/markdown/commonmark.rs new file mode 100644 index 000000000000..7685d6478f7c --- /dev/null +++ b/xtask/coverage/src/markdown/commonmark.rs @@ -0,0 +1,149 @@ +// CommonMark spec compliance tests (https://spec.commonmark.org/) +// +// Spec provenance: +// Version: 0.31.2 +// URL: https://spec.commonmark.org/0.31.2/spec.json +// Downloaded: 2026-01-24 +// Examples: 652 +// +// To update the spec, run: +// just update-commonmark-spec +// +// After updating, verify with: +// just test-markdown-conformance + +use crate::runner::{TestCase, TestRunOutcome, TestSuite}; +use biome_markdown_parser::{document_to_html, parse_markdown}; +use biome_markdown_syntax::MdDocument; +use biome_parser::diagnostic::ParseDiagnostic; +use biome_rowan::{AstNode, TextRange}; +use serde::Deserialize; +use std::io; +use std::path::Path; + +const SPEC_JSON: &str = include_str!("spec.json"); + +#[derive(Debug, Deserialize)] +struct SpecTest { + markdown: String, + html: String, + example: u32, + section: String, +} + +struct CommonMarkTestCase { + name: String, + markdown: String, + expected_html: String, +} + +impl TestCase for CommonMarkTestCase { + fn name(&self) -> &str { + &self.name + } + + fn run(&self) -> TestRunOutcome { + let parsed = parse_markdown(&self.markdown); + + let Some(document) = MdDocument::cast(parsed.syntax()) else { + return TestRunOutcome::IncorrectlyErrored { + errors: vec![ParseDiagnostic::new( + format!("Bogus node: {:?}", parsed.syntax().kind()), + TextRange::empty(0.into()), + )], + files: Default::default(), + }; + }; + + let actual = document_to_html( + &document, + parsed.list_tightness(), + parsed.list_item_indents(), + parsed.quote_indents(), + ); + + let expected = normalize_html(&self.expected_html); + let actual_normalized = normalize_html(&actual); + + if expected == actual_normalized { + TestRunOutcome::Passed(Default::default()) + } else { + TestRunOutcome::IncorrectlyErrored { + errors: vec![ParseDiagnostic::new( + format!( + "HTML mismatch\nExpected:\n{}\nActual:\n{}", + self.expected_html, actual + ), + TextRange::empty(0.into()), + )], + files: Default::default(), + } + } + } +} + +// Normalize HTML for comparison, preserving whitespace inside
 blocks.
+fn normalize_html(html: &str) -> String {
+    let mut result = Vec::new();
+    let mut in_pre = false;
+
+    for line in html.lines() {
+        if line.contains("") {
+            in_pre = false;
+        }
+    }
+
+    result.join("\n").trim().to_string() + "\n"
+}
+
+pub(crate) struct CommonMarkTestSuite;
+
+impl TestSuite for CommonMarkTestSuite {
+    fn name(&self) -> &str {
+        "markdown/commonmark"
+    }
+
+    fn base_path(&self) -> &str {
+        "xtask/coverage/src/markdown"
+    }
+
+    fn checkout(&self) -> io::Result<()> {
+        Ok(())
+    }
+
+    fn is_test(&self, _path: &Path) -> bool {
+        false
+    }
+
+    fn load_test(&self, _path: &Path) -> Option> {
+        None
+    }
+
+    fn load_all(&self) -> Option>> {
+        let tests: Vec =
+            serde_json::from_str(SPEC_JSON).expect("Failed to parse spec.json");
+
+        let cases = tests
+            .into_iter()
+            .map(|spec| {
+                Box::new(CommonMarkTestCase {
+                    name: format!("example_{:03}_{}", spec.example, spec.section),
+                    markdown: spec.markdown,
+                    expected_html: spec.html,
+                }) as Box
+            })
+            .collect();
+
+        Some(cases)
+    }
+}
diff --git a/xtask/coverage/src/markdown/mod.rs b/xtask/coverage/src/markdown/mod.rs
new file mode 100644
index 000000000000..0fa90a17d950
--- /dev/null
+++ b/xtask/coverage/src/markdown/mod.rs
@@ -0,0 +1 @@
+pub mod commonmark;
diff --git a/crates/biome_markdown_parser/tests/spec.json b/xtask/coverage/src/markdown/spec.json
similarity index 100%
rename from crates/biome_markdown_parser/tests/spec.json
rename to xtask/coverage/src/markdown/spec.json
diff --git a/xtask/coverage/src/runner.rs b/xtask/coverage/src/runner.rs
index 32db86fbe1d3..339a2406abb2 100644
--- a/xtask/coverage/src/runner.rs
+++ b/xtask/coverage/src/runner.rs
@@ -106,7 +106,7 @@ pub(crate) fn create_bogus_node_in_tree_diagnostic(node: JsSyntaxNode) -> ParseD
     .with_hint( "This bogus node is present in the parsed tree but the parser didn't emit a diagnostic for it. Change the parser to either emit a diagnostic, fix the grammar, or the parsing.")
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub(crate) struct TestCaseFiles {
     files: Vec,
 }
@@ -185,6 +185,11 @@ pub(crate) trait TestSuite: Send + Sync {
     fn is_test(&self, path: &Path) -> bool;
     fn load_test(&self, path: &Path) -> Option>;
     fn checkout(&self) -> io::Result<()>;
+
+    /// Override to provide tests without filesystem scanning.
+    fn load_all(&self) -> Option>> {
+        None
+    }
 }
 
 pub(crate) struct TestSuiteInstance {
@@ -341,6 +346,17 @@ pub(crate) fn run_test_suite(
 }
 
 fn load_tests(suite: &dyn TestSuite, context: &mut TestRunContext) -> TestSuiteInstance {
+    if let Some(mut tests) = suite.load_all() {
+        if let Some(filter) = &context.filter {
+            tests.retain(|test| test.name().contains(filter.as_str()));
+        }
+        context.reporter.tests_discovered(suite, tests.len());
+        for _ in &tests {
+            context.reporter.test_loaded();
+        }
+        return TestSuiteInstance::new(suite, tests);
+    }
+
     let paths = WalkDir::new(suite.base_path())
         .into_iter()
         .filter_map(Result::ok)