test(parse): add normative argv grammar and conformance corpus

Cargo.toml

@@ -3,6 +3,7 @@ resolver = "2"
 members = [
     "clap_usage",
     "cli",
+    "conformance",
     "lib",
 ]
 

conformance/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "usage-conformance"
+description = "Harness for the argv conformance corpus"
+publish = false
+version = "0.0.0"
+edition = "2021"
+rust-version = "1.80.0"
+homepage = { workspace = true }
+documentation = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+usage-lib = { workspace = true }
+
+[[bin]]
+name = "oracle"
+path = "src/bin/oracle.rs"

conformance/src/bin/oracle.rs

@@ -0,0 +1,87 @@
+//! Report what usage-lib does with each corpus vector.
+//!
+//! Authoring aid, not a test. `cargo run -p usage-conformance --bin oracle` prints
+//! every vector's observed result next to its expectation, which is how the
+//! `reference` field on each vector gets filled in with a measurement rather than
+//! a guess. `--json` emits the same thing machine-readably.
+//!
+//! The test suite (`conformance/tests/reference.rs`) is what actually enforces
+//! agreement in CI.
+
+use usage_conformance::reference::run;
+use usage_conformance::{load, Reference};
+
+fn main() -> Result<(), String> {
+    let json = std::env::args().any(|a| a == "--json");
+    let filter = std::env::args()
+        .skip(1)
+        .find(|a| !a.starts_with("--"))
+        .unwrap_or_default();
+
+    let files = load(usage_conformance::corpus_dir())?;
+    let mut rows = Vec::new();
+
+    for file in &files {
+        for vector in &file.vectors {
+            if !filter.is_empty() && !vector.id.contains(&filter) {
+                continue;
+            }
+            let observed = run(vector);
+            let agrees = observed.matches(&vector.expect);
+            let declared_agrees = matches!(vector.reference, Reference::Agrees);
+            rows.push((
+                file.section.clone(),
+                vector.id.clone(),
+                agrees,
+                declared_agrees,
+                format!("{observed:?}"),
+                format!("{:?}", vector.expect),
+            ));
+        }
+    }
+
+    if json {
+        let out: Vec<_> = rows
+            .iter()
+            .map(|(section, id, agrees, declared, observed, expect)| {
+                serde_json::json!({
+                    "section": section,
+                    "id": id,
+                    "reference_agrees": agrees,
+                    "declared_agrees": declared,
+                    "observed": observed,
+                    "expected": expect,
+                })
+            })
+            .collect();
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&out).map_err(|e| e.to_string())?
+        );
+        return Ok(());
+    }
+
+    let mut mismatched = 0;
+    for (section, id, agrees, declared, observed, expect) in &rows {
+        let mark = match (agrees, declared) {
+            (true, true) => "ok  ",
+            (false, false) => "div ",
+            _ => {
+                mismatched += 1;
+                "MISM"
+            }
+        };
+        println!("{mark} {section}/{id}");
+        if agrees != declared {
+            println!("       expected: {expect}");
+            println!("       observed: {observed}");
+        }
+    }
+
+    println!(
+        "\n{} vectors, {} declared divergences, {mismatched} mislabeled",
+        rows.len(),
+        rows.iter().filter(|r| !r.3).count()
+    );
+    Ok(())
+}

conformance/src/lib.rs

@@ -0,0 +1,180 @@
+//! The argv conformance corpus: its format, a loader, and the reference runner.
+//!
+//! The corpus is the executable half of [the argv grammar]. Each vector pairs a
+//! spec and an `argv` with the result that parsing one against the other must
+//! produce. The files are plain JSON so implementations in other languages can
+//! run the same cases without reimplementing a test format, and so the grammar
+//! has a mechanical definition rather than only a prose one.
+//!
+//! [the argv grammar]: https://usage.jdx.dev/spec/argv
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+pub mod reference;
+
+/// One `corpus/*.json` file: a themed group of vectors.
+///
+/// Unknown fields are rejected throughout the corpus types. A misspelled
+/// `reference` would otherwise default to [`Reference::Agrees`], and a misspelled
+/// `flags` would become an empty expectation — both of which would let a
+/// malformed vector load and pass while testing nothing.
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct VectorFile {
+    /// Which part of the grammar this file covers, e.g. `"short-flags"`.
+    pub section: String,
+    /// What the group establishes, plus anything a reader needs in order to
+    /// judge whether these expectations are the right ones.
+    pub about: String,
+    pub vectors: Vec<Vector>,
+}
+
+/// A single case: parse `argv` against `spec` and you must get `expect`.
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(deny_unknown_fields)]
+pub struct Vector {
+    /// Stable identifier, unique across the corpus. Failures and
+    /// cross-implementation reports quote it, so renaming one breaks anybody
+    /// tracking known failures.
+    pub id: String,
+    /// What behavior this pins down, in one sentence.
+    pub doc: String,
+    /// A complete spec, as KDL.
+    pub spec: String,
+    /// The command line, excluding the program name.
+    pub argv: Vec<String>,
+    /// The environment the parse sees. Only vectors about `env` fallback set it;
+    /// the harness never consults the real environment, so no vector's result
+    /// can depend on the machine running it.
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub env: BTreeMap<String, String>,
+    pub expect: Expect,
+    /// Whether usage-lib, the reference implementation, agrees with `expect`.
+    ///
+    /// Recorded per vector rather than assumed. The corpus describes the grammar
+    /// the spec intends; usage-lib is one implementation of it, and where the two
+    /// differ that is worth writing down instead of hiding. These notes are the
+    /// compatibility matrix any new implementation has to read.
+    #[serde(default)]
+    pub reference: Reference,
+}
+
+/// The result of a parse: a binding, or a class of failure.
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum Expect {
+    Ok(Parsed),
+    /// Only the *class* of error is pinned, never its wording. Message text is a
+    /// diagnostics concern and is expected to differ between implementations.
+    Error(ErrorCode),
+}
+
+/// What a successful parse binds.
+///
+/// Keyed by the name the spec gives each flag and argument rather than by the
+/// token that set it, so `-j`, `--jobs`, and `JOBS=8` all land under `jobs`.
+/// Anything left unset is omitted rather than recorded as null.
+#[derive(Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct Parsed {
+    /// The subcommand path selected, outermost first; empty for the root.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub cmd: Vec<String>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub flags: BTreeMap<String, Value>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub args: BTreeMap<String, Value>,
+}
+
+/// A bound value.
+///
+/// Deliberately small. The grammar decides which tokens bind where, not what
+/// they mean: turning `"8"` into a number is the caller's business, so the
+/// corpus records the string.
+#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(untagged)]
+pub enum Value {
+    Bool(bool),
+    Str(String),
+    Bools(Vec<bool>),
+    Strs(Vec<String>),
+}
+
+/// The classes of failure the grammar distinguishes.
+///
+/// Coarse on purpose. An implementation should produce something far more
+/// specific; what the corpus pins is that a command line fails for a given
+/// *reason*, which is what lets a strict parser and a lenient one be told apart
+/// mechanically.
+#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum ErrorCode {
+    /// A token looked like a flag, but no flag by that name is in scope here.
+    UnknownFlag,
+    /// A flag needing a value was last, or was followed by something that cannot
+    /// be its value.
+    MissingFlagValue,
+    /// A required flag never appeared.
+    MissingRequiredFlag,
+    /// A required positional was never filled.
+    MissingRequiredArg,
+    /// More positionals were given than the command accepts.
+    UnexpectedArg,
+    /// A value was given that is not among the declared choices.
+    InvalidChoice,
+    /// A positional declared `double_dash="required"` was given before `--`.
+    ArgRequiresDoubleDash,
+    /// A variadic got fewer values than `var_min`.
+    VarTooFew,
+    /// A variadic got more values than `var_max`.
+    VarTooMany,
+}
+
+/// Whether the reference implementation matches a vector's expectation.
+#[derive(Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum Reference {
+    /// usage-lib produces exactly `expect`.
+    #[default]
+    Agrees,
+    /// usage-lib produces something else. The note says what, and why the corpus
+    /// keeps its own expectation regardless.
+    Diverges(String),
+}
+
+/// Load every `*.json` file in a corpus directory, sorted by file name.
+pub fn load(dir: impl AsRef<Path>) -> Result<Vec<VectorFile>, String> {
+    let dir = dir.as_ref();
+    // An unreadable entry is an error rather than something to skip: silently
+    // dropping one would let CI validate a partial corpus and still pass.
+    let mut paths: Vec<_> = std::fs::read_dir(dir)
+        .map_err(|e| format!("reading {}: {e}", dir.display()))?
+        .map(|entry| {
+            entry
+                .map(|e| e.path())
+                .map_err(|e| format!("reading an entry of {}: {e}", dir.display()))
+        })
+        .collect::<Result<Vec<_>, String>>()?
+        .into_iter()
+        .filter(|p| p.extension().is_some_and(|x| x == "json"))
+        .collect();
+    paths.sort();
+
+    paths
+        .iter()
+        .map(|p| {
+            let text =
+                std::fs::read_to_string(p).map_err(|e| format!("reading {}: {e}", p.display()))?;
+            serde_json::from_str(&text).map_err(|e| format!("parsing {}: {e}", p.display()))
+        })
+        .collect()
+}
+
+/// The corpus directory, resolved against this crate rather than the process's
+/// working directory.
+pub fn corpus_dir() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR")).join("../corpus")
+}