diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md index 5f208eb..5f2b988 100644 --- a/.github/agents/developer.agent.md +++ b/.github/agents/developer.agent.md @@ -19,7 +19,7 @@ Perform software development tasks by determining and applying appropriate stand If one exists, fetch it as the starting point; adjust placeholder names and heading depth to match the target path before writing the file - For each modified file, identify which companion artifacts need updating - (requirements, design docs, tests, review-sets) + (requirements, design docs, verification docs, tests, review-sets, README.md, user guides) - Include companion artifact updates in the work plan 4. **Execute work** following standards requirements and quality checks 5. **Formatting**: Run `pwsh ./fix.ps1` to silently apply all @@ -35,8 +35,7 @@ Perform software development tasks by determining and applying appropriate stand # Developer Agent Report **Result**: (SUCCEEDED|FAILED) - -## Work Summary +**Report**: `.agent-logs/developer-{subject}-{unique-id}.md` - **Files Modified**: {List of files created/modified/deleted} - **Languages Detected**: {Languages identified} diff --git a/.github/agents/formal-review.agent.md b/.github/agents/formal-review.agent.md index 7dd8e84..14e487a 100644 --- a/.github/agents/formal-review.agent.md +++ b/.github/agents/formal-review.agent.md @@ -44,6 +44,7 @@ standards from the selection matrix in AGENTS.md. # Formal Review Report **Result**: (SUCCEEDED|FAILED) +**Report**: `.agent-logs/formal-review-{subject}-{unique-id}.md` ## Review Summary diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md index 7cc0352..d94e880 100644 --- a/.github/agents/implementation.agent.md +++ b/.github/agents/implementation.agent.md @@ -28,36 +28,18 @@ The state-transitions include retrying a limited number of times: ## PLANNING State (start) -Call the **explore** agent as a sub-agent (built-in agent type) with: +Call the **planning** agent as a sub-agent (custom agent from `.github/agents/`) with: - **context**: the user's request + any previous quality findings + retry context -- **goal**: produce a verified implementation plan through these steps: - - 1. Investigate the codebase and develop a concrete implementation plan that - addresses the request - 2. **Identify companion artifact deliverables**: for every code change in the - plan, list the requirements files, design documents, and review-set entries - that must be created or updated - traceability must flow requirements → - design → code, so these are mandatory deliverables, not optional extras - 3. Review the plan for assumptions, weaknesses, and gaps - identify up to 5 - key assumptions and rate each as: - - **VERIFIED**: confirmed by codebase evidence - - **LIKELY**: consistent with codebase patterns but not directly confirmed - - **UNVERIFIED**: not confirmed by any evidence - 4. For any assumption rated UNVERIFIED or LIKELY, attempt to resolve it - through additional investigation and revise the plan to address identified - weaknesses - repeat the critique-and-strengthen cycle up to 2 additional - times if unresolved issues remain, but stop as soon as the plan is stable - 5. List up to 5 risks to the implementation - 6. Assess feasibility: can this be implemented in a single development pass? - 7. State a **recommendation**: GO or INCOMPLETE - GO if the plan is sound, or - INCOMPLETE if critical unknowns remain that only the user can resolve - -Once the explore sub-agent finishes: - -- IF recommendation is INCOMPLETE: Transition to REPORT with Result: INCOMPLETE, +- **goal**: produce a verified implementation plan, or a targeted plan to address + the identified quality issues if this is a retry + +Once the planning sub-agent finishes: + +- IF Result is FAILED: Transition to REPORT with Result: FAILED +- IF Result is INCOMPLETE: Transition to REPORT with Result: INCOMPLETE, listing the unknowns and what CAN be implemented once they are resolved -- OTHERWISE (GO): Transition to DEVELOPMENT +- OTHERWISE (SUCCEEDED): Transition to DEVELOPMENT ## DEVELOPMENT State @@ -76,7 +58,8 @@ Once the developer sub-agent finishes: Call the **quality** agent as a sub-agent (custom agent from `.github/agents/`) with: -- **context**: the user's request + development summary + files changed + previous issues (if any) +- **context**: the user's request + development summary + files changed + planning companion artifact table + + previous issues (if any) - **goal**: check the quality of the work performed for any issues Once the quality sub-agent finishes: @@ -92,6 +75,9 @@ Once the quality sub-agent finishes: this agent may report INCOMPLETE when the request cannot be implemented without information only the user can provide. +For full planning details (assumptions, risks, feasibility), read the planning +report file referenced in the planning agent's response. + Generate the completion report using the template below, then save it to `.agent-logs/{agent-name}-{subject}-{unique-id}.md` per the AGENTS.md reporting requirements, and return the summary to the caller. @@ -102,19 +88,20 @@ requirements, and return the summary to the caller. # Implementation Orchestration Report **Result**: (SUCCEEDED|FAILED|INCOMPLETE) -**Final State**: (PLANNING|DEVELOPMENT|QUALITY|REPORT) +**Report**: `.agent-logs/implementation-{subject}-{unique-id}.md` +**Last Active State**: (PLANNING|DEVELOPMENT|QUALITY) **Retry Count**: ## State Machine Execution -- **Planning Results**: {Implementation plan, assumption ratings, risks, and recommendation} +- **Planning Results**: {Planning report path; plan summary and SUCCEEDED/INCOMPLETE/FAILED result} - **Development Results**: {Summary of developer agent results} - **Quality Results**: {Summary of quality agent results} - **State Transitions**: {Log of state changes and decisions} ## Sub-Agent Coordination -- **Explore Agent (Planning)**: {Plan, assumption verdicts, top risks, GO/INCOMPLETE recommendation} +- **Planning Agent**: {Report file path, SUCCEEDED/INCOMPLETE/FAILED result, plan summary} - **Developer Agent**: {Development status and files modified} - **Quality Agent**: {Validation results and compliance status} @@ -123,4 +110,9 @@ requirements, and return the summary to the caller. - **Implementation Success**: {Overall completion status} - **Quality Compliance**: {Final quality validation status} - **Issues Resolved**: {Problems encountered and resolution attempts} + +## Unknowns (only when Result is INCOMPLETE) + +- **Unresolved Questions**: {List each question the user must answer} +- **What Can Proceed**: {Work that can be done without the missing information} ``` diff --git a/.github/agents/lint-fix.agent.md b/.github/agents/lint-fix.agent.md index 549e751..36d3ca1 100644 --- a/.github/agents/lint-fix.agent.md +++ b/.github/agents/lint-fix.agent.md @@ -68,8 +68,7 @@ submission, not during normal development. # Lint Fix Report **Result**: (SUCCEEDED|FAILED) - -## Summary +**Report**: `.agent-logs/lint-fix-{subject}-{unique-id}.md` - **Iterations**: {Number of fix-loop iterations performed} - **Files Modified**: {List of all files changed} diff --git a/.github/agents/planning.agent.md b/.github/agents/planning.agent.md new file mode 100644 index 0000000..20e75ee --- /dev/null +++ b/.github/agents/planning.agent.md @@ -0,0 +1,134 @@ +--- +name: planning +description: Planning agent that investigates the codebase, develops a verified implementation plan, and identifies all companion artifact deliverables. +user-invocable: true +--- + +# Planning Agent + +Investigate the codebase and produce a verified implementation plan with all +companion artifact deliverables. + +## Step 1 — Load Standards + +Read the relevant standards from `.github/standards/` using the selection matrix +in `AGENTS.md` based on the artifact types in scope for the request (requirements, +design, verification, documentation, code). + +## Step 2 — Investigate and Plan + +Read `docs/design/introduction.md` first (if present), then investigate the +codebase to develop a concrete implementation plan: + +- Identify all files to create, modify, or delete +- Describe the change required for each file + +## Step 3 — Identify Companion Artifact Deliverables + +For each planned change, assess the mandatory companion artifacts below (create/update/N/A +with justification): + +- **Requirements** — functional changes require a requirement entry +- **Design Documentation** — new or changed components require design docs +- **Verification Documentation** — new or changed components require verification docs +- **Tests** — functional changes require test coverage +- **Review Sets** — changes to the software item hierarchy (units or subsystems + added, removed, or reorganized) require review-set updates +- **README.md** — user-facing changes require README updates +- **User Guide** — user-facing features require user guide updates + +## Step 4 — Critique and Strengthen + +Identify up to 5 key assumptions and rate each: + +- **VERIFIED**: confirmed by codebase evidence +- **LIKELY**: consistent with codebase patterns but not directly confirmed +- **UNVERIFIED**: not confirmed by any evidence + +For UNVERIFIED or LIKELY assumptions, investigate further and revise the plan. +Repeat up to 2 more times, stopping when the plan is stable. + +## Step 5 — Risk Assessment + +List up to 5 risks with a brief mitigation for each. + +## Step 6 — Feasibility Assessment + +State whether this can be implemented in a single development pass and any +preconditions that affect feasibility. + +## Step 7 — Recommendation + +- **SUCCEEDED** — the plan is sound and the developer agent can proceed +- **INCOMPLETE** — critical unknowns remain that only the user can resolve; + list each unknown explicitly +- **FAILED** — investigation could not produce a viable plan + +# REPORT Phase + +Save the full analysis to `.agent-logs/planning-{subject}-{unique-id}.md` per +the AGENTS.md reporting requirements. + +Then respond to the caller with ONLY the lean structured summary below. + +# Report Template + +```markdown +# Planning Report + +**Result**: (SUCCEEDED|INCOMPLETE|FAILED) +**Request Summary**: {Brief restatement of the task as understood} +**Report**: `.agent-logs/planning-{subject}-{unique-id}.md` + +## Implementation Plan + +| File | Action | Description | +|------|--------|-------------| +| {path} | create/modify/delete | {what changes and why} | + +## Companion Artifact Deliverables + +| Category | File | Action | +|----------|------|--------| +| Requirements | {path} | create/update/N/A — {justification} | +| Design Documentation | {path} | create/update/N/A — {justification} | +| Verification Documentation | {path} | create/update/N/A — {justification} | +| Tests | {path} | create/update/N/A — {justification} | +| Review Sets | {path} | create/update/N/A — {justification} | +| README.md | {path} | create/update/N/A — {justification} | +| User Guide | {path} | create/update/N/A — {justification} | + +## Assumption Analysis + +| # | Assumption | Rating | Resolution | +|---|-----------|--------|------------| +| 1 | {assumption} | VERIFIED/LIKELY/UNVERIFIED | {resolution or N/A} | + +## Risk Assessment + +1. **[severity]** {risk} — {mitigation} + +## Feasibility Assessment + +{Single-pass or not, and why. Any preconditions.} + +## Unknowns + +{Only present when Result is INCOMPLETE. List each question the user must +resolve before implementation can proceed.} +``` + +# Lean Structured Response (returned to caller) + +```markdown +**Result**: (SUCCEEDED|INCOMPLETE|FAILED) +**Report**: `.agent-logs/planning-{subject}-{unique-id}.md` + +**Plan**: +{Repeat the Implementation Plan table} + +**Companion Artifacts**: +{Repeat the Companion Artifact Deliverables table} + +**Unknowns**: {Only if INCOMPLETE — list questions for the user} +``` diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md index 380d11f..26fd251 100644 --- a/.github/agents/quality.agent.md +++ b/.github/agents/quality.agent.md @@ -13,14 +13,23 @@ Grade and validate software development work by ensuring compliance with project 1. **Analyze the task request AND completed work** to determine scope: identify which artifact categories were changed, and which *should have been changed* given the task - new user-visible features always require requirements, - design, and review-set coverage regardless of whether those files were touched; - test-only additions (corner-case tests, defensive boundary tests, regression - tests) do not require a corresponding requirement + design, verification docs, and README/user guide updates regardless of + whether those files were touched; Review Sets are always in scope when + the software item hierarchy changes (units or subsystems added, removed, or + reorganized); test-only additions (corner-case tests, defensive boundary + tests, regression tests) do not require a corresponding requirement; if a + planning companion artifact table is provided in context, cross-reference it + — any artifact listed as create/update must be covered in the evaluation and + FAIL if the artifact was not produced 2. **Read relevant standards** using the selection matrix in AGENTS.md 3. **Evaluate all in-scope categories** - N/A only when the task genuinely cannot affect a category; if the task introduces new user-visible features or - structural changes then Requirements, Design Documentation, and Review - Management are always in scope and FAIL if the artifacts were not updated + structural changes then Requirements, Design Documentation, and Verification + Documentation are always in scope and FAIL if the artifacts were not updated; + Documentation (README/user guide) is always in scope for user-facing changes + and FAIL if not updated; Review Sets are always in scope when the + software item hierarchy changes (units or subsystems added, removed, or + reorganized) and FAIL if review-sets were not updated 4. **Validate tool compliance** using ReqStream, ReviewMark, and build tools 5. **Generate focused quality report** per the AGENTS.md reporting requirements - save to `.agent-logs/{agent-name}-{subject}-{unique-id}.md` and return the summary to the caller @@ -36,6 +45,7 @@ For each checklist item in the template below, record as `(PASS|FAIL|N/A) - {one # Quality Assessment Report **Result**: (SUCCEEDED|FAILED) +**Report**: `.agent-logs/quality-{subject}-{unique-id}.md` **Overall Grade**: (PASS|FAIL) ## Required Fixes (only when Result is FAILED) @@ -50,13 +60,13 @@ Priority-ordered list of issues that MUST be resolved for the next retry: - **Evaluated**: {List sections assessed and why} - **Skipped**: {One-line per skipped section with reason, e.g., "Design - Documentation: N/A - no design files modified"} + Documentation: N/A - no component behavior, structure, or interface changed"} ## Requirements Compliance: (PASS|FAIL|N/A) - Were requirements created/updated for all functional changes? - Were source filters applied for platform-specific requirements? -- Is requirements traceability maintained to tests? +- Is forward traceability from requirements to verification artifacts preserved? ## Design Documentation Compliance: (PASS|FAIL|N/A) @@ -80,7 +90,14 @@ Priority-ordered list of issues that MUST be resolved for the next retry: - Are cross-hierarchy test dependencies documented in design docs? - Do all tests pass? -## Review Management Compliance: (PASS|FAIL|N/A) +## Verification Documentation Compliance: (PASS|FAIL|N/A) + +- Were verification documents created/updated for all new or changed components? +- Do verification documents include all mandatory sections (Verification Approach, Test Environment, + Acceptance Criteria, Test Scenarios)? +- Is requirements-to-test coverage tracked via the ReqStream trace matrix (not embedded in verification docs)? + +## Review Sets Compliance: (PASS|FAIL|N/A) - Were review-sets updated for structural changes? - Is review scope appropriate for change magnitude? diff --git a/.github/agents/template-sync.agent.md b/.github/agents/template-sync.agent.md index c013503..df4d488 100644 --- a/.github/agents/template-sync.agent.md +++ b/.github/agents/template-sync.agent.md @@ -22,12 +22,18 @@ Delegate each group to a sub-agent. - **Root config files** - all non-collection files at the repository root - **One group per flat `docs/` folder** - e.g. `docs/build_notes/`, `docs/user_guide/` +- **One group for root files in each of `docs/design/`, `docs/verification/`, + `docs/reqstream/`** - e.g. `docs/design/introduction.md` — separate from the + system subtrees beneath them - **One group per system subtree** in `docs/design/`, `docs/verification/`, `docs/reqstream/` - each subtree and all its descendants is one group # Orchestration -For each group intersecting the requested scope, call a sub-agent with: +For Audit mode, call an **explore** sub-agent (built-in) per group. +For Sync, Scaffold, and Recreate modes, call a **general-purpose** sub-agent (built-in) per group. + +For each group intersecting the requested scope, call the appropriate sub-agent with: - **context**: - Group scope and template URL from the `# Reference Template` section in `AGENTS.md` @@ -40,7 +46,10 @@ For each group intersecting the requested scope, call a sub-agent with: each item is a subsystem or unit, then select the appropriate template (`subsystem-name.*` or `unit-name.*`) regardless of the item's folder depth — do not infer item type from path depth alone - - If a template counterpart cannot be fetched, skip the file and report it + - If a file has no template counterpart, skip it and report it as + "No template found" — this is not a failure + - If a file appears in `repository-map.md` but its template cannot be fetched, + report Result: FAILED and list the affected files - **goal**: - Based on the given mode: - **Audit** - fetch each template counterpart; compare headings; report missing @@ -78,9 +87,25 @@ For each group intersecting the requested scope, call a sub-agent with: `TODO:` placeholders in YAML string values (e.g. `title:`, `justification:`) are content placeholders — always resolve them to real content; infer from README, related files, sibling docs, and path; if confident write directly; - if ambiguous offer 2–3 concrete options and ask the user; keep asking until - they answer - never leave a TODO or TEMPLATE-DIRECTIVE in the output unless - the user explicitly requests it + if ambiguous, **do not ask interactively** — return the unresolved questions + in the result so the orchestrator can ask the user and re-invoke; never leave + a TODO or TEMPLATE-DIRECTIVE in the output unless the user explicitly requests it + - Return results in this format for each file in the group: + + ```markdown + ### {file-path} + + - **Template**: {template path or "not found"} + - **Missing sections**: {list or "none"} + - **Heading depth issues**: {list or "none"} + - **Content format issues**: {list or "none"} *(Recreate only)* + - **Action**: (Reported | Sections added | Created | Rebuilt | No template found) + - **Unresolved Questions**: {list or "none"} + ``` + +If any sub-agent returns unresolved questions, collect them, ask the user, then +re-invoke the affected sub-agent(s) with the answers before assembling the final report. +If questions remain unresolved after asking the user, report Result: INCOMPLETE. Collect sub-agent results and assemble the final report. @@ -89,7 +114,8 @@ Collect sub-agent results and assemble the final report. ```markdown # Template Sync Report -**Result**: (SUCCEEDED|FAILED) +**Result**: (SUCCEEDED|FAILED|INCOMPLETE) +**Report**: `.agent-logs/template-sync-{subject}-{unique-id}.md` **Mode**: (Audit|Sync|Scaffold|Recreate) ## Files @@ -102,8 +128,13 @@ Collect sub-agent results and assemble the final report. - **Content format issues**: {list of sections where intra-section content did not match the template comment's prescribed format, or "none"} *(Recreate only)* - **Action**: (Reported | Sections added | Created | Rebuilt | No template found) +- **Unresolved Questions**: {list or "none"} ## Summary - **Conformant**: {count} | **Deviations**: {count} | **Updated**: {count} + +## Unknowns (only when Result is INCOMPLETE) + +- **Unresolved Questions**: {List each placeholder or ambiguity the user must resolve} ``` diff --git a/.github/standards/coding-principles.md b/.github/standards/coding-principles.md index 9e67fbb..6797c61 100644 --- a/.github/standards/coding-principles.md +++ b/.github/standards/coding-principles.md @@ -3,11 +3,6 @@ name: Coding Principles description: Follow these standards when developing any software code. --- -# Coding Principles Standards - -This document defines universal coding principles and quality standards for software development within -Continuous Compliance environments. - # Core Principles ## Literate Coding @@ -20,10 +15,9 @@ All code MUST follow literate programming principles: matches design intent without reading the full codebase - **Logical Separation**: Complex functions use block comments to separate and describe logical steps within the implementation -- **Full Symbol Documentation**: ALL symbols have comprehensive documentation - because reviewers and auditors must verify every implementation detail, not - just the public interface - access-level specifics (public, protected, - private, internal, etc.) vary by language; see the language-specific standard +- **Full Symbol Documentation**: ALL symbols have comprehensive documentation — + not just the public interface, because reviewers and auditors must verify every + implementation detail. Access-level specifics vary by language; see the language-specific standard. - **Clarity Over Cleverness**: Code should be immediately understandable by team members ## API Documentation @@ -79,13 +73,13 @@ interface correctly without reading the implementation: ## Universal Anti-Patterns -- **Skip Literate Coding**: Don't skip literate programming comments - they are required for maintainability -- **Ignore Compiler Warnings**: Don't ignore compiler warnings - they exist for quality enforcement +- **Skip Literate Coding**: Don't skip literate programming comments +- **Ignore Compiler Warnings**: Don't ignore compiler warnings - **Hidden Dependencies**: Don't create untestable code with hidden dependencies - **Hidden Functionality**: Don't implement functionality without requirement traceability because untraced functionality cannot be validated during audits - **Monolithic Functions**: Don't write monolithic functions with multiple responsibilities -- **Overcomplicated Solutions**: Don't make solutions more complex than necessary - favor simplicity and clarity +- **Overcomplicated Solutions**: Don't make solutions more complex than necessary - **Premature Optimization**: Don't optimize for performance before establishing correctness - **Copy-Paste Programming**: Don't duplicate logic - extract common functionality into reusable components - **Magic Numbers**: Don't use unexplained constants - either name them or add clear comments diff --git a/.github/standards/csharp-language.md b/.github/standards/csharp-language.md index 6df39cd..ec05a25 100644 --- a/.github/standards/csharp-language.md +++ b/.github/standards/csharp-language.md @@ -12,9 +12,6 @@ Read these standards first before applying this standard: # API Documentation and Literate Coding Example -The example below demonstrates good XmlDoc API documentation combined with -literate coding comments. - ```csharp /// /// Converts a raw sensor reading into a validated measurement ready for downstream consumers. diff --git a/.github/standards/csharp-testing.md b/.github/standards/csharp-testing.md index 181de02..1f93b72 100644 --- a/.github/standards/csharp-testing.md +++ b/.github/standards/csharp-testing.md @@ -66,8 +66,6 @@ These are non-obvious v3 behaviors that differ from v2 or common assumptions: # Quality Checks -Before submitting C# tests, verify: - - [ ] All tests follow AAA pattern with clear section comments - [ ] Test names follow hierarchical naming pattern above - [ ] Each test verifies single, specific behavior (no shared state between tests) diff --git a/.github/standards/design-documentation.md b/.github/standards/design-documentation.md index 156edfd..e5b7bf9 100644 --- a/.github/standards/design-documentation.md +++ b/.github/standards/design-documentation.md @@ -28,10 +28,9 @@ docs/design/ └── {package-name}.md # heading depth ## ``` -Subsystems may nest recursively, so nested paths may end in either a subsystem document -(`docs/design/{system-name}[/{subsystem-name}...]/{subsystem-name}.md`) or a unit document -(`docs/design/{system-name}[/{subsystem-name}...]/{unit-name}.md`). -Each file's heading depth equals its folder depth under `docs/design/`. +All sections in every file are mandatory; write "N/A - {justification}" rather than removing any. +Determine subsystem vs. unit classification from `docs/design/introduction.md` — folder depth does not determine classification. +Do not record version numbers anywhere in design documentation — version information is managed in SBOMs. # introduction.md (MANDATORY) @@ -46,8 +45,7 @@ Must include: # System Design (MANDATORY) -Create `{system-name}.md` (`#` heading) and `{system-name}/` folder. All sections mandatory; -write "N/A - {justification}" rather than removing any section: +Create `{system-name}.md` (`#` heading) and `{system-name}/` folder: - **Architecture**: software items, relationships, and collaboration - **External Interfaces**: name, direction, format, constraints @@ -58,11 +56,7 @@ write "N/A - {justification}" rather than removing any section: # Subsystem Design (MANDATORY) -Place `{subsystem-name}.md` in the **parent** folder; create `{subsystem-name}/` for children. -**Important**: A file at `{system-name}/**/*.md` may be either a subsystem or a unit. Always -determine the correct classification from `docs/design/introduction.md` — folder depth does not -determine classification. -All sections mandatory; write "N/A - {justification}" rather than removing any section: +Place `{subsystem-name}.md` in the **parent** folder; create `{subsystem-name}/` for children: - **Overview**: responsibility, boundaries, contained units - **Interfaces**: what it exposes and consumes @@ -70,31 +64,34 @@ All sections mandatory; write "N/A - {justification}" rather than removing any s # Unit Design (MANDATORY) -Place `{unit-name}.md` in the **parent** folder. -**Important**: A file at `{system-name}/**/*.md` may be either a subsystem or a unit. Always -determine the correct classification from `docs/design/introduction.md` — folder depth does not -determine classification. -All sections mandatory; write "N/A - {justification}" rather than removing any section: +Place `{unit-name}.md` in the **parent** folder: - **Purpose**: single responsibility - **Data Model**: fields, properties, types, invariants (IEC 62304 §5.4.2) - **Key Methods**: name, purpose, algorithm, preconditions, postconditions, parameter types - **Error Handling**: detection and handling; what is propagated vs. handled locally -- **Interactions**: dependencies on other units/subsystems/OTS; who calls this unit +- **Dependencies**: other units, subsystems, OTS items, and shared packages used +- **Callers**: units or subsystems that call or consume this unit # OTS Integration Design (when OTS items exist) Create `docs/design/ots.md` (`#` heading) covering the overall OTS integration strategy. -For each OTS item, create `docs/design/ots/{ots-name}.md` (`##` heading) covering: -why chosen, which features/APIs used, integration patterns, version constraints. +For each OTS item, create `docs/design/ots/{ots-name}.md` (`##` heading) with sections: + +- **Purpose**: why chosen and what it provides to the local system +- **Features Used**: which specific features, APIs, or capabilities are consumed +- **Integration Pattern**: how it is consumed; initialization, configuration, disposal requirements # Shared Package Integration Design (when Shared Packages exist) Create `docs/design/shared.md` (`#` heading) covering the overall consumption strategy. -For each Shared Package, create `docs/design/shared/{package-name}.md` (`##` heading) covering: -which advertised features are consumed, integration pattern, configuration/initialization. +For each Shared Package, create `docs/design/shared/{package-name}.md` (`##` heading) with sections: + +- **Advertised Features Consumed**: which features the local system relies on +- **Integration Pattern**: how the package is referenced, initialized, and consumed +- **Assumptions**: any assumptions the local system makes about the package's behavior # Writing Guidelines @@ -111,7 +108,7 @@ which advertised features are consumed, integration pattern, configuration/initi - [ ] System design includes all mandatory sections (Architecture, External Interfaces, Dependencies, Risk Control Measures, Data Flow, Design Constraints) - [ ] Subsystem design includes all mandatory sections (Overview, Interfaces, Design) -- [ ] Unit design includes all mandatory sections (Purpose, Data Model, Key Methods, Error Handling, Interactions) +- [ ] Unit design includes all mandatory sections (Purpose, Data Model, Key Methods, Error Handling, Dependencies, Callers) - [ ] Non-applicable mandatory sections contain "N/A - {justification}" - [ ] `docs/design/ots.md` and `docs/design/ots/{ots-name}.md` exist when OTS items are present - [ ] `docs/design/shared.md` and `docs/design/shared/{package-name}.md` exist when Shared Packages are present diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md index 4921a55..b521433 100644 --- a/.github/standards/reviewmark-usage.md +++ b/.github/standards/reviewmark-usage.md @@ -88,9 +88,6 @@ When constructing review-sets, follow these principles to maintain manageable sc # Review-Set Organization -Organize review-sets using these standard patterns to ensure comprehensive coverage -while keeping each review manageable in scope: - **Naming conventions**: Placeholders in documentation, requirements, design, and verification file paths are kebab-case (e.g., `{system-name}`). Placeholders in source and test file paths may use the casing conventional for the project's @@ -236,17 +233,6 @@ Before submitting ReviewMark configuration, verify: - [ ] `.reviewmark.yaml` exists at repository root with proper structure - [ ] Review-set organization follows the standard hierarchy patterns -- [ ] Purpose review-set includes README.md, user guide, system requirements, design introduction, and system design files -- [ ] System-level reviews follow hierarchical scope principle (exclude subsystem/unit details) -- [ ] Subsystem reviews follow hierarchical scope principle (exclude unit source code) -- [ ] Only unit reviews include actual source code files -- [ ] Architecture review-sets include system verification design alongside system design -- [ ] Design review-sets include all system design files -- [ ] Verification review-sets include all system verification files -- [ ] Subsystem review-sets include subsystem verification design -- [ ] Unit review-sets include unit verification design -- [ ] OTS review-sets include OTS requirements, integration design, and verification evidence -- [ ] Shared Package review-sets include Shared Package requirements, integration design, and verification evidence - [ ] Each review-set focuses on a single compliance question (single focus principle) - [ ] File patterns use correct glob syntax and match intended files - [ ] Review-set file counts remain manageable (context management principle) diff --git a/.github/standards/software-items.md b/.github/standards/software-items.md index 444928c..6c29525 100644 --- a/.github/standards/software-items.md +++ b/.github/standards/software-items.md @@ -3,12 +3,6 @@ name: Software Items description: Follow these standards when categorizing software components. --- -# Software Items Definition Standards - -This document defines standards for categorizing software items within -Continuous Compliance environments because proper categorization determines -requirements management approach, testing strategy, and review scope. - # Software Item Categories Categorize all software into six primary groups: @@ -63,8 +57,6 @@ dash-separated IDs. Examples covering all three forms: # Categorization Guidelines -Choose the appropriate category based on scope and testability: - ## Software Package - Represents one distributable artifact @@ -102,7 +94,7 @@ Choose the appropriate category based on scope and testability: - Examples: System.Text.Json, Entity Framework, third-party APIs - **Artifact locations** (OTS items have no internal design documentation): - Requirements: `docs/reqstream/ots/{ots-name}.yaml` - - Design: `docs/design/ots/{ots-name}.md` (integration/usage design - how the local system uses this item) + - Design: `docs/design/ots/{ots-name}.md` (integration/usage design) - Verification: `docs/verification/ots/{ots-name}.md` - These folders sit parallel to system folders (not inside any system folder) - System design documentation records which OTS items each system depends on @@ -122,10 +114,9 @@ Choose the appropriate category based on scope and testability: downstream integration tests that transitively prove the advertised features are functional - **Artifact locations** (no internal design documentation in the consuming repository): - Requirements: `docs/reqstream/shared/{package-name}.yaml` - - Design: `docs/design/shared/{package-name}.md` (integration/usage design - which features are consumed and how) + - Design: `docs/design/shared/{package-name}.md` (integration/usage design) - Verification: `docs/verification/shared/{package-name}.md` - These folders sit parallel to system and OTS folders -- System design documentation records which Shared Packages each system depends on # Software Item Artifact Model diff --git a/.github/standards/technical-documentation.md b/.github/standards/technical-documentation.md index 0dc4455..23893bd 100644 --- a/.github/standards/technical-documentation.md +++ b/.github/standards/technical-documentation.md @@ -6,9 +6,6 @@ globs: ["docs/**/*.md", "README.md", "!docs/**/generated/**"] # Technical Documentation Standards -This document defines standards for technical documentation within Continuous -Compliance environments. - # Core Principles Technical documentation serves as compliance evidence and must be structured @@ -40,8 +37,7 @@ docs/{collection}/ Without `title.txt` and `definition.yaml` the pipeline cannot generate the document. When creating a new document collection, create these three files together and use -the existing collections under `docs/` as templates - they share a consistent -structure across all collections. +the existing collections under `docs/` as templates. The `generated/` folder is **never committed** to the repository - it is created locally and in CI by the build pipeline. Do not flag its absence as a conformance @@ -85,8 +81,7 @@ elsewhere causes duplicate sections in the compiled PDF. ## Document Ordering -List documents in logical reading order in Pandoc configuration because -readers need coherent information flow from general to specific topics. +List documents in logical reading order in `definition.yaml`. ## Heading Depth Rule (MANDATORY) @@ -110,15 +105,10 @@ available - keep internal structure flat to avoid excessive nesting. Write technical documentation for clarity and compliance verification: - **Clear and Concise**: Use direct language and avoid unnecessary complexity. - Regulatory reviewers must understand content quickly. -- **Structured Sections**: Use consistent heading hierarchy and section - organization. Enables automated processing and review. -- **Specific Examples**: Include concrete examples with actual values rather - than placeholders. Supports implementation verification. +- **Structured Sections**: Use consistent heading hierarchy and section organization. +- **Specific Examples**: Include concrete examples with actual values rather than placeholders. - **Current Information**: Keep documentation synchronized with code changes. - Outdated documentation invalidates compliance evidence. -- **Traceable Content**: Link documentation to requirements and implementation - where applicable for audit trails. +- **Traceable Content**: Link documentation to requirements and implementation where applicable. ## References Sections @@ -138,26 +128,16 @@ Instead use **verbal references** - plain prose that identifies the target by na > > Refer to the *System Requirements* document for the full specification. -Verbal references are readable by both AI agents and humans in any rendering environment. - # Markdown Format Requirements -Markdown documentation in this repository must follow the formatting standards -defined in `.markdownlint-cli2.yaml` (subject to any exclusions configured there) -for consistency and professional presentation: - -- **120 Character Line Limit**: Keep lines 120 characters or fewer for readability. - Break long lines naturally at punctuation or logical breaks. -- **No Trailing Whitespace**: Remove all trailing spaces and tabs from line - endings to prevent formatting inconsistencies. -- **Blank Lines Around Headings**: Include a blank line both before and after - each heading to improve document structure and readability. -- **Blank Lines Around Lists**: Include a blank line both before and after - numbered and bullet lists to ensure proper rendering and visual separation. -- **ATX-Style Headers**: Use `#` syntax for headers instead of underline style - for consistency across all documentation. -- **Consistent List Indentation**: Use 2-space indentation for nested list - items to maintain uniform formatting. +Follow `.markdownlint-cli2.yaml` formatting standards: + +- **120 Character Line Limit**: Keep lines 120 characters or fewer; break at punctuation or logical breaks. +- **No Trailing Whitespace**: Remove all trailing spaces and tabs. +- **Blank Lines Around Headings**: Include a blank line before and after each heading. +- **Blank Lines Around Lists**: Include a blank line before and after numbered and bullet lists. +- **ATX-Style Headers**: Use `#` syntax, not underline style. +- **Consistent List Indentation**: Use 2-space indentation for nested list items. # Auto-Generated Content (CRITICAL) @@ -168,8 +148,6 @@ build outputs that are overwritten on every CI run: respective `docs/` sections, or in `docs/generated/` for final release artifacts - **Source Modification**: Update source files (requirements YAML, `.reviewmark.yaml`, tool configuration) instead of generated output -- **Tool Integration**: Generated content integrates with CI/CD pipelines and - manual changes disrupt automation # README.md Best Practices @@ -192,20 +170,12 @@ Structure README.md for both human readers and AI agent processing: - **Code Block Languages**: Specify language for syntax highlighting and tool processing - **Clear Prerequisites**: List exact version requirements and dependencies -## Quality Guidelines - -- **Scannable Structure**: Use bullet points, headings, and short paragraphs -- **Current Examples**: Verify all code examples work with current version -- **Link Validation**: Ensure all external links are accessible and current -- **Consistent Tone**: Professional, helpful tone appropriate for technical audience - # Quality Checks Before submitting technical documentation, verify: - [ ] Documentation organized under `docs/` following standard folder structure - [ ] Pandoc collections include `introduction.md` with Purpose and Scope sections -- [ ] Content follows clear and concise writing guidelines with specific examples - [ ] No modifications made to auto-generated markdown files in compliance folders - [ ] README.md includes all required sections with absolute URLs and concrete examples - [ ] Documentation integrated into ReviewMark review-sets for formal review diff --git a/.github/standards/testing-principles.md b/.github/standards/testing-principles.md index 73974ff..917463e 100644 --- a/.github/standards/testing-principles.md +++ b/.github/standards/testing-principles.md @@ -3,11 +3,6 @@ name: Testing Principles description: Follow these standards when developing any software tests. --- -# Testing Principles Standards - -This document defines universal testing principles and quality standards for test development within -Continuous Compliance environments. - # Test Dependency Boundaries (MANDATORY) Respect software item hierarchy boundaries to ensure review-sets can validate proper architectural scope. diff --git a/.github/standards/verification-documentation.md b/.github/standards/verification-documentation.md index d29da5f..494e40f 100644 --- a/.github/standards/verification-documentation.md +++ b/.github/standards/verification-documentation.md @@ -28,10 +28,8 @@ docs/verification/ └── {package-name}.md # heading depth ## ``` -Subsystems may nest recursively: -`docs/verification/{system-name}[/{subsystem-name}...]/{subsystem-name}.md` or -`docs/verification/{system-name}[/{subsystem-name}...]/{unit-name}.md`. -Each file's heading depth equals its folder depth under `docs/verification/`. +All sections in every file are mandatory; write "N/A - {justification}" rather than removing any. +Determine subsystem vs. unit classification from `docs/design/introduction.md` — folder depth does not determine classification. # introduction.md (MANDATORY) @@ -44,56 +42,44 @@ Must include: # System Verification Design (MANDATORY) -Create `{system-name}.md` (`#` heading) and `{system-name}/` folder. All sections mandatory; -write "N/A - {justification}" rather than removing any section: +Create `{system-name}.md` (`#` heading) and `{system-name}/` folder: -- **Verification Strategy**: test types (unit, integration, end-to-end), framework, project structure +- **Verification Approach**: test types (unit, integration, end-to-end), framework, project structure - **Test Environment**: OS, runtime, external services, files, or configuration required - **Acceptance Criteria**: what constitutes a passing system test (IEC 62304 §5.7.2) -- **System-Level Test Scenarios**: named scenarios for each system requirement -- **Requirements Coverage**: requirement → scenario(s) → test method(s) mapping +- **Test Scenarios**: named scenarios for each system requirement # Subsystem Verification Design (MANDATORY) -Place `{subsystem-name}.md` in the **parent** folder; create `{subsystem-name}/` for children. -**Important**: A file at `{system-name}/**/*.md` may be either a -subsystem or a unit. Always determine the correct classification from -`docs/design/introduction.md` — folder depth does not determine classification. -All sections mandatory; write "N/A - {justification}" rather than removing any section: +Place `{subsystem-name}.md` in the **parent** folder; create `{subsystem-name}/` for children: -- **Verification Strategy**: integration test approach and mocking at subsystem boundary +- **Verification Approach**: integration test approach and mocking at subsystem boundary - **Test Environment**: any environment setup beyond the standard test runner - **Acceptance Criteria**: what constitutes a passing subsystem test (IEC 62304 §5.5.2) - **Test Scenarios**: named scenarios including boundary conditions, error paths, and normal operation -- **Requirements Coverage**: requirement → scenario(s) → test method(s) mapping # Unit Verification Design (MANDATORY) -Place `{unit-name}.md` in the **parent** folder. -**Important**: A file at `{system-name}/**/*.md` may be either a -subsystem or a unit. Always determine the correct classification from -`docs/design/introduction.md` — folder depth does not determine classification. -All sections mandatory; write "N/A - {justification}" rather than removing any section: +Place `{unit-name}.md` in the **parent** folder: - **Verification Approach**: what is mocked/stubbed and why; injected vs. real dependencies - **Test Environment**: any environment setup beyond the standard test runner - **Acceptance Criteria**: what constitutes passing unit tests (IEC 62304 §5.5.2) - **Test Scenarios**: named scenarios including boundary values, error paths, and normal operation -- **Requirements Coverage**: requirement → scenario(s) → test method(s) mapping # OTS Verification Evidence (when OTS items exist) Create `docs/verification/ots.md` (`#` heading) covering the overall OTS verification strategy. For each OTS item, create `docs/verification/ots/{ots-name}.md` (`##` heading) covering: -verification approach (self-validation, integration tests, vendor evidence) and requirements coverage. +verification approach (self-validation, integration tests, vendor evidence). # Shared Package Verification Evidence (when Shared Packages exist) Create `docs/verification/shared.md` (`#` heading) covering the overall Shared Package verification strategy. For each Shared Package, create `docs/verification/shared/{package-name}.md` (`##` heading) covering: -verification approach and requirements coverage. +verification approach. # Writing Guidelines @@ -106,14 +92,10 @@ verification approach and requirements coverage. - [ ] `introduction.md` includes Companion Artifact Structure - [ ] Each file's heading depth matches its folder depth - [ ] All folders use kebab-case mirroring source structure -- [ ] System verification includes all mandatory sections (Verification Strategy, Test Environment, - Acceptance Criteria, System-Level Test Scenarios, Requirements Coverage) -- [ ] Subsystem verification includes all mandatory sections (Verification Strategy, Test Environment, - Acceptance Criteria, Test Scenarios, Requirements Coverage) -- [ ] Unit verification includes all mandatory sections (Verification Approach, Test Environment, - Acceptance Criteria, Test Scenarios, Requirements Coverage) +- [ ] Each system/subsystem/unit file includes all mandatory sections (Verification Approach, + Test Environment, Acceptance Criteria, Test Scenarios) - [ ] Non-applicable mandatory sections contain "N/A - {justification}" -- [ ] Every requirement is mapped to at least one named test scenario +- [ ] Requirements-to-test coverage is tracked via the ReqStream trace matrix, not in these documents - [ ] `docs/verification/ots.md` and `docs/verification/ots/{ots-name}.md` exist when OTS items are present - [ ] `docs/verification/shared.md` and `docs/verification/shared/{package-name}.md` exist when Shared Packages are present - [ ] Documents are integrated into ReviewMark review-sets diff --git a/.reviewmark.yaml b/.reviewmark.yaml index 43c4639..448f916 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -27,7 +27,7 @@ evidence-source: # Each review-set focuses on a single compliance question with manageable file counts. reviews: - # Purpose Review (only one per repository) + # Purpose - id: Purpose title: Review that Advertised Features Match System Design paths: @@ -37,20 +37,21 @@ reviews: - docs/design/introduction.md - docs/design/sarifmark.md - # SarifMark-Architecture Review + # SarifMark - Specials - id: SarifMark-Architecture title: Review that SarifMark Architecture Satisfies Requirements paths: - docs/reqstream/sarifmark.yaml - docs/design/introduction.md - docs/design/sarifmark.md + - docs/design/ots.md + - docs/design/shared.md - docs/verification/introduction.md - docs/verification/sarifmark.md - test/**/IntegrationTests.cs - test/**/Runner.cs - test/**/AssemblyInfo.cs - # SarifMark-Design Review - id: SarifMark-Design title: Review that SarifMark Design is Consistent and Complete paths: @@ -60,7 +61,6 @@ reviews: - docs/design/sarifmark.md - docs/design/sarifmark/**/*.md - # SarifMark-Verification Review - id: SarifMark-Verification title: Review that SarifMark Verification is Consistent and Complete paths: @@ -69,8 +69,8 @@ reviews: - docs/verification/sarifmark.md - docs/verification/sarifmark/**/*.md - docs/verification/ots.md + - docs/verification/shared.md - # SarifMark-AllRequirements Review - id: SarifMark-AllRequirements title: Review that All SarifMark Requirements are Complete paths: @@ -79,7 +79,7 @@ reviews: - docs/reqstream/sarifmark/**/*.yaml - docs/reqstream/ots/**/*.yaml - # SarifMark-Program Review + # SarifMark - Program - id: SarifMark-Program title: Review that SarifMark Program Implementation is Correct paths: @@ -89,7 +89,7 @@ reviews: - src/**/Program.cs - test/**/ProgramTests.cs - # SarifMark-Cli Review + # SarifMark - Cli - id: SarifMark-Cli title: Review that SarifMark Cli Satisfies Subsystem Requirements paths: @@ -98,7 +98,6 @@ reviews: - docs/verification/sarifmark/cli.md - test/**/Cli/CliTests.cs - # SarifMark-Cli-Context Review - id: SarifMark-Cli-Context title: Review that SarifMark Cli Context Implementation is Correct paths: @@ -108,17 +107,15 @@ reviews: - src/**/Cli/Context.cs - test/**/Cli/ContextTests.cs - # SarifMark-Sarif Review + # SarifMark - Sarif - id: SarifMark-Sarif title: Review that SarifMark Sarif Satisfies Subsystem Requirements paths: - docs/reqstream/sarifmark/sarif.yaml - - docs/reqstream/sarifmark/sarif/report.yaml - docs/design/sarifmark/sarif.md - docs/verification/sarifmark/sarif.md - test/**/Sarif/SarifTests.cs - # SarifMark-Sarif-SarifFinding Review - id: SarifMark-Sarif-SarifFinding title: Review that SarifMark Sarif SarifFinding Implementation is Correct paths: @@ -128,7 +125,6 @@ reviews: - src/**/Sarif/SarifFinding.cs - test/**/Sarif/SarifFindingTests.cs - # SarifMark-Sarif-SarifRun Review - id: SarifMark-Sarif-SarifRun title: Review that SarifMark Sarif SarifRun Implementation is Correct paths: @@ -138,7 +134,6 @@ reviews: - src/**/Sarif/SarifRun.cs - test/**/Sarif/SarifRunTests.cs - # SarifMark-Sarif-SarifResults Review - id: SarifMark-Sarif-SarifResults title: Review that SarifMark Sarif SarifResults Implementation is Correct paths: @@ -148,7 +143,7 @@ reviews: - src/**/Sarif/SarifResults.cs - test/**/Sarif/SarifResultsTests.cs - # SarifMark-SelfTest Review + # SarifMark - SelfTest - id: SarifMark-SelfTest title: Review that SarifMark SelfTest Satisfies Subsystem Requirements paths: @@ -157,7 +152,6 @@ reviews: - docs/verification/sarifmark/self-test.md - test/**/SelfTest/SelfTestTests.cs - # SarifMark-SelfTest-Validation Review - id: SarifMark-SelfTest-Validation title: Review that SarifMark SelfTest Validation Implementation is Correct paths: @@ -167,7 +161,7 @@ reviews: - src/**/SelfTest/Validation.cs - test/**/SelfTest/ValidationTests.cs - # SarifMark-Utilities Review + # SarifMark - Utilities - id: SarifMark-Utilities title: Review that SarifMark Utilities Satisfies Subsystem Requirements paths: @@ -176,7 +170,6 @@ reviews: - docs/verification/sarifmark/utilities.md - test/**/Utilities/UtilitiesTests.cs - # SarifMark-Utilities-PathHelpers Review - id: SarifMark-Utilities-PathHelpers title: Review that SarifMark Utilities PathHelpers Implementation is Correct paths: @@ -186,7 +179,7 @@ reviews: - src/**/Utilities/PathHelpers.cs - test/**/Utilities/PathHelpersTests.cs - # OTS Reviews + # OTS Items - id: OTS-BuildMark title: Review that BuildMark Provides Required Functionality paths: @@ -249,3 +242,18 @@ reviews: - docs/reqstream/ots/weasyprint.yaml - docs/design/ots/weasyprint.md - docs/verification/ots/weasyprint.md + + - id: SarifMark-OTS-TestResults + title: Review that TestResults Provides Required Functionality + paths: + - docs/reqstream/ots/test-results.yaml + - docs/design/ots/test-results.md + - docs/verification/ots/test-results.md + + # Shared Packages + - id: SarifMark-Shared-SarifMark + title: Review that Shared SarifMark Package Provides Required Functionality + paths: + - docs/reqstream/shared/sarifmark.yaml + - docs/design/shared/sarifmark.md + - docs/verification/shared/sarifmark.md diff --git a/AGENTS.md b/AGENTS.md index 891bd53..400bf91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ commit messages, and reports). This repository follows a reference template for structure and file conventions. -- **Template URL**: `https://github.com/demaconsulting/Agents/raw/refs/heads/template` +- **template-url**: `https://github.com/demaconsulting/Agents/raw/refs/heads/template` - **Repository map**: `{template-url}/repository-map.md` - **Template files**: `{template-url}/{file-path}` for files described in the map @@ -87,12 +87,15 @@ The default agent should handle simple, straightforward tasks directly. Delegate to specialized agents only for specific scenarios: - **Pre-PR lint cleanup** (fix all lint issues before pull request) → Call the lint-fix agent -- **Light development work** (small fixes, simple features) → Call the developer agent +- **Scoped fixes with no new user-visible behavior** (PR review comments, doc + corrections, known bug fixes with defined root cause) → Call the developer agent - **Light quality checking** (basic validation) → Call the quality agent -- **Formal feature implementation** (complex, multi-step) → Call the implementation agent -- **Formal bug resolution** (complex debugging, systematic fixes) → Call the implementation agent +- **Any change introducing new user-visible behavior** (features, enhancements, + new commands or options) → Call the implementation agent +- **Formal bug resolution** (complex debugging, unknown root cause) → Call the implementation agent - **Formal reviews** (compliance verification, detailed analysis) → Call the formal-review agent - **Structural audit**: (repository layout vs. template) → Call the template-sync agent +- **Implementation planning only** (review a plan before committing to implementation) → Call the planning agent # Agent Reporting (Specialized Agents Must Follow) @@ -101,16 +104,16 @@ Specialized agents MUST generate a completion report: 1. Save to `.agent-logs/{agent-name}-{subject}-{unique-id}.md` where `{subject}` is a kebab-case task summary (max 5 words) and `{unique-id}` is a short unique suffix (e.g., 8-char hex or timestamp) -2. Start with `**Result**: (SUCCEEDED|FAILED)` as the first metadata field +2. Start with `**Result**: (SUCCEEDED|FAILED|INCOMPLETE)` as the first metadata field 3. Include the agent-specific report sections defined in each agent's prompt 4. Return the summary to the caller Result semantics for orchestrator decision-making: -- **SUCCEEDED**: Work completed and all applicable quality gates met +- **SUCCEEDED**: Work completed and all quality gates applicable to that agent's scope met - **FAILED**: Work could not be completed or quality gates not met - **INCOMPLETE**: Work cannot proceed without information only the user can - provide (implementation agent only) + provide (implementation, planning, and template-sync agents) # Formatting (After Making Changes) diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml index b66a0a1..50ccea3 100644 --- a/docs/design/definition.yaml +++ b/docs/design/definition.yaml @@ -7,6 +7,7 @@ resource-path: - docs/design/sarifmark/self-test - docs/design/sarifmark/utilities - docs/design/ots + - docs/design/shared - docs/template input-files: - docs/design/title.txt @@ -19,6 +20,7 @@ input-files: - docs/design/sarifmark/sarif/sarif-finding.md - docs/design/sarifmark/sarif/sarif-run.md - docs/design/sarifmark/sarif/sarif-results.md + - docs/design/sarifmark/self-test.md - docs/design/sarifmark/self-test/validation.md - docs/design/sarifmark/utilities.md @@ -33,6 +35,9 @@ input-files: - docs/design/ots/versionmark.md - docs/design/ots/weasyprint.md - docs/design/ots/xunit-v3.md + - docs/design/ots/test-results.md + - docs/design/shared.md + - docs/design/shared/sarifmark.md template: template.html table-of-contents: true number-sections: true diff --git a/docs/design/introduction.md b/docs/design/introduction.md index 49d486b..0822f00 100644 --- a/docs/design/introduction.md +++ b/docs/design/introduction.md @@ -3,7 +3,8 @@ SarifMark is a .NET command-line tool that generates markdown reports from SARIF (Static Analysis Results Interchange Format) 2.1.0 files. This document describes the design of the SarifMark software system, covering the architecture and detailed design of all local software items — system, subsystems, and -units — and the integration and usage design for all OTS software items used in the project pipeline. +units — the integration and usage design for all OTS software items used in the project pipeline, and +the integration and usage design for all shared package dependencies. ## Purpose @@ -25,78 +26,66 @@ Local items: OTS items: -- **BuildMark**, **FileAssert**, **Pandoc**, **ReqStream**, **ReviewMark**, **SonarMark**, +- **BuildMark**, **DemaConsulting.TestResults**, **FileAssert**, **Pandoc**, **ReqStream**, **ReviewMark**, **SonarMark**, **VersionMark**, **WeasyPrint**, **xUnit v3**: integration and usage design for each OTS software item used in the project pipeline. +Shared packages: + +- **SarifMark (released version)**: integration and usage design for the released version of SarifMark + consumed in the CI pipeline to generate the CodeQL quality report. + This document does not cover test projects, the CI/CD pipeline configuration, installation procedures, end-user usage patterns, or the internal design of OTS items. ## Software Structure -The following tree shows how the SarifMark software items are organized across the system, subsystem, -and unit levels: - -```text -SarifMark (System) -├── Program (Unit) -├── Cli (Subsystem) -│ └── Context (Unit) -├── Sarif (Subsystem) -│ ├── SarifFinding (Unit) -│ ├── SarifRun (Unit) -│ └── SarifResults (Unit) -├── SelfTest (Subsystem) -│ └── Validation (Unit) -└── Utilities (Subsystem) - └── PathHelpers (Unit) - -OTS Dependencies: -├── BuildMark (OTS) -├── FileAssert (OTS) -├── Pandoc (OTS) -├── ReqStream (OTS) -├── ReviewMark (OTS) -├── SonarMark (OTS) -├── VersionMark (OTS) -├── WeasyPrint (OTS) -└── xUnit v3 (OTS) -``` +- **SarifMark** (System) - .NET CLI tool that generates markdown reports from SARIF 2.1.0 files + - Program (Unit) - system-level entry point and execution dispatcher + - **Cli** (Subsystem) - command-line argument parsing and execution context + - Context (Unit) - argument parser, I/O owner, and exit-code manager + - **Sarif** (Subsystem) - SARIF file reading and markdown report generation + - SarifFinding (Unit) - immutable record for a single analysis finding + - SarifRun (Unit) - immutable record for a single tool run + - SarifResults (Unit) - SARIF file reader and markdown report generator + - **SelfTest** (Subsystem) - end-to-end self-validation of tool capabilities + - Validation (Unit) - self-validation test runner + - **Utilities** (Subsystem) - shared path-safety helpers + - PathHelpers (Unit) - safe path combination utilities + +**OTS Dependencies:** + +- BuildMark (OTS) +- DemaConsulting.TestResults (OTS) +- FileAssert (OTS) +- Pandoc (OTS) +- ReqStream (OTS) +- ReviewMark (OTS) +- SonarMark (OTS) +- VersionMark (OTS) +- WeasyPrint (OTS) +- xUnit v3 (OTS) + +**Shared Package Dependencies:** + +- SarifMark (Shared Package) ## Folder Layout -The source code folder structure mirrors the top-level subsystem breakdown, giving reviewers an -explicit navigation aid from design to code. All source files use the root namespace -`DemaConsulting.SarifMark` regardless of their subdirectory location. This is an intentional -flat-namespace convention: subdirectory names reflect design structure only, not namespace -hierarchy. - -```text -src/DemaConsulting.SarifMark/ -├── Program.cs — entry point and execution orchestrator -├── Cli/ -│ └── Context.cs — command-line argument parser and I/O owner -├── Sarif/ -│ ├── SarifFinding.cs — immutable record for a single analysis finding -│ ├── SarifRun.cs — immutable record for a single tool run -│ └── SarifResults.cs — SARIF file reading and markdown report generation -├── SelfTest/ -│ └── Validation.cs — self-validation test runner -└── Utilities/ - └── PathHelpers.cs — safe path combination utilities - -test/DemaConsulting.SarifMark.Tests/ -├── Cli/ -│ └── ContextTests.cs — Context unit tests -├── Sarif/ -│ ├── SarifFindingTests.cs — SarifFinding unit tests -│ ├── SarifRunTests.cs — SarifRun unit tests -│ └── SarifResultsTests.cs — SarifResults unit tests -├── SelfTest/ -│ └── ValidationTests.cs — Validation unit tests -└── Utilities/ - └── PathHelpersTests.cs — PathHelpers unit tests -``` +- **src/** - source files and projects + - **DemaConsulting.SarifMark/** - main project source + - **Cli/** - command-line argument parsing + - **Sarif/** - SARIF reading and report generation + - **SelfTest/** - self-validation test runner + - **Utilities/** - path-safety helpers +- **test/** - test projects + - **DemaConsulting.SarifMark.Tests/** - unit and integration tests + - **Cli/** - tests for the Cli subsystem + - **Sarif/** - tests for the Sarif subsystem + - **SelfTest/** - tests for the SelfTest subsystem + - **Utilities/** - tests for the Utilities subsystem +- **docs/design/ots/** - OTS item integration and usage design documents +- **docs/design/shared/** - shared package integration and usage design documents ## Companion Artifact Structure @@ -117,6 +106,12 @@ OTS items have integration and usage design documentation parallel to system fol - Design: `docs/design/ots/{ots-name}.md` - Verification: `docs/verification/ots/{ots-name}.md` +Shared package items have integration and usage design documentation parallel to system and OTS folders: + +- Requirements: `docs/reqstream/shared/{name}.yaml` +- Design: `docs/design/shared/{name}.md` +- Verification: `docs/verification/shared/{name}.md` + Review-sets: defined in `.reviewmark.yaml` ## References diff --git a/docs/design/ots.md b/docs/design/ots.md index 88b0266..6aafa45 100644 --- a/docs/design/ots.md +++ b/docs/design/ots.md @@ -1,7 +1,7 @@ ## OTS Dependencies -SarifMark uses nine OTS software items: eight DEMA Consulting pipeline tools and the -xUnit v3 testing framework. All nine items are consumed as .NET tools or NuGet packages +SarifMark uses ten OTS software items: nine DEMA Consulting pipeline tools and the +xUnit v3 testing framework. All ten items are consumed as .NET tools or NuGet packages and are managed through the local tool manifest and the project dependency lock files. Per-item integration designs are documented in the `ots/` sub-folder. @@ -13,7 +13,7 @@ All OTS packages must carry an OSI-approved open-source license that is compatib the MIT License under which SarifMark is distributed. Each package must demonstrate active maintenance through regular releases and publicly available source code. -DEMA Consulting pipeline tools (BuildMark, FileAssert, Pandoc, ReqStream, ReviewMark, +DEMA Consulting pipeline tools (BuildMark, DemaConsulting.TestResults, FileAssert, Pandoc, ReqStream, ReviewMark, SonarMark, VersionMark, WeasyPrint) are preferred because they are designed specifically for the Continuous Compliance workflow and provide documented compliance evidence that integrates directly with the project's traceability pipeline. diff --git a/docs/design/ots/buildmark.md b/docs/design/ots/buildmark.md index ccf9faa..d773398 100644 --- a/docs/design/ots/buildmark.md +++ b/docs/design/ots/buildmark.md @@ -30,5 +30,16 @@ test steps have completed: supplied as environment variables from the CI context. 3. BuildMark writes the build-notes markdown file to the configured output path. 4. The generated markdown file is published as a release artifact. +5. The `--validate` flag runs BuildMark's built-in self-validation suite, producing a + self-test report that confirms all advertised features are operational in the installed + environment. The CI pipeline invokes `dotnet buildmark --validate --results {path}` to + collect self-validation evidence. No application-level code in SarifMark references BuildMark directly. + +### Error Handling + +BuildMark is invoked as a CI pipeline step; a nonzero exit code from any `dotnet buildmark` +invocation causes the CI step to fail and stops the pipeline immediately. No special wrapper +or retry logic is applied. Failures are surfaced directly through the GitHub Actions step +status and must be investigated by inspecting the step log. diff --git a/docs/design/ots/fileassert.md b/docs/design/ots/fileassert.md index 31429c4..456c7bd 100644 --- a/docs/design/ots/fileassert.md +++ b/docs/design/ots/fileassert.md @@ -38,5 +38,8 @@ generation steps: output file, checking existence, size, page count, and content strings. 4. A non-zero exit code from FileAssert causes the CI step to fail, blocking the pipeline. +5. The `--validate` flag runs FileAssert's built-in self-validation suite, confirming all + advertised features are operational. The CI pipeline invokes + `dotnet fileassert --validate --results {path}` to collect self-validation evidence. No application-level code in SarifMark references FileAssert directly. diff --git a/docs/design/ots/reqstream.md b/docs/design/ots/reqstream.md index b701117..f3551e5 100644 --- a/docs/design/ots/reqstream.md +++ b/docs/design/ots/reqstream.md @@ -38,5 +38,8 @@ completed and the TRX result files are available: 4. A non-zero exit code blocks the pipeline and requires the developer to add tests or justifications before merging. 5. The generated reports are published as release artifacts. +6. The `--validate` flag runs ReqStream's built-in self-validation suite, confirming all + advertised features are operational. The CI pipeline invokes + `dotnet reqstream --validate --results {path}` to collect self-validation evidence. No application-level code in SarifMark references ReqStream directly. diff --git a/docs/design/ots/reviewmark.md b/docs/design/ots/reviewmark.md index d4a635c..b132ef1 100644 --- a/docs/design/ots/reviewmark.md +++ b/docs/design/ots/reviewmark.md @@ -22,6 +22,8 @@ compliance documents in the SarifMark release bundle. reviews with reviewer, date, and outcome information. - **Review coverage enforcement** — when run with enforcement flags, fails the pipeline if required files have not been reviewed. +- **Review depth configuration** — the `--depth` flag controls heading depth in review + plan and report output. ### Integration Pattern @@ -32,6 +34,9 @@ ReviewMark is invoked as a .NET tool from CI pipeline steps: the review evidence store. 3. ReviewMark writes the review plan and review report to the configured output paths. 4. The generated documents are published as release artifacts. +5. The `--validate` flag runs ReviewMark's built-in self-validation suite, confirming all + advertised features are operational. The CI pipeline invokes + `dotnet reviewmark --validate --results artifacts/reviewmark-self-validation.trx` to collect self-validation evidence. The review configuration in `.reviewmark.yaml` defines the review sets and their member files. No application-level code in SarifMark references ReviewMark directly. diff --git a/docs/design/ots/sonarmark.md b/docs/design/ots/sonarmark.md index 444ee92..b389c01 100644 --- a/docs/design/ots/sonarmark.md +++ b/docs/design/ots/sonarmark.md @@ -33,5 +33,9 @@ analysis has completed: 3. SonarMark queries the SonarCloud API and writes the quality report to the configured output path. 4. The generated report is published as a release artifact. +5. The `--validate` flag runs SonarMark's built-in self-validation suite using mock API + data, confirming all advertised features are operational without requiring a live + SonarCloud connection. The CI pipeline invokes + `dotnet sonarmark --validate --results artifacts/sonarmark-self-validation.trx` to collect self-validation evidence. No application-level code in SarifMark references SonarMark directly. diff --git a/docs/design/ots/test-results.md b/docs/design/ots/test-results.md new file mode 100644 index 0000000..33351e9 --- /dev/null +++ b/docs/design/ots/test-results.md @@ -0,0 +1,48 @@ +## DemaConsulting.TestResults + +### Purpose + +`DemaConsulting.TestResults` is a NuGet package produced by DEMA Consulting. It provides a +lightweight, format-agnostic model for collecting and serializing test results inside a running +process. SarifMark uses it in the SelfTest subsystem to record the outcomes of its three +self-validation scenarios and to write those outcomes to TRX or JUnit XML files when the user +supplies the `--results` flag alongside `--validate`. + +The package was chosen because it produces the TRX and JUnit XML formats that standard CI/CD +result consumers (Azure DevOps, GitHub Actions, Jenkins) expect, without introducing a dependency +on a full test framework. + +### Classification + +Although `DemaConsulting.TestResults` is produced by DEMA Consulting — the same organization that +produces SarifMark — it is an independently developed and separately released package with its own +requirements, versioning, and release lifecycle. It is not part of the SarifMark product and its +internal design requirements do not drive SarifMark requirements. It is therefore classified as an +OTS software item. + +### Features Used + +- **`TestResults`** — mutable collection that accumulates `TestResult` records during a validation + run. Passed to `WriteResultsFile` after all three self-validation scenarios have executed. +- **`TestResult`** — immutable record holding a test name, `TestOutcome`, and optional failure + message. One record is appended per scenario inside `RunValidationTest`. +- **`TestOutcome`** — enumeration of `Passed` and `Failed` values used to mark each test record. +- **`TestResultsIO`** — static serializer class (namespace `DemaConsulting.TestResults.IO`) + providing: + - `WriteTrx(TestResults, string)` — serializes to TRX format for `.trx` output files. + - `WriteJUnit(TestResults, string)` — serializes to JUnit XML format for `.xml` output files. + +### Integration Pattern + +`DemaConsulting.TestResults` is referenced as a `PackageReference` in the production project +`src/DemaConsulting.SarifMark/DemaConsulting.SarifMark.csproj`. No initialization or +configuration is required; all types are used directly. + +Inside `Validation.Run`, a `TestResults` instance is created at the start of the validation +pass and passed into each call to `RunValidationTest`. After all three scenarios complete, +`WriteResultsFile` inspects the file extension of `context.ResultsFile` and delegates to +`TestResultsIO.WriteTrx` (`.trx`) or `TestResultsIO.WriteJUnit` (`.xml`) accordingly. Any +serialization failure is caught and reported via `context.WriteError`; it does not propagate +to the caller. + +There are no global initialization, thread-affinity, or disposal requirements. diff --git a/docs/design/ots/versionmark.md b/docs/design/ots/versionmark.md index f7ef88b..9cff32a 100644 --- a/docs/design/ots/versionmark.md +++ b/docs/design/ots/versionmark.md @@ -30,5 +30,8 @@ VersionMark is invoked as a .NET tool from a CI pipeline step: 3. VersionMark interrogates each tool and writes the version report to the configured output path. 4. The generated report is published as a release artifact. +5. The `--validate` flag runs VersionMark's built-in self-validation suite, confirming all + advertised features are operational. The CI pipeline invokes + `dotnet versionmark --validate --results {path}` to collect self-validation evidence. No application-level code in SarifMark references VersionMark directly. diff --git a/docs/design/ots/xunit-v3.md b/docs/design/ots/xunit-v3.md index daee2fc..5c8db49 100644 --- a/docs/design/ots/xunit-v3.md +++ b/docs/design/ots/xunit-v3.md @@ -28,8 +28,8 @@ xUnit v3 is consumed as a NuGet package referenced by the test project: 1. `xunit.v3` is declared as a `PackageReference` in `test/DemaConsulting.SarifMark.Tests/DemaConsulting.SarifMark.Tests.csproj`. -2. `xunit.runner.visualstudio` is declared as a `PackageReference` in the same project - to enable VSTest-compatible execution and TRX output. +2. `xunit.runner.visualstudio` and `Microsoft.NET.Test.Sdk` are declared as `PackageReference` entries in the same + project to enable VSTest-compatible execution, test discovery, and TRX output. 3. Tests are executed by `dotnet test` as part of the CI pipeline build step. 4. The TRX output file is written to the configured results path and subsequently consumed by ReqStream for traceability enforcement. diff --git a/docs/design/sarifmark.md b/docs/design/sarifmark.md index 3f2f8c3..58444cb 100644 --- a/docs/design/sarifmark.md +++ b/docs/design/sarifmark.md @@ -50,6 +50,9 @@ provides path-safety helpers shared across the system. - *Role*: Provider (the tool accepts arguments from the shell) - *Contract*: Accepts flags and parameters (`--sarif`, `--report`, `--depth`, `--heading`, `--validate`, `--results`, `--enforce`, `--log`, `--silent`, `--version`, `--help`). + `--report-depth` is a **deprecated** alias for `--depth` and `--result` is a **deprecated** alias + for `--results`; both are accepted identically to their canonical forms but are intentionally + omitted from the `--help` output. Exits with code 0 on success and 1 on any error or when `--enforce` detects issues. - *Constraints*: Unrecognized arguments cause an `ArgumentException`, producing exit code 1 with an error message. Value-bearing flags require a following token. @@ -102,6 +105,8 @@ provides path-safety helpers shared across the system. see *VersionMark Integration Design* - **WeasyPrint**: converts HTML documents to PDF — see *WeasyPrint Integration Design* +- **DemaConsulting.TestResults**: the OTS package used by the self-validation subsystem to collect, format, and + serialize test results — see *TestResults Integration Design* ## Risk Control Measures @@ -159,3 +164,54 @@ conditions. All other exception types indicate programming errors or environment should not be silently swallowed. See the unit-level Error Handling sections in the *Program*, *Context*, *SarifResults*, and *Validation* unit design documents for per-subsystem error-handling detail. + +## Report Format + +The report format is the system's primary output contract. It defines the structure +and content of the UTF-8 markdown file written when `--report` is specified. + +A generated report consists of one or more run sections. For a single-run SARIF file +the output is the run section directly. For a multi-run SARIF file, run sections are +concatenated in order without an extra blank line between them (each run section +already ends with a trailing newline). + +Each run section contains the following elements in order: + +**Heading line**: A markdown heading at the configured depth (`#` × `depth`), +followed by either the custom heading text (when `--heading` is supplied) or the +default `"[ToolName] Analysis"` label. For multi-run SARIF files each heading is +suffixed with `" (#N)"` where `N` is the 1-based run index. + +**Tool attribution line**: `"**Tool:** ToolName ToolVersion"` on its own line, +where `ToolName` and `ToolVersion` are taken from the SARIF `tool.driver` object. + +**File count line**: `"**Files:** N"` where `N` is the count of entries in the SARIF +`artifacts` array for that run. `N` is `0` when the array is absent. + +**Issues sub-heading**: A markdown heading at `depth + 1` (capped at `6`) titled +`"Issues"`. + +**Issues count summary**: `"Found no issues"`, `"Found 1 issue"`, or +`"Found N issues"` reflecting the number of non-suppressed results in that run. + +**Result lines**: One line per non-suppressed finding in the form +`"location: severity [ruleId] message "` (two trailing spaces as a markdown hard +line break). The location prefix follows these rules: + +- `"(no location)"` — when the result URI is null, empty, or whitespace +- `"uri"` — when the URI is set but no start line is present +- `"uri(startLine)"` — when both URI and start line are present + +Suppressed results — those whose SARIF `suppressions` array is non-empty — are +excluded from the count and do not appear as result lines. + +The heading depth parameter must be an integer in `[1, 6]`; values outside this +range cause an `ArgumentOutOfRangeException` reported as an argument error with +exit code 1. The optional custom heading replaces the default +`"[ToolName] Analysis"` label when supplied. + +The report format contract is implemented by `SarifResults` (entry point; depth +validation; single-run delegation; multi-run iteration with indexed headings) and +`SarifRun` (per-run formatter; heading, tool attribution, file count, issues +sub-heading, count summary, and one result line per non-suppressed finding) in the +Sarif subsystem. diff --git a/docs/design/sarifmark/cli/context.md b/docs/design/sarifmark/cli/context.md index 432bf69..2edebc0 100644 --- a/docs/design/sarifmark/cli/context.md +++ b/docs/design/sarifmark/cli/context.md @@ -31,13 +31,16 @@ not provided. `null` when not provided. **Depth**: `int` — Heading depth for the generated report supplied via `--depth` or the -legacy alias `--report-depth`; must be a positive integer; default `1`. +deprecated alias `--report-depth`; must be an integer between 1 and 6 inclusive; default `1`. The valid range is +enforced at parse time. The deprecated +alias is accepted for backwards compatibility but is intentionally omitted from `--help` output. **Heading**: `string?` — Custom heading text supplied via `--heading`; `null` when not provided. When null, the report heading defaults to `"[ToolName] Analysis"`. **ResultsFile**: `string?` — Path for self-validation results supplied via `--results` or the -legacy alias `--result`; `null` when not provided. +deprecated alias `--result`; `null` when not provided. The deprecated alias is accepted for +backwards compatibility but is intentionally omitted from `--help` output. **ExitCode**: `int` — Returns `0` until `WriteError` is called; returns `1` thereafter. Derived from the internal `_hasErrors` flag. @@ -83,7 +86,7 @@ and calls `OpenLogFile` when a log path was specified. #### Error Handling `Create` throws `ArgumentException` for unrecognized tokens and for malformed value-bearing -flags (e.g., `--depth` not followed by a positive integer, or a string flag at end of args). +flags (e.g., `--depth` not followed by an integer between 1 and 6 inclusive, or a string flag at end of args). It throws `InvalidOperationException` if the log file cannot be opened. `ArgumentNullException` is thrown immediately if `args` is null. These exceptions propagate to `Program.Main`, which translates them to exit code 1. @@ -92,11 +95,20 @@ The private `ArgumentParser` inner class throws `ArgumentException` on any unrec token. Value-bearing string flags throw `ArgumentException` when they appear as the last token without a following value. +Value-bearing flags accept the immediately following token as their value without inspecting it further. A token +that begins with `--` (such as `--help`) is treated as a valid value, not as a flag. This is intentional: such +tokens are valid filenames on all supported platforms, and rejecting them would prevent users from writing output +to files whose names happen to match flag names. + #### Dependencies - **.NET base class library** — `Console`, `StreamWriter`, `Path`, `ArgumentException`, `ArgumentNullException`, `IDisposable`. +`Context` itself does not call `PathHelpers` directly. `PathHelpers.SafePathCombine` is used +by `ContextTests` (in the test project) to construct safe file paths for test fixtures, but +that usage is confined to the test boundary and is not part of the `Context` runtime contract. + #### Callers - **Program** — constructs `Context.Create(args)` inside a `using` block and passes the diff --git a/docs/design/sarifmark/program.md b/docs/design/sarifmark/program.md index 85d02c9..d81ac0b 100644 --- a/docs/design/sarifmark/program.md +++ b/docs/design/sarifmark/program.md @@ -10,7 +10,7 @@ subsystem based on the parsed flags, and handles top-level exception translation **Version**: `string` — The assembly informational version string; derived at runtime from `AssemblyInformationalVersionAttribute`, falling back to `AssemblyVersion`, then to `"0.0.0"`. -This property is read-only and is computed on first access. +This property is read-only and is recomputed on every access (no caching). ### Key Methods diff --git a/docs/design/sarifmark/self-test/validation.md b/docs/design/sarifmark/self-test/validation.md index e1fe3c1..8bc28b3 100644 --- a/docs/design/sarifmark/self-test/validation.md +++ b/docs/design/sarifmark/self-test/validation.md @@ -87,6 +87,8 @@ graceful cleanup in constrained environments. directory path itself from a GUID-based name. - **Context** — used both as the external output channel passed to `Run` and as internally constructed test contexts passed to `Program.Run` during each test. +- **DemaConsulting.TestResults** — `TestResults`, `TestResult`, `TrxSerializer`, and `JUnitSerializer` types are used + to collect test results and serialize them in TRX or JUnit XML format. #### Callers diff --git a/docs/design/shared.md b/docs/design/shared.md new file mode 100644 index 0000000..a4b1073 --- /dev/null +++ b/docs/design/shared.md @@ -0,0 +1,45 @@ +## Shared Package Dependencies + +SarifMark consumes one shared package: a released version of SarifMark itself. This +self-referential dependency exists because the CI pipeline invokes a released version +of the product to generate the CodeQL quality report that is published alongside each +release's compliance documents. + +A shared package, as defined in the project's software categorization standard, is a +software package produced within the same program and consumed as a dependency. It is +referenced by its advertised features rather than its internal design or source code. + +### Selection Criteria + +SarifMark is used in its own CI pipeline because it is the tool specifically designed +to generate markdown quality reports from SARIF files — precisely the task required to +document the CodeQL analysis results. Using a released version ensures that the quality +report in the release artifacts was produced by a stable, independently verified build +rather than the in-development version. + +### Version Management Policy + +The shared package version is managed through the local tool manifest +(`.config/dotnet-tools.json`), which pins the installed version. Upgrades are applied +by updating the manifest entry and re-running `dotnet tool restore`. Version numbers +are not recorded in design documentation; version information is captured in the project +SBOM produced by the CI pipeline. + +### General Integration Approach + +The shared package is consumed as a .NET tool invoked from a CI pipeline step. No +application-level code references the shared package directly. The pipeline step +supplies all required parameters — SARIF file path, report output path, custom heading, +and report depth — on the command line. Errors are propagated through non-zero exit +codes and surfaced as CI pipeline step failures. + +### Qualification Strategy + +Because the shared package is a released version of the same product, the product's +own self-validation test suite constitutes primary qualification evidence. All features +consumed in the CI pipeline — SARIF reading, markdown report generation, custom heading +configuration, and report depth configuration — are directly covered by integration +tests in `test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs`. A passing CI build +is itself evidence that the shared package executed correctly within the pipeline for +the built version. Per-package qualification details are documented in +`docs/design/shared/sarifmark.md`. diff --git a/docs/design/shared/sarifmark.md b/docs/design/shared/sarifmark.md new file mode 100644 index 0000000..f8d386b --- /dev/null +++ b/docs/design/shared/sarifmark.md @@ -0,0 +1,52 @@ +## SarifMark (Shared Package) + +SarifMark (`DemaConsulting.SarifMark`) is used as a shared package in its own CI build pipeline. +A released version of SarifMark is invoked to generate the CodeQL quality report that is +published as part of each release's documentation artifacts. + +### Advertised Features Consumed + +- **SARIF file reading** — accepts a SARIF 2.1.0 input file via the `--sarif` parameter and + parses all tool runs and findings contained in the file. +- **Markdown report generation** — writes a structured markdown quality report to the output + path specified by the `--report` parameter. +- **Custom heading configuration** — accepts a custom heading string via the `--heading` + parameter that is used as the top-level section title of the generated report. +- **Report depth configuration** — accepts a heading depth value via the `--report-depth` + parameter that controls the markdown heading level (`#`, `##`, `###`, etc.) used in the + generated report. + +### Integration Pattern + +SarifMark is invoked as a .NET tool from a CI pipeline step after the CodeQL analysis has +completed and produced a SARIF output file: + +1. The pipeline installs the released version of SarifMark via `dotnet tool restore`. +2. A pipeline step invokes `dotnet sarifmark` with the SARIF file path, report output path, + custom heading, and report depth: + + ```text + dotnet sarifmark + --sarif artifacts/csharp.sarif + --report docs/code_quality/generated/codeql-quality.md + --heading "SarifMark CodeQL Analysis" + --report-depth 1 + ``` + +3. SarifMark reads the SARIF file, processes all tool runs and findings, and writes the + markdown quality report to the configured output path. +4. The generated report is published as a release artifact alongside the other compliance + documents. + +No application-level code in SarifMark references the shared package directly; it is consumed +exclusively through the CI pipeline step. + +### Assumptions + +The self-referential nature of this dependency is intentional: the shared package is an earlier +released version of the same product. All features consumed in the CI pipeline are part of +SarifMark's own advertised feature set and are fully covered by the product's self-validation +test suite. Accordingly, no separate qualification effort is required — the product's own +passing test suite constitutes sufficient evidence that the consumed features behave as specified. +It is assumed that the released version behaves identically to the version under test for the +features exercised in the CI pipeline step. diff --git a/docs/reqstream/ots/buildmark.yaml b/docs/reqstream/ots/buildmark.yaml index 2c399b9..4ea01fb 100644 --- a/docs/reqstream/ots/buildmark.yaml +++ b/docs/reqstream/ots/buildmark.yaml @@ -6,14 +6,58 @@ sections: sections: - title: BuildMark Requirements requirements: - - id: SarifMark-OTS-BuildMark - title: BuildMark shall generate build-notes documentation from GitHub - Actions metadata. + - id: SarifMark-OTS-BuildMark-MarkdownReport + title: BuildMark shall generate a markdown report from repository and + issue-tracking data. justification: | - DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and - renders them as a markdown build-notes document included in the release artifacts. - It runs as part of the same CI pipeline that produces the TRX test results, so a - successful pipeline run is evidence that BuildMark executed without error. + DemaConsulting.BuildMark generates a build-notes markdown document that + aggregates Git history, GitHub issues, and pull request information for a + given build version. The self-validation test BuildMark_MarkdownReportGeneration + confirms the tool generates a correct report from mock data, proving the + feature works in the installed environment. tags: [ots] tests: - BuildMark_MarkdownReportGeneration + + - id: SarifMark-OTS-BuildMark-GitIntegration + title: BuildMark shall read version tags and commits from the Git repository. + justification: | + BuildMark reads the local Git repository to identify the baseline version + tag and collects commits between versions. The self-validation test + BuildMark_GitIntegration confirms Git repository connector functionality + in the installed environment. + tags: [ots] + tests: + - BuildMark_GitIntegration + + - id: SarifMark-OTS-BuildMark-IssueTracking + title: BuildMark shall retrieve issues and pull requests from GitHub. + justification: | + BuildMark connects to the GitHub API to retrieve issue and pull request + data for inclusion in the build notes. The self-validation test + BuildMark_IssueTracking confirms GitHub integration works correctly. + tags: [ots] + tests: + - BuildMark_IssueTracking + + - id: SarifMark-OTS-BuildMark-KnownIssues + title: BuildMark shall include known issues in the report when requested. + justification: | + When invoked with --include-known-issues, BuildMark appends a Known Issues + section to the build notes. The self-validation test + BuildMark_KnownIssuesReporting confirms this optional feature is + functioning correctly. + tags: [ots] + tests: + - BuildMark_KnownIssuesReporting + + - id: SarifMark-OTS-BuildMark-RulesRouting + title: BuildMark shall route items to report sections by configured rules. + justification: | + BuildMark supports label- and type-based routing rules that assign + issues and pull requests to named report sections. The self-validation + test BuildMark_RulesRouting confirms routing assigns items to the + correct sections. + tags: [ots] + tests: + - BuildMark_RulesRouting diff --git a/docs/reqstream/ots/fileassert.yaml b/docs/reqstream/ots/fileassert.yaml index 8cccf45..75cfa01 100644 --- a/docs/reqstream/ots/fileassert.yaml +++ b/docs/reqstream/ots/fileassert.yaml @@ -6,16 +6,53 @@ sections: sections: - title: FileAssert Requirements requirements: - - id: SarifMark-OTS-FileAssert - title: FileAssert shall validate generated documents against acceptance - criteria. + - id: SarifMark-OTS-FileAssert-VersionDisplay + title: FileAssert shall display its version when invoked with --version. justification: | - DemaConsulting.FileAssert validates HTML and PDF documents produced during the - build, asserting that each document exists, has a non-trivial size, is structurally - valid, and contains expected content. It provides OTS evidence for Pandoc and - WeasyPrint and independently confirms file assertion is functioning. Self-validation - proves the tool itself is operational before ReqStream consumes the results. + Version display confirms the tool is correctly installed and the version + string is accessible. The self-validation test FileAssert_VersionDisplay + confirms this feature works in the installed environment. tags: [ots] tests: - FileAssert_VersionDisplay + + - id: SarifMark-OTS-FileAssert-HelpDisplay + title: FileAssert shall display usage information when invoked with --help. + justification: | + Help display confirms the CLI interface is operational and the tool's + features are discoverable. The self-validation test FileAssert_HelpDisplay + confirms this feature works in the installed environment. + tags: [ots] + tests: - FileAssert_HelpDisplay + + - id: SarifMark-OTS-FileAssert-Results + title: FileAssert shall write test results to file with pass and fail + outcomes. + justification: | + Writing test results to TRX or JUnit XML format is required for CI pipeline + test reporting integration. The self-validation test FileAssert_Results + confirms result file generation works correctly. + tags: [ots] + tests: + - FileAssert_Results + + - id: SarifMark-OTS-FileAssert-Exists + title: FileAssert shall verify file existence using glob patterns. + justification: | + File existence assertions via glob patterns are the core use case for + FileAssert in the SarifMark pipeline. The self-validation test + FileAssert_Exists confirms file existence assertion works correctly. + tags: [ots] + tests: + - FileAssert_Exists + + - id: SarifMark-OTS-FileAssert-Contains + title: FileAssert shall verify file content with contains assertions. + justification: | + Content assertions confirm that generated documents contain expected text. + The self-validation test FileAssert_Contains confirms content assertion + works correctly in the installed environment. + tags: [ots] + tests: + - FileAssert_Contains diff --git a/docs/reqstream/ots/pandoc.yaml b/docs/reqstream/ots/pandoc.yaml index 8651978..011192e 100644 --- a/docs/reqstream/ots/pandoc.yaml +++ b/docs/reqstream/ots/pandoc.yaml @@ -7,13 +7,16 @@ sections: - title: Pandoc Requirements requirements: - id: SarifMark-OTS-Pandoc - title: Pandoc shall convert Markdown documents to valid HTML. + title: Pandoc shall convert Markdown documents to HTML. justification: | DemaConsulting.PandocTool converts Markdown source documents to HTML as part of the documentation build pipeline. FileAssert validates that each generated HTML file exists, has a non-trivial size, contains a valid HTML title element, and includes expected document content. Passing FileAssert assertions for each document type - proves Pandoc executed correctly and produced meaningful output. + proves Pandoc executed correctly and produced meaningful output. The requirements + report and trace matrix HTML documents are excluded from the OTS evidence + list because they are generated by the compliance toolchain itself — using them as evidence + would create a self-referential dependency. tags: [ots] tests: - Pandoc_BuildNotesHtml @@ -22,3 +25,4 @@ sections: - Pandoc_ReviewReportHtml - Pandoc_DesignHtml - Pandoc_UserGuideHtml + - Pandoc_VerificationHtml diff --git a/docs/reqstream/ots/reqstream.yaml b/docs/reqstream/ots/reqstream.yaml index 7dd74b1..8a40289 100644 --- a/docs/reqstream/ots/reqstream.yaml +++ b/docs/reqstream/ots/reqstream.yaml @@ -6,15 +6,68 @@ sections: sections: - title: ReqStream Requirements requirements: - - id: SarifMark-OTS-ReqStream + - id: SarifMark-OTS-ReqStream-Processing + title: ReqStream shall process all included requirements files and build + a complete requirements model. + justification: | + ReqStream reads requirements.yaml and all included YAML files to build + the complete requirements model used for enforcement and report generation. + The self-validation test ReqStream_RequirementsProcessing confirms this + core parsing feature works correctly in the installed environment. + tags: [ots] + tests: + - ReqStream_RequirementsProcessing + + - id: SarifMark-OTS-ReqStream-TraceMatrix + title: ReqStream shall generate a traceability matrix linking requirements + to tests. + justification: | + ReqStream generates a traceability matrix document that links every + requirement to its linked test evidence. The self-validation test + ReqStream_TraceMatrix confirms the matrix generation feature works correctly. + tags: [ots] + tests: + - ReqStream_TraceMatrix + + - id: SarifMark-OTS-ReqStream-ReportExport + title: ReqStream shall export requirements reports and justification documents. + justification: | + ReqStream generates a requirements report and a justifications document + as release artifacts. The self-validation test ReqStream_ReportExport + confirms report and justification export works correctly. + tags: [ots] + tests: + - ReqStream_ReportExport + + - id: SarifMark-OTS-ReqStream-TagsFiltering + title: ReqStream shall filter requirements by tag when tags are specified. + justification: | + Tag filtering allows subsets of requirements to be processed or enforced + selectively. The self-validation test ReqStream_TagsFiltering confirms + tag-based filtering works correctly. + tags: [ots] + tests: + - ReqStream_TagsFiltering + + - id: SarifMark-OTS-ReqStream-Enforcement title: ReqStream shall enforce that every requirement is linked to passing test evidence. justification: | - DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to - produce a requirements report, justifications document, and traceability matrix. When - run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, - making unproven requirements a build-breaking condition. A successful pipeline run with - --enforce proves all requirements are covered and that ReqStream is functioning. + When run with --enforce, ReqStream exits with a non-zero code if any + requirement lacks test evidence, making unproven requirements a + build-breaking condition. A successful self-validation run with + ReqStream_EnforcementMode proves enforcement is functioning correctly. tags: [ots] tests: - ReqStream_EnforcementMode + + - id: SarifMark-OTS-ReqStream-Lint + title: ReqStream shall validate requirements files and report structural + issues. + justification: | + The --lint mode validates requirements YAML files for structural and + semantic issues without processing test results. The self-validation + test ReqStream_Lint confirms lint mode works correctly. + tags: [ots] + tests: + - ReqStream_Lint diff --git a/docs/reqstream/ots/reviewmark.yaml b/docs/reqstream/ots/reviewmark.yaml index 309d11f..ef619ba 100644 --- a/docs/reqstream/ots/reviewmark.yaml +++ b/docs/reqstream/ots/reviewmark.yaml @@ -6,15 +6,109 @@ sections: sections: - title: ReviewMark Requirements requirements: - - id: SarifMark-OTS-ReviewMark - title: ReviewMark shall generate a review plan and review report from - the review configuration. + - id: SarifMark-OTS-ReviewMark-VersionDisplay + title: ReviewMark shall display its version when invoked with --version. justification: | - DemaConsulting.ReviewMark reads the .reviewmark.yaml configuration and the review - evidence store to produce a review plan and review report documenting file review - coverage and currency. It runs in the same CI pipeline that produces the TRX test - results, so a successful pipeline run is evidence that ReviewMark executed without error. + Version display confirms the tool is correctly installed and the version + string is accessible. The self-validation test ReviewMark_VersionDisplay + confirms this feature works in the installed environment. + tags: [ots] + tests: + - ReviewMark_VersionDisplay + + - id: SarifMark-OTS-ReviewMark-HelpDisplay + title: ReviewMark shall display usage information when invoked with --help. + justification: | + Help display confirms the CLI interface is operational and the tool's + features are discoverable. The self-validation test ReviewMark_HelpDisplay + confirms this feature works in the installed environment. + tags: [ots] + tests: + - ReviewMark_HelpDisplay + + - id: SarifMark-OTS-ReviewMark-Plan + title: ReviewMark shall generate a review plan from the review configuration. + justification: | + DemaConsulting.ReviewMark reads the .reviewmark.yaml configuration and the + review evidence store to produce a review plan documenting file review + coverage. The self-validation test ReviewMark_ReviewPlanGeneration confirms + plan generation works correctly in the installed environment. tags: [ots] tests: - ReviewMark_ReviewPlanGeneration + + - id: SarifMark-OTS-ReviewMark-Report + title: ReviewMark shall generate a review report from the review configuration. + justification: | + DemaConsulting.ReviewMark reads the .reviewmark.yaml configuration and the + review evidence store to produce a review report documenting file review + currency. The self-validation test ReviewMark_ReviewReportGeneration confirms + report generation works correctly in the installed environment. + tags: [ots] + tests: - ReviewMark_ReviewReportGeneration + + - id: SarifMark-OTS-ReviewMark-IndexScan + title: ReviewMark shall scan PDF evidence files and write an index. + justification: | + The --index mode scans a directory of PDF evidence files and writes an + up-to-date index.json catalogue. The self-validation test + ReviewMark_IndexScan confirms index scanning works correctly. + tags: [ots] + tests: + - ReviewMark_IndexScan + + - id: SarifMark-OTS-ReviewMark-WorkingDirectory + title: ReviewMark shall support overriding the working directory for file + operations. + justification: | + The --dir flag overrides the working directory used for default paths + and glob resolution. The self-validation test + ReviewMark_WorkingDirectoryOverride confirms this feature works correctly. + tags: [ots] + tests: + - ReviewMark_WorkingDirectoryOverride + + - id: SarifMark-OTS-ReviewMark-Enforce + title: ReviewMark shall exit non-zero when reviews have issues. + justification: | + The --enforce flag causes ReviewMark to exit with a non-zero code when + any review-set is stale, missing, or failed, or any file is uncovered. + The self-validation test ReviewMark_Enforce confirms enforcement mode + functions correctly. + tags: [ots] + tests: + - ReviewMark_Enforce + + - id: SarifMark-OTS-ReviewMark-Elaborate + title: ReviewMark shall print a markdown elaboration of a review set. + justification: | + The --elaborate flag prints the ID, title, fingerprint, and file list for + a named review set. The self-validation test ReviewMark_Elaborate confirms + elaboration works correctly. + tags: [ots] + tests: + - ReviewMark_Elaborate + + - id: SarifMark-OTS-ReviewMark-Lint + title: ReviewMark shall validate the review configuration file and report + issues. + justification: | + The --lint mode validates the .reviewmark.yaml definition file for + structural and semantic issues. The self-validation test ReviewMark_Lint + confirms lint validation works correctly. + tags: [ots] + tests: + - ReviewMark_Lint + + - id: SarifMark-OTS-ReviewMark-DepthFlag + title: ReviewMark shall support configuring review depth via the --depth + flag. + justification: | + The --depth flag controls how many heading levels are generated in review + plan and report output. The self-validation test ReviewMark_DepthFlag + confirms the flag is accepted and influences report output in the installed + environment. + tags: [ots] + tests: + - ReviewMark_DepthFlag diff --git a/docs/reqstream/ots/sonarmark.yaml b/docs/reqstream/ots/sonarmark.yaml index 1226bc7..b90487d 100644 --- a/docs/reqstream/ots/sonarmark.yaml +++ b/docs/reqstream/ots/sonarmark.yaml @@ -6,16 +6,42 @@ sections: sections: - title: SonarMark Requirements requirements: - - id: SarifMark-OTS-SonarMark - title: SonarMark shall generate a SonarCloud quality report. + - id: SarifMark-OTS-SonarMark-QualityGate + title: SonarMark shall retrieve quality-gate status from SonarCloud. justification: | - DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and - renders it as a markdown document included in the release artifacts. It runs in the - same CI pipeline that produces the TRX test results, so a successful pipeline run is - evidence that SonarMark executed without error. + SonarMark queries the SonarCloud API for the quality-gate status of the + configured project. The self-validation test SonarMark_QualityGateRetrieval + confirms quality-gate retrieval works correctly in the installed environment. tags: [ots] tests: - SonarMark_QualityGateRetrieval + + - id: SarifMark-OTS-SonarMark-Issues + title: SonarMark shall retrieve the issues list from SonarCloud. + justification: | + SonarMark retrieves open issues by severity from the SonarCloud API and + renders them in the quality report. The self-validation test + SonarMark_IssuesRetrieval confirms issues retrieval works correctly. + tags: [ots] + tests: - SonarMark_IssuesRetrieval + + - id: SarifMark-OTS-SonarMark-HotSpots + title: SonarMark shall retrieve hot spots from SonarCloud. + justification: | + SonarMark retrieves open security hot spots from the SonarCloud API and + renders them in the quality report. The self-validation test + SonarMark_HotSpotsRetrieval confirms hot spots retrieval works correctly. + tags: [ots] + tests: - SonarMark_HotSpotsRetrieval + + - id: SarifMark-OTS-SonarMark-MarkdownReport + title: SonarMark shall generate a markdown quality report. + justification: | + SonarMark renders all retrieved SonarCloud data into a structured markdown + quality report. The self-validation test SonarMark_MarkdownReportGeneration + confirms markdown report generation works correctly using mock data. + tags: [ots] + tests: - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots/test-results.yaml b/docs/reqstream/ots/test-results.yaml new file mode 100644 index 0000000..d04a2d5 --- /dev/null +++ b/docs/reqstream/ots/test-results.yaml @@ -0,0 +1,42 @@ +--- +# DemaConsulting.TestResults OTS requirements +sections: + - title: OTS Software Requirements + sections: + - title: DemaConsulting.TestResults Requirements + requirements: + - id: SarifMark-OTS-TestResults-Collect + title: DemaConsulting.TestResults shall accumulate test results during + a validation run. + justification: | + The SelfTest subsystem must record the outcome of each self-validation scenario. + The TestResults collection type provides a mutable accumulator that each scenario + appends to, so results can be serialized in a single step after all scenarios + complete. Using this package avoids implementing a custom accumulation mechanism. + tags: [ots] + tests: + - Validation_Run_ValidContext_PrintsSummary + + - id: SarifMark-OTS-TestResults-WriteTrx + title: DemaConsulting.TestResults shall serialize test results to TRX + format when the output file has a .trx extension. + justification: | + TRX is the standard MSTest/Azure DevOps test result format consumed by Azure + DevOps pipelines and other CI systems. Writing TRX output lets SarifMark + validation results be published as native test artifacts in those environments + without requiring additional conversion tools. + tags: [ots] + tests: + - Validation_Run_WithTrxResultsFile_WritesResultsFile + + - id: SarifMark-OTS-TestResults-WriteJUnit + title: DemaConsulting.TestResults shall serialize test results to JUnit + XML format when the output file has a .xml extension. + justification: | + JUnit XML is the de-facto standard test result format consumed by Jenkins, + GitHub Actions test summaries, and many other CI systems. Writing JUnit XML + output lets SarifMark validation results be published as native test artifacts + in those environments without requiring additional conversion tools. + tags: [ots] + tests: + - Validation_Run_WithXmlResultsFile_WritesResultsFile diff --git a/docs/reqstream/ots/versionmark.yaml b/docs/reqstream/ots/versionmark.yaml index 00dd8f7..55e3f1d 100644 --- a/docs/reqstream/ots/versionmark.yaml +++ b/docs/reqstream/ots/versionmark.yaml @@ -6,14 +6,49 @@ sections: sections: - title: VersionMark Requirements requirements: - - id: SarifMark-OTS-VersionMark - title: VersionMark shall publish captured tool-version information. + - id: SarifMark-OTS-VersionMark-Capture + title: VersionMark shall capture tool version information from configured + commands. justification: | - DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the - pipeline and writes a versions markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that VersionMark executed without error. + VersionMark reads version metadata for each tool defined in .versionmark.yaml + by executing the configured command and extracting the version via regex. + The self-validation test VersionMark_CapturesVersions confirms version + capture works correctly in the installed environment. tags: [ots] tests: - VersionMark_CapturesVersions + + - id: SarifMark-OTS-VersionMark-Publish + title: VersionMark shall generate a markdown report from captured version + information. + justification: | + VersionMark reads captured version JSON files and renders a markdown + tool-versions document. The self-validation test + VersionMark_GeneratesMarkdownReport confirms markdown report generation + works correctly in the installed environment. + tags: [ots] + tests: - VersionMark_GeneratesMarkdownReport + + - id: SarifMark-OTS-VersionMark-LintValid + title: VersionMark shall pass lint validation for a well-formed configuration + file. + justification: | + The --lint mode validates .versionmark.yaml configuration files and exits + with code 0 for valid files. The self-validation test + VersionMark_LintPassesForValidConfig confirms that valid configuration + files are accepted without errors. + tags: [ots] + tests: + - VersionMark_LintPassesForValidConfig + + - id: SarifMark-OTS-VersionMark-LintInvalid + title: VersionMark shall report errors and exit non-zero for a malformed + configuration file. + justification: | + The --lint mode must reject configuration files that are missing required + fields. The self-validation test VersionMark_LintReportsErrorsForInvalidConfig + confirms that invalid configuration files are rejected with a non-zero exit code. + tags: [ots] + tests: + - VersionMark_LintReportsErrorsForInvalidConfig diff --git a/docs/reqstream/ots/weasyprint.yaml b/docs/reqstream/ots/weasyprint.yaml index 60c0656..69b6229 100644 --- a/docs/reqstream/ots/weasyprint.yaml +++ b/docs/reqstream/ots/weasyprint.yaml @@ -14,6 +14,9 @@ sections: exists, has a non-trivial size, contains at least one page, and includes expected document content in the rendered text. Passing FileAssert assertions for each document type proves WeasyPrint executed correctly and produced meaningful output. + The requirements report and trace matrix PDF documents are excluded from the OTS evidence + list because they are generated by the compliance toolchain itself — using them as + evidence would create a self-referential dependency. tags: [ots] tests: - WeasyPrint_BuildNotesPdf @@ -22,3 +25,4 @@ sections: - WeasyPrint_ReviewReportPdf - WeasyPrint_DesignPdf - WeasyPrint_UserGuidePdf + - WeasyPrint_VerificationPdf diff --git a/docs/reqstream/ots/xunit-v3.yaml b/docs/reqstream/ots/xunit-v3.yaml index 8377013..0d38fc3 100644 --- a/docs/reqstream/ots/xunit-v3.yaml +++ b/docs/reqstream/ots/xunit-v3.yaml @@ -7,29 +7,41 @@ sections: - title: xUnit v3 Requirements requirements: - id: SarifMark-OTS-XUnitV3-Discovery - title: xUnit v3 shall discover and execute unit tests. + title: xUnit v3 shall discover test methods marked with [Fact] or [Theory] + attributes. justification: | - xunit.v3 is the unit-testing framework used by the project. It discovers and runs - all test methods marked with [Fact] or [Theory] attributes, confirming the framework - is functioning correctly and all tests pass. + xunit.v3 is the unit-testing framework used by the project. Test discovery + confirms the framework correctly identifies all test methods across the test + project, which is a prerequisite for test execution. tags: [ots] tests: - SarifResults_Read_NoResults_ReturnsValidResults - SarifResults_Read_WithResults_ReturnsValidResults - - SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults - Context_Create_VersionFlag_SetsVersionTrue - Context_Create_HelpFlag_SetsHelpTrue + - id: SarifMark-OTS-XUnitV3-Execution + title: xUnit v3 shall execute discovered tests and report pass/fail results. + justification: | + Execution of all discovered tests confirms xUnit v3 can run the test suite + to completion across all test classes, producing reliable pass/fail results + for requirements traceability. + tags: [ots] + tests: + - SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults + - Context_Create_HelpFlag_SetsHelpTrue + - Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent + - id: SarifMark-OTS-XUnitV3-TrxOutput - title: xUnit v3 shall write TRX result files for traceability. + title: >- + The test execution pipeline shall produce TRX result files + containing passing test names. justification: | - xunit.runner.visualstudio provides the VSTest adapter that writes TRX result files - when tests are executed via `dotnet test`. These TRX files are consumed by ReqStream - for requirements traceability enforcement, linking passing tests to requirement IDs. + The combination of xunit.v3, xunit.runner.visualstudio, and `dotnet test + --results-directory` produces TRX result files during test execution. + These TRX files are consumed by ReqStream for requirements traceability + enforcement, linking passing tests to requirement IDs. tags: [ots] tests: - SarifResults_Read_NoResults_ReturnsValidResults - - SarifResults_Read_WithResults_ReturnsValidResults - - SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue diff --git a/docs/reqstream/sarifmark.yaml b/docs/reqstream/sarifmark.yaml index e1af67c..39452b0 100644 --- a/docs/reqstream/sarifmark.yaml +++ b/docs/reqstream/sarifmark.yaml @@ -65,10 +65,8 @@ sections: - SarifMark_NonExistentSarifFile_ShowsError - id: SarifMark-System-SarifSummary - title: >- - The tool shall report the tool name, version, and result count for each - run - after processing. + title: The tool shall generate a summary section for each SARIF analysis run + in the report. tags: [public] justification: >- Reporting tool identity and result count after reading confirms successful @@ -77,7 +75,7 @@ sections: findings were found. children: - - SarifMark-Program-SarifReading + - SarifMark-Program-SarifSummary tests: - SarifMark_ValidSarifFile_ProcessesSuccessfully @@ -211,16 +209,123 @@ sections: - id: SarifMark-System-SafePaths title: >- - The tool shall use safe path-handling functions to protect against - path-traversal vulnerabilities. + The tool shall reject any file path argument that contains + path-traversal sequences or specifies an absolute path where + a relative path is required. tags: [public] justification: >- - Centralizing path-safety logic prevents path-traversal vulnerabilities from - being - independently reimplemented and ensures consistent protection wherever external - inputs are combined with base paths. + Rejecting path-traversal sequences and absolute paths at all external + input boundaries prevents file operations from escaping designated + directories, ensuring the tool cannot be directed to read or write + outside permitted locations regardless of how arguments are constructed. children: - SarifMark-Utilities-SafePathHandling tests: - Utilities_SafePathHandling_PathTraversal_ThrowsException - Utilities_SafePathHandling_AbsolutePath_ThrowsException + + # Report format requirements — public-facing system-level contract + - id: SarifMark-Report-Markdown + title: The tool shall generate markdown reports from SARIF files. + justification: >- + Markdown report generation transforms machine-readable SARIF into + human-readable documentation, + enabling better understanding and communication of analysis results + across development teams. + tags: [public] + children: + - SarifMark-SarifResults-ToMarkdown + tests: + - Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent + + - id: SarifMark-Report-Depth + title: The tool shall support configurable markdown heading depth. + justification: >- + Configurable heading depth allows reports to integrate seamlessly + into existing documentation + hierarchies, preventing heading level conflicts when reports are + embedded in larger documents. + tags: [public] + children: + - SarifMark-SarifResults-ValidateDepth + tests: + - Sarif_GenerateReport_ReportDepth_IsConfigurable + + - id: SarifMark-Report-Counts + title: The tool shall display result counts in reports. + justification: >- + Result counts provide a quick summary of analysis findings, helping + users understand the scope of + issues at a glance and track improvements over time. + tags: [public] + children: + - SarifMark-SarifRun-FormatCount + tests: + - Sarif_GenerateReport_ResultCount_ContainsResultCount + + - id: SarifMark-Report-Locations + title: The tool shall display location information for results. + justification: >- + Displaying location information enables developers to quickly navigate + to the source of issues, + making the reports actionable and reducing time spent searching + for problematic code. + tags: [public] + children: + - SarifMark-SarifRun-FormatLocation + tests: + - Sarif_GenerateReport_LocationInfo_ContainsLocationInfo + + - id: SarifMark-Report-Headings + title: The tool shall support custom headings in reports. + justification: >- + Custom headings allow users to tailor reports to their documentation + standards and naming + conventions, improving consistency across project documentation. + tags: [public] + children: + - SarifMark-SarifResults-ToMarkdown + tests: + - Sarif_GenerateReport_CustomHeading_UsesCustomHeading + + - id: SarifMark-Report-LineBreaks + title: The tool shall format multiple results with proper line breaks. + justification: >- + Proper line breaks between results ensure readability and visual + separation in generated markdown, + making it easier to distinguish between different findings in the + report. + tags: [public] + children: + - SarifMark-SarifResults-ToMarkdown + tests: + - Sarif_GenerateReport_MultipleResults_FormatsWithLineBreaks + + - id: SarifMark-Report-FileCount + title: The tool shall include the file count in generated reports. + justification: >- + Including the file count in the report header provides evidence + that analysis was actually + performed on the expected files, which is particularly valuable + in regulated or audited + development environments. + tags: [public] + children: + - SarifMark-SarifResults-FileCount + tests: + - Sarif_GenerateReport_FileCount_ContainsFileCount + + - id: SarifMark-Report-NoIssues + title: >- + The SarifMark system shall produce a valid markdown report when + the SARIF file contains no findings. + justification: >- + A clean SARIF file with zero findings is the expected output of a + passing analysis. The report must still be generated with the tool + attribution and file count, confirming the analysis completed + successfully even when no issues are found. + tags: [public] + children: + - SarifMark-SarifResults-ToMarkdown + tests: + - SarifMark_ValidSarif_NoIssues_GeneratesReport diff --git a/docs/reqstream/sarifmark/cli.yaml b/docs/reqstream/sarifmark/cli.yaml index 28d6a86..1a285c6 100644 --- a/docs/reqstream/sarifmark/cli.yaml +++ b/docs/reqstream/sarifmark/cli.yaml @@ -6,8 +6,9 @@ sections: - title: CLI Requirements requirements: - id: SarifMark-Cli-Interface - title: The CLI subsystem shall accept command-line arguments and route - output. + title: >- + The CLI subsystem shall parse command-line arguments into a + structured execution context. justification: >- A command-line interface enables automation, scripting, and integration into CI/CD pipelines, @@ -19,6 +20,19 @@ sections: - Cli_Create_VersionFlag_SetsVersionFlag - Cli_Create_HelpFlag_SetsHelpFlag + - id: SarifMark-Cli-Output + title: >- + The CLI subsystem shall route console output through the execution + context. + justification: >- + Routing all output through the execution context allows the silent + and log-file settings to uniformly suppress or redirect output + without requiring each operation to manage its own output destination. + tags: [public] + tests: + - Cli_Create_SilentFlag_SuppressesOutput + - Cli_Create_LogFile_WritesOutputToFile + - id: SarifMark-Cli-Version title: The CLI shall recognize the --version flag. justification: >- @@ -84,16 +98,30 @@ sections: - Cli_Create_EnforceFlag_SetsEnforceFlag - id: SarifMark-Cli-WriteError - title: The CLI shall write errors to stderr and set a non-zero exit code. + title: The CLI shall write error messages to the standard error stream. + justification: >- + Writing errors to stderr separates diagnostic output from + regular console output, enabling callers and pipelines to + distinguish between tool output and error information. + tags: [public] + children: + - SarifMark-Context-WriteError-Stderr + - SarifMark-Context-WriteError-Log + - SarifMark-Context-WriteError-ExitCode + tests: + - Cli_WriteError_WithMessage_SetsExitCodeToOne + + - id: SarifMark-Cli-WriteError-ExitCode + title: The CLI shall set the process exit code to 1 when an error is written. justification: >- - Writing errors to stderr separates diagnostic output from the regular - output stream, and - automatically setting a non-zero exit code ensures that callers receive - a failure signal - without requiring explicit exit code management throughout the codebase. + Automatically updating the exit code when an error is written + ensures that callers receive a failure signal without requiring + explicit exit-code management throughout the codebase. tags: [public] children: - - SarifMark-Context-WriteError + - SarifMark-Context-WriteError-Stderr + - SarifMark-Context-WriteError-Log + - SarifMark-Context-WriteError-ExitCode tests: - Cli_WriteError_WithMessage_SetsExitCodeToOne diff --git a/docs/reqstream/sarifmark/cli/context.yaml b/docs/reqstream/sarifmark/cli/context.yaml index 04da547..ad0b08b 100644 --- a/docs/reqstream/sarifmark/cli/context.yaml +++ b/docs/reqstream/sarifmark/cli/context.yaml @@ -141,26 +141,66 @@ sections: - Context_Create_ReportWithoutValue_ThrowsArgumentException - id: SarifMark-Context-ReportDepthParam - title: >- - The Context unit shall accept --depth (and --report-depth for legacy - compatibility) as a - positive integer heading depth, defaulting to 1 when omitted. + title: The Context unit shall accept the --depth parameter as the + heading depth for the generated report. justification: >- - Validating the depth value at parse time prevents invalid heading - levels from propagating into - markdown generation, and defaulting to 1 ensures sensible behavior - when the parameter is omitted. + Accepting a named depth parameter decouples heading-level selection + from + argument parsing, allowing downstream report generation to receive + a + validated integer depth without inspecting raw argument strings. tags: [internal] tests: - Context_Create_DepthParameter_SetsDepth - Context_Create_DepthWithoutValue_ThrowsArgumentException - - Context_Create_DepthInvalidValue_ThrowsArgumentException - - Context_Create_DepthZero_ThrowsArgumentException + + - id: SarifMark-Context-ReportDepthAlias + title: The Context unit shall accept --report-depth as a legacy alias + for --depth. + justification: >- + Accepting the legacy --report-depth alias preserves backwards compatibility + for existing scripts and workflows that used the earlier parameter + name, + preventing breaking changes when users upgrade. + tags: [internal] + tests: - Context_Create_ReportDepthParameter_SetsReportDepth - Context_Create_ReportDepthWithoutValue_ThrowsArgumentException + + - id: SarifMark-Context-ReportDepthValidation + title: The Context unit shall reject depth values outside the range + 1 to 6 inclusive with an error. + justification: >- + Restricting depth to the valid markdown heading range of 1 to 6 + at + parse time prevents invalid heading levels from propagating into + markdown + generation, ensuring the output is always accepted by standard markdown + renderers. + tags: [internal] + tests: + - Context_Create_DepthInvalidValue_ThrowsArgumentException + - Context_Create_DepthZero_ThrowsArgumentException + - Context_Create_DepthNegative_ThrowsArgumentException + - Context_Create_DepthAboveMax_ThrowsArgumentException + - Context_Create_DepthAtMinimum_SetsDepth + - Context_Create_DepthAtMaximum_SetsDepth - Context_Create_ReportDepthInvalidValue_ThrowsArgumentException - Context_Create_ReportDepthZero_ThrowsArgumentException + - id: SarifMark-Context-ReportDepthDefault + title: The Context unit shall use a heading depth of 1 when the --depth + parameter is not provided. + justification: >- + Defaulting to depth 1 ensures sensible report heading output when + the + caller omits the parameter, generating top-level headings that are + valid + in any markdown document. + tags: [internal] + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - id: SarifMark-Context-HeadingParam title: The Context unit shall accept a --heading parameter value as the custom report heading text. @@ -287,21 +327,6 @@ sections: - Context_WriteLine_WithLogFile_WritesToLog - Context_WriteLine_SilentModeWithLogFile_WritesToLog - - id: SarifMark-Context-WriteError - title: The WriteError method shall write errors to stderr and set - a non-zero exit code. - justification: >- - Writing errors to stderr and setting a non-zero exit code ensures - that callers receive - a failure signal without requiring explicit exit code management. - tags: [internal] - children: - - SarifMark-Context-WriteError-Stderr - - SarifMark-Context-WriteError-Log - - SarifMark-Context-WriteError-ExitCode - tests: - - Context_WriteError_WritesToErrorAndSetsExitCode - - id: SarifMark-Context-WriteError-Stderr title: >- The WriteError method shall write the error message to the standard diff --git a/docs/reqstream/sarifmark/program.yaml b/docs/reqstream/sarifmark/program.yaml index c20ea2e..4022b30 100644 --- a/docs/reqstream/sarifmark/program.yaml +++ b/docs/reqstream/sarifmark/program.yaml @@ -19,20 +19,49 @@ sections: tests: - Program_Main_VersionFlag_DisplaysVersionOnly - - id: SarifMark-Program-Main + - id: SarifMark-Program-ParseArgs title: >- - The Program unit shall parse arguments, execute the requested operation, - and return the result as an exit code. + The Program unit shall parse command-line arguments into a + structured execution context. justification: >- - Keeping argument parsing, command dispatch, and exit code reporting - as separate - responsibilities ensures each can be independently tested and maintained. + Parsing arguments into a typed context object decouples the + entry point from both the CLI argument format and the + downstream handlers, allowing each to be tested and changed + independently. tags: [internal] tests: - Program_Main_NoArguments_ReturnsError - Program_Main_VersionFlag_DisplaysVersionOnly - Program_Main_HelpFlag_DisplaysHelp + - id: SarifMark-Program-Dispatch + title: >- + The Program unit shall dispatch to the appropriate operation + handler based on the parsed execution context. + justification: >- + Dispatching to handlers via a parsed context keeps the entry + point as a thin coordinator, so each handler can be invoked + and verified in isolation. + tags: [internal] + tests: + - Program_Main_VersionFlag_DisplaysVersionOnly + - Program_Main_HelpFlag_DisplaysHelp + - Program_Main_ValidateFlag_RunsValidation + - Program_Main_ValidSarifFile_ProcessesSuccessfully + + - id: SarifMark-Program-ExitCode + title: >- + The Program unit shall return the operation result as the process + exit code. + justification: >- + Propagating the operation result to the process exit code lets + callers and CI/CD pipelines observe success or failure without + parsing console output. + tags: [internal] + tests: + - Program_Main_VersionFlag_DisplaysVersionOnly + - Program_Main_HelpFlag_DisplaysHelp + - id: SarifMark-Program-Main-Exceptions title: The Program unit shall return exit code 1 when argument or processing errors occur. @@ -110,13 +139,25 @@ sections: - id: SarifMark-Program-SarifReading title: >- - The Program unit shall read the specified SARIF file and report the - tool name, - version, and result count. + The Program unit shall read and parse the SARIF file specified + by the --sarif argument. + justification: >- + Reading the specified SARIF file is the prerequisite for all + downstream analysis; verifying this step in isolation confirms + that file access and deserialization succeed before any reporting + or enforcement logic runs. + tags: [internal] + tests: + - Program_Main_ValidSarifFile_ProcessesSuccessfully + + - id: SarifMark-Program-SarifSummary + title: >- + The Program unit shall report the tool name, version, and result + count after successfully reading the SARIF file. justification: >- - Reporting tool identity and result count after reading confirms successful - processing and - gives users immediate feedback about what was analyzed. + Reporting tool identity and result count after reading confirms + successful processing and gives users immediate feedback about + what was analyzed and how many findings were found. tags: [internal] tests: - Program_Main_ValidSarifFile_ProcessesSuccessfully diff --git a/docs/reqstream/sarifmark/sarif.yaml b/docs/reqstream/sarifmark/sarif.yaml index 339ddf5..b3d20d3 100644 --- a/docs/reqstream/sarifmark/sarif.yaml +++ b/docs/reqstream/sarifmark/sarif.yaml @@ -27,7 +27,9 @@ sections: children: - SarifMark-SarifResults-ValidatePathArgument - SarifMark-SarifResults-ValidatePathExists - - SarifMark-SarifResults-ValidateStructure + - SarifMark-SarifResults-ValidateJson + - SarifMark-SarifResults-ValidateVersion + - SarifMark-SarifResults-ValidateRuns tests: - Sarif_Read_NonExistentFile_ThrowsFileNotFoundException - Sarif_Read_InvalidSarifFile_ThrowsInvalidOperationException @@ -129,21 +131,3 @@ sections: - SarifMark-SarifResults-MultiRunMarkdown tests: - Sarif_Read_MultiRunSarifFile_ProcessesAllRuns - - - id: SarifMark-Sarif-Report - title: The tool shall generate markdown reports from SARIF files. - justification: >- - Report generation transforms machine-readable SARIF into human-readable - documentation, enabling better understanding and communication of - analysis results across development teams. - tags: [public] - children: - - SarifMark-Report-Markdown - - SarifMark-Report-Depth - - SarifMark-Report-Counts - - SarifMark-Report-Locations - - SarifMark-Report-Headings - - SarifMark-Report-LineBreaks - - SarifMark-Report-FileCount - tests: - - Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent diff --git a/docs/reqstream/sarifmark/sarif/report.yaml b/docs/reqstream/sarifmark/sarif/report.yaml deleted file mode 100644 index a02e0f8..0000000 --- a/docs/reqstream/sarifmark/sarif/report.yaml +++ /dev/null @@ -1,98 +0,0 @@ ---- -# Report unit requirements -sections: - - title: 'SarifMark Requirements' - sections: - - title: Sarif Requirements - sections: - - title: Report Requirements - requirements: - - id: SarifMark-Report-Markdown - title: The tool shall generate markdown reports from SARIF files. - justification: >- - Markdown report generation transforms machine-readable SARIF into - human-readable documentation, - enabling better understanding and communication of analysis results - across development teams. - tags: [public] - children: - - SarifMark-SarifResults-ToMarkdown - tests: - - Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent - - - id: SarifMark-Report-Depth - title: The tool shall support configurable markdown heading depth. - justification: >- - Configurable heading depth allows reports to integrate seamlessly - into existing documentation - hierarchies, preventing heading level conflicts when reports are - embedded in larger documents. - tags: [public] - children: - - SarifMark-SarifResults-ValidateDepth - tests: - - Sarif_GenerateReport_ReportDepth_IsConfigurable - - - id: SarifMark-Report-Counts - title: The tool shall display result counts in reports. - justification: >- - Result counts provide a quick summary of analysis findings, helping - users understand the scope of - issues at a glance and track improvements over time. - tags: [public] - children: - - SarifMark-SarifRun-FormatCount - tests: - - Sarif_GenerateReport_ResultCount_ContainsResultCount - - - id: SarifMark-Report-Locations - title: The tool shall display location information for results. - justification: >- - Displaying location information enables developers to quickly navigate - to the source of issues, - making the reports actionable and reducing time spent searching - for problematic code. - tags: [public] - children: - - SarifMark-SarifRun-FormatLocation - tests: - - Sarif_GenerateReport_LocationInfo_ContainsLocationInfo - - - id: SarifMark-Report-Headings - title: The tool shall support custom headings in reports. - justification: >- - Custom headings allow users to tailor reports to their documentation - standards and naming - conventions, improving consistency across project documentation. - tags: [public] - children: - - SarifMark-SarifResults-ToMarkdown - tests: - - Sarif_GenerateReport_CustomHeading_UsesCustomHeading - - - id: SarifMark-Report-LineBreaks - title: The tool shall format multiple results with proper line breaks. - justification: >- - Proper line breaks between results ensure readability and visual - separation in generated markdown, - making it easier to distinguish between different findings in the - report. - tags: [public] - children: - - SarifMark-SarifResults-ToMarkdown - tests: - - Sarif_GenerateReport_MultipleResults_FormatsWithLineBreaks - - - id: SarifMark-Report-FileCount - title: The tool shall include the file count in generated reports. - justification: >- - Including the file count in the report header provides evidence - that analysis was actually - performed on the expected files, which is particularly valuable - in regulated or audited - development environments. - tags: [public] - children: - - SarifMark-SarifResults-FileCount - tests: - - Sarif_GenerateReport_FileCount_ContainsFileCount diff --git a/docs/reqstream/sarifmark/sarif/sarif-finding.yaml b/docs/reqstream/sarifmark/sarif/sarif-finding.yaml index 09df64b..e2cfd24 100644 --- a/docs/reqstream/sarifmark/sarif/sarif-finding.yaml +++ b/docs/reqstream/sarifmark/sarif/sarif-finding.yaml @@ -8,22 +8,36 @@ sections: sections: - title: SarifFinding Requirements requirements: - - id: SarifMark-SarifFinding-Properties - title: >- - The SarifFinding unit shall preserve the rule identifier, severity - level, message, - optional file location, and optional line number provided at creation. + - id: SarifMark-SarifFinding-RuleId + title: The SarifFinding unit shall preserve the rule identifier provided + at creation. + justification: >- + The rule identifier links each finding to the analysis rule that + raised + it, enabling report consumers to look up rule documentation and + understand the category of the issue. + tags: [internal] + tests: + - SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties + + - id: SarifMark-SarifFinding-Message + title: The SarifFinding unit shall preserve the message provided at + creation. + justification: >- + The message carries the human-readable description of the finding + that appears in the generated report, making the issue understandable + without referring back to the raw SARIF file. + tags: [internal] + tests: + - SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties + + - id: SarifMark-SarifFinding-Severity + title: The SarifFinding unit shall preserve the severity level provided + at creation. justification: >- - Capturing all five pieces of data on a single record provides a - self-contained representation of - one SARIF finding, making it straightforward to pass findings between - parsing and formatting - stages without repeated JSON traversal. The five properties form - an indivisible record - contract: a finding is only meaningful when all five slots (three - required, two optional) are - present together, so they are specified as a single requirement - rather than five separate ones. + The severity level classifies findings by impact (error, warning, + note, etc.), allowing report consumers to prioritize remediation + and apply enforcement rules based on severity thresholds. tags: [internal] tests: - SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties diff --git a/docs/reqstream/sarifmark/sarif/sarif-results.yaml b/docs/reqstream/sarifmark/sarif/sarif-results.yaml index 12cc099..a13d37b 100644 --- a/docs/reqstream/sarifmark/sarif/sarif-results.yaml +++ b/docs/reqstream/sarifmark/sarif/sarif-results.yaml @@ -37,21 +37,43 @@ sections: tests: - SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException - - id: SarifMark-SarifResults-ValidateStructure - title: >- - The SARIF results unit shall reject SARIF documents that do not - contain - the required version and runs fields. + - id: SarifMark-SarifResults-ValidateJson + title: The SARIF results unit shall reject files that do not contain + valid JSON. justification: >- - Structural validation at read time ensures that downstream processing - always operates on - well-formed data, preventing cryptic errors and producing informative - messages that guide - users to fix malformed input files. + Rejecting malformed JSON at read time produces a clear diagnostic + before + any structural checks are attempted, making it immediately apparent + that + the input file is corrupted or incorrectly formatted. tags: [internal] tests: - SarifResults_Read_InvalidJson_ThrowsInvalidOperationException + + - id: SarifMark-SarifResults-ValidateVersion + title: The SARIF results unit shall reject SARIF documents that do + not contain the required version field. + justification: >- + Checking for the version field at read time ensures that downstream + processing always operates on a recognizable SARIF document, producing + an informative message that guides users to fix malformed input + files. + tags: [internal] + tests: - SarifResults_Read_MissingVersion_ThrowsInvalidOperationException + + - id: SarifMark-SarifResults-ValidateRuns + title: The SARIF results unit shall reject SARIF documents that do + not contain a non-empty runs array. + justification: >- + Checking for a non-empty runs array at read time ensures that report + generation always has at least one run to process, preventing empty + or + degenerate outputs and producing informative messages that guide + users + to fix malformed input files. + tags: [internal] + tests: - SarifResults_Read_MissingRuns_ThrowsInvalidOperationException - SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException @@ -127,15 +149,12 @@ sections: - id: SarifMark-SarifResults-ToMarkdown title: >- The SARIF results unit shall generate a complete markdown report - for - a single run or a combined report for multiple runs. + for a single-run SARIF file. justification: >- For a single run, delegating directly to the run's report generation - preserves the output format. - For multiple runs, concatenating each run's output with indexed - headings produces a - complete combined report. In both cases the caller needs only one - method call. + preserves the output format and provides a simple, cohesive output + document + for a single tool run without additional indexing overhead. tags: [internal] tests: - SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput @@ -150,6 +169,19 @@ sections: - SarifResults_ToMarkdown_ResultWithoutLocation_ShowsNoLocation - SarifResults_ToMarkdown_ResultWithUriNoLine_ShowsUriOnly + - id: SarifMark-SarifResults-ToMarkdownMultiRun + title: >- + The SARIF results unit shall render a multi-run SARIF file by + concatenating individually indexed run sections. + justification: >- + For multiple runs, concatenating each run's output with indexed + headings produces a complete combined report. The caller needs only + one method call regardless of whether the SARIF file contains one + or many runs. + tags: [internal] + tests: + - SarifResults_ToMarkdown_MultipleRuns_IncludesRunIndices + - id: SarifMark-SarifResults-ValidateDepth title: >- The SARIF results unit shall require a heading depth between 1 and diff --git a/docs/reqstream/sarifmark/sarif/sarif-run.yaml b/docs/reqstream/sarifmark/sarif/sarif-run.yaml index f40d1e0..68b769d 100644 --- a/docs/reqstream/sarifmark/sarif/sarif-run.yaml +++ b/docs/reqstream/sarifmark/sarif/sarif-run.yaml @@ -33,14 +33,20 @@ sections: - SarifRun_InternalConstructor_CreatesValidInstance - id: SarifMark-SarifRun-Results - title: The SarifRun unit shall expose the collection of non-suppressed - findings and their total count. + title: The SarifRun unit shall expose the collection of parsed findings. justification: >- - Providing the full results collection enables report formatters - to iterate and render - individual findings, while the derived result count avoids repeated - collection traversal - across callers. + Exposing the full results collection enables report formatters + to iterate and render individual findings for each tool run. + tags: [internal] + tests: + - SarifRun_InternalConstructor_CreatesValidInstance + + - id: SarifMark-SarifRun-ResultCount + title: The SarifRun unit shall expose the total count of parsed findings. + justification: >- + Providing a derived result count avoids repeated collection traversal + across callers and enables report formatters to render count summaries + efficiently. tags: [internal] tests: - SarifRun_InternalConstructor_CreatesValidInstance diff --git a/docs/reqstream/sarifmark/self-test.yaml b/docs/reqstream/sarifmark/self-test.yaml index 425ee30..0510d44 100644 --- a/docs/reqstream/sarifmark/self-test.yaml +++ b/docs/reqstream/sarifmark/self-test.yaml @@ -83,3 +83,19 @@ sections: - SarifMark-Validation-EnforcementTest tests: - SelfTest_EnforcementTest_RunsWithinValidation + - SelfTest_EnforceFlag_WithIssues_ReturnsNonZeroExitCode + + - id: SarifMark-SelfTest-DepthParam + title: The SelfTest subsystem shall accept the --depth parameter to configure + the heading depth used in self-validation report generation. + justification: >- + Passing the depth parameter through to self-validation ensures that + the + self-test produces reports at the same heading level as normal analysis + mode, confirming that depth-aware report generation is exercised end-to-end + during validation. + tags: [public] + children: + - SarifMark-Validation-ReportGenerationTest + tests: + - SelfTest_DepthParameter_AffectsSelfValidationReport diff --git a/docs/reqstream/sarifmark/self-test/validation.yaml b/docs/reqstream/sarifmark/self-test/validation.yaml index ee17840..8266bb9 100644 --- a/docs/reqstream/sarifmark/self-test/validation.yaml +++ b/docs/reqstream/sarifmark/self-test/validation.yaml @@ -10,19 +10,31 @@ sections: requirements: - id: SarifMark-Validation-Run title: >- - The Validation unit shall execute the validation header, all validation - tests, - a pass/fail summary, and results file output in sequence. + The Validation unit shall execute all configured validation + tests in the suite. justification: >- - Sequencing these four phases in a fixed order ensures that every - validation run produces a - consistent, complete report regardless of individual test outcomes, - giving users a reliable - picture of the tool's health in their environment. + Executing every test in the configured suite in a fixed order + ensures that every validation run produces a consistent, + complete result regardless of individual test outcomes, giving + users a reliable picture of the tool's health in their environment. tags: [internal] tests: - Validation_Run_ValidContext_RunsAllTests + - id: SarifMark-Validation-ResultsOutput + title: >- + The Validation unit shall write the validation results to a + file when a results file path is configured. + justification: >- + Writing results to a file only when a path is configured + makes the output step explicit and optional, enabling integration + with CI/CD test result infrastructure without imposing file I/O + on callers that do not require it. + tags: [internal] + tests: + - Validation_Run_WithTrxResultsFile_WritesResultsFile + - Validation_Run_WithXmlResultsFile_WritesResultsFile + - id: SarifMark-Validation-NullCheck title: The Validation unit shall reject a null execution context with a clear error. @@ -49,6 +61,65 @@ sections: tests: - Validation_Run_ValidContext_PrintsValidationHeader + - id: SarifMark-Validation-Header-ToolVersion + title: The Validation unit shall include the tool version in the validation + header. + justification: >- + Including the tool version in the header allows validation results + to + be attributed to a specific release, enabling precise reproduction + and + comparison across versions. + tags: [internal] + tests: + - Validation_Run_ValidContext_PrintsValidationHeader + + - id: SarifMark-Validation-Header-MachineName + title: The Validation unit shall include the machine name in the validation + header. + justification: >- + Including the machine name allows validation results to be attributed + to a specific host, enabling identification of environment-specific + failures. + tags: [internal] + tests: + - Validation_Run_ValidContext_PrintsValidationHeader + + - id: SarifMark-Validation-Header-OsVersion + title: The Validation unit shall include the OS version in the validation + header. + justification: >- + Including the OS version contextualizes validation results with + the + operating system in which they were collected, supporting cross-platform + comparison and reproducibility analysis. + tags: [internal] + tests: + - Validation_Run_ValidContext_PrintsValidationHeader + + - id: SarifMark-Validation-Header-Runtime + title: The Validation unit shall include the .NET runtime version + in the validation header. + justification: >- + Including the .NET runtime version contextualizes validation results + with the specific runtime in use, enabling identification of + runtime-specific failures across supported .NET versions. + tags: [internal] + tests: + - Validation_Run_ValidContext_PrintsValidationHeader + + - id: SarifMark-Validation-Header-Timestamp + title: The Validation unit shall include the timestamp in the validation + header. + justification: >- + Including the timestamp establishes when validation was performed, + enabling chronological ordering of validation records and detection + of + time-sensitive issues. + tags: [internal] + tests: + - Validation_Run_ValidContext_PrintsValidationHeader + - id: SarifMark-Validation-SarifReadingTest title: The Validation unit shall include a SARIF reading test in the validation suite. diff --git a/docs/reqstream/shared/sarifmark.yaml b/docs/reqstream/shared/sarifmark.yaml new file mode 100644 index 0000000..f76e495 --- /dev/null +++ b/docs/reqstream/shared/sarifmark.yaml @@ -0,0 +1,61 @@ +--- +# SarifMark shared package requirements + +sections: + - title: Shared Package Requirements + sections: + - title: SarifMark Requirements + requirements: + - id: SarifMark-Shared-SarifMark-ReadSarif + title: SarifMark shall read a SARIF input file specified by the --sarif + parameter. + justification: | + The CI pipeline step invokes SarifMark with --sarif artifacts/csharp.sarif to + supply the CodeQL analysis output. SarifMark must accept the file path and + successfully parse the SARIF content. The self-validation test + SarifMark_ValidSarifFile_ProcessesSuccessfully confirms that SarifMark reads + a valid SARIF file and processes it without error, as this is a core feature + verified by the product's own test suite. + tags: [shared] + tests: + - SarifMark_ValidSarifFile_ProcessesSuccessfully + + - id: SarifMark-Shared-SarifMark-GenerateReport + title: SarifMark shall generate a markdown report at the path specified + by the --report parameter. + justification: | + The CI pipeline step invokes SarifMark with --report docs/code_quality/generated/codeql-quality.md + to write the CodeQL quality report into the release artifacts. SarifMark must create + the markdown file at the specified path. The self-validation test + SarifMark_GenerateReport_CreatesReportFile confirms that SarifMark creates a + report file at the given output path, and SarifMark_ValidSarif_NoIssues_GeneratesReport + confirms report generation also succeeds when the SARIF file contains no issues. + tags: [shared] + tests: + - SarifMark_GenerateReport_CreatesReportFile + - SarifMark_ValidSarif_NoIssues_GeneratesReport + + - id: SarifMark-Shared-SarifMark-ConfigureHeading + title: SarifMark shall use a custom heading in the generated report when + specified by the --heading parameter. + justification: | + The CI pipeline step supplies --heading "SarifMark CodeQL Analysis" so that the + generated report carries the correct section title in the release documentation bundle. + The self-validation test SarifMark_CustomHeading_AppearsInReport confirms that the + specified heading text appears as the top-level heading in the generated markdown report. + tags: [shared] + tests: + - SarifMark_CustomHeading_AppearsInReport + + - id: SarifMark-Shared-SarifMark-ConfigureReportDepth + title: SarifMark shall apply the heading depth specified by the --report-depth + parameter. + justification: | + The CI pipeline step supplies --report-depth 1 so that the generated report uses + the correct heading level when embedded in the release documentation. The + self-validation test SarifMark_LegacyReportDepth_IsAccepted confirms that the + --report-depth parameter is accepted and applied correctly to the heading depth + of the generated markdown report. + tags: [shared] + tests: + - SarifMark_LegacyReportDepth_IsAccepted diff --git a/docs/user_guide/usage.md b/docs/user_guide/usage.md index aa438d6..5812b00 100644 --- a/docs/user_guide/usage.md +++ b/docs/user_guide/usage.md @@ -2,7 +2,10 @@ ## Getting Started -After installation the `sarifmark` command is available from any terminal. +After a **global** installation (`dotnet tool install --global`) the `sarifmark` command is +available from any terminal without a path qualifier. After a **local** installation the tool +must be invoked with `dotnet tool run sarifmark` (or the equivalent `dotnet sarifmark` shorthand, +which is a standard .NET local-tool invocation available in any shell or terminal where the .NET SDK is installed). Generate a markdown report from a SARIF file: @@ -35,8 +38,10 @@ sarifmark --help | `--log ` | Write console output to a log file | | `--sarif ` | SARIF file to process (required for analysis) | | `--report ` | Export analysis results to a markdown file | -| `--depth ` | Markdown header depth for the report (default: `1`) | +| `--depth ` | Markdown header depth for the report (default: `1`; accepted range: `1`–`6`, corresponding to Markdown heading levels `#` through `######`) | | `--heading ` | Custom heading for the report (default: `[ToolName] Analysis`) | +| `--report-depth ` | Deprecated alias for `--depth` (supported for backwards compatibility) | +| `--result ` | Deprecated alias for `--results` (supported for backwards compatibility) | ## Common Usage Patterns diff --git a/docs/verification/definition.yaml b/docs/verification/definition.yaml index 44ccd90..faf8aed 100644 --- a/docs/verification/definition.yaml +++ b/docs/verification/definition.yaml @@ -7,6 +7,7 @@ resource-path: - docs/verification/sarifmark/self-test - docs/verification/sarifmark/utilities - docs/verification/ots + - docs/verification/shared - docs/template input-files: - docs/verification/title.txt @@ -19,6 +20,7 @@ input-files: - docs/verification/sarifmark/sarif/sarif-finding.md - docs/verification/sarifmark/sarif/sarif-run.md - docs/verification/sarifmark/sarif/sarif-results.md + - docs/verification/sarifmark/self-test.md - docs/verification/sarifmark/self-test/validation.md - docs/verification/sarifmark/utilities.md @@ -33,6 +35,9 @@ input-files: - docs/verification/ots/sonarmark.md - docs/verification/ots/versionmark.md - docs/verification/ots/weasyprint.md + - docs/verification/ots/test-results.md + - docs/verification/shared.md + - docs/verification/shared/sarifmark.md template: template.html table-of-contents: true number-sections: true diff --git a/docs/verification/introduction.md b/docs/verification/introduction.md index 5355342..298603d 100644 --- a/docs/verification/introduction.md +++ b/docs/verification/introduction.md @@ -11,8 +11,13 @@ without reading source code. ## Scope This document covers all in-house software items comprising the SarifMark system — including all subsystems and units — -and all OTS software items used in the build pipeline and test infrastructure. It does not cover installation -procedures, end-user guides, or CI/CD pipeline configuration. +all OTS software items used in the build pipeline, and all shared package dependencies +(released version of SarifMark used in the CI pipeline to generate the CodeQL quality report). +OTS packages used by the test infrastructure (xUnit v3, DemaConsulting.TestResults) are also covered because they are +third-party items whose correct operation must be independently confirmed; the test project itself is out of scope. +It does not cover installation procedures, end-user guides, or CI/CD pipeline configuration. + +Out of scope: build pipeline scripts and the test project itself (`test/DemaConsulting.SarifMark.Tests/`). ## Companion Artifact Structure @@ -33,11 +38,11 @@ OTS items have integration/usage design documentation parallel to system folders - Design: `docs/design/ots/{ots-name}.md` - Verification: `docs/verification/ots/{ots-name}.md` -Shared Packages have integration/usage design documentation parallel to system and OTS folders: +Shared package items have integration/usage design documentation parallel to system and OTS folders: -- Requirements: `docs/reqstream/shared/{package-name}.yaml` -- Design: `docs/design/shared/{package-name}.md` -- Verification: `docs/verification/shared/{package-name}.md` +- Requirements: `docs/reqstream/shared/{name}.yaml` +- Design: `docs/design/shared/{name}.md` +- Verification: `docs/verification/shared/{name}.md` Review-sets are defined in `.reviewmark.yaml`. diff --git a/docs/verification/ots.md b/docs/verification/ots.md index 96fd117..56e5597 100644 --- a/docs/verification/ots.md +++ b/docs/verification/ots.md @@ -4,11 +4,14 @@ Each OTS item is verified using one of three evidence categories matched to its role in the pipeline: -1. **Self-validation output**: Tools that expose a `--validate` or version/help flag (FileAssert) are exercised in - the CI pipeline self-validation step; passing output confirms the tool is installed and operational. -2. **Successful CI pipeline completion**: Tools without a self-test mechanism (BuildMark, ReviewMark, SonarMark, - VersionMark, ReqStream, Pandoc, WeasyPrint) are verified by successful CI pipeline execution — each tool produces - an artefact (document, report, or exit-code assertion) that confirms functional operation. +1. **Self-validation output**: Tools that expose a `--validate` flag (BuildMark, FileAssert, ReqStream, ReviewMark, + VersionMark — verified through SarifMark's self-validation tests) are exercised through the `--validate` + self-validation mechanism; passing output confirms the tool is installed and all advertised features are + operational. DemaConsulting.TestResults is a NuGet package (not a CLI tool) and does not expose `--validate`; + it is verified through SarifMark's integration and self-validation tests. +2. **Successful CI pipeline completion**: Tools verified by successful CI pipeline execution — each tool produces + an artifact (document, report, or exit-code assertion) that confirms functional operation. Pandoc and + WeasyPrint are verified via FileAssert assertions on generated HTML and PDF files. 3. **Framework operation**: xUnit v3 (the test framework) is verified implicitly — the test suite runs and produces passing results, confirming the framework discovers and executes tests correctly. diff --git a/docs/verification/ots/buildmark.md b/docs/verification/ots/buildmark.md index b33df2f..cbc0ba0 100644 --- a/docs/verification/ots/buildmark.md +++ b/docs/verification/ots/buildmark.md @@ -2,18 +2,55 @@ ### Verification Approach -BuildMark is used in the SarifMark CI pipeline to generate a build-notes markdown document containing build metadata, -tool versions, and pipeline artefact listings. BuildMark has no self-test mechanism. Verification evidence is provided -by successful CI pipeline execution: the pipeline step that invokes BuildMark completes without error and produces the -build-notes markdown file, which is subsequently converted to HTML and PDF by Pandoc and WeasyPrint. +BuildMark is verified through its built-in `--validate` self-validation mechanism. Running +`dotnet buildmark --validate` executes five internal test scenarios that confirm the tool +is installed and functioning correctly in the current environment. + +Most scenarios exercise specific advertised features using mock data or the local Git repository +and do not require live GitHub API access. The exception is the `BuildMark_IssueTracking` +scenario, which connects to the GitHub API and therefore requires a valid GitHub API token +(sourced from `GH_TOKEN`, `GITHUB_TOKEN`, or the `gh` CLI) and network access to +`api.github.com`. Environments without API access must skip or exclude that scenario. + +### Test Environment + +Tests require: + +- A local Git repository (used by `BuildMark_GitIntegration` to read version tags). +- Network access to the GitHub API (used by `BuildMark_IssueTracking`); the test uses + a token sourced from `GH_TOKEN`, `GITHUB_TOKEN`, or the `gh` CLI. +- A writable temporary directory for output files. + +### Acceptance Criteria + +All five self-validation scenarios must pass with exit code 0 and zero failures in an environment +that provides GitHub API access. In environments without API access, the four scenarios that do +not require live network access (`BuildMark_MarkdownReportGeneration`, `BuildMark_GitIntegration`, +`BuildMark_KnownIssuesReporting`, `BuildMark_RulesRouting`) must pass; `BuildMark_IssueTracking` +may be skipped or excluded. ### Test Scenarios -**BuildMark_MarkdownReportGeneration**: The CI pipeline step that invokes BuildMark completes without error and -produces the build-notes markdown document containing build metadata, tool versions, and pipeline artefact listings, -confirming BuildMark executed successfully and generated the expected output. -This scenario is verified by successful completion of the BuildMark pipeline step in CI. +**BuildMark_MarkdownReportGeneration**: Invokes `dotnet buildmark --validate`; the +`BuildMark_MarkdownReportGeneration` scenario generates a markdown build-notes document +from mock data and confirms the report contains the expected content. +This scenario is tested by `BuildMark_MarkdownReportGeneration`. + +**BuildMark_GitIntegration**: The `BuildMark_GitIntegration` self-validation scenario reads +version tags and commits from the local Git repository and confirms the Git connector +returns expected data. +This scenario is tested by `BuildMark_GitIntegration`. + +**BuildMark_IssueTracking**: The `BuildMark_IssueTracking` self-validation scenario connects +to the GitHub API, retrieves issue and pull request data, and confirms the integration +returns expected results. +This scenario is tested by `BuildMark_IssueTracking`. -### Requirements Coverage +**BuildMark_KnownIssuesReporting**: The `BuildMark_KnownIssuesReporting` self-validation +scenario generates a report with the `--include-known-issues` flag and confirms the Known +Issues section is included in the output. +This scenario is tested by `BuildMark_KnownIssuesReporting`. -- **`SarifMark-OTS-BuildMark`**: Build notes generated in CI — `BuildMark_MarkdownReportGeneration` (CI pipeline step) +**BuildMark_RulesRouting**: The `BuildMark_RulesRouting` self-validation scenario applies +routing rules and confirms that items are assigned to the correct report sections. +This scenario is tested by `BuildMark_RulesRouting`. diff --git a/docs/verification/ots/fileassert.md b/docs/verification/ots/fileassert.md index 616c191..15dca18 100644 --- a/docs/verification/ots/fileassert.md +++ b/docs/verification/ots/fileassert.md @@ -2,31 +2,45 @@ ### Verification Approach -FileAssert is used in the SarifMark CI pipeline to assert that generated output files (HTML documents, PDF documents) -exist, have non-trivial size, contain valid structural elements, and include expected content. FileAssert exposes a -`--version` and `--help` flag. Self-validation tests confirm FileAssert is installed and operational before it is used -to validate generated documents. Functional verification is provided by the successful execution of file assertion -steps in the CI pipeline. +FileAssert is verified through its built-in `--validate` self-validation mechanism. Running +`fileassert --validate` executes five internal test scenarios that confirm the tool is +installed and all advertised features are functioning correctly: version display, help display, +result file generation, file existence assertion, and content assertion. -### Test Scenarios +### Test Environment -**FileAssert_VersionDisplay**: The CI self-validation step invokes `fileassert --version`; the tool responds with its -version string and exits with code 0, confirming FileAssert is installed and operational. -This scenario is verified by the self-validation CI pipeline step. +Tests require: -**FileAssert_HelpDisplay**: The CI self-validation step invokes `fileassert --help`; the tool responds with its usage -information and exits with code 0, confirming the CLI interface is functioning as expected. -This scenario is verified by the self-validation CI pipeline step. +- No network access; all scenarios operate on temporary file fixtures. +- A writable temporary directory for output files. -**FileAssert_HtmlDocumentAssertions**: FileAssert validates generated HTML documents in the CI pipeline, asserting -that each HTML output file exists, has non-trivial size, contains a valid `` element, and includes expected -document content. This scenario is verified by the file-assertion CI pipeline steps that validate Pandoc HTML outputs. +### Acceptance Criteria -**FileAssert_PdfDocumentAssertions**: FileAssert validates generated PDF documents in the CI pipeline, asserting -that each PDF output file exists, has non-trivial size, contains at least one page, and includes expected rendered -text. This scenario is verified by the file-assertion CI pipeline steps that validate WeasyPrint PDF outputs. +All five self-validation scenarios must pass with exit code 0 and zero failures, confirming +that FileAssert is correctly installed and all advertised features are operational. -### Requirements Coverage +### Test Scenarios -- **`SarifMark-OTS-FileAssert`**: CI file assertions pass — `FileAssert_VersionDisplay`, `FileAssert_HelpDisplay`, - `FileAssert_HtmlDocumentAssertions`, `FileAssert_PdfDocumentAssertions` +**FileAssert_VersionDisplay**: The `FileAssert_VersionDisplay` self-validation scenario +invokes `--version` and confirms the tool outputs a valid version string and exits with +code 0. +This scenario is tested by `FileAssert_VersionDisplay`. + +**FileAssert_HelpDisplay**: The `FileAssert_HelpDisplay` self-validation scenario invokes +`--help` and confirms usage information and available options are displayed. +This scenario is tested by `FileAssert_HelpDisplay`. + +**FileAssert_Results**: The `FileAssert_Results` self-validation scenario runs test +assertions that produce pass and fail outcomes and writes results to a TRX file, confirming +result file generation with mixed outcomes works correctly. +This scenario is tested by `FileAssert_Results`. + +**FileAssert_Exists**: The `FileAssert_Exists` self-validation scenario runs a file +existence assertion via glob pattern and confirms the assertion passes when the expected +file is present. +This scenario is tested by `FileAssert_Exists`. + +**FileAssert_Contains**: The `FileAssert_Contains` self-validation scenario runs a content +assertion that checks a file contains expected text and confirms the assertion passes when +the content is present. +This scenario is tested by `FileAssert_Contains`. diff --git a/docs/verification/ots/pandoc.md b/docs/verification/ots/pandoc.md index f91cd06..5174498 100644 --- a/docs/verification/ots/pandoc.md +++ b/docs/verification/ots/pandoc.md @@ -3,10 +3,15 @@ ### Verification Approach Pandoc is used in the SarifMark build pipeline to convert Markdown document collections to HTML using a custom HTML -template. It processes the design document, user guide, build notes, code-quality report, review plan, and review -report. Verification evidence is provided by FileAssert assertions confirming that each HTML output file exists, has +template. It processes the design document, verification document, user guide, build notes, code-quality report, review plan, +and review report. Verification evidence is provided by FileAssert assertions confirming that each HTML output file exists, has non-trivial size, contains a valid `<title>` element, and includes expected document content. +The requirements report and trace matrix HTML documents are excluded from the verification +scenarios because they are generated by the compliance toolchain itself (ReqStream and Pandoc +together) — using them as verification evidence for Pandoc would create a self-referential +dependency where the tool being verified is also responsible for producing the evidence. + ### Test Scenarios **Pandoc_BuildNotesHtml**: Pandoc converts the build-notes Markdown collection to an HTML file; FileAssert assertions @@ -33,8 +38,6 @@ content. This scenario is verified by FileAssert assertions in the CI pipeline. confirm the output file exists, has non-trivial size, contains a valid `<title>` element, and includes expected user-guide content. This scenario is verified by FileAssert assertions in the CI pipeline. -### Requirements Coverage - -- **`SarifMark-OTS-Pandoc`**: Pandoc converts each document collection to valid HTML — `Pandoc_BuildNotesHtml`, - `Pandoc_CodeQualityHtml`, `Pandoc_ReviewPlanHtml`, `Pandoc_ReviewReportHtml`, `Pandoc_DesignHtml`, - `Pandoc_UserGuideHtml` +**Pandoc_VerificationHtml**: Pandoc converts the verification document Markdown collection to an HTML file; FileAssert +assertions confirm the output file exists, has non-trivial size, contains a valid `<title>` element, and includes +expected verification content. This scenario is verified by FileAssert assertions in the CI pipeline. diff --git a/docs/verification/ots/reqstream.md b/docs/verification/ots/reqstream.md index 2df8ebd..665c870 100644 --- a/docs/verification/ots/reqstream.md +++ b/docs/verification/ots/reqstream.md @@ -2,19 +2,49 @@ ### Verification Approach -ReqStream is used in the SarifMark CI pipeline to enforce requirements traceability. It reads requirements YAML files -and TRX test result files, verifies that every requirement maps to at least one passing test, and fails the pipeline if -any requirement is untested. Verification evidence is provided by successful CI pipeline execution with `--enforce` -mode: the pipeline step completes with exit code 0, confirming that ReqStream parsed all requirements, matched them to -passing test results, and found no untested requirements. +ReqStream is verified through its built-in `--validate` self-validation mechanism. Running +`reqstream --validate` executes six internal test scenarios that confirm the tool is +installed and functioning correctly in the current environment, including requirements +processing, traceability matrix generation, report export, tag filtering, enforcement mode, +and lint validation. + +### Test Environment + +Tests require: + +- No network access; all scenarios operate on temporary file fixtures. +- A writable temporary directory for output files. + +### Acceptance Criteria + +All six self-validation scenarios must pass with exit code 0 and zero failures, confirming +that ReqStream is correctly installed and all advertised features are operational. ### Test Scenarios -**ReqStream_EnforcementMode**: The CI pipeline step invokes ReqStream with `--enforce`; the step completes with exit -code 0, confirming that ReqStream parsed all requirements YAML files, matched every requirement to at least one passing -test result in the TRX files, and found no untested requirements. -This scenario is verified by successful completion of the ReqStream enforcement pipeline step in CI. +**ReqStream_RequirementsProcessing**: The `ReqStream_RequirementsProcessing` self-validation +scenario processes a requirements YAML file and confirms all requirements are parsed into +the complete model. +This scenario is tested by `ReqStream_RequirementsProcessing`. + +**ReqStream_TraceMatrix**: The `ReqStream_TraceMatrix` self-validation scenario generates a +traceability matrix document and confirms the matrix correctly links requirements to tests. +This scenario is tested by `ReqStream_TraceMatrix`. + +**ReqStream_ReportExport**: The `ReqStream_ReportExport` self-validation scenario exports a +requirements report and a justifications document and confirms both output files are +created with expected content. +This scenario is tested by `ReqStream_ReportExport`. + +**ReqStream_TagsFiltering**: The `ReqStream_TagsFiltering` self-validation scenario filters +requirements by tag and confirms only matching requirements are included in the output. +This scenario is tested by `ReqStream_TagsFiltering`. -### Requirements Coverage +**ReqStream_EnforcementMode**: The `ReqStream_EnforcementMode` self-validation scenario +invokes ReqStream with `--enforce` against a requirements set where all requirements are +covered, confirming exit code 0 and that enforcement mode correctly signals success. +This scenario is tested by `ReqStream_EnforcementMode`. -- **`SarifMark-OTS-ReqStream`**: Enforcement mode passes in CI — `ReqStream_EnforcementMode` (CI pipeline step) +**ReqStream_Lint**: The `ReqStream_Lint` self-validation scenario invokes `--lint` against +a requirements YAML file and confirms the file is structurally valid with no errors reported. +This scenario is tested by `ReqStream_Lint`. diff --git a/docs/verification/ots/reviewmark.md b/docs/verification/ots/reviewmark.md index 40c985e..61c4b7d 100644 --- a/docs/verification/ots/reviewmark.md +++ b/docs/verification/ots/reviewmark.md @@ -2,24 +2,70 @@ ### Verification Approach -ReviewMark is used in the SarifMark CI pipeline to generate a review plan and a review report from the -`.reviewmark.yaml` review-set configuration. Verification evidence is provided by successful CI pipeline execution: -the pipeline steps that invoke ReviewMark complete without error and produce both the review plan and review report -markdown files, which are subsequently converted to HTML and PDF. +ReviewMark is verified through its built-in `--validate` self-validation mechanism. Running +`reviewmark --validate` executes ten internal test scenarios that confirm the tool is +installed and all advertised features are functioning correctly in the current environment. +Self-validation covers version display, help display, review plan generation, review report +generation, index scanning, working directory override, enforcement mode, elaboration, +lint validation, and review depth configuration. + +### Test Environment + +Tests require: + +- No network access; all scenarios operate on temporary file fixtures. +- A writable temporary directory for output files. + +### Acceptance Criteria + +All ten self-validation scenarios must pass with exit code 0 and zero failures, confirming +that ReviewMark is correctly installed and all advertised features are operational. ### Test Scenarios -**ReviewMark_ReviewPlanGeneration**: The CI pipeline step that invokes ReviewMark to generate the review plan -completes without error and produces the review plan markdown file, confirming ReviewMark read the -`.reviewmark.yaml` configuration and generated the expected output. -This scenario is verified by successful completion of the ReviewMark plan pipeline step in CI. +**ReviewMark_VersionDisplay**: The `ReviewMark_VersionDisplay` self-validation scenario +invokes `--version` and confirms the tool outputs a valid version string. +This scenario is tested by `ReviewMark_VersionDisplay`. + +**ReviewMark_HelpDisplay**: The `ReviewMark_HelpDisplay` self-validation scenario invokes +`--help` and confirms usage information and available options are displayed. +This scenario is tested by `ReviewMark_HelpDisplay`. + +**ReviewMark_ReviewPlanGeneration**: The `ReviewMark_ReviewPlanGeneration` self-validation +scenario generates a review plan from a temporary definition file and confirms the output +contains the expected plan structure. +This scenario is tested by `ReviewMark_ReviewPlanGeneration`. + +**ReviewMark_ReviewReportGeneration**: The `ReviewMark_ReviewReportGeneration` self-validation +scenario generates a review report from a temporary definition file and confirms the output +contains the expected report structure. +This scenario is tested by `ReviewMark_ReviewReportGeneration`. + +**ReviewMark_IndexScan**: The `ReviewMark_IndexScan` self-validation scenario scans a +directory of PDF evidence files with `--index` and confirms the index.json catalogue is +written with the correct content. +This scenario is tested by `ReviewMark_IndexScan`. + +**ReviewMark_WorkingDirectoryOverride**: The `ReviewMark_WorkingDirectoryOverride` +self-validation scenario uses `--dir` to override the working directory and confirms file +operations resolve correctly relative to the specified directory. +This scenario is tested by `ReviewMark_WorkingDirectoryOverride`. + +**ReviewMark_Enforce**: The `ReviewMark_Enforce` self-validation scenario invokes +`--enforce` against a review configuration with known issues and confirms exit code is +non-zero, proving enforcement mode correctly rejects stale or missing reviews. +This scenario is tested by `ReviewMark_Enforce`. -**ReviewMark_ReviewReportGeneration**: The CI pipeline step that invokes ReviewMark to generate the review report -completes without error and produces the review report markdown file, confirming ReviewMark processed the -review-set configuration and generated the expected report output. -This scenario is verified by successful completion of the ReviewMark report pipeline step in CI. +**ReviewMark_Elaborate**: The `ReviewMark_Elaborate` self-validation scenario invokes +`--elaborate {id}` and confirms the tool prints the ID, title, fingerprint, and file list +for the specified review set. +This scenario is tested by `ReviewMark_Elaborate`. -### Requirements Coverage +**ReviewMark_Lint**: The `ReviewMark_Lint` self-validation scenario exercises `--lint` against both a valid definition +file (confirming exit code 0) and an invalid definition file (confirming non-zero exit code and error reporting). The +valid-config case is the primary scenario referenced here. +This scenario is tested by `ReviewMark_Lint`. -- **`SarifMark-OTS-ReviewMark`**: Generates review plan and report in CI — `ReviewMark_ReviewPlanGeneration`, - `ReviewMark_ReviewReportGeneration` +**ReviewMark_DepthFlag**: The `ReviewMark_DepthFlag` self-validation scenario invokes ReviewMark with `--depth` +and confirms the flag is accepted and affects the heading depth in generated output. +This scenario is tested by `ReviewMark_DepthFlag`. diff --git a/docs/verification/ots/sonarmark.md b/docs/verification/ots/sonarmark.md index 6f8966a..7bd03ad 100644 --- a/docs/verification/ots/sonarmark.md +++ b/docs/verification/ots/sonarmark.md @@ -2,33 +2,42 @@ ### Verification Approach -SonarMark is used in the SarifMark CI pipeline to retrieve quality gate status, issues, and hotspots from SonarCloud -and generate a markdown code-quality report. Verification evidence is provided by successful CI pipeline execution: -the pipeline step that invokes SonarMark completes without error and produces the code-quality markdown document, -confirming SonarMark connected to SonarCloud and rendered the report correctly. +SonarMark is verified through its built-in `--validate` self-validation mechanism. Running +`sonarmark --validate` executes four internal test scenarios that confirm quality-gate +retrieval, issues retrieval, hot spots retrieval, and markdown report generation are all +functioning correctly using mock API responses — no live SonarCloud connection is required +for self-validation. -### Test Scenarios +### Test Environment -**SonarMark_QualityGateRetrieval**: The CI pipeline step that invokes SonarMark completes without error and produces -the code-quality markdown document, confirming SonarMark successfully connected to SonarCloud and rendered the quality -gate status, issues, and hotspots into the expected report format. -This scenario is verified by successful completion of the SonarMark pipeline step in CI. +Tests require: -**SonarMark_IssuesRetrieval**: The CI pipeline step that invokes SonarMark completes without error, confirming -SonarMark successfully retrieved the issues list from SonarCloud and rendered it in the code-quality markdown report. -This scenario is verified by successful completion of the SonarMark pipeline step in CI. +- No network access; all self-validation scenarios use mock API data. +- A writable temporary directory for output files. -**SonarMark_HotSpotsRetrieval**: The CI pipeline step that invokes SonarMark completes without error, confirming -SonarMark successfully retrieved the hot spots list from SonarCloud and rendered it in the code-quality markdown -report. This scenario is verified by successful completion of the SonarMark pipeline step in CI. +### Acceptance Criteria -**SonarMark_MarkdownReportGeneration**: The CI pipeline step that invokes SonarMark produces the code-quality -markdown document, confirming SonarMark successfully generated a complete markdown report containing quality-gate, -issues, and hot spots sections. -This scenario is verified by successful completion of the SonarMark pipeline step in CI. +All four self-validation scenarios must pass with exit code 0 and zero failures, confirming +that SonarMark is correctly installed and all advertised features are operational. -### Requirements Coverage +### Test Scenarios -- **`SarifMark-OTS-SonarMark`**: Retrieves quality data and generates markdown report — - `SonarMark_QualityGateRetrieval`, `SonarMark_IssuesRetrieval`, `SonarMark_HotSpotsRetrieval`, - `SonarMark_MarkdownReportGeneration` +**SonarMark_QualityGateRetrieval**: The `SonarMark_QualityGateRetrieval` self-validation +scenario retrieves mock quality-gate data and confirms the quality-gate status is correctly +parsed and available for report generation. +This scenario is tested by `SonarMark_QualityGateRetrieval`. + +**SonarMark_IssuesRetrieval**: The `SonarMark_IssuesRetrieval` self-validation scenario +retrieves mock issues data and confirms the issues list is correctly parsed and available +for report generation. +This scenario is tested by `SonarMark_IssuesRetrieval`. + +**SonarMark_HotSpotsRetrieval**: The `SonarMark_HotSpotsRetrieval` self-validation scenario +retrieves mock hot spots data and confirms the hot spots list is correctly parsed and +available for report generation. +This scenario is tested by `SonarMark_HotSpotsRetrieval`. + +**SonarMark_MarkdownReportGeneration**: The `SonarMark_MarkdownReportGeneration` +self-validation scenario generates a markdown quality report from mock data and confirms +the report contains the expected quality-gate, issues, and hot spots sections. +This scenario is tested by `SonarMark_MarkdownReportGeneration`. diff --git a/docs/verification/ots/test-results.md b/docs/verification/ots/test-results.md new file mode 100644 index 0000000..29d5cab --- /dev/null +++ b/docs/verification/ots/test-results.md @@ -0,0 +1,46 @@ +## DemaConsulting.TestResults + +### Verification Approach + +`DemaConsulting.TestResults` is verified through integration tests in the SarifMark test suite +that exercise the package through the SelfTest subsystem. No mocking is applied; the verification +calls the real serialization code to produce actual TRX and JUnit XML files. The resulting files +are then inspected to confirm the expected content is present. + +The tests verify that: + +- `TestResults` accumulates test outcomes written by `Validation.Run` without error. +- `TestResultsIO.WriteTrx` produces a valid `.trx` file containing the SarifMark self-validation + test suite content when the `--results` flag specifies a `.trx` path. +- `TestResultsIO.WriteJUnit` produces a valid JUnit XML file containing the expected `testsuite` + element when the `--results` flag specifies a `.xml` path. + +### Test Environment + +Tests require: + +- No network access; verification is entirely in-process. +- A writable temporary directory for the output results files. +- The `DemaConsulting.TestResults` NuGet package installed as a package reference in the + production project (`src/DemaConsulting.SarifMark/DemaConsulting.SarifMark.csproj`). + +### Acceptance Criteria + +The OTS integration is accepted when all linked tests pass with zero failures and the output +files contain the expected content markers (`SarifMark Self-Validation` in TRX output and +`testsuite` in JUnit XML output). + +### Test Scenarios + +**TestResultsCollection**: Runs the full self-validation pipeline with `--validate --silent` and +confirms exit code 0, proving that `TestResults` accumulates all three scenario outcomes without +error. Tested by `Validation_Run_ValidContext_PrintsSummary`. + +**TrxSerializationOutput**: Runs the self-validation pipeline with a `.trx` results path and +confirms the file is created and contains `SarifMark Self-Validation`, proving that +`TestResultsIO.WriteTrx` serializes correctly. Tested by `Validation_Run_WithTrxResultsFile_WritesResultsFile`. + +**JUnitSerializationOutput**: Runs the self-validation pipeline with a `.xml` results path and +confirms the file is created and contains a `testsuite` element, proving that +`TestResultsIO.WriteJUnit` serializes correctly. +Tested by `Validation_Run_WithXmlResultsFile_WritesResultsFile`. diff --git a/docs/verification/ots/versionmark.md b/docs/verification/ots/versionmark.md index e7a8a87..87db5c6 100644 --- a/docs/verification/ots/versionmark.md +++ b/docs/verification/ots/versionmark.md @@ -2,22 +2,46 @@ ### Verification Approach -VersionMark is used in the SarifMark CI pipeline to capture the versions of all tools used in the pipeline and -generate a markdown versions document. Verification evidence is provided by successful CI pipeline execution: the -pipeline step that invokes VersionMark completes without error and produces the versions markdown document, confirming -VersionMark executed and captured all configured tool versions. +VersionMark is verified through its built-in `--validate` self-validation mechanism. Running +`versionmark --validate` executes four internal test scenarios that confirm the tool is +installed and all advertised features are functioning correctly. Self-validation covers version +capture, markdown report generation, and lint validation for both valid and invalid configuration +files. + +### Test Environment + +Tests require: + +- `dotnet --version` is accessible on the PATH (used by `VersionMark_CapturesVersions` to + capture a real tool version). +- A writable temporary directory for output files. +- No network access. + +### Acceptance Criteria + +All four self-validation scenarios must pass with exit code 0 and zero failures, confirming +that VersionMark is correctly installed and all advertised features are operational. ### Test Scenarios -**VersionMark_CapturesVersions**: The CI pipeline step that invokes VersionMark completes without error, confirming -VersionMark executed and captured the versions of all configured tools used in the pipeline. -This scenario is verified by successful completion of the VersionMark pipeline step in CI. +**VersionMark_CapturesVersions**: The `VersionMark_CapturesVersions` self-validation scenario +creates a minimal `.versionmark.yaml` that captures the `dotnet` version, runs the capture +command, and confirms the output JSON file contains the expected version key. +This scenario is tested by `VersionMark_CapturesVersions`. -**VersionMark_GeneratesMarkdownReport**: The VersionMark pipeline step produces the tool-versions markdown document, -confirming that VersionMark generated the expected output file containing the captured version information. -This scenario is verified by the presence of the versions markdown file in CI pipeline artifacts. +**VersionMark_GeneratesMarkdownReport**: The `VersionMark_GeneratesMarkdownReport` +self-validation scenario creates two version JSON input files, runs the publish command, and +confirms the generated markdown report contains the expected tool version entries. +This scenario is tested by `VersionMark_GeneratesMarkdownReport`. -### Requirements Coverage +**VersionMark_LintPassesForValidConfig**: The `VersionMark_LintPassesForValidConfig` +self-validation scenario runs `--lint` against a valid `.versionmark.yaml` configuration file +and confirms exit code 0 with no errors, proving the linter correctly accepts well-formed +configuration. +This scenario is tested by `VersionMark_LintPassesForValidConfig`. -- **`SarifMark-OTS-VersionMark`**: Tool versions captured and markdown report generated — - `VersionMark_CapturesVersions`, `VersionMark_GeneratesMarkdownReport` +**VersionMark_LintReportsErrorsForInvalidConfig**: The `VersionMark_LintReportsErrorsForInvalidConfig` +self-validation scenario runs `--lint` against a configuration file missing the required `regex` +field and confirms exit code is non-zero, proving the linter correctly rejects malformed +configuration. +This scenario is tested by `VersionMark_LintReportsErrorsForInvalidConfig`. diff --git a/docs/verification/ots/weasyprint.md b/docs/verification/ots/weasyprint.md index 59d1817..589bd69 100644 --- a/docs/verification/ots/weasyprint.md +++ b/docs/verification/ots/weasyprint.md @@ -6,6 +6,12 @@ WeasyPrint is used in the SarifMark CI pipeline to convert HTML documents to PDF produced by Pandoc in the pipeline. Verification evidence is provided by FileAssert assertions confirming that each PDF output file exists, has non-trivial size, contains at least one page, and includes expected rendered text. +The requirements report and trace matrix PDF documents are excluded from the verification +scenarios because they are generated by the compliance toolchain itself (ReqStream, Pandoc, +and WeasyPrint together) — using them as verification evidence for WeasyPrint would create +a self-referential dependency where the tool being verified is also responsible for producing +the evidence. + ### Test Scenarios **WeasyPrint_BuildNotesPdf**: WeasyPrint converts the build-notes HTML document to a PDF file; FileAssert assertions @@ -32,8 +38,6 @@ This scenario is verified by FileAssert assertions in the CI pipeline. confirm the output file exists, has non-trivial size, contains at least one page, and includes expected rendered text. This scenario is verified by FileAssert assertions in the CI pipeline. -### Requirements Coverage - -- **`SarifMark-OTS-WeasyPrint`**: Converts each document type to a valid PDF — `WeasyPrint_BuildNotesPdf`, - `WeasyPrint_CodeQualityPdf`, `WeasyPrint_ReviewPlanPdf`, `WeasyPrint_ReviewReportPdf`, `WeasyPrint_DesignPdf`, - `WeasyPrint_UserGuidePdf` +**WeasyPrint_VerificationPdf**: WeasyPrint converts the verification document HTML to a PDF file; FileAssert +assertions confirm the output file exists, has non-trivial size, contains at least one page, and includes expected +rendered text. This scenario is verified by FileAssert assertions in the CI pipeline. diff --git a/docs/verification/ots/xunit-v3.md b/docs/verification/ots/xunit-v3.md index 3a365bb..b97f247 100644 --- a/docs/verification/ots/xunit-v3.md +++ b/docs/verification/ots/xunit-v3.md @@ -11,20 +11,16 @@ of the result files produced during `dotnet test --results-directory` invocation ### Test Scenarios -**XUnitV3_TestDiscoveryAndExecution**: The test suite runs with `dotnet test` and discovers and executes tests across -all test classes, including `SarifResultsTests`, `ContextTests`, and integration tests, producing passing results. -This confirms xUnit v3 framework discovery and execution are operational. -This scenario is tested by representative test methods across all test classes in -`test/DemaConsulting.SarifMark.Tests/`. +**XUnitV3_TestDiscovery**: xUnit v3 discovers all test methods marked with [Fact] or [Theory] across all test classes +in `test/DemaConsulting.SarifMark.Tests/`, confirming the framework's test discovery mechanism is operational. +This scenario is evidenced by the successful collection and reporting of all test method names in `dotnet test` output. + +**XUnitV3_TestExecution**: xUnit v3 executes all discovered test methods and produces pass/fail results, confirming +that test execution and result reporting work correctly across all test categories (unit, subsystem, and integration +tests). +This scenario is evidenced by representative passing tests across all test classes. **XUnitV3_TrxOutputGeneration**: The `dotnet test --results-directory` invocation produces TRX result files in the specified directory, confirming that the `xunit.runner.visualstudio` adapter serializes test results in TRX format as required by ReqStream for traceability enforcement. This scenario is verified by the presence of TRX result files in the CI pipeline test-results artifacts. - -### Requirements Coverage - -- **`SarifMark-OTS-XUnitV3-Discovery`**: xUnit v3 discovers and executes tests across all test classes — - `XUnitV3_TestDiscoveryAndExecution` -- **`SarifMark-OTS-XUnitV3-TrxOutput`**: xUnit v3 writes TRX result files — - `XUnitV3_TrxOutputGeneration` diff --git a/docs/verification/sarifmark.md b/docs/verification/sarifmark.md index 9f47063..bc819bc 100644 --- a/docs/verification/sarifmark.md +++ b/docs/verification/sarifmark.md @@ -8,7 +8,8 @@ tests reside in `test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs` and in `dotnet {dllPath} {args}` as a subprocess and captures exit codes and console output for assertion. No mocking is used at the system level; the full compiled binary is exercised against real SARIF test-data files (`sample.sarif`, `multi-result.sarif`, `multi-run.sarif`, `invalid.sarif`) stored in -`test/DemaConsulting.SarifMark.Tests/TestData/`. Unit tests exercise individual classes directly with console streams +`test/DemaConsulting.SarifMark.Tests/TestData/`. The `DemaConsulting.TestResults` OTS package is verified through +integration tests in `ValidationTests.cs` that exercise the package within the self-validation pipeline. Unit tests exercise individual classes directly with console streams redirected via `StringWriter`. The test framework is xUnit v3, executed via `dotnet test`. Three additional named scenarios (`SarifMark_SarifReading`, `SarifMark_MarkdownReportGeneration`, `SarifMark_Enforcement`) are self-validation tests invoked through the tool's own `--validate` flag; they are not xUnit test methods but named scenarios reported @@ -110,56 +111,64 @@ This scenario is tested by `SarifMark_CustomHeading_AppearsInReport`. **SarifMark_MultiRunSarifFile_CreatesReport**: Invoke the tool with `--sarif multi-run.sarif --report {path}` and assert the report file is created and contains sections for both runs (Tool1 and Tool2), confirming that multi-run SARIF files produce a combined report. This scenario is tested by `SarifMark_MultiRunSarifFile_CreatesReport`. -## Requirements Coverage - -The following list maps each system-level requirement to the named test scenario(s) that provide verification -evidence. - -- **`SarifMark-System-Version`**: Tool displays version on `--version` — - `SarifMark_VersionFlag_OutputsVersion` -- **`SarifMark-System-Help`**: Tool displays help on `--help` — - `SarifMark_HelpFlag_OutputsUsageInformation` -- **`SarifMark-System-Validate`**: Tool supports `--validate` mode — - `SarifMark_ValidateFlag_RunsSelfValidation` -- **`SarifMark-System-SarifRequired`**: Tool requires `--sarif` for analysis — - `SarifMark_MissingSarifParameter_ShowsError` -- **`SarifMark-System-SarifAnalysis`**: Tool reads and analyses SARIF files — - `SarifMark_ValidSarifFile_ProcessesSuccessfully`, - `SarifMark_NonExistentSarifFile_ShowsError` -- **`SarifMark-System-SarifSummary`**: Tool reports tool name, version, and result count — - `SarifMark_ValidSarifFile_ProcessesSuccessfully` -- **`SarifMark-System-Report`**: Tool generates markdown reports — - `SarifMark_GenerateReport_CreatesReportFile` -- **`SarifMark-System-Enforce`**: Non-zero exit code in enforcement mode — - `SarifMark_EnforceFlagWithIssues_ReturnsError` -- **`SarifMark-System-Silent`**: `--silent` suppresses console output — - `SarifMark_SilentFlag_SuppressesOutput` -- **`SarifMark-System-LogFile`**: `--log` writes output to file — - `SarifMark_LogFile_WritesOutputToFile` -- **`SarifMark-System-InvalidArgs`**: Unknown arguments rejected with error — - `SarifMark_UnknownArgument_ShowsError` -- **`SarifMark-System-ReportDepth`**: Configurable heading depth — - `SarifMark_ReportDepth_IsConfigurable`, `SarifMark_LegacyReportDepth_IsAccepted` -- **`SarifMark-System-ReportHeading`**: Configurable custom heading for generated reports — - `SarifMark_CustomHeading_AppearsInReport` -- **`SarifMark-System-ValidateResults`**: Write self-validation results to file — - `SelfTest_ResultsFile_TrxPath_WritesTrxFile`, `SelfTest_ResultsFile_XmlPath_WritesJUnitFile` -- **`SarifMark-System-Platform`**: Runs on Windows, Linux, and macOS — - `windows@SarifMark_VersionFlag_OutputsVersion`, `ubuntu@SarifMark_VersionFlag_OutputsVersion`, - `macos@SarifMark_VersionFlag_OutputsVersion` -- **`SarifMark-Plt-Windows`**: Runs on Windows — - `SarifMark_VersionFlag_OutputsVersion` (Windows runner) -- **`SarifMark-Plt-Linux`**: Runs on Linux (Ubuntu) — - `SarifMark_VersionFlag_OutputsVersion` (Ubuntu runner) -- **`SarifMark-Plt-MacOS`**: Runs on macOS — - `SarifMark_VersionFlag_OutputsVersion` (macOS runner) -- **`SarifMark-Plt-Net8`**: Supports .NET 8 runtime — - `SarifMark_SarifReading`, `SarifMark_MarkdownReportGeneration` (.NET 8) -- **`SarifMark-Plt-Net9`**: Supports .NET 9 runtime — - `SarifMark_SarifReading`, `SarifMark_MarkdownReportGeneration` (.NET 9) -- **`SarifMark-Plt-Net10`**: Supports .NET 10 runtime — - `SarifMark_SarifReading`, `SarifMark_MarkdownReportGeneration` (.NET 10) -- **`SarifMark-System-MultiRunSarif`**: Combined report from multi-run SARIF files — - `SarifMark_MultiRunSarifFile_CreatesReport` -- **`SarifMark-System-SafePaths`**: Uses safe path-handling functions — - `SarifMark_ValidSarifFile_ProcessesSuccessfully`, `SarifMark_ValidateFlag_RunsSelfValidation` +**Utilities_SafePathHandling_PathTraversal_ThrowsException**: Call `PathHelpers.SafePathCombine` with a relative path +containing path-traversal segments (e.g. `../etc/passwd`); assert that `ArgumentException` is thrown, confirming +that path-traversal attempts are rejected before any file-system access occurs, satisfying the safe-path-handling +system requirement. +This scenario is tested by `Utilities_SafePathHandling_PathTraversal_ThrowsException`. + +**Utilities_SafePathHandling_AbsolutePath_ThrowsException**: Call `PathHelpers.SafePathCombine` with an absolute path +as the relative argument (e.g. `/etc/passwd`); assert that `ArgumentException` is thrown, confirming that +absolute-path escape attempts are rejected by the centralized safe-path logic. +This scenario is tested by `Utilities_SafePathHandling_AbsolutePath_ThrowsException`. + +**Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent**: Read `sample.sarif` and +call `ToMarkdown(1)`; assert the returned string is non-empty and contains +`"# TestTool Analysis"`, confirming that the tool generates a markdown report from a +valid SARIF file at the default heading depth. +This scenario is tested by `Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent`. + +**Sarif_GenerateReport_ReportDepth_IsConfigurable**: Read `sample.sarif` and call +`ToMarkdown(3)`; assert the output contains `"### TestTool Analysis"`, confirming +that the heading depth parameter controls the depth of the generated report heading. +This scenario is tested by `Sarif_GenerateReport_ReportDepth_IsConfigurable`. + +**Sarif_GenerateReport_ResultCount_ContainsResultCount**: Read `sample.sarif` and call +`ToMarkdown(1)`; assert the output contains `"Found 1 issue"`, confirming that the +result count is included in the generated report and that singular grammar is applied +for a count of one. +This scenario is tested by `Sarif_GenerateReport_ResultCount_ContainsResultCount`. + +**Sarif_GenerateReport_LocationInfo_ContainsLocationInfo**: Read `sample.sarif` and +call `ToMarkdown(1)`; assert the output contains `"file:///path/to/file.cs"`, +confirming that location information from SARIF results is included in the generated +report. +This scenario is tested by `Sarif_GenerateReport_LocationInfo_ContainsLocationInfo`. + +**Sarif_GenerateReport_CustomHeading_UsesCustomHeading**: Read `sample.sarif` and call +`ToMarkdown(1, "Custom Analysis Heading")`; assert the output contains the custom +heading text, confirming that the optional heading parameter replaces the default +tool-name label. +This scenario is tested by `Sarif_GenerateReport_CustomHeading_UsesCustomHeading`. + +**Sarif_GenerateReport_MultipleResults_FormatsWithLineBreaks**: Read +`multi-result.sarif` and call `ToMarkdown(1)`; assert the output contains +`"Found 2 issues"` and that result lines for `first.cs` and `second.cs` each end +with two trailing spaces (markdown hard line break), confirming that multiple results +are formatted with proper line breaks. +This scenario is tested by `Sarif_GenerateReport_MultipleResults_FormatsWithLineBreaks`. + +**Sarif_GenerateReport_FileCount_ContainsFileCount**: Read `sample.sarif` and call +`ToMarkdown(1)`; assert the output contains `"**Files:** 2"`, confirming that the +file count from the SARIF `artifacts` array is included in the generated report. +This scenario is tested by `Sarif_GenerateReport_FileCount_ContainsFileCount`. + +**SarifMark_ValidSarif_NoIssues_GeneratesReport**: Invoke the tool with `--sarif` pointing to a SARIF file with no +results and `--report {path}`; assert exit code is 0, the report file is created, and the report contains the tool +attribution line (`**Tool:**`), confirming the tool generates a valid report for a clean SARIF file with no findings. +This scenario is tested by `SarifMark_ValidSarif_NoIssues_GeneratesReport`. + +**SarifMark_ValidateResultsParameter_WritesResultsFile**: Invoke the tool with `--validate --results {path.trx}`; +assert exit code is 0 and the TRX results file is created and contains a `<TestRun` element, confirming that the +`--results` parameter causes self-validation results to be written to the specified file. +This scenario is tested by `SarifMark_ValidateResultsParameter_WritesResultsFile`. diff --git a/docs/verification/sarifmark/cli.md b/docs/verification/sarifmark/cli.md index 01d01b0..8840eab 100644 --- a/docs/verification/sarifmark/cli.md +++ b/docs/verification/sarifmark/cli.md @@ -78,21 +78,18 @@ This scenario is tested by `Cli_Create_ReportDepthParameter_SetsReportDepth`. alias is accepted and `ResultsFile` is set. This scenario is tested by `Cli_Create_ResultLegacyAlias_SetsResultsFilePath`. -### Requirements Coverage - -- **`SarifMark-Cli-Interface`**: `Cli_Create_VersionFlag_SetsVersionFlag`, `Cli_Create_HelpFlag_SetsHelpFlag` -- **`SarifMark-Cli-Version`**: `Cli_Create_VersionFlag_SetsVersionFlag` -- **`SarifMark-Cli-Help`**: `Cli_Create_HelpFlag_SetsHelpFlag` -- **`SarifMark-Cli-Silent`**: `Cli_Create_SilentFlag_SuppressesOutput` -- **`SarifMark-Cli-Log`**: `Cli_Create_LogFile_WritesOutputToFile` -- **`SarifMark-Cli-Enforce`**: `Cli_Create_EnforceFlag_SetsEnforceFlag` -- **`SarifMark-Cli-WriteError`**: `Cli_WriteError_WithMessage_SetsExitCodeToOne` -- **`SarifMark-Cli-InvalidArgs`**: `Cli_Create_UnknownArgument_ThrowsArgumentException` -- **`SarifMark-Cli-Validate`**: `Cli_Create_ValidateFlag_SetsValidateFlag` -- **`SarifMark-Cli-Sarif`**: `Cli_Create_SarifParameter_SetsSarifFilePath` -- **`SarifMark-Cli-Report`**: `Cli_Create_ReportParameter_SetsReportFilePath` -- **`SarifMark-Cli-ReportDepth`**: `Cli_Create_DepthParameter_SetsDepth` -- **`SarifMark-Cli-Heading`**: `Cli_Create_HeadingParameter_SetsCustomHeading` -- **`SarifMark-Cli-Results`**: `Cli_Create_ResultsParameter_SetsResultsFilePath` -- **`SarifMark-Cli-ReportDepthLegacyAlias`**: `Cli_Create_ReportDepthParameter_SetsReportDepth` -- **`SarifMark-Cli-ResultLegacyAlias`**: `Cli_Create_ResultLegacyAlias_SetsResultsFilePath` +**Cli_Create_DepthInvalidValue_ThrowsArgumentException**: Pass `--depth abc` (a non-integer value) to +`Context.Create`; assert `ArgumentException` is thrown with a message indicating `--depth requires an integer +between 1 and 6`, confirming that invalid depth values are rejected before reaching the report formatter. +This scenario is tested by `Cli_Create_DepthInvalidValue_ThrowsArgumentException`. + +**Cli_Create_LogWithoutSilent_WritesToConsoleAndLogFile**: Pass `--log {path}` without `--silent` to `Context.Create` +and write a message; assert the message appears both on the console and in the log file, confirming that log-file +output is additive rather than replacing console output when silent mode is not active. +This scenario is tested by `Cli_Create_LogWithoutSilent_WritesToConsoleAndLogFile`. + +**Cli_Create_DepthZero_ThrowsArgumentException**: Pass `--depth 0` to `Context.Create`; assert `ArgumentException` is thrown with a message indicating `--depth requires an integer between 1 and 6`, confirming that a depth of zero is rejected. +This scenario is tested by `Cli_Create_DepthZero_ThrowsArgumentException`. + +**Cli_Create_DepthNegative_ThrowsArgumentException**: Pass `--depth -1` to `Context.Create`; assert `ArgumentException` is thrown with a message indicating `--depth requires an integer between 1 and 6`, confirming that negative depth values are rejected. +This scenario is tested by `Cli_Create_DepthNegative_ThrowsArgumentException`. diff --git a/docs/verification/sarifmark/cli/context.md b/docs/verification/sarifmark/cli/context.md index e8707a6..1cc1ed2 100644 --- a/docs/verification/sarifmark/cli/context.md +++ b/docs/verification/sarifmark/cli/context.md @@ -7,6 +7,11 @@ All dependencies are standard .NET BCL types (`Console`, `File`) — no mocking are redirected via `StringWriter` for output assertions. Temporary files are used for log-file tests and are always cleaned up in `finally` blocks. +Several log-file test methods construct temporary file paths using `PathHelpers.SafePathCombine` rather than +`Path.Combine`, which means the tests indirectly exercise `PathHelpers` as a cross-unit dependency. `PathHelpers` +is documented as an explicit dependency of `Context` in the design documentation; its own correctness is verified +separately in the PathHelpers unit verification document. + #### Test Environment Standard xUnit v3 test runner with `dotnet test`. Temporary files are created in the OS temporary directory @@ -59,6 +64,23 @@ This scenario is tested by `Context_Create_ReportWithoutValue_ThrowsArgumentExce the provided value. This scenario is tested by `Context_Create_DepthParameter_SetsDepth`. +**Context_Create_DepthNegative_ThrowsArgumentException**: Pass `--depth -1`; assert `ArgumentException` is thrown +with a message indicating the depth must be between 1 and 6. +This scenario is tested by `Context_Create_DepthNegative_ThrowsArgumentException`. + +**Context_Create_DepthAboveMax_ThrowsArgumentException**: Pass `--depth 7`; assert `ArgumentException` is thrown +with a message indicating the depth must be between 1 and 6, confirming that depth values exceeding the maximum +heading level are rejected. +This scenario is tested by `Context_Create_DepthAboveMax_ThrowsArgumentException`. + +**Context_Create_DepthAtMinimum_SetsDepth**: Pass `--depth 1`; assert `Depth` is 1, confirming the lower boundary +value is accepted. +This scenario is tested by `Context_Create_DepthAtMinimum_SetsDepth`. + +**Context_Create_DepthAtMaximum_SetsDepth**: Pass `--depth 6`; assert `Depth` is 6, confirming the upper boundary +value is accepted. +This scenario is tested by `Context_Create_DepthAtMaximum_SetsDepth`. + **Context_Create_HeadingArgument_SetsHeading**: Pass `--heading {text}`; assert `Heading` is set. This scenario is tested by `Context_Create_HeadingArgument_SetsHeading`. @@ -133,51 +155,3 @@ This scenario is tested by `Context_WriteError_SilentMode_DoesNotWriteToConsoleB **Context_WriteLine_SilentModeWithLogFile_WritesToLog**: Create context with `--silent` and `--log {path}`, call `WriteLine`; assert the message appears in the log file even though silent mode suppresses console output. This scenario is tested by `Context_WriteLine_SilentModeWithLogFile_WritesToLog`. - -### Requirements Coverage - -- **`SarifMark-Context-Create`**: `Context_Create_NoArguments_ReturnsDefaultContext` -- **`SarifMark-Context-VersionFlag`**: `Context_Create_VersionFlag_SetsVersionTrue`, - `Context_Create_ShortVersionFlag_SetsVersionTrue` -- **`SarifMark-Context-HelpFlag`**: `Context_Create_HelpFlag_SetsHelpTrue`, - `Context_Create_QuestionMarkHelpFlag_SetsHelpTrue`, - `Context_Create_ShortHelpFlag_SetsHelpTrue` -- **`SarifMark-Context-SilentFlag`**: `Context_Create_SilentFlag_SetsSilentTrue`, - `Context_WriteLine_SilentMode_DoesNotWriteToConsole`, - `Context_WriteError_SilentMode_DoesNotWriteToConsoleButSetsExitCode` -- **`SarifMark-Context-ValidateFlag`**: `Context_Create_ValidateFlag_SetsValidateTrue` -- **`SarifMark-Context-EnforceFlag`**: `Context_Create_EnforceFlag_SetsEnforceTrue` -- **`SarifMark-Context-SarifParam`**: `Context_Create_SarifParameter_SetsSarifFile` -- **`SarifMark-Context-SarifParam-MissingValue`**: `Context_Create_SarifWithoutValue_ThrowsArgumentException` -- **`SarifMark-Context-ReportParam`**: `Context_Create_ReportParameter_SetsReportFile` -- **`SarifMark-Context-ReportParam-MissingValue`**: `Context_Create_ReportWithoutValue_ThrowsArgumentException` -- **`SarifMark-Context-ReportDepthParam`**: `Context_Create_DepthParameter_SetsDepth`, - `Context_Create_DepthWithoutValue_ThrowsArgumentException`, - `Context_Create_DepthInvalidValue_ThrowsArgumentException`, - `Context_Create_DepthZero_ThrowsArgumentException`, - `Context_Create_ReportDepthParameter_SetsReportDepth`, - `Context_Create_ReportDepthWithoutValue_ThrowsArgumentException`, - `Context_Create_ReportDepthInvalidValue_ThrowsArgumentException`, - `Context_Create_ReportDepthZero_ThrowsArgumentException` -- **`SarifMark-Context-HeadingParam`**: `Context_Create_HeadingArgument_SetsHeading`, - `Context_Create_HeadingWithoutValue_ThrowsArgumentException` -- **`SarifMark-Context-ResultsParam`**: `Context_Create_ResultsParameter_SetsResultsFile` -- **`SarifMark-Context-ResultLegacyAlias`**: `Context_Create_ResultLegacyAlias_SetsResultsFile` -- **`SarifMark-Context-ResultsParam-MissingValue`**: `Context_Create_ResultsWithoutValue_ThrowsArgumentException` -- **`SarifMark-Context-LogParam`**: `Context_Create_LogFile_OpensFileSuccessfully`, - `Context_Create_InvalidLogFilePath_ThrowsInvalidOperationException` -- **`SarifMark-Context-LogParam-MissingValue`**: `Context_Create_LogWithoutValue_ThrowsArgumentException` -- **`SarifMark-Context-UnknownArgs`**: `Context_Create_UnknownArgument_ThrowsArgumentException` -- **`SarifMark-Context-WriteLine-Console`**: `Context_WriteLine_WritesToConsole`, - `Context_WriteLine_SilentMode_DoesNotWriteToConsole` -- **`SarifMark-Context-WriteLine-Log`**: `Context_WriteLine_WithLogFile_WritesToLog`, - `Context_WriteLine_SilentModeWithLogFile_WritesToLog` -- **`SarifMark-Context-WriteError`**: `Context_WriteError_WritesToErrorAndSetsExitCode` -- **`SarifMark-Context-WriteError-Stderr`**: `Context_WriteError_WritesToErrorAndSetsExitCode`, - `Context_WriteError_SilentMode_DoesNotWriteToConsoleButSetsExitCode` -- **`SarifMark-Context-WriteError-Log`**: `Context_WriteError_WritesToErrorAndSetsExitCode`, - `Context_WriteError_WithLogFile_WritesToLog` -- **`SarifMark-Context-WriteError-ExitCode`**: `Context_WriteError_WritesToErrorAndSetsExitCode`, - `Context_WriteError_SilentMode_DoesNotWriteToConsoleButSetsExitCode` -- **`SarifMark-Context-ExitCode`**: `Context_ExitCode_StartsAtZero_ChangesToOneAfterError` -- **`SarifMark-Context-Dispose`**: `Context_Dispose_ProperlyClosesLogFile` diff --git a/docs/verification/sarifmark/program.md b/docs/verification/sarifmark/program.md index 6a9e517..f1971bc 100644 --- a/docs/verification/sarifmark/program.md +++ b/docs/verification/sarifmark/program.md @@ -14,8 +14,9 @@ configuration are required. ### Acceptance Criteria -All `ProgramTests` test methods pass, confirming that every execution path — version display, help display, SARIF -processing, error handling, report generation, and enforcement — produces the expected exit code and console output. +All `ProgramTests` test methods pass, confirming that every execution path — argument parsing, operation dispatch, +version display, help display, SARIF processing, error handling, report generation, and enforcement — produces the +expected exit code and console output. No `Program` unit requirement may remain without at least one named test scenario (IEC 62304 §5.5.2). ### Test Scenarios @@ -35,8 +36,9 @@ This scenario is tested by `Program_Main_HelpFlag_DisplaysHelp`. **Program_Main_UnknownArgument_ReturnsError**: Invoke with an unrecognized argument; assert exit code is 1. This scenario is tested by `Program_Main_UnknownArgument_ReturnsError`. -**Program_Main_ValidateFlag_RunsValidation**: Invoke with `--validate`; assert self-validation runs and exit code -is 0. +**Program_Main_ValidateFlag_RunsValidation**: Invoke with `--validate`; assert self-validation runs and exits with +code 0, and the TRX results file is created and contains a `<TestRun` element, confirming that the validation +pipeline executed to completion. This scenario is tested by `Program_Main_ValidateFlag_RunsValidation`. **Program_Main_ValidSarifFile_ProcessesSuccessfully**: Invoke with `--sarif {valid path}`; assert analysis output @@ -50,21 +52,3 @@ This scenario is tested by `Program_Main_EnforceFlagWithIssues_ReturnsError`. **Program_Main_ReportFile_CreatesReport**: Invoke with `--sarif` and `--report {path}`; assert the report file is created on disk. This scenario is tested by `Program_Main_ReportFile_CreatesReport`. - -### Requirements Coverage - -- **`SarifMark-Program-Version`**: `Program_Main_VersionFlag_DisplaysVersionOnly` -- **`SarifMark-Program-Main`**: `Program_Main_NoArguments_ReturnsError`, - `Program_Main_VersionFlag_DisplaysVersionOnly`, - `Program_Main_HelpFlag_DisplaysHelp` -- **`SarifMark-Program-Main-Exceptions`**: `Program_Main_NoArguments_ReturnsError`, - `Program_Main_UnknownArgument_ReturnsError` -- **`SarifMark-Program-Run`**: `Program_Main_VersionFlag_DisplaysVersionOnly`, - `Program_Main_HelpFlag_DisplaysHelp` -- **`SarifMark-Program-Banner`**: `Program_Main_HelpFlag_DisplaysHelp` -- **`SarifMark-Program-Help`**: `Program_Main_HelpFlag_DisplaysHelp` -- **`SarifMark-Program-Validation`**: `Program_Main_ValidateFlag_RunsValidation` -- **`SarifMark-Program-SarifArgument`**: `Program_Main_NoArguments_ReturnsError` -- **`SarifMark-Program-SarifReading`**: `Program_Main_ValidSarifFile_ProcessesSuccessfully` -- **`SarifMark-Program-EnforcementCheck`**: `Program_Main_EnforceFlagWithIssues_ReturnsError` -- **`SarifMark-Program-ReportGeneration`**: `Program_Main_ReportFile_CreatesReport` diff --git a/docs/verification/sarifmark/sarif.md b/docs/verification/sarifmark/sarif.md index a68c7fc..d92f5d4 100644 --- a/docs/verification/sarifmark/sarif.md +++ b/docs/verification/sarifmark/sarif.md @@ -65,25 +65,3 @@ This scenario is tested by `Sarif_GenerateReport_ResultCount_ContainsResultCount **Sarif_GenerateReport_CustomHeading_UsesCustomHeading**: Read `sample.sarif` and call `ToMarkdown(1, "Custom Analysis Heading")`; assert the custom heading string appears in the output instead of the default tool-name heading. This scenario is tested by `Sarif_GenerateReport_CustomHeading_UsesCustomHeading`. - -### Requirements Coverage - -- **`SarifMark-Sarif-Reading`**: `Sarif_Read_ValidSarifFile_ProcessesSuccessfully` -- **`SarifMark-Sarif-Validation`**: `Sarif_Read_NonExistentFile_ThrowsFileNotFoundException`, - `Sarif_Read_InvalidSarifFile_ThrowsInvalidOperationException` -- **`SarifMark-Sarif-ToolInfo`**: `Sarif_Read_ValidSarifFile_ProcessesSuccessfully` -- **`SarifMark-Sarif-Results`**: `Sarif_Read_ValidSarifFile_ProcessesSuccessfully`, - `Sarif_GenerateReport_ResultCount_ContainsResultCount` -- **`SarifMark-Sarif-Locations`**: `Sarif_GenerateReport_LocationInfo_ContainsLocationInfo` -- **`SarifMark-Sarif-FilePaths`**: `Sarif_Read_NonExistentFile_ThrowsFileNotFoundException` -- **`SarifMark-Sarif-Processing`**: `Sarif_Read_ValidSarifFile_ProcessesSuccessfully`, - `Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent` -- **`SarifMark-Sarif-FileCount`**: `Sarif_GenerateReport_FileCount_ContainsFileCount` -- **`SarifMark-Sarif-MultiRun`**: `Sarif_Read_MultiRunSarifFile_ProcessesAllRuns` -- **`SarifMark-Report-Depth`**: `Sarif_GenerateReport_ReportDepth_IsConfigurable` -- **`SarifMark-Report-LineBreaks`**: `Sarif_GenerateReport_MultipleResults_FormatsWithLineBreaks` -- **`SarifMark-Report-Headings`**: `Sarif_GenerateReport_CustomHeading_UsesCustomHeading` -- **`SarifMark-Report-Markdown`**: `Sarif_GenerateReport_DefaultDepth_ProducesMarkdownContent` -- **`SarifMark-Report-Counts`**: `Sarif_GenerateReport_ResultCount_ContainsResultCount` -- **`SarifMark-Report-Locations`**: `Sarif_GenerateReport_LocationInfo_ContainsLocationInfo` -- **`SarifMark-Report-FileCount`**: `Sarif_GenerateReport_FileCount_ContainsFileCount` diff --git a/docs/verification/sarifmark/sarif/sarif-finding.md b/docs/verification/sarifmark/sarif/sarif-finding.md index b8a9d03..bdba3e6 100644 --- a/docs/verification/sarifmark/sarif/sarif-finding.md +++ b/docs/verification/sarifmark/sarif/sarif-finding.md @@ -13,9 +13,10 @@ test input is constructed inline within each test method. #### Acceptance Criteria -All `SarifFindingTests` test methods pass, confirming that the internal constructor stores all properties correctly -and that nullable properties correctly accept `null` values. No `SarifFinding` unit requirement may remain without -at least one named test scenario (IEC 62304 §5.5.2). +All `SarifFindingTests` test methods pass, confirming that the internal constructor stores all properties correctly, +nullable properties correctly accept `null` values, and that access to the internal constructor requires the +`InternalsVisibleTo` grant. No `SarifFinding` unit requirement may remain without at least one named test scenario +(IEC 62304 §5.5.2). #### Test Scenarios @@ -30,9 +31,10 @@ This scenario is tested by `SarifFinding_Constructor_NullUri_UriPropertyIsNull`. `StartLine` property is `null`. This scenario is tested by `SarifFinding_Constructor_NullStartLine_StartLinePropertyIsNull`. -### Requirements Coverage - -- **`SarifMark-SarifFinding-Properties`**: `SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties` -- **`SarifMark-SarifFinding-Uri`**: `SarifFinding_Constructor_NullUri_UriPropertyIsNull` -- **`SarifMark-SarifFinding-StartLine`**: `SarifFinding_Constructor_NullStartLine_StartLinePropertyIsNull` -- **`SarifMark-SarifFinding-Internal`**: `SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties` +**SarifFinding_InternalConstructor_AccessRestriction**: The `SarifFinding` constructor has +`internal` access, enforced at compile time. The `InternalsVisibleTo` attribute on the +SarifMark assembly grants test-only access to the test project. This restriction is verified +by the fact that the test project requires `InternalsVisibleTo` to compile — without it, the +`SarifFinding_Constructor_AllPropertiesProvided_StoresAllProperties` test would fail to compile. +The compiler enforcement is passive verification; no separate runtime test is required. +This scenario is verified by compiler enforcement and the `InternalsVisibleTo` declaration. diff --git a/docs/verification/sarifmark/sarif/sarif-results.md b/docs/verification/sarifmark/sarif/sarif-results.md index 56b1826..d47cf30 100644 --- a/docs/verification/sarifmark/sarif/sarif-results.md +++ b/docs/verification/sarifmark/sarif/sarif-results.md @@ -24,30 +24,114 @@ No `SarifResults` unit requirement may remain without at least one named test sc thrown. This scenario is tested by `SarifResults_Read_NullPath_ThrowsArgumentException`. +**SarifResults_Read_EmptyPath_ThrowsArgumentException**: Pass an empty string as the path; assert `ArgumentException` +is thrown. +This scenario is tested by `SarifResults_Read_EmptyPath_ThrowsArgumentException`. + +**SarifResults_Read_WhitespacePath_ThrowsArgumentException**: Pass a whitespace-only string as the path; assert +`ArgumentException` is thrown. +This scenario is tested by `SarifResults_Read_WhitespacePath_ThrowsArgumentException`. + **SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException**: Pass a non-existent path; assert `FileNotFoundException` is thrown. This scenario is tested by `SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException`. +**SarifResults_Read_InvalidJson_ThrowsInvalidOperationException**: Pass a file containing invalid JSON; assert +`InvalidOperationException` is thrown. +This scenario is tested by `SarifResults_Read_InvalidJson_ThrowsInvalidOperationException`. + +**SarifResults_Read_MissingVersion_ThrowsInvalidOperationException**: Pass SARIF missing the required `version` +field; assert `InvalidOperationException` is thrown. +This scenario is tested by `SarifResults_Read_MissingVersion_ThrowsInvalidOperationException`. + +**SarifResults_Read_MissingRuns_ThrowsInvalidOperationException**: Pass SARIF missing the `runs` array; assert +`InvalidOperationException` is thrown. +This scenario is tested by `SarifResults_Read_MissingRuns_ThrowsInvalidOperationException`. + **SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException**: Pass SARIF with an empty `runs` array; assert `InvalidOperationException` is thrown. This scenario is tested by `SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException`. +**SarifResults_Read_MissingTool_ThrowsInvalidOperationException**: Pass SARIF with a run that has no `tool` field; +assert `InvalidOperationException` is thrown. +This scenario is tested by `SarifResults_Read_MissingTool_ThrowsInvalidOperationException`. + +**SarifResults_Read_MissingDriver_ThrowsInvalidOperationException**: Pass SARIF with a `tool` object that has no +`driver` field; assert `InvalidOperationException` is thrown. +This scenario is tested by `SarifResults_Read_MissingDriver_ThrowsInvalidOperationException`. + **SarifResults_Read_MissingToolName_UsesUnknown**: Pass SARIF with no tool name field; assert tool name defaults to `"Unknown"`. This scenario is tested by `SarifResults_Read_MissingToolName_UsesUnknown`. +**SarifResults_Read_MissingToolVersion_UsesUnknown**: Pass SARIF with no tool version field; assert tool version +defaults to `"Unknown"`. +This scenario is tested by `SarifResults_Read_MissingToolVersion_UsesUnknown`. + **SarifResults_Read_AllVersionFields_PrioritizesVersion**: Pass SARIF with multiple version fields populated; assert the correct priority order is applied. This scenario is tested by `SarifResults_Read_AllVersionFields_PrioritizesVersion`. +**SarifResults_Read_SemanticVersionField_ReturnsSemanticVersion**: Pass SARIF with only the semantic version field +populated; assert the semantic version string is returned. +This scenario is tested by `SarifResults_Read_SemanticVersionField_ReturnsSemanticVersion`. + +**SarifResults_Read_DottedQuadFileVersionField_ReturnsDottedQuadFileVersion**: Pass SARIF with only the dotted-quad +file version field populated; assert the dotted-quad version string is returned. +This scenario is tested by `SarifResults_Read_DottedQuadFileVersionField_ReturnsDottedQuadFileVersion`. + +**SarifResults_Read_VersionAndSemanticVersion_PrioritizesVersion**: Pass SARIF with both version and semantic version +fields; assert the `version` field takes priority. +This scenario is tested by `SarifResults_Read_VersionAndSemanticVersion_PrioritizesVersion`. + +**SarifResults_Read_SemanticAndDottedQuad_PrioritizesSemanticVersion**: Pass SARIF with both semantic version and +dotted-quad file version; assert the semantic version takes priority. +This scenario is tested by `SarifResults_Read_SemanticAndDottedQuad_PrioritizesSemanticVersion`. + +**SarifResults_Read_EmptyVersionField_FallsBackToSemanticVersion**: Pass SARIF with an empty `version` field and a +populated semantic version; assert the semantic version is used as the fallback. +This scenario is tested by `SarifResults_Read_EmptyVersionField_FallsBackToSemanticVersion`. + **SarifResults_Read_NoResults_ReturnsValidResults**: Pass SARIF with no results array; assert a valid empty `SarifResults` object is returned. This scenario is tested by `SarifResults_Read_NoResults_ReturnsValidResults`. +**SarifResults_Read_EmptyResults_ReturnsValidResults**: Pass SARIF with an empty results array; assert a valid +`SarifResults` object with zero findings is returned. +This scenario is tested by `SarifResults_Read_EmptyResults_ReturnsValidResults`. + +**SarifResults_Read_WithResults_ReturnsValidResults**: Pass SARIF with results; assert a valid `SarifResults` object +containing the expected findings is returned. +This scenario is tested by `SarifResults_Read_WithResults_ReturnsValidResults`. + **SarifResults_Read_EmptySuppressions_DoesNotExcludeResult**: Pass a result with an empty suppression list; assert the result is included (not filtered out). This scenario is tested by `SarifResults_Read_EmptySuppressions_DoesNotExcludeResult`. +**SarifResults_Read_WithSuppressedResults_ExcludesSuppressedResults**: Pass SARIF containing results with non-empty +suppressions arrays; assert those results are excluded from the output. +This scenario is tested by `SarifResults_Read_WithSuppressedResults_ExcludesSuppressedResults`. + +**SarifResults_Read_WithLocations_ReturnsResultsWithLocationData**: Pass SARIF with location data on results; assert +the parsed findings contain the correct URI and line number information. +This scenario is tested by `SarifResults_Read_WithLocations_ReturnsResultsWithLocationData`. + +**SarifResults_Read_NoArtifacts_ReturnsZeroFileCount**: Pass SARIF with no `artifacts` section; assert `FileCount` +is 0. +This scenario is tested by `SarifResults_Read_NoArtifacts_ReturnsZeroFileCount`. + +**SarifResults_Read_WithArtifacts_ReturnsFileCount**: Pass SARIF with an `artifacts` array; assert `FileCount` +equals the number of entries in the array. +This scenario is tested by `SarifResults_Read_WithArtifacts_ReturnsFileCount`. + +**SarifResults_Read_MultipleRuns_EachRunHasOwnFileCount**: Pass a multi-run SARIF where each run has a different +number of artifacts; assert each run reports its own correct file count. +This scenario is tested by `SarifResults_Read_MultipleRuns_EachRunHasOwnFileCount`. + +**SarifResults_Read_MultipleRuns_ReturnsAllRuns**: Pass a multi-run SARIF; assert all runs are present in the +returned `SarifResults` object. +This scenario is tested by `SarifResults_Read_MultipleRuns_ReturnsAllRuns`. + **SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput**: Call `ToMarkdown(1)`; assert the output uses `#` headings and contains the expected structure. This scenario is tested by `SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput`. @@ -56,10 +140,6 @@ This scenario is tested by `SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput headings at maximum depth. This scenario is tested by `SarifResults_ToMarkdown_Depth6_ProducesCorrectOutput`. -**SarifResults_Read_NoArtifacts_ReturnsZeroFileCount**: Pass SARIF with no `artifacts` section; assert `FileCount` -is 0. -This scenario is tested by `SarifResults_Read_NoArtifacts_ReturnsZeroFileCount`. - **SarifResults_Runs_SingleRun_ReturnsSingleRun**: Pass a single-run SARIF; assert exactly one `SarifRun` is returned. This scenario is tested by `SarifResults_Runs_SingleRun_ReturnsSingleRun`. @@ -95,25 +175,42 @@ This scenario is tested by `SarifResults_ToMarkdown_MultipleResults_EnforcesLine report header. This scenario is tested by `SarifResults_ToMarkdown_ShowsFileCount`. -#### Requirements Coverage - -- **`SarifMark-SarifResults-ValidatePathArgument`**: `SarifResults_Read_NullPath_ThrowsArgumentException` -- **`SarifMark-SarifResults-ValidatePathExists`**: `SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException` -- **`SarifMark-SarifResults-ValidateStructure`**: `SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException` -- **`SarifMark-SarifResults-ExtractTool`**: `SarifResults_Read_MissingToolName_UsesUnknown` -- **`SarifMark-SarifResults-VersionPriority`**: `SarifResults_Read_AllVersionFields_PrioritizesVersion` -- **`SarifMark-SarifResults-ParseResults`**: `SarifResults_Read_NoResults_ReturnsValidResults` -- **`SarifMark-SarifResults-FilterSuppressions`**: `SarifResults_Read_EmptySuppressions_DoesNotExcludeResult` -- **`SarifMark-SarifResults-ToMarkdown`**: `SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput` -- **`SarifMark-SarifResults-ValidateDepth`**: `SarifResults_ToMarkdown_Depth6_ProducesCorrectOutput` -- **`SarifMark-SarifResults-FileCount`**: `SarifResults_Read_NoArtifacts_ReturnsZeroFileCount` -- **`SarifMark-SarifResults-Runs`**: `SarifResults_Runs_SingleRun_ReturnsSingleRun` -- **`SarifMark-SarifResults-HasIssues`**: `SarifResults_HasIssues_NoIssues_ReturnsFalse` -- **`SarifMark-SarifResults-MultiRunMarkdown`**: `SarifResults_ToMarkdown_MultipleRuns_IncludesRunIndices` -- **`SarifMark-Report-Markdown`**: `SarifResults_ToMarkdown_Depth1_ProducesCorrectOutput` -- **`SarifMark-Report-Depth`**: `SarifResults_ToMarkdown_Depth3_UsesCorrectHeadingLevels` -- **`SarifMark-Report-Counts`**: `SarifResults_ToMarkdown_OneResult_UsesSingularForm` -- **`SarifMark-Report-Locations`**: `SarifResults_ToMarkdown_ResultWithUriNoLine_ShowsUriOnly` -- **`SarifMark-Report-Headings`**: `SarifResults_ToMarkdown_CustomHeading_UsesProvidedHeading` -- **`SarifMark-Report-LineBreaks`**: `SarifResults_ToMarkdown_MultipleResults_EnforcesLineBreaks` -- **`SarifMark-Report-FileCount`**: `SarifResults_ToMarkdown_ShowsFileCount` +**SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults**: Call `ToMarkdown` against SARIF with zero results; assert +the output contains `"Found no issues"`. +This scenario is tested by `SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults`. + +**SarifResults_ToMarkdown_DepthLessThan1_ThrowsArgumentOutOfRangeException**: Call `ToMarkdown` with a depth value +less than 1; assert `ArgumentOutOfRangeException` is thrown. +This scenario is tested by `SarifResults_ToMarkdown_DepthLessThan1_ThrowsArgumentOutOfRangeException`. + +**SarifResults_ToMarkdown_DepthGreaterThan6_ThrowsArgumentOutOfRangeException**: Call `ToMarkdown` with a depth +value greater than 6; assert `ArgumentOutOfRangeException` is thrown. +This scenario is tested by `SarifResults_ToMarkdown_DepthGreaterThan6_ThrowsArgumentOutOfRangeException`. + +**SarifResults_ToMarkdown_ResultWithoutLocation_ShowsNoLocation**: Pass a result with no location information; +assert the output contains `"(no location)"`. +This scenario is tested by `SarifResults_ToMarkdown_ResultWithoutLocation_ShowsNoLocation`. + +**SarifResults_ToMarkdown_NullHeading_UsesDefaultHeading**: Pass `null` as the heading parameter; assert the +default `"[ToolName] Analysis"` heading is used. +This scenario is tested by `SarifResults_ToMarkdown_NullHeading_UsesDefaultHeading`. + +**SarifResults_ToMarkdown_NoHeadingParameter_UsesDefaultHeading**: Call `ToMarkdown` without supplying a heading +parameter; assert the default `"[ToolName] Analysis"` heading is used. +This scenario is tested by `SarifResults_ToMarkdown_NoHeadingParameter_UsesDefaultHeading`. + +**SarifResults_ToMarkdown_ZeroFileCount_ShowsZero**: Call `ToMarkdown` against SARIF with no artifacts; assert the +output contains `"**Files:** 0"`. +This scenario is tested by `SarifResults_ToMarkdown_ZeroFileCount_ShowsZero`. + +**SarifResults_InternalConstructor_ExposesRunsAndHasIssues**: Construct a `SarifResults` instance via the internal +constructor with known runs; assert `Runs` contains the expected run objects and `HasIssues` reflects the expected +state. +This scenario is tested by `SarifResults_InternalConstructor_ExposesRunsAndHasIssues`. + +**SarifResults_HasIssues_WithIssues_ReturnsTrue**: Pass SARIF with findings; assert `HasIssues` is `true`. +This scenario is tested by `SarifResults_HasIssues_WithIssues_ReturnsTrue`. + +**SarifResults_HasIssues_AnyRunHasIssues_ReturnsTrue**: Pass a multi-run SARIF where at least one run has findings; +assert `HasIssues` is `true`. +This scenario is tested by `SarifResults_HasIssues_AnyRunHasIssues_ReturnsTrue`. diff --git a/docs/verification/sarifmark/sarif/sarif-run.md b/docs/verification/sarifmark/sarif/sarif-run.md index f13fb8c..8a88a66 100644 --- a/docs/verification/sarifmark/sarif/sarif-run.md +++ b/docs/verification/sarifmark/sarif/sarif-run.md @@ -43,19 +43,48 @@ This scenario is tested by `SarifRun_ToMarkdown_DepthGreaterThan6_ThrowsArgument assert the location section is absent from the output. This scenario is tested by `SarifRun_ToMarkdown_ResultWithoutLocation_ShowsNoLocation`. +**SarifRun_HasIssues_WithResults_ReturnsTrue**: Construct a `SarifRun` with a non-empty results collection; assert +`HasIssues` is `true`, covering the path where a run contains findings. +This scenario is tested by `SarifRun_HasIssues_WithResults_ReturnsTrue`. + +**SarifRun_ToMarkdown_WithResults_ShowsResults**: Call `ToMarkdown(1)` on a run containing one result with location +data; assert the output contains the result count, rule ID, message, and formatted location string. +This scenario is tested by `SarifRun_ToMarkdown_WithResults_ShowsResults`. + +**SarifRun_ToMarkdown_ShowsFileCount**: Call `ToMarkdown(1)` on a run with a non-zero file count; assert the output +contains the `**Files:**` line with the correct count, confirming file count reporting is included in the report. +This scenario is tested by `SarifRun_ToMarkdown_ShowsFileCount`. + +**SarifRun_ToMarkdown_NullHeading_UsesDefaultHeading**: Call `ToMarkdown(1, null)`; assert the output uses the +default heading derived from the tool name (e.g. `# MyTool Analysis`), confirming that a null heading argument +falls back to the auto-generated default. +This scenario is tested by `SarifRun_ToMarkdown_NullHeading_UsesDefaultHeading`. + +**SarifRun_ToMarkdown_WhitespaceHeading_UsesDefaultHeading**: Call `ToMarkdown(1, " ")`; assert the output uses +the default heading derived from the tool name, confirming that a whitespace-only heading argument is treated as +absent and falls back to the auto-generated default. +This scenario is tested by `SarifRun_ToMarkdown_WhitespaceHeading_UsesDefaultHeading`. + **SarifRun_ToMarkdown_OneResult_UsesSingularForm**: Generate output for a run with one result; assert the result count uses the singular form. This scenario is tested by `SarifRun_ToMarkdown_OneResult_UsesSingularForm`. -#### Requirements Coverage - -- **`SarifMark-SarifRun-ToolName`**: `SarifRun_InternalConstructor_CreatesValidInstance` -- **`SarifMark-SarifRun-ToolVersion`**: `SarifRun_InternalConstructor_CreatesValidInstance` -- **`SarifMark-SarifRun-Results`**: `SarifRun_InternalConstructor_CreatesValidInstance` -- **`SarifMark-SarifRun-FileCount`**: `SarifRun_InternalConstructor_CreatesValidInstance` -- **`SarifMark-SarifRun-HasIssues`**: `SarifRun_HasIssues_NoResults_ReturnsFalse` -- **`SarifMark-SarifRun-ToMarkdown`**: `SarifRun_ToMarkdown_Depth1_ProducesCorrectOutput` -- **`SarifMark-SarifRun-ValidateDepth`**: `SarifRun_ToMarkdown_DepthLessThan1_ThrowsArgumentOutOfRangeException`, - `SarifRun_ToMarkdown_DepthGreaterThan6_ThrowsArgumentOutOfRangeException` -- **`SarifMark-SarifRun-FormatLocation`**: `SarifRun_ToMarkdown_ResultWithoutLocation_ShowsNoLocation` -- **`SarifMark-SarifRun-FormatCount`**: `SarifRun_ToMarkdown_OneResult_UsesSingularForm` +**SarifRun_ToMarkdown_MultipleResults_UsesPluralForm**: Call `ToMarkdown(1)` on a run with three results; assert the +output contains `Found 3 issues` (plural) and does not contain `Found 3 issue`, confirming that the plural form +is used when more than one result is present. +This scenario is tested by `SarifRun_ToMarkdown_MultipleResults_UsesPluralForm`. + +**SarifRun_ToMarkdown_Depth6_IssuesHeadingCappedAtSix**: Call `ToMarkdown(6)`; assert the output uses `######` for +both the tool heading and the Issues sub-heading, confirming that the Issues sub-heading depth is capped at 6 +(i.e. `Math.Min(depth + 1, 6)`) and no seven-hash heading is generated. +This scenario is tested by `SarifRun_ToMarkdown_Depth6_IssuesHeadingCappedAtSix`. + +**SarifRun_ToMarkdown_CustomHeading_UsesProvidedHeading**: Call `ToMarkdown(1, "Custom Heading")`; assert the output +uses the provided custom heading text instead of the default tool-name label, confirming that the heading parameter +is correctly applied. +This scenario is tested by `SarifRun_ToMarkdown_CustomHeading_UsesProvidedHeading`. + +**SarifRun_ToMarkdown_ResultWithUriNoLine_ShowsUriOnly**: Call `ToMarkdown(1)` on a run containing a result with a +URI but no start-line number; assert the output contains only the URI without a line number suffix, confirming that +partial location information is gracefully rendered. +This scenario is tested by `SarifRun_ToMarkdown_ResultWithUriNoLine_ShowsUriOnly`. diff --git a/docs/verification/sarifmark/self-test.md b/docs/verification/sarifmark/self-test.md index 8dc3872..1018e58 100644 --- a/docs/verification/sarifmark/self-test.md +++ b/docs/verification/sarifmark/self-test.md @@ -2,16 +2,19 @@ ### Verification Approach -The `SelfTest` subsystem is verified through tests that invoke the subsystem via the `--validate` code path of the -compiled DLL using the `Runner.Run` helper. Tests assert that self-validation runs, produces the expected output, writes -result files in the requested format, and exercises the enforcement scenario. Tests are defined in +The `SelfTest` subsystem is verified through unit-level tests that create a `Context` by calling +`Context.Create(string[])` with the `--validate` flag and then invoke `Validation.Run(context)` directly in-process. +No subprocess is spawned and no compiled DLL path is required; the subsystem is exercised within the xUnit test runner +using the same process. Tests assert that self-validation runs with exit code 0, produces the expected log output +(including `Total Tests:` and named scenario results), writes result files in the requested format (TRX and JUnit XML), +and that the depth parameter influences the self-validation report. Tests are defined in `test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs` using the xUnit v3 framework. ### Test Environment -Standard xUnit v3 test runner with `dotnet test`. The compiled DLL must be available to the `Runner.Run` helper. -Temporary result files (`.trx`, `.xml`) are created in the OS temporary directory and cleaned up after each test. No -external services or network configuration are required. +Standard xUnit v3 test runner with `dotnet test`. Temporary log files and result files (`.trx`, `.xml`) are created in +the OS temporary directory and cleaned up after each test in `finally` blocks. No external services, compiled DLL +paths, or network configuration are required. ### Acceptance Criteria @@ -36,15 +39,16 @@ file is created with a `<testsuite` element. This scenario is tested by `SelfTest_ResultsFile_XmlPath_WritesJUnitFile`. **SelfTest_EnforcementTest_RunsWithinValidation**: Invoke `--validate`; assert the enforcement scenario -(`SarifMark_Enforcement`) runs within the self-validation suite and is reported in the output. +(`SarifMark_Enforcement`) runs within the self-validation suite and is reported as passed, confirming that enforcement +mode detects issues and returns a non-zero exit code when issues are present. This scenario is tested by `SelfTest_EnforcementTest_RunsWithinValidation`. -### Requirements Coverage +**SelfTest_DepthParameter_AffectsSelfValidationReport**: Invoke `--validate --depth 2`; assert self-validation passes +and the markdown report generation scenario (`SarifMark_MarkdownReportGeneration`) is reported as passed, confirming +the depth parameter is accepted and propagated to the self-validation report generation test. +This scenario is tested by `SelfTest_DepthParameter_AffectsSelfValidationReport`. -- **`SarifMark-Validate-Mode`**: `SelfTest_ValidateFlag_RunsSelfValidation` -- **`SarifMark-Validate-ResultFiles`**: `SelfTest_ResultsFile_TrxPath_WritesTrxFile`, - `SelfTest_ResultsFile_XmlPath_WritesJUnitFile` -- **`SarifMark-Validate-TrxFormat`**: `SelfTest_ResultsFile_TrxPath_WritesTrxFile` -- **`SarifMark-Validate-JUnitFormat`**: `SelfTest_ResultsFile_XmlPath_WritesJUnitFile` -- **`SarifMark-Enforce-Mode`**: `SelfTest_EnforcementTest_RunsWithinValidation` -- **`SarifMark-Enforce-ExitCode`**: `SelfTest_EnforcementTest_RunsWithinValidation` +**SelfTest_EnforceFlag_WithIssues_ReturnsNonZeroExitCode**: Invoke `Program.Main` with `--sarif sample.sarif --enforce` +where `sample.sarif` contains findings; assert the exit code is 1, confirming that enforcement mode returns a non-zero +exit code when issues are found. +This scenario is tested by `SelfTest_EnforceFlag_WithIssues_ReturnsNonZeroExitCode`. diff --git a/docs/verification/sarifmark/self-test/validation.md b/docs/verification/sarifmark/self-test/validation.md index 4eaba9c..3db9955 100644 --- a/docs/verification/sarifmark/self-test/validation.md +++ b/docs/verification/sarifmark/self-test/validation.md @@ -75,22 +75,3 @@ This scenario is tested by `Validation_TemporaryDirectory_Create_DirectoryExists **Validation_TemporaryDirectory_Dispose_DirectoryDeleted**: Dispose a `TemporaryDirectory` instance; assert the directory is deleted from disk. This scenario is tested by `Validation_TemporaryDirectory_Dispose_DirectoryDeleted`. - -#### Requirements Coverage - -- **`SarifMark-Validation-Run`**: `Validation_Run_ValidContext_RunsAllTests` -- **`SarifMark-Validation-NullCheck`**: `Validation_Run_NullContext_ThrowsArgumentNullException` -- **`SarifMark-Validation-Header`**: `Validation_Run_ValidContext_PrintsValidationHeader` -- **`SarifMark-Validation-SarifReadingTest`**: `Validation_Run_ValidContext_RunsAllTests`, - `Validation_Run_ValidContext_VerifiesSarifReadingOutput` -- **`SarifMark-Validation-ReportGenerationTest`**: `Validation_Run_ValidContext_RunsAllTests`, - `Validation_Run_ValidContext_VerifiesReportGenerationOutput` -- **`SarifMark-Validation-EnforcementTest`**: `Validation_Run_ValidContext_RunsAllTests`, - `Validation_Run_ValidContext_VerifiesEnforcementOutput` -- **`SarifMark-Validation-Summary`**: `Validation_Run_ValidContext_PrintsSummary` -- **`SarifMark-Validation-TrxResultsFile`**: `Validation_Run_WithTrxResultsFile_WritesResultsFile` -- **`SarifMark-Validation-XmlResultsFile`**: `Validation_Run_WithXmlResultsFile_WritesResultsFile` -- **`SarifMark-Validation-UnsupportedResultsFile`**: `Validation_Run_WithUnsupportedResultsFileExtension_WritesError` -- **`SarifMark-Validation-WriteResultsFile-IOFailure`**: `Validation_Run_WithNonExistentResultsDirectory_WritesError` -- **`SarifMark-Validation-TempDir`**: `Validation_TemporaryDirectory_Create_DirectoryExists`, - `Validation_TemporaryDirectory_Dispose_DirectoryDeleted` diff --git a/docs/verification/sarifmark/utilities.md b/docs/verification/sarifmark/utilities.md index 2a0b848..614749c 100644 --- a/docs/verification/sarifmark/utilities.md +++ b/docs/verification/sarifmark/utilities.md @@ -32,17 +32,12 @@ at the subsystem boundary. This scenario is tested by `Utilities_SafePathHandling_PathTraversal_ThrowsException`. **Utilities_SafePathHandling_AbsolutePath_ThrowsException**: Call `PathHelpers.SafePathCombine` with an absolute child -path (`/etc/passwd`); assert `ArgumentException` is thrown, confirming absolute path injection is rejected. +path (`/etc/passwd`); assert `ArgumentException` is thrown, confirming absolute path injection is rejected. The +Windows-specific case (`C:\Windows\System32`) is only exercised on Windows; the Unix-style absolute path +(`/etc/passwd`) is exercised on all platforms. This scenario is tested by `Utilities_SafePathHandling_AbsolutePath_ThrowsException`. **Utilities_SafePathHandling_NullRelativePath_ThrowsException**: Call `PathHelpers.SafePathCombine` with a `null` relative path; assert `ArgumentNullException` is thrown, confirming null-argument validation works at the subsystem boundary. This scenario is tested by `Utilities_SafePathHandling_NullRelativePath_ThrowsException`. - -### Requirements Coverage - -- **`SarifMark-Utilities-SafePathHandling`**: `Utilities_SafePathHandling_ValidPaths_CombinesSuccessfully`, - `Utilities_SafePathHandling_PathTraversal_ThrowsException`, - `Utilities_SafePathHandling_AbsolutePath_ThrowsException`, - `Utilities_SafePathHandling_NullRelativePath_ThrowsException` diff --git a/docs/verification/sarifmark/utilities/path-helpers.md b/docs/verification/sarifmark/utilities/path-helpers.md index a0784f5..f1a29c3 100644 --- a/docs/verification/sarifmark/utilities/path-helpers.md +++ b/docs/verification/sarifmark/utilities/path-helpers.md @@ -17,8 +17,8 @@ configuration are required. #### Acceptance Criteria All `PathHelpersTests` test methods pass, confirming that valid relative path combinations return the correct result -and that boundary conditions (null base path, absolute path escape) throw the correct exceptions. No `PathHelpers` -unit requirement may remain without at least one named test scenario (IEC 62304 §5.5.2). +and that boundary conditions (null base path, absolute path escape, directory traversal) throw the correct exceptions. +No `PathHelpers` unit requirement may remain without at least one named test scenario (IEC 62304 §5.5.2). #### Test Scenarios @@ -26,16 +26,42 @@ unit requirement may remain without at least one named test scenario (IEC 62304 path; assert the combined result equals the expected absolute path. This scenario is tested by `PathHelpers_SafePathCombine_ValidPaths_CombinesSuccessfully`. +**PathHelpers_SafePathCombine_SimpleFilename_CombinesSuccessfully**: Call with a valid base path and a simple filename +(e.g. `file.txt`); assert the combined result equals the expected path, confirming that single-component relative +paths are accepted. +This scenario is tested by `PathHelpers_SafePathCombine_SimpleFilename_CombinesSuccessfully`. + +**PathHelpers_SafePathCombine_PathWithSubdirectories_CombinesSuccessfully**: Call with a valid base path and a +multi-component relative path (e.g. `documents/work/report.pdf`); assert the combined result equals the expected path, +confirming that relative paths with multiple directory components are accepted. +This scenario is tested by `PathHelpers_SafePathCombine_PathWithSubdirectories_CombinesSuccessfully`. + +**PathHelpers_SafePathCombine_GuidBasedFilename_CombinesSuccessfully**: Call with a temp directory base path and a +GUID-based relative filename (e.g. `test-{guid}.tmp`); assert the combined result equals the expected path, +confirming that dynamically generated filenames are accepted. +This scenario is tested by `PathHelpers_SafePathCombine_GuidBasedFilename_CombinesSuccessfully`. + +**PathHelpers_SafePathCombine_FilenameWithEmbeddedDots_CombinesSuccessfully**: Call with a valid base path and a +relative path containing embedded `..` as a substring (e.g. `v1..0.sarif`); assert the combined result equals the +expected path, confirming that filenames containing `..` as part of the name (not as a traversal segment) are +accepted. +This scenario is tested by `PathHelpers_SafePathCombine_FilenameWithEmbeddedDots_CombinesSuccessfully`. + **PathHelpers_SafePathCombine_NullBasePath_ThrowsArgumentNullException**: Pass `null` as the base path; assert `ArgumentNullException` is thrown. This scenario is tested by `PathHelpers_SafePathCombine_NullBasePath_ThrowsArgumentNullException`. -**PathHelpers_SafePathCombine_AbsoluteChildPath_ThrowsException**: Pass a child segment that resolves outside the -base path (e.g. an absolute path); assert an exception is thrown to prevent path traversal. -This scenario is tested by `PathHelpers_SafePathCombine_AbsoluteChildPath_ThrowsException`. +**PathHelpers_SafePathCombine_NullRelativePath_ThrowsArgumentNullException**: Pass `null` as the relative path; +assert `ArgumentNullException` is thrown with `ParamName` equal to `relativePath`. +This scenario is tested by `PathHelpers_SafePathCombine_NullRelativePath_ThrowsArgumentNullException`. -#### Requirements Coverage +**PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException**: Pass a child segment that is an absolute path +(e.g. `/etc/passwd` on Unix or `C:\Windows\System32` on Windows); assert `ArgumentException` is thrown to prevent +escape from the base directory, confirming that absolute-path inputs are rejected. +This scenario is tested by `PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException`. -- **`SarifMark-PathHelpers-SafeCombine`**: `PathHelpers_SafePathCombine_ValidPaths_CombinesSuccessfully` -- **`SarifMark-PathHelpers-NullCheck`**: `PathHelpers_SafePathCombine_NullBasePath_ThrowsArgumentNullException` -- **`SarifMark-PathHelpers-PostCombineCheck`**: `PathHelpers_SafePathCombine_AbsoluteChildPath_ThrowsException` +**PathHelpers_SafePathCombine_PathWithParentDirectory_ThrowsArgumentException**: Pass a relative child +path that uses `..` segments to escape the base directory (e.g., `../etc/passwd`); assert +`ArgumentException` is thrown with `ParamName` equal to `relativePath` and a message containing +`"Invalid path component"`, confirming that directory traversal outside the base directory is rejected. +This scenario is tested by `PathHelpers_SafePathCombine_PathWithParentDirectory_ThrowsArgumentException`. diff --git a/docs/verification/shared.md b/docs/verification/shared.md new file mode 100644 index 0000000..2767034 --- /dev/null +++ b/docs/verification/shared.md @@ -0,0 +1,39 @@ +# Shared Package Verification + +## Verification Strategy + +Each shared package is verified through the product's own self-validation test suite, +supplemented by confirmation that the CI pipeline step completes successfully. Because +the one shared package consumed by SarifMark is a released version of the same product, +every feature consumed in the CI pipeline is directly covered by the integration tests +in `test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs`. + +Self-validation tests exercise SarifMark end-to-end against local SARIF test data files +and confirm that SARIF reading, markdown report generation, custom heading application, +and report depth configuration all function correctly without requiring any external +services. The CI pipeline step additionally confirms that the released version operates +correctly in the pipeline environment by completing with exit code 0 and producing the +expected report file. + +## Qualification Evidence + +For the SarifMark shared package, the following evidence is collected: + +- **Self-validation integration tests**: The integration tests in + `test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs` cover all features consumed + by the CI pipeline step. All relevant test methods must pass in every CI run, and the + TRX result files are linked to requirements via the ReqStream trace matrix. +- **CI pipeline step completion**: The `Generate CodeQL Quality Report with SarifMark` + pipeline step must complete with exit code 0. The generated file + `docs/code_quality/generated/codeql-quality.md` must be present and non-empty in the + pipeline workspace, confirming the shared package read the SARIF input and wrote the + markdown report successfully. + +## Regression Approach + +On any shared package version upgrade, the CI pipeline is run in full. All self-validation +integration tests must pass. The pipeline step producing the CodeQL quality report must +complete without error and the output file must be present. If a version upgrade changes +the CLI interface or output format in a way that affects the pipeline step, the +corresponding documentation (`docs/verification/shared/sarifmark.md`) is reviewed and +updated before the upgrade is accepted into the main branch. diff --git a/docs/verification/shared/sarifmark.md b/docs/verification/shared/sarifmark.md new file mode 100644 index 0000000..89b393e --- /dev/null +++ b/docs/verification/shared/sarifmark.md @@ -0,0 +1,60 @@ +## SarifMark (Shared Package) + +### Verification Approach + +SarifMark is verified as a shared package through the product's own self-validation test suite. +Because the shared package is a released version of the same product, every feature consumed +in the CI pipeline is directly covered by the integration tests in +`test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs`. These tests exercise SarifMark +end-to-end against real SARIF input files and confirm that SARIF reading, report generation, +custom heading application, and report depth configuration all function correctly. + +In addition to the test suite, the CI pipeline step itself serves as operational confirmation: +the step must complete with exit code 0 and the generated report must be present at the +expected output path before the pipeline proceeds to publish release artifacts. + +### Test Environment + +The self-validation tests require: + +- No external services; all tests operate on local SARIF test data files bundled with the + test project. +- A writable temporary directory for output report files. +- A built version of `DemaConsulting.SarifMark.dll` in the test assembly's output directory. + +The CI pipeline step additionally requires that the CodeQL SARIF output file +`artifacts/csharp.sarif` is present before the SarifMark step executes. + +### Acceptance Criteria + +All relevant self-validation integration tests must pass with exit code 0 and zero failures. +The CI pipeline step must complete with exit code 0 and the file +`docs/code_quality/generated/codeql-quality.md` must be present and non-empty in the +pipeline workspace after the step runs. + +### Test Scenarios + +**SarifMark_ValidSarifFile_ProcessesSuccessfully**: SarifMark is invoked with a valid SARIF +input file via `--sarif`. The test confirms the process exits with code 0, reads the SARIF +file successfully, and reports the tool name and result count without error. +This scenario is tested by `SarifMark_ValidSarifFile_ProcessesSuccessfully`. + +**SarifMark_GenerateReport_CreatesReportFile**: SarifMark is invoked with `--sarif` and +`--report` pointing to a temporary output path. The test confirms the process exits with +code 0, creates the report file, and the file contains the expected tool heading. +This scenario is tested by `SarifMark_GenerateReport_CreatesReportFile`. + +**SarifMark_ValidSarif_NoIssues_GeneratesReport**: SarifMark is invoked with a SARIF file +that contains no findings. The test confirms the process exits with code 0 and the generated +report correctly indicates that no issues were found. +This scenario is tested by `SarifMark_ValidSarif_NoIssues_GeneratesReport`. + +**SarifMark_CustomHeading_AppearsInReport**: SarifMark is invoked with `--sarif`, `--report`, +and `--heading "Custom Analysis"`. The test confirms the process exits with code 0 and the +generated report contains the specified custom heading as the top-level section title. +This scenario is tested by `SarifMark_CustomHeading_AppearsInReport`. + +**SarifMark_LegacyReportDepth_IsAccepted**: SarifMark is invoked with `--sarif`, `--report`, +and `--report-depth 3`. The test confirms the process exits with code 0, the report file is +created, and the heading in the report uses the `###` level corresponding to depth 3. +This scenario is tested by `SarifMark_LegacyReportDepth_IsAccepted`. diff --git a/requirements.yaml b/requirements.yaml index fdf9fac..60eebc1 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -4,7 +4,6 @@ includes: - docs/reqstream/sarifmark/cli.yaml - docs/reqstream/sarifmark/cli/context.yaml - docs/reqstream/sarifmark/sarif.yaml - - docs/reqstream/sarifmark/sarif/report.yaml - docs/reqstream/sarifmark/sarif/sarif-run.yaml - docs/reqstream/sarifmark/sarif/sarif-finding.yaml - docs/reqstream/sarifmark/sarif/sarif-results.yaml @@ -23,3 +22,5 @@ includes: - docs/reqstream/ots/pandoc.yaml - docs/reqstream/ots/weasyprint.yaml - docs/reqstream/ots/fileassert.yaml + - docs/reqstream/ots/test-results.yaml + - docs/reqstream/shared/sarifmark.yaml diff --git a/src/DemaConsulting.SarifMark/Cli/Context.cs b/src/DemaConsulting.SarifMark/Cli/Context.cs index a11f629..004ae25 100644 --- a/src/DemaConsulting.SarifMark/Cli/Context.cs +++ b/src/DemaConsulting.SarifMark/Cli/Context.cs @@ -106,7 +106,7 @@ private Context() /// that can only be assigned during object-initializer syntax. The factory method /// delegates the actual parsing to the private <see cref="ArgumentParser"/> helper, /// validates and transforms the parsed values, and then constructs the immutable - /// <see cref="Context"/> record in a single initializer block. This pattern keeps + /// <see cref="Context"/> instance in a single initializer block. This pattern keeps /// the public surface of <see cref="Context"/> read-only while still allowing /// thorough validation before any property is set. /// </remarks> @@ -252,7 +252,7 @@ public void ParseArguments(string[] args) /// <param name="args">All arguments.</param> /// <param name="index">Current index.</param> /// <returns>Updated index.</returns> - /// <exception cref="ArgumentException">Thrown when <paramref name="arg"/> is not a recognized argument, or a value-bearing argument is the last token without a following value, or a depth value is not a positive integer.</exception> + /// <exception cref="ArgumentException">Thrown when <paramref name="arg"/> is not a recognized argument, or a value-bearing argument is the last token without a following value, or a depth value is not an integer between 1 and 6.</exception> private int ParseArgument(string arg, string[] args, int index) { switch (arg) @@ -331,13 +331,14 @@ private static string GetRequiredStringArgument(string arg, string[] args, int i } /// <summary> - /// Gets a required positive integer argument value. + /// Gets a required integer argument value that must be between 1 and 6 inclusive, + /// corresponding to the six Markdown heading levels. /// </summary> /// <param name="arg">Argument name.</param> /// <param name="args">All arguments.</param> /// <param name="index">Current index.</param> /// <returns>The argument value.</returns> - /// <exception cref="ArgumentException">Thrown when <paramref name="arg"/> is the last token in the argument list, or its value is not a positive integer.</exception> + /// <exception cref="ArgumentException">Thrown when <paramref name="arg"/> is the last token in the argument list, or its value is not an integer between 1 and 6.</exception> private static int GetRequiredIntArgument(string arg, string[] args, int index) { if (index >= args.Length) @@ -345,9 +346,9 @@ private static int GetRequiredIntArgument(string arg, string[] args, int index) throw new ArgumentException($"{arg} requires a depth argument", nameof(args)); } - if (!int.TryParse(args[index], out var value) || value < 1) + if (!int.TryParse(args[index], out var value) || value < 1 || value > 6) { - throw new ArgumentException($"{arg} requires a positive integer", nameof(args)); + throw new ArgumentException($"{arg} requires an integer between 1 and 6", nameof(args)); } return value; diff --git a/src/DemaConsulting.SarifMark/Program.cs b/src/DemaConsulting.SarifMark/Program.cs index 08b779c..2d4e62c 100644 --- a/src/DemaConsulting.SarifMark/Program.cs +++ b/src/DemaConsulting.SarifMark/Program.cs @@ -149,6 +149,12 @@ private static void PrintBanner(Context context) /// <summary> /// Prints usage information. /// </summary> + /// <remarks> + /// Deprecated aliases (<c>--report-depth</c> for <c>--depth</c> and <c>--result</c> for + /// <c>--results</c>) are intentionally omitted from the help output to encourage use of the + /// canonical flag names. Both aliases remain accepted by <see cref="Context.Create"/> for + /// backwards compatibility with existing scripts. + /// </remarks> /// <param name="context">The context for output.</param> private static void PrintHelp(Context context) { @@ -164,7 +170,7 @@ private static void PrintHelp(Context context) context.WriteLine(" --log <file> Write output to log file"); context.WriteLine(" --sarif <file> SARIF file to process"); context.WriteLine(" --report <file> Export analysis results to markdown file"); - context.WriteLine(" --depth <depth> Markdown header depth for report (default: 1)"); + context.WriteLine(" --depth <depth> Markdown header depth for report (1-6, default: 1)"); context.WriteLine(" --heading <text> Custom heading for report (default: [ToolName] Analysis)"); } @@ -176,10 +182,11 @@ private static void PrintHelp(Context context) /// <see cref="Context.SarifFile"/> and optionally writing a markdown report to /// <see cref="Context.ReportFile"/>. The following exception types are absorbed and /// routed through <see cref="Context.WriteError"/> rather than propagated: - /// <see cref="FileNotFoundException"/>, <see cref="InvalidOperationException"/> (SARIF - /// read failures), <see cref="IOException"/>, <see cref="UnauthorizedAccessException"/>, - /// <see cref="ArgumentException"/>, and <see cref="NotSupportedException"/> (report - /// write failures). + /// <see cref="FileNotFoundException"/> and <see cref="InvalidOperationException"/> (SARIF + /// read failures — I/O and access errors are wrapped as <see cref="InvalidOperationException"/> + /// by <see cref="SarifResults.Read"/>), and <see cref="IOException"/>, + /// <see cref="UnauthorizedAccessException"/>, <see cref="ArgumentException"/>, and + /// <see cref="NotSupportedException"/> (report write failures). /// </remarks> /// <param name="context">The context containing command line arguments and program state.</param> private static void ProcessSarifAnalysis(Context context) @@ -212,7 +219,7 @@ private static void ProcessSarifAnalysis(Context context) } catch (InvalidOperationException ex) { - context.WriteError($"Error: Failed to read SARIF file: {ex.Message}"); + context.WriteError($"Error: {ex.Message}"); return; } diff --git a/src/DemaConsulting.SarifMark/Sarif/SarifResults.cs b/src/DemaConsulting.SarifMark/Sarif/SarifResults.cs index 8b27af9..575c4c7 100644 --- a/src/DemaConsulting.SarifMark/Sarif/SarifResults.cs +++ b/src/DemaConsulting.SarifMark/Sarif/SarifResults.cs @@ -54,7 +54,7 @@ internal SarifResults(IReadOnlyList<SarifRun> runs) /// <returns>A SarifResults record containing the extracted information.</returns> /// <exception cref="ArgumentException">Thrown when the file path is null or empty.</exception> /// <exception cref="FileNotFoundException">Thrown when the file does not exist.</exception> - /// <exception cref="InvalidOperationException">Thrown when the SARIF file is invalid or malformed.</exception> + /// <exception cref="InvalidOperationException">Thrown when the SARIF file cannot be read or is invalid or malformed.</exception> public static SarifResults Read(string filePath) { if (string.IsNullOrWhiteSpace(filePath)) @@ -85,6 +85,17 @@ public static SarifResults Read(string filePath) return new SarifResults(runs); } + catch (FileNotFoundException) + { + // FileNotFoundException is a subclass of IOException; re-throw explicitly so the + // documented FileNotFoundException contract is preserved in the race-condition case + // where the file is deleted between the File.Exists check and File.ReadAllText. + throw; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + throw new InvalidOperationException($"Could not read SARIF file '{filePath}': {ex.Message}", ex); + } catch (JsonException ex) { throw new InvalidOperationException($"Invalid JSON in SARIF file: {ex.Message}", ex); diff --git a/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs b/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs index 3990dab..62745d0 100644 --- a/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs @@ -321,7 +321,7 @@ public void Cli_Create_DepthInvalidValue_ThrowsArgumentException() var ex = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "abc"])); // Assert - Assert.Contains("--depth requires a positive integer", ex.Message); + Assert.Contains("--depth requires an integer between 1 and 6", ex.Message); } /// <summary> @@ -360,4 +360,36 @@ public void Cli_Create_LogWithoutSilent_WritesToConsoleAndLogFile() } } } + + /// <summary> + /// Test that --depth 0 is rejected with an error. + /// </summary> + [Fact] + public void Cli_Create_DepthZero_ThrowsArgumentException() + { + // Arrange + // (no setup required) + + // Act + var ex = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "0"])); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", ex.Message); + } + + /// <summary> + /// Test that --depth -1 is rejected with an error. + /// </summary> + [Fact] + public void Cli_Create_DepthNegative_ThrowsArgumentException() + { + // Arrange + // (no setup required) + + // Act + var ex = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "-1"])); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", ex.Message); + } } diff --git a/test/DemaConsulting.SarifMark.Tests/Cli/ContextTests.cs b/test/DemaConsulting.SarifMark.Tests/Cli/ContextTests.cs index b54834d..2b5ce2c 100644 --- a/test/DemaConsulting.SarifMark.Tests/Cli/ContextTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Cli/ContextTests.cs @@ -118,8 +118,13 @@ public void Context_Create_ShortHelpFlag_SetsHelpTrue() [Fact] public void Context_Create_UnknownArgument_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--unknown"])); + + // Assert Assert.Contains("Unsupported argument", exception.Message); } @@ -197,8 +202,13 @@ public void Context_Create_HeadingArgument_SetsHeading() [Fact] public void Context_Create_HeadingWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--heading"])); + + // Assert Assert.Contains("--heading requires", exception.Message); } @@ -314,8 +324,10 @@ public void Context_Create_InvalidLogFilePath_ThrowsInvalidOperationException() // Arrange - Use an invalid path (directory that doesn't exist) var invalidPath = PathHelpers.SafePathCombine("/nonexistent/directory", "test.log"); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => Context.Create(["--log", invalidPath])); + + // Assert Assert.Contains("Failed to open log file", exception.Message); } @@ -386,7 +398,10 @@ public void Context_ExitCode_StartsAtZero_ChangesToOneAfterError() [Fact] public void Context_Create_MultipleFlags_SetsAllPropertiesCorrectly() { - // Arrange & Act + // Arrange + // (no setup required) + + // Act using var context = Context.Create(["--silent", "--enforce", "--sarif", "test.sarif"]); // Assert @@ -442,8 +457,13 @@ public void Context_Create_DepthParameter_SetsDepth() [Fact] public void Context_Create_DepthWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--depth"])); + + // Assert Assert.Contains("--depth requires", exception.Message); } @@ -453,9 +473,14 @@ public void Context_Create_DepthWithoutValue_ThrowsArgumentException() [Fact] public void Context_Create_DepthInvalidValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "invalid"])); - Assert.Contains("--depth requires a positive integer", exception.Message); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", exception.Message); } /// <summary> @@ -464,9 +489,14 @@ public void Context_Create_DepthInvalidValue_ThrowsArgumentException() [Fact] public void Context_Create_DepthZero_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "0"])); - Assert.Contains("--depth requires a positive integer", exception.Message); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", exception.Message); } /// <summary> @@ -488,8 +518,13 @@ public void Context_Create_ReportDepthParameter_SetsReportDepth() [Fact] public void Context_Create_ReportDepthWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--report-depth"])); + + // Assert Assert.Contains("--report-depth requires", exception.Message); } @@ -499,9 +534,14 @@ public void Context_Create_ReportDepthWithoutValue_ThrowsArgumentException() [Fact] public void Context_Create_ReportDepthInvalidValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--report-depth", "invalid"])); - Assert.Contains("--report-depth requires a positive integer", exception.Message); + + // Assert + Assert.Contains("--report-depth requires an integer between 1 and 6", exception.Message); } /// <summary> @@ -510,9 +550,14 @@ public void Context_Create_ReportDepthInvalidValue_ThrowsArgumentException() [Fact] public void Context_Create_ReportDepthZero_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--report-depth", "0"])); - Assert.Contains("--report-depth requires a positive integer", exception.Message); + + // Assert + Assert.Contains("--report-depth requires an integer between 1 and 6", exception.Message); } /// <summary> @@ -673,8 +718,13 @@ public void Context_WriteError_WithLogFile_WritesToLog() [Fact] public void Context_Create_SarifWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--sarif"])); + + // Assert Assert.Contains("--sarif requires", exception.Message); } @@ -684,8 +734,13 @@ public void Context_Create_SarifWithoutValue_ThrowsArgumentException() [Fact] public void Context_Create_ReportWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--report"])); + + // Assert Assert.Contains("--report requires", exception.Message); } @@ -695,8 +750,13 @@ public void Context_Create_ReportWithoutValue_ThrowsArgumentException() [Fact] public void Context_Create_ResultsWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--results"])); + + // Assert Assert.Contains("--results requires", exception.Message); } @@ -706,8 +766,77 @@ public void Context_Create_ResultsWithoutValue_ThrowsArgumentException() [Fact] public void Context_Create_LogWithoutValue_ThrowsArgumentException() { - // Act & Assert + // Arrange + // (no setup required) + + // Act var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--log"])); + + // Assert Assert.Contains("--log requires", exception.Message); } + + /// <summary> + /// Test that creating a context with --depth and negative value throws exception. + /// </summary> + [Fact] + public void Context_Create_DepthNegative_ThrowsArgumentException() + { + // Arrange + // (no setup required) + + // Act + var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "-1"])); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", exception.Message); + } + + /// <summary> + /// Test that creating a context with --depth and a value above the maximum throws exception. + /// </summary> + [Fact] + public void Context_Create_DepthAboveMax_ThrowsArgumentException() + { + // Arrange + // (no setup required) + + // Act + var exception = Assert.Throws<ArgumentException>(() => Context.Create(["--depth", "7"])); + + // Assert + Assert.Contains("--depth requires an integer between 1 and 6", exception.Message); + } + + /// <summary> + /// Test that creating a context with --depth 1 (lower boundary) is accepted. + /// </summary> + [Fact] + public void Context_Create_DepthAtMinimum_SetsDepth() + { + // Arrange + // (no setup required) + + // Act + using var context = Context.Create(["--depth", "1"]); + + // Assert + Assert.Equal(1, context.Depth); + } + + /// <summary> + /// Test that creating a context with --depth 6 (upper boundary) is accepted. + /// </summary> + [Fact] + public void Context_Create_DepthAtMaximum_SetsDepth() + { + // Arrange + // (no setup required) + + // Act + using var context = Context.Create(["--depth", "6"]); + + // Assert + Assert.Equal(6, context.Depth); + } } diff --git a/test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs b/test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs index 55b068c..5b63069 100644 --- a/test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/IntegrationTests.cs @@ -520,4 +520,45 @@ public void SarifMark_MultiRunSarifFile_CreatesReport() } } } + + /// <summary> + /// Test that a SARIF file with no issues generates a valid report. + /// </summary> + [Fact] + public void SarifMark_ValidSarif_NoIssues_GeneratesReport() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "no-issues.sarif"); + Assert.True(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-no-issues-report-{Guid.NewGuid()}.md"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--report", reportFile); + + // Assert + Assert.Equal(0, exitCode); + Assert.True(File.Exists(reportFile), "Report file was not created"); + + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("**Tool:**", reportContent); + Assert.Contains("Found no issues", reportContent); + } + finally + { + // Clean up the temporary report file + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } } + diff --git a/test/DemaConsulting.SarifMark.Tests/ProgramTests.cs b/test/DemaConsulting.SarifMark.Tests/ProgramTests.cs index 65ab156..d0d33ca 100644 --- a/test/DemaConsulting.SarifMark.Tests/ProgramTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/ProgramTests.cs @@ -155,11 +155,27 @@ public void Program_Main_UnknownArgument_ReturnsError() [Fact] public void Program_Main_ValidateFlag_RunsValidation() { - // Act - run in silent mode to suppress all console output - var result = Program.Main(["--validate", "--silent"]); + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"validate-results-{Guid.NewGuid()}.trx"); + + try + { + // Act - run in silent mode with a results file to prove validation ran + var result = Program.Main(["--validate", "--silent", "--results", resultsFile]); - // Assert - Assert.Equal(0, result); + // Assert + Assert.Equal(0, result); + Assert.True(File.Exists(resultsFile), "Results file was not created — validation did not run"); + var content = File.ReadAllText(resultsFile); + Assert.Contains("<TestRun", content); + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } } /// <summary> diff --git a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifResultsTests.cs b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifResultsTests.cs index 632a425..432ace3 100644 --- a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifResultsTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifResultsTests.cs @@ -54,8 +54,14 @@ public void Dispose() [Fact] public void SarifResults_Read_NullPath_ThrowsArgumentException() { - // Act & Assert - Assert.Throws<ArgumentException>(() => SarifResults.Read(null!)); + // Arrange + // (no setup required) + + // Act + var exception = Assert.Throws<ArgumentException>(() => SarifResults.Read(null!)); + + // Assert + Assert.NotNull(exception); } /// <summary> @@ -64,8 +70,14 @@ public void SarifResults_Read_NullPath_ThrowsArgumentException() [Fact] public void SarifResults_Read_EmptyPath_ThrowsArgumentException() { - // Act & Assert - Assert.Throws<ArgumentException>(() => SarifResults.Read(string.Empty)); + // Arrange + // (no setup required) + + // Act + var exception = Assert.Throws<ArgumentException>(() => SarifResults.Read(string.Empty)); + + // Assert + Assert.NotNull(exception); } /// <summary> @@ -74,8 +86,14 @@ public void SarifResults_Read_EmptyPath_ThrowsArgumentException() [Fact] public void SarifResults_Read_WhitespacePath_ThrowsArgumentException() { - // Act & Assert - Assert.Throws<ArgumentException>(() => SarifResults.Read(" ")); + // Arrange + // (no setup required) + + // Act + var exception = Assert.Throws<ArgumentException>(() => SarifResults.Read(" ")); + + // Assert + Assert.NotNull(exception); } /// <summary> @@ -87,8 +105,11 @@ public void SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException() // Arrange var filePath = PathHelpers.SafePathCombine(_testDirectory, "nonexistent.sarif"); - // Act & Assert - Assert.Throws<FileNotFoundException>(() => SarifResults.Read(filePath)); + // Act + var exception = Assert.Throws<FileNotFoundException>(() => SarifResults.Read(filePath)); + + // Assert + Assert.NotNull(exception); } /// <summary> @@ -101,8 +122,10 @@ public void SarifResults_Read_InvalidJson_ThrowsInvalidOperationException() var filePath = PathHelpers.SafePathCombine(_testDirectory, "invalid.sarif"); File.WriteAllText(filePath, "{ invalid json }"); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("Invalid JSON", exception.Message); } @@ -120,8 +143,10 @@ public void SarifResults_Read_MissingVersion_ThrowsInvalidOperationException() } """); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("missing 'version'", exception.Message); } @@ -139,8 +164,10 @@ public void SarifResults_Read_MissingRuns_ThrowsInvalidOperationException() } """); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("missing or invalid 'runs'", exception.Message); } @@ -159,8 +186,10 @@ public void SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException() } """); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("'runs' array is empty", exception.Message); } @@ -181,8 +210,10 @@ public void SarifResults_Read_MissingTool_ThrowsInvalidOperationException() } """); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("missing 'tool'", exception.Message); } @@ -205,8 +236,10 @@ public void SarifResults_Read_MissingDriver_ThrowsInvalidOperationException() } """); - // Act & Assert + // Act var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(filePath)); + + // Assert Assert.Contains("missing 'driver'", exception.Message); } @@ -773,8 +806,10 @@ public void SarifResults_ToMarkdown_DepthLessThan1_ThrowsArgumentOutOfRangeExcep // Arrange var results = new SarifResults([new SarifRun("TestTool", "1.0.0", [])]); - // Act & Assert + // Act var exception = Assert.Throws<ArgumentOutOfRangeException>(() => results.ToMarkdown(0)); + + // Assert Assert.Contains("Depth must be between 1 and 6", exception.Message); } @@ -787,8 +822,10 @@ public void SarifResults_ToMarkdown_DepthGreaterThan6_ThrowsArgumentOutOfRangeEx // Arrange var results = new SarifResults([new SarifRun("TestTool", "1.0.0", [])]); - // Act & Assert + // Act var exception = Assert.Throws<ArgumentOutOfRangeException>(() => results.ToMarkdown(7)); + + // Assert Assert.Contains("Depth must be between 1 and 6", exception.Message); } @@ -1337,5 +1374,6 @@ public void SarifResults_ToMarkdown_MultipleRuns_IncludesRunIndices() Assert.Contains("(#1)", md); Assert.Contains("(#2)", md); } + } diff --git a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs index 45388d7..83a837f 100644 --- a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs @@ -63,8 +63,11 @@ public void Sarif_Read_NonExistentFile_ThrowsFileNotFoundException() // Arrange var nonExistentPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.sarif"); - // Act / Assert - Assert.Throws<FileNotFoundException>(() => SarifResults.Read(nonExistentPath)); + // Act + var exception = Assert.Throws<FileNotFoundException>(() => SarifResults.Read(nonExistentPath)); + + // Assert + Assert.NotNull(exception); } /// <summary> @@ -113,8 +116,11 @@ public void Sarif_Read_InvalidSarifFile_ThrowsInvalidOperationException() var sarifFile = Path.Combine(_testDataPath, "invalid.sarif"); Assert.True(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - // Act / Assert - Assert.Throws<InvalidOperationException>(() => SarifResults.Read(sarifFile)); + // Act + var exception = Assert.Throws<InvalidOperationException>(() => SarifResults.Read(sarifFile)); + + // Assert + Assert.NotNull(exception); } /// <summary> diff --git a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs index 69b915c..306bff3 100644 --- a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs @@ -180,4 +180,35 @@ public void SelfTest_EnforcementTest_RunsWithinValidation() } } } + + /// <summary> + /// Test that enforcement mode returns a non-zero exit code when issues are found. + /// </summary> + [Fact] + public void SelfTest_EnforceFlag_WithIssues_ReturnsNonZeroExitCode() + { + // Arrange + var sarifFile = Path.Combine(AppContext.BaseDirectory, "TestData", "sample.sarif"); + Assert.True(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + var originalOut = Console.Out; + var originalError = Console.Error; + try + { + using var outWriter = new StringWriter(); + using var errWriter = new StringWriter(); + Console.SetOut(outWriter); + Console.SetError(errWriter); + + // Act + var exitCode = Program.Main(["--sarif", sarifFile, "--enforce", "--silent"]); + + // Assert + Assert.Equal(1, exitCode); + } + finally + { + Console.SetOut(originalOut); + Console.SetError(originalError); + } + } } diff --git a/test/DemaConsulting.SarifMark.Tests/TestData/no-issues.sarif b/test/DemaConsulting.SarifMark.Tests/TestData/no-issues.sarif new file mode 100644 index 0000000..00b414b --- /dev/null +++ b/test/DemaConsulting.SarifMark.Tests/TestData/no-issues.sarif @@ -0,0 +1,22 @@ +{ + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "CleanTool", + "version": "1.0.0" + } + }, + "artifacts": [ + { + "location": { + "uri": "file:///path/to/file.cs" + } + } + ], + "results": [] + } + ] +} diff --git a/test/DemaConsulting.SarifMark.Tests/Utilities/PathHelpersTests.cs b/test/DemaConsulting.SarifMark.Tests/Utilities/PathHelpersTests.cs index d415ccf..41fac0b 100644 --- a/test/DemaConsulting.SarifMark.Tests/Utilities/PathHelpersTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Utilities/PathHelpersTests.cs @@ -202,4 +202,5 @@ public void PathHelpers_SafePathCombine_GuidBasedFilename_CombinesSuccessfully() // Assert Assert.Equal(Path.Combine(basePath, relativePath), result); } + }