diff --git a/.github/agents/code-review.agent.md b/.github/agents/code-review.agent.md index cee797f..bb48e5c 100644 --- a/.github/agents/code-review.agent.md +++ b/.github/agents/code-review.agent.md @@ -15,7 +15,7 @@ Formal reviews are a quality enforcement mechanism, and as such MUST be performe 1. Download the to get the checklist to fill in -2. Use `dotnet reviewmark --elaborate [review-set]` to get the files to review +2. Use `dotnet reviewmark --elaborate {review-set}` to get the files to review 3. Review the files all together 4. Populate the checklist with the findings to `.agent-logs/reviews/review-report-{review-set}.md` of the project. @@ -41,33 +41,34 @@ of the project consisting of: ## Review Summary -- **Review Set**: [Review set name/identifier] -- **Review Report File**: [Name of detailed review report generated] -- **Files Reviewed**: [Count and list of files reviewed] -- **Review Template Used**: [Template source and version] +- **Review Set**: {Review set name/identifier} +- **Review Report File**: {Name of detailed review report generated} +- **Files Reviewed**: {Count and list of files reviewed} +- **Review Template Used**: {Template source and version} ## Review Results -- **Overall Conclusion**: [Summary of review results] -- **Critical Issues**: [Count of critical findings] -- **High Issues**: [Count of high severity findings] -- **Medium Issues**: [Count of medium severity findings] -- **Low Issues**: [Count of low severity findings] +- **Overall Conclusion**: {Summary of review results} +- **Critical Issues**: {Count of critical findings} +- **High Issues**: {Count of high severity findings} +- **Medium Issues**: {Count of medium severity findings} +- **Low Issues**: {Count of low severity findings} ## Issue Details -[For each issue found, include:] -- **File**: [File name and line number where applicable] -- **Issue Type**: [Security, logic error, compliance violation, etc.] -- **Severity**: [Critical/High/Medium/Low] -- **Description**: [Issue description] -- **Recommendation**: [Specific remediation recommendation] +For each issue found, include: + +- **File**: {File name and line number where applicable} +- **Issue Type**: {Security, logic error, compliance violation, etc.} +- **Severity**: {Critical/High/Medium/Low} +- **Description**: {Issue description} +- **Recommendation**: {Specific remediation recommendation} ## Compliance Status -- **Review Status**: [Complete/Incomplete with reasoning] -- **Quality Gates**: [Status of review checklist items] -- **Approval Status**: [Approved/Rejected with justification] +- **Review Status**: {Complete/Incomplete with reasoning} +- **Quality Gates**: {Status of review checklist items} +- **Approval Status**: {Approved/Rejected with justification} ``` Return summary to caller. diff --git a/.github/agents/developer.agent.md b/.github/agents/developer.agent.md index d936129..2671008 100644 --- a/.github/agents/developer.agent.md +++ b/.github/agents/developer.agent.md @@ -31,20 +31,20 @@ of the project consisting of: ## Work Summary -- **Files Modified**: [List of files created/modified/deleted] -- **Languages Detected**: [Languages identified] -- **Standards Applied**: [Standards files consulted] +- **Files Modified**: {List of files created/modified/deleted} +- **Languages Detected**: {Languages identified} +- **Standards Applied**: {Standards files consulted} ## Tooling Executed -- **Language Tools**: [Compilers, linters, formatters used] -- **Compliance Tools**: [ReqStream, ReviewMark tools used] -- **Validation Results**: [Tool execution results] +- **Language Tools**: {Compilers, linters, formatters used} +- **Compliance Tools**: {ReqStream, ReviewMark tools used} +- **Validation Results**: {Tool execution results} ## Compliance Status -- **Quality Checks**: [Standards quality checks status] -- **Issues Resolved**: [Any problems encountered and resolved] +- **Quality Checks**: {Standards quality checks status} +- **Issues Resolved**: {Any problems encountered and resolved} ``` Return this summary to the caller. diff --git a/.github/agents/implementation.agent.md b/.github/agents/implementation.agent.md index 35cc1c8..03603a4 100644 --- a/.github/agents/implementation.agent.md +++ b/.github/agents/implementation.agent.md @@ -72,22 +72,22 @@ of the project consisting of: ## State Machine Execution -- **Research Results**: [Summary of explore agent findings] -- **Development Results**: [Summary of developer agent results] -- **Quality Results**: [Summary of quality agent results] -- **State Transitions**: [Log of state changes and decisions] +- **Research Results**: {Summary of explore agent findings} +- **Development Results**: {Summary of developer agent results} +- **Quality Results**: {Summary of quality agent results} +- **State Transitions**: {Log of state changes and decisions} ## Sub-Agent Coordination -- **Explore Agent**: [Research findings and context] -- **Developer Agent**: [Development status and files modified] -- **Quality Agent**: [Validation results and compliance status] +- **Explore Agent**: {Research findings and context} +- **Developer Agent**: {Development status and files modified} +- **Quality Agent**: {Validation results and compliance status} ## Final Status -- **Implementation Success**: [Overall completion status] -- **Quality Compliance**: [Final quality validation status] -- **Issues Resolved**: [Problems encountered and resolution attempts] +- **Implementation Success**: {Overall completion status} +- **Quality Compliance**: {Final quality validation status} +- **Issues Resolved**: {Problems encountered and resolution attempts} ``` Return this summary to the caller. diff --git a/.github/agents/quality.agent.md b/.github/agents/quality.agent.md index 691a17d..18fc7c6 100644 --- a/.github/agents/quality.agent.md +++ b/.github/agents/quality.agent.md @@ -41,95 +41,95 @@ This ensures orchestrators properly halt workflows when quality gates fail. ## Assessment Summary -- **Work Reviewed**: [Description of work assessed] -- **Standards Applied**: [Standards files used for assessment] -- **Categories Evaluated**: [Quality check categories assessed] +- **Work Reviewed**: {Description of work assessed} +- **Standards Applied**: {Standards files used for assessment} +- **Categories Evaluated**: {Quality check categories assessed} ## Requirements Compliance: (PASS|FAIL|N/A) -- Were requirements updated to reflect functional changes? (PASS|FAIL|N/A) - [Evidence] -- Were new requirements created for new features? (PASS|FAIL|N/A) - [Evidence] -- Do requirement IDs follow semantic naming standards? (PASS|FAIL|N/A) - [Evidence] -- Do requirement files follow kebab-case naming convention? (PASS|FAIL|N/A) - [Evidence] -- Are requirement files organized under `docs/reqstream/` with proper folder structure? (PASS|FAIL|N/A) - [Evidence] -- Are OTS requirements properly placed in `docs/reqstream/ots/` subfolder? (PASS|FAIL|N/A) - [Evidence] -- Were source filters applied appropriately for platform-specific requirements? (PASS|FAIL|N/A) - [Evidence] -- Does ReqStream enforcement pass without errors? (PASS|FAIL|N/A) - [Evidence] -- Is requirements traceability maintained to tests? (PASS|FAIL|N/A) - [Evidence] +- Were requirements updated to reflect functional changes? (PASS|FAIL|N/A) - {Evidence} +- Were new requirements created for new features? (PASS|FAIL|N/A) - {Evidence} +- Do requirement IDs follow semantic naming standards? (PASS|FAIL|N/A) - {Evidence} +- Do requirement files follow kebab-case naming convention? (PASS|FAIL|N/A) - {Evidence} +- Are requirement files organized under `docs/reqstream/` with proper folder structure? (PASS|FAIL|N/A) - {Evidence} +- Are OTS requirements properly placed in `docs/reqstream/ots/` subfolder? (PASS|FAIL|N/A) - {Evidence} +- Were source filters applied appropriately for platform-specific requirements? (PASS|FAIL|N/A) - {Evidence} +- Does ReqStream enforcement pass without errors? (PASS|FAIL|N/A) - {Evidence} +- Is requirements traceability maintained to tests? (PASS|FAIL|N/A) - {Evidence} ## Design Documentation Compliance: (PASS|FAIL|N/A) -- Were design documents updated for architectural changes? (PASS|FAIL|N/A) - [Evidence] -- Were new design artifacts created for new components? (PASS|FAIL|N/A) - [Evidence] -- Do design folder names use kebab-case convention matching source structure? (PASS|FAIL|N/A) - [Evidence] -- Are design files properly named ({subsystem-name}.md, {unit-name}.md patterns)? (PASS|FAIL|N/A) - [Evidence] -- Is `docs/design/introduction.md` present with required Software Structure section? (PASS|FAIL|N/A) - [Evidence] -- Are design decisions documented with rationale? (PASS|FAIL|N/A) - [Evidence] -- Is system/subsystem/unit categorization maintained? (PASS|FAIL|N/A) - [Evidence] -- Is design-to-implementation traceability preserved? (PASS|FAIL|N/A) - [Evidence] +- Were design documents updated for architectural changes? (PASS|FAIL|N/A) - {Evidence} +- Were new design artifacts created for new components? (PASS|FAIL|N/A) - {Evidence} +- Do design folder names use kebab-case convention matching source structure? (PASS|FAIL|N/A) - {Evidence} +- Are design files properly named ({subsystem-name}.md, {unit-name}.md patterns)? (PASS|FAIL|N/A) - {Evidence} +- Is `docs/design/introduction.md` present with required Software Structure section? (PASS|FAIL|N/A) - {Evidence} +- Are design decisions documented with rationale? (PASS|FAIL|N/A) - {Evidence} +- Is system/subsystem/unit categorization maintained? (PASS|FAIL|N/A) - {Evidence} +- Is design-to-implementation traceability preserved? (PASS|FAIL|N/A) - {Evidence} ## Code Quality Compliance: (PASS|FAIL|N/A) -- Are language-specific standards followed (from applicable standards files)? (PASS|FAIL|N/A) - [Evidence] -- Are quality checks from standards files satisfied? (PASS|FAIL|N/A) - [Evidence] -- Is code properly categorized (system/subsystem/unit/OTS)? (PASS|FAIL|N/A) - [Evidence] -- Is appropriate separation of concerns maintained? (PASS|FAIL|N/A) - [Evidence] -- Was language-specific tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] +- Are language-specific standards followed (from applicable standards files)? (PASS|FAIL|N/A) - {Evidence} +- Are quality checks from standards files satisfied? (PASS|FAIL|N/A) - {Evidence} +- Is code properly categorized (system/subsystem/unit/OTS)? (PASS|FAIL|N/A) - {Evidence} +- Is appropriate separation of concerns maintained? (PASS|FAIL|N/A) - {Evidence} +- Was language-specific tooling executed and passing? (PASS|FAIL|N/A) - {Evidence} ## Testing Compliance: (PASS|FAIL|N/A) -- Were tests created/updated for all functional changes? (PASS|FAIL|N/A) - [Evidence] -- Is test coverage maintained for all requirements? (PASS|FAIL|N/A) - [Evidence] -- Are testing standards followed (AAA pattern, etc.)? (PASS|FAIL|N/A) - [Evidence] -- Does test categorization align with code structure? (PASS|FAIL|N/A) - [Evidence] -- Do all tests pass without failures? (PASS|FAIL|N/A) - [Evidence] +- Were tests created/updated for all functional changes? (PASS|FAIL|N/A) - {Evidence} +- Is test coverage maintained for all requirements? (PASS|FAIL|N/A) - {Evidence} +- Are testing standards followed (AAA pattern, etc.)? (PASS|FAIL|N/A) - {Evidence} +- Does test categorization align with code structure? (PASS|FAIL|N/A) - {Evidence} +- Do all tests pass without failures? (PASS|FAIL|N/A) - {Evidence} ## Review Management Compliance: (PASS|FAIL|N/A) -- Were review-sets updated to include new/modified files? (PASS|FAIL|N/A) - [Evidence] -- Do file patterns follow include-then-exclude approach? (PASS|FAIL|N/A) - [Evidence] -- Is review scope appropriate for change magnitude? (PASS|FAIL|N/A) - [Evidence] -- Was ReviewMark tooling executed and passing? (PASS|FAIL|N/A) - [Evidence] -- Were review artifacts generated correctly? (PASS|FAIL|N/A) - [Evidence] +- Were review-sets updated for structural changes (new/deleted systems, subsystems, or units)? (PASS|FAIL|N/A) - {Evidence} +- Do file patterns follow include-then-exclude approach? (PASS|FAIL|N/A) - {Evidence} +- Is review scope appropriate for change magnitude? (PASS|FAIL|N/A) - {Evidence} +- Was ReviewMark tooling executed and passing? (PASS|FAIL|N/A) - {Evidence} +- Were review artifacts generated correctly? (PASS|FAIL|N/A) - {Evidence} ## Documentation Compliance: (PASS|FAIL|N/A) -- Was README.md updated for user-facing changes? (PASS|FAIL|N/A) - [Evidence] -- Were user guides updated for feature changes? (PASS|FAIL|N/A) - [Evidence] -- Does API documentation reflect code changes? (PASS|FAIL|N/A) - [Evidence] -- Was compliance documentation generated? (PASS|FAIL|N/A) - [Evidence] -- Does documentation follow standards formatting? (PASS|FAIL|N/A) - [Evidence] -- Is documentation organized under `docs/` following standard folder structure? (PASS|FAIL|N/A) - [Evidence] -- Do Pandoc collections include proper `introduction.md` with Purpose and Scope sections? (PASS|FAIL|N/A) - [Evidence] -- Are auto-generated markdown files left unmodified? (PASS|FAIL|N/A) - [Evidence] -- Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - [Evidence] -- Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - [Evidence] +- Was README.md updated for user-facing changes? (PASS|FAIL|N/A) - {Evidence} +- Were user guides updated for feature changes? (PASS|FAIL|N/A) - {Evidence} +- Does API documentation reflect code changes? (PASS|FAIL|N/A) - {Evidence} +- Was compliance documentation generated? (PASS|FAIL|N/A) - {Evidence} +- Does documentation follow standards formatting? (PASS|FAIL|N/A) - {Evidence} +- Is documentation organized under `docs/` following standard folder structure? (PASS|FAIL|N/A) - {Evidence} +- Do Pandoc collections include proper `introduction.md` with Purpose and Scope sections? (PASS|FAIL|N/A) - {Evidence} +- Are auto-generated markdown files left unmodified? (PASS|FAIL|N/A) - {Evidence} +- Do README.md files use absolute URLs and include concrete examples? (PASS|FAIL|N/A) - {Evidence} +- Is documentation integrated into ReviewMark review-sets for formal review? (PASS|FAIL|N/A) - {Evidence} ## Software Item Completeness: (PASS|FAIL|N/A) -- Does every identified software unit have its own requirements file? (PASS|FAIL|N/A) - [Evidence] -- Does every identified software unit have its own design document? (PASS|FAIL|N/A) - [Evidence] -- Does every identified subsystem have its own requirements file? (PASS|FAIL|N/A) - [Evidence] -- Does every identified subsystem have its own design document? (PASS|FAIL|N/A) - [Evidence] +- Does every identified software unit have its own requirements file? (PASS|FAIL|N/A) - {Evidence} +- Does every identified software unit have its own design document? (PASS|FAIL|N/A) - {Evidence} +- Does every identified subsystem have its own requirements file? (PASS|FAIL|N/A) - {Evidence} +- Does every identified subsystem have its own design document? (PASS|FAIL|N/A) - {Evidence} ## Process Compliance: (PASS|FAIL|N/A) -- Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - [Evidence] -- Did all quality gates execute successfully? (PASS|FAIL|N/A) - [Evidence] -- Were appropriate tools used for validation? (PASS|FAIL|N/A) - [Evidence] -- Were standards consistently applied across work? (PASS|FAIL|N/A) - [Evidence] -- Was compliance evidence generated and preserved? (PASS|FAIL|N/A) - [Evidence] +- Was Continuous Compliance workflow followed? (PASS|FAIL|N/A) - {Evidence} +- Did all quality gates execute successfully? (PASS|FAIL|N/A) - {Evidence} +- Were appropriate tools used for validation? (PASS|FAIL|N/A) - {Evidence} +- Were standards consistently applied across work? (PASS|FAIL|N/A) - {Evidence} +- Was compliance evidence generated and preserved? (PASS|FAIL|N/A) - {Evidence} ## Overall Findings -- **Critical Issues**: [Count and description of critical findings] -- **Recommendations**: [Suggested improvements and next steps] -- **Tools Executed**: [Quality tools used for validation] +- **Critical Issues**: {Count and description of critical findings} +- **Recommendations**: {Suggested improvements and next steps} +- **Tools Executed**: {Quality tools used for validation} ## Compliance Status -- **Standards Adherence**: [Overall compliance rating with specific standards] -- **Quality Gates**: [Status of automated quality checks with tool outputs] +- **Standards Adherence**: {Overall compliance rating with specific standards} +- **Quality Gates**: {Status of automated quality checks with tool outputs} ``` Return this summary to the caller. diff --git a/.github/agents/repo-consistency.agent.md b/.github/agents/repo-consistency.agent.md index b0f93d2..b623895 100644 --- a/.github/agents/repo-consistency.agent.md +++ b/.github/agents/repo-consistency.agent.md @@ -52,29 +52,29 @@ of the project consisting of: ## Consistency Analysis -- **Template PRs Analyzed**: [Number and timeframe of PRs reviewed] -- **Template Changes Identified**: [Count and types of template improvements] -- **Applicable Updates**: [Changes determined suitable for this repository] -- **Project Customizations Preserved**: [Valid differences maintained] +- **Template PRs Analyzed**: {Number and timeframe of PRs reviewed} +- **Template Changes Identified**: {Count and types of template improvements} +- **Applicable Updates**: {Changes determined suitable for this repository} +- **Project Customizations Preserved**: {Valid differences maintained} ## Template Evolution Applied -- **Files Modified**: [List of files updated for template consistency] -- **Improvements Adopted**: [Specific template enhancements implemented] -- **Configuration Updates**: [Tool configurations, workflows, or standards updated] +- **Files Modified**: {List of files updated for template consistency} +- **Improvements Adopted**: {Specific template enhancements implemented} +- **Configuration Updates**: {Tool configurations, workflows, or standards updated} ## Consistency Status -- **Template Alignment**: [Overall consistency rating with template] -- **Customization Respect**: [How project-specific needs were preserved] -- **Functionality Validation**: [Verification that changes don't break existing features] -- **Future Consistency**: [Recommendations for ongoing template alignment] +- **Template Alignment**: {Overall consistency rating with template} +- **Customization Respect**: {How project-specific needs were preserved} +- **Functionality Validation**: {Verification that changes don't break existing features} +- **Future Consistency**: {Recommendations for ongoing template alignment} ## Issues Resolved -- **Drift Corrections**: [Template drift issues addressed] -- **Enhancement Adoptions**: [Template improvements successfully integrated] -- **Validation Results**: [Testing and validation outcomes] +- **Drift Corrections**: {Template drift issues addressed} +- **Enhancement Adoptions**: {Template improvements successfully integrated} +- **Validation Results**: {Testing and validation outcomes} ``` Return this summary to the caller. diff --git a/.github/standards/csharp-testing.md b/.github/standards/csharp-testing.md index 2f26520..f96a3c3 100644 --- a/.github/standards/csharp-testing.md +++ b/.github/standards/csharp-testing.md @@ -13,14 +13,11 @@ requirements. [TestMethod] public void ServiceName_MethodName_Scenario_ExpectedBehavior() { - // Arrange - (description) - // TODO: Set up test data, mocks, and system under test. + // Arrange: description of setup (omit if nothing to set up) - // Act - (description) - // TODO: Execute the action being tested + // Act: description of action (can combine with Assert when action occurs within assertion) - // Assert - (description) - // TODO: Verify expected outcomes and interactions + // Assert: description of verification } ``` @@ -28,7 +25,9 @@ public void ServiceName_MethodName_Scenario_ExpectedBehavior() Use descriptive test names because test names appear in requirements traceability matrices and compliance reports. -- **Pattern**: `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- **System tests**: `{SystemName}_{Functionality}_{Scenario}_{ExpectedBehavior}` +- **Subsystem tests**: `{SubsystemName}_{Functionality}_{Scenario}_{ExpectedBehavior}` +- **Unit tests**: `{ClassName}_{MethodUnderTest}_{Scenario}_{ExpectedBehavior}` - **Descriptive Scenarios**: Clearly describe the input condition being tested - **Expected Behavior**: State the expected outcome or exception @@ -110,7 +109,7 @@ Use `Assert.StartsWith` instead, as it produces clearer failure messages: Before submitting C# tests, verify: - [ ] All tests follow AAA pattern with clear section comments -- [ ] Test names follow `ClassName_MethodUnderTest_Scenario_ExpectedBehavior` +- [ ] Test names follow hierarchical patterns defined in Test Naming Standards section - [ ] Each test verifies single, specific behavior (no shared state) - [ ] Both success and failure scenarios covered including edge cases - [ ] External dependencies mocked with NSubstitute or equivalent diff --git a/.github/standards/reqstream-usage.md b/.github/standards/reqstream-usage.md index bd8c739..ff3bc95 100644 --- a/.github/standards/reqstream-usage.md +++ b/.github/standards/reqstream-usage.md @@ -58,6 +58,17 @@ only flow downward in the hierarchy to maintain clear traceability: This prevents circular dependencies and ensures clear hierarchical relationships for compliance auditing. +# Test Linkage Hierarchy + +Requirements MUST link to tests at their own level to maintain proper test scope: + +- **System requirements** → link ONLY to system-level integration tests +- **Subsystem requirements** → link ONLY to subsystem-level tests +- **Unit requirements** → link ONLY to unit-level tests + +Lower-level tests validate implementation details, while higher-level requirements +are validated through integration behavior at their architectural level. + # Requirements File Format ```yaml @@ -69,7 +80,9 @@ sections: justification: | Business rationale explaining why this requirement exists. Include regulatory or standard references where applicable. - tests: + children: # Links to child requirements (optional) + - ChildSystem-Feature-Behavior + tests: # Links to test methods (required) - TestMethodName - windows@PlatformSpecificTest # Source filter for platform evidence ``` @@ -158,6 +171,7 @@ Before submitting requirements, verify: - [ ] Files organized under `docs/reqstream/` following folder structure patterns - [ ] Subsystem folders use kebab-case naming matching source code - [ ] OTS requirements placed in `ots/` subfolder +- [ ] Every software unit has requirements file, design doc, and tests - [ ] Valid YAML syntax passes yamllint validation - [ ] ReqStream enforcement passes: `dotnet reqstream --enforce` - [ ] Test result formats compatible (TRX, JUnit XML) diff --git a/.github/standards/reviewmark-usage.md b/.github/standards/reviewmark-usage.md index 2fdaa19..e2e380a 100644 --- a/.github/standards/reviewmark-usage.md +++ b/.github/standards/reviewmark-usage.md @@ -8,7 +8,7 @@ review, organizes them into review-sets, and generates review plans and reports. ## Key Commands - **Lint Configuration**: `dotnet reviewmark --lint` -- **Elaborate Review-Set**: `dotnet reviewmark --elaborate [review-set]` +- **Elaborate Review-Set**: `dotnet reviewmark --elaborate {review-set}` - **Generate Plan**: `dotnet reviewmark --plan docs/code_review_plan/plan.md` - **Generate Report**: `dotnet reviewmark --report docs/code_review_report/report.md` @@ -29,7 +29,7 @@ Configure reviews in `.reviewmark.yaml` at repository root: needs-review: # Include source code (adjust file extensions for your repo) - "**/*.cs" # C# source files - - "**/*.cpp" # C++ source files + - "**/*.cpp" # C++ source files - "**/*.hpp" # C++ header files - "!**/bin/**" # Generated source in build outputs - "!**/obj/**" # Generated source in build intermediates @@ -48,72 +48,121 @@ evidence-source: type: none ``` +# Review-Set Design Principles + +When constructing review-sets, follow these principles to maintain manageable scope and effective compliance evidence: + +- **Hierarchical Scope**: Higher-level reviews exclude lower-level implementation details, relying instead on design + documents to describe what components they use. System reviews exclude subsystem/unit details, subsystem reviews + exclude unit source code, only unit reviews include actual implementation. +- **Single Focus**: Each review-set proves one specific compliance question (user promises, system architecture, + design consistency, etc.) +- **Context Management**: Keep file counts manageable to prevent context overflow while maintaining complete coverage + through the hierarchy + # Review-Set Organization -Organize review-sets using standard patterns to ensure comprehensive coverage -and consistent review processes: +Organize review-sets using these standard patterns to ensure comprehensive coverage +while keeping each review manageable in scope: + +**Note**: File path patterns shown below use C# naming conventions (PascalCase, `.cs` extensions). +Other languages should adapt these patterns to their conventions (e.g., C++ might use +`snake_case` with `.cpp`/`.hpp` extensions). + +## `Purpose` Review (only one per repository) + +Reviews user-facing capabilities and system promises: + +- **Purpose**: Proves that the systems provide the capabilities the user is being told about +- **Title**: "Review that Advertised Features Match System Design" +- **Scope**: Excludes subsystem and unit files, relying on system-level design documents + to describe what subsystems and units they use +- **File Path Patterns**: + - README: `README.md` + - User guide: `docs/user_guide/**/*.md` + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Design introduction: `docs/design/introduction.md` + - System design: `docs/design/{system-name}/{system-name}.md` -## [System]-Architecture Review (one per system) +## `{System}-Architecture` Review (one per system) Reviews system architecture and operational validation: -- **Files**: System requirements (`docs/reqstream/{system-name}/{system-name}.yaml`), design introduction - (`docs/design/introduction.md`), system design (`docs/design/{system-name}/{system-name}.md`), - integration tests -- **Purpose**: Validates system operates as designed and meets overall requirements -- **Example**: `SomeSystem-Architecture` +- **Purpose**: Proves that the system is designed and tested to satisfy its requirements +- **Title**: "Review that {System} Architecture Satisfies Requirements" +- **Scope**: Excludes subsystem and unit files, relying on system-level design to describe + what subsystems and units it uses +- **File Path Patterns**: + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Design introduction: `docs/design/introduction.md` + - System design: `docs/design/{system-name}/{system-name}.md` + - System integration tests: `test/{SystemName}.Tests/{SystemName}Tests.cs` -## [System]-Design Review +## `{System}-Design` Review (one per system) Reviews architectural and design consistency: -- **Files**: System requirements, platform requirements, all design documents under `docs/design/` -- **Purpose**: Ensures design completeness and architectural coherence -- **Example**: `SomeSystem-Design` +- **Purpose**: Proves the system design is consistent and complete +- **Title**: "Review that {System} Design is Consistent and Complete" +- **Scope**: Only brings in top-level requirements and relies on brevity of design documentation +- **File Path Patterns**: + - System requirements: `docs/reqstream/{system-name}/{system-name}.yaml` + - Platform requirements: `docs/reqstream/{system-name}/platform-requirements.yaml` + - Design introduction: `docs/design/introduction.md` + - System design files: `docs/design/{system-name}/**/*.md` -## [System]-AllRequirements Review +## `{System}-AllRequirements` Review (one per system) Reviews requirements quality and traceability: -- **Files**: All requirement files including root `requirements.yaml` and all files under `docs/reqstream/{system-name}/` -- **Purpose**: Validates requirements structure, IDs, justifications, and test linkage -- **Example**: `SomeSystem-AllRequirements` +- **Purpose**: Proves the requirements are consistent and complete +- **Title**: "Review that All {System} Requirements are Complete" +- **Scope**: Only brings in requirements files to keep review manageable +- **File Path Patterns**: + - Root requirements: `requirements.yaml` + - System requirements: `docs/reqstream/{system-name}/**/*.yaml` + - OTS requirements: `docs/reqstream/ots/**/*.yaml` (if applicable) -## [System]-[Subsystem] Review +## `{System}-{Subsystem}` Review (one per subsystem) Reviews subsystem architecture and interfaces: -- **Files**: Subsystem requirements, design documents, integration tests (usually no source code) -- **Purpose**: Validates subsystem behavior and interface compliance -- **File Path Pattern**: +- **Purpose**: Proves that the subsystem is designed and tested to satisfy its requirements +- **Title**: "Review that {System} {Subsystem} Satisfies Subsystem Requirements" +- **Scope**: Excludes units under the subsystem, relying on subsystem design to describe + what units it uses +- **File Path Patterns**: - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{subsystem-name}.yaml` - Design: `docs/design/{system-name}/{subsystem-name}/{subsystem-name}.md` - - Tests: `test/{SystemName}.Tests/{SubsystemName}/{SubsystemName}*` or similar -- **Example**: `SomeSystem-Authentication`, `SomeSystem-DataLayer` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{SubsystemName}Tests.cs` -## [System]-[Subsystem]-[Unit] Review +## `{System}-{Subsystem}-{Unit}` Review (one per unit) Reviews individual software unit implementation: -- **Files**: Unit requirements, design documents, source code, unit tests -- **Purpose**: Validates unit meets requirements and is properly implemented -- **File Path Pattern**: - - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{unit-name}.yaml` or `docs/reqstream/{system-name}/{unit-name}.yaml` - - Design: `docs/design/{system-name}/{subsystem-name}/{unit-name}.md` or `docs/design/{system-name}/{unit-name}.md` - - Source: `src/{SystemName}/{SubsystemName}/{UnitName}.cs` - - Tests: `test/{SystemName}.Tests/{SubsystemName}/{UnitName}Tests.cs` -- **Example**: `SomeSystem-Authentication-PasswordValidator`, `SomeSystem-DataLayer-ConfigParser` +- **Purpose**: Proves the unit is designed, implemented, and tested to satisfy its requirements +- **Title**: "Review that {System} {Subsystem} {Unit} Implementation is Correct" +- **Scope**: Complete unit review including all artifacts +- **File Path Patterns**: + - Requirements: `docs/reqstream/{system-name}/{subsystem-name}/{unit-name}.yaml` or + `docs/reqstream/{system-name}/{unit-name}.yaml` + - Design: `docs/design/{system-name}/{subsystem-name}/{unit-name}.md` or + `docs/design/{system-name}/{unit-name}.md` + - Source: `src/{SystemName}/{SubsystemName}/{UnitName}.cs` or `src/{SystemName}/{UnitName}.cs` + - Tests: `test/{SystemName}.Tests/{SubsystemName}/{UnitName}Tests.cs` or + `test/{SystemName}.Tests/{UnitName}Tests.cs` # Quality Checks Before submitting ReviewMark configuration, verify: - [ ] `.reviewmark.yaml` exists at repository root with proper structure -- [ ] `needs-review` patterns cover requirements, design, code, and tests with proper exclusions -- [ ] Each review-set has unique `id` and groups architecturally related files +- [ ] Review-set organization follows the standard hierarchy patterns +- [ ] Purpose review-set includes README.md, user guide, system requirements, design introduction, and system design files +- [ ] System-level reviews follow hierarchical scope principle (exclude subsystem/unit details) +- [ ] Subsystem reviews follow hierarchical scope principle (exclude unit source code) +- [ ] Only unit reviews include actual source code files +- [ ] Each review-set focuses on a single compliance question (single focus principle) - [ ] File patterns use correct glob syntax and match intended files -- [ ] File paths reflect current naming conventions (kebab-case design/requirements folders, PascalCase source folders) +- [ ] Review-set file counts remain manageable (context management principle) - [ ] Evidence source properly configured (`none` for dev, `url` for production) -- [ ] Environment variables used for credentials (never hardcoded) -- [ ] Generated documents accessible for compliance auditing -- [ ] Review-set organization follows standard patterns ([System]-[Subsystem], [System]-Design, etc.) diff --git a/.github/standards/software-items.md b/.github/standards/software-items.md index 7991add..ce7e328 100644 --- a/.github/standards/software-items.md +++ b/.github/standards/software-items.md @@ -18,6 +18,11 @@ Categorize all software into four primary groups: - **OTS Software Item**: Third-party component (library, framework, tool) providing functionality not developed in-house +**Naming**: When names collide in hierarchy, add descriptive suffix to higher-level entity: + +- System: Application/Library/System (e.g. TestResults → TestResultsLibrary) +- Subsystem: Subsystem (e.g. Linter → LinterSubsystem) + # Categorization Guidelines Choose the appropriate category based on scope and testability: diff --git a/.reviewmark.yaml b/.reviewmark.yaml index 1143e73..ebbe1b4 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -6,8 +6,7 @@ # Patterns identifying all files that require review. # Processed in order; prefix a pattern with '!' to exclude. needs-review: - - "**/*.cs" # All C# test files - - "!src/**/*.cs" # Source code reviewed through code review process + - "**/*.cs" # All C# source and test files - "requirements.yaml" # Root requirements file - "docs/reqstream/**/*.yaml" # Requirements files - "docs/design/**/*.md" # Design documents @@ -23,62 +22,75 @@ evidence-source: location: https://raw.githubusercontent.com/demaconsulting/SarifMark/reviews/index.json # Review sets grouping files by logical unit of review. -# Subsystem reviews cover the externally-visible behaviour of each feature subsystem: -# subsystem requirements + subsystem design + subsystem test suite. -# Software unit reviews cover the internal design of each individual class: -# unit requirements + unit design + unit test suite. +# Purpose review: proves advertised features match the system design. +# Architecture reviews: prove the system satisfies its requirements. +# Design review: proves design is consistent and complete. +# AllRequirements review: proves requirements are consistent and complete. +# Subsystem reviews: prove each subsystem satisfies its requirements. +# Unit reviews: prove each unit is designed, implemented, and tested correctly. reviews: # --------------------------------------------------------------------------- - # Architecture Review + # Purpose Review # --------------------------------------------------------------------------- - - id: SarifMark-Architecture - title: Review of SarifMark Architecture + - id: SarifMark-Purpose + title: Review that Advertised Features Match System Design paths: + - "README.md" + - "docs/user_guide/**/*.md" - "docs/reqstream/sarifmark/sarifmark.yaml" - - "docs/reqstream/sarifmark/platform-requirements.yaml" - "docs/design/introduction.md" - "docs/design/sarifmark/sarifmark.md" - - "test/**/IntegrationTests.cs" - - "test/**/Runner.cs" - - "test/**/AssemblyInfo.cs" # --------------------------------------------------------------------------- - # All Requirements Review + # Architecture Review # --------------------------------------------------------------------------- - - id: SarifMark-AllRequirements - title: Review of All SarifMark Requirements + - id: SarifMark-Architecture + title: Review that SarifMark Architecture Satisfies Requirements paths: - - "requirements.yaml" - - "docs/reqstream/**/*.yaml" + - "docs/reqstream/sarifmark/sarifmark.yaml" + - "docs/design/introduction.md" + - "docs/design/sarifmark/sarifmark.md" + - "test/**/IntegrationTests.cs" + - "test/**/Runner.cs" + - "test/**/AssemblyInfo.cs" # --------------------------------------------------------------------------- # Design Review # --------------------------------------------------------------------------- - id: SarifMark-Design - title: Review of SarifMark Design Documentation + title: Review that SarifMark Design is Consistent and Complete paths: - - "README.md" - "docs/reqstream/sarifmark/sarifmark.yaml" - "docs/reqstream/sarifmark/platform-requirements.yaml" - "docs/design/**/*.md" + # --------------------------------------------------------------------------- + # All Requirements Review + # --------------------------------------------------------------------------- + + - id: SarifMark-AllRequirements + title: Review that All SarifMark Requirements are Complete + paths: + - "requirements.yaml" + - "docs/reqstream/**/*.yaml" + # --------------------------------------------------------------------------- # Subsystem Reviews # --------------------------------------------------------------------------- - id: SarifMark-Cli - title: Review of SarifMark Command-Line Interface Subsystem + title: Review that SarifMark Cli Satisfies Subsystem Requirements paths: - "docs/reqstream/sarifmark/cli/cli.yaml" - "docs/design/sarifmark/cli/cli.md" - "test/**/Cli/CliTests.cs" - id: SarifMark-Sarif - title: Review of SarifMark SARIF Reading Subsystem + title: Review that SarifMark Sarif Satisfies Subsystem Requirements paths: - "docs/reqstream/sarifmark/sarif/sarif.yaml" - "docs/reqstream/sarifmark/sarif/report.yaml" @@ -86,14 +98,14 @@ reviews: - "test/**/Sarif/SarifTests.cs" - id: SarifMark-SelfTest - title: Review of SarifMark Self-Validation Subsystem + title: Review that SarifMark SelfTest Satisfies Subsystem Requirements paths: - "docs/reqstream/sarifmark/self-test/self-test.yaml" - "docs/design/sarifmark/self-test/self-test.md" - "test/**/SelfTest/SelfTestTests.cs" - id: SarifMark-Utilities - title: Review of SarifMark Utilities Subsystem + title: Review that SarifMark Utilities Satisfies Subsystem Requirements paths: - "docs/reqstream/sarifmark/utilities/utilities.yaml" - "docs/design/sarifmark/utilities/utilities.md" @@ -104,43 +116,49 @@ reviews: # --------------------------------------------------------------------------- - id: SarifMark-Program - title: Review of SarifMark Program Software Unit + title: Review that SarifMark Program Implementation is Correct paths: - "docs/reqstream/sarifmark/program.yaml" - "docs/design/sarifmark/program.md" + - "src/**/Program.cs" - "test/**/ProgramTests.cs" - id: SarifMark-Cli-Context - title: Review of SarifMark Context Software Unit + title: Review that SarifMark Cli Context Implementation is Correct paths: - "docs/reqstream/sarifmark/cli/context.yaml" - "docs/design/sarifmark/cli/context.md" + - "src/**/Cli/Context.cs" - "test/**/Cli/ContextTests.cs" - id: SarifMark-Sarif-SarifResult - title: Review of SarifMark SarifResult Software Unit + title: Review that SarifMark Sarif SarifResult Implementation is Correct paths: - "docs/reqstream/sarifmark/sarif/sarif-result.yaml" - "docs/design/sarifmark/sarif/sarif-result.md" + - "src/**/Sarif/SarifResult.cs" - "test/**/Sarif/SarifResultsTests.cs" - id: SarifMark-Sarif-SarifResults - title: Review of SarifMark SarifResults Software Unit + title: Review that SarifMark Sarif SarifResults Implementation is Correct paths: - "docs/reqstream/sarifmark/sarif/sarif-results.yaml" - "docs/design/sarifmark/sarif/sarif-results.md" + - "src/**/Sarif/SarifResults.cs" - "test/**/Sarif/SarifResultsTests.cs" - id: SarifMark-SelfTest-Validation - title: Review of SarifMark Validation Software Unit + title: Review that SarifMark SelfTest Validation Implementation is Correct paths: - "docs/reqstream/sarifmark/self-test/validation.yaml" - "docs/design/sarifmark/self-test/validation.md" + - "src/**/SelfTest/Validation.cs" - "test/**/SelfTest/ValidationTests.cs" - id: SarifMark-Utilities-PathHelpers - title: Review of SarifMark PathHelpers Software Unit + title: Review that SarifMark Utilities PathHelpers Implementation is Correct paths: - "docs/reqstream/sarifmark/utilities/path-helpers.yaml" - "docs/design/sarifmark/utilities/path-helpers.md" + - "src/**/Utilities/PathHelpers.cs" - "test/**/Utilities/PathHelpersTests.cs" diff --git a/AGENTS.md b/AGENTS.md index 87fc5c7..b6668ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,7 +81,7 @@ Delegate to specialized agents only for specific scenarios: 1. **Markdown Auto-fix**: `npx markdownlint-cli2 --fix **/*.md` (fixes most markdown issues except line length) 2. **Dotnet Auto-fix**: `dotnet format` (reformats .NET languages) -3. **Run full check**: `lint.bat` (Windows) or `lint.sh` (Unix) +3. **Run full check**: `lint.bat` (Windows) or `lint.sh` (Unix) 4. **Fix remaining**: Address line length, spelling, YAML syntax manually 5. **Verify clean**: Re-run until 0 errors before quality validation diff --git a/docs/reqstream/sarifmark/cli/cli.yaml b/docs/reqstream/sarifmark/cli/cli.yaml index c29787f..36cbd4c 100644 --- a/docs/reqstream/sarifmark/cli/cli.yaml +++ b/docs/reqstream/sarifmark/cli/cli.yaml @@ -26,7 +26,6 @@ sections: - SarifMark-Context-VersionFlag tests: - Cli_VersionFlag_OutputsVersion - - Context_Create_VersionFlag_SetsVersionTrue - id: SarifMark-Cli-Help title: The tool shall display help information when requested. @@ -38,7 +37,6 @@ sections: - SarifMark-Context-HelpFlag tests: - Cli_HelpFlag_OutputsUsageInformation - - Context_Create_HelpFlag_SetsHelpTrue - id: SarifMark-Cli-Silent title: The tool shall support silent mode to suppress console output. @@ -71,7 +69,6 @@ sections: children: - SarifMark-Context-EnforceFlag tests: - - SarifMark_Enforcement - Cli_EnforceFlagWithIssues_ReturnsError - id: SarifMark-Cli-InvalidArgs diff --git a/docs/reqstream/sarifmark/sarif/report.yaml b/docs/reqstream/sarifmark/sarif/report.yaml index 2739946..bcb61f9 100644 --- a/docs/reqstream/sarifmark/sarif/report.yaml +++ b/docs/reqstream/sarifmark/sarif/report.yaml @@ -14,7 +14,6 @@ sections: - SarifMark-SarifResults-ToMarkdown tests: - Sarif_GenerateReport_CreatesReportFile - - SarifMark_MarkdownReportGeneration - id: SarifMark-Report-Depth title: The tool shall support configurable markdown heading depth. @@ -36,8 +35,7 @@ sections: children: - SarifMark-SarifResults-FormatCount tests: - - SarifResults_ToMarkdown_NoResults_ShowsFoundNoResults - - SarifResults_ToMarkdown_OneResult_UsesSingularForm + - Sarif_Report_ContainsResultCount - id: SarifMark-Report-Locations title: The tool shall display location information for results. @@ -48,8 +46,7 @@ sections: children: - SarifMark-SarifResults-FormatLocation tests: - - SarifResults_ToMarkdown_ResultWithoutLocation_ShowsNoLocation - - SarifResults_ToMarkdown_ResultWithUriNoLine_ShowsUriOnly + - Sarif_Report_ContainsLocationInfo - id: SarifMark-Report-Headings title: The tool shall support custom headings in reports. @@ -60,10 +57,7 @@ sections: children: - SarifMark-SarifResults-ToMarkdown tests: - - SarifResults_ToMarkdown_CustomHeading_UsesProvidedHeading - - SarifResults_ToMarkdown_NullHeading_UsesDefaultHeading - - SarifResults_ToMarkdown_NoHeadingParameter_UsesDefaultHeading - - Context_Create_HeadingArgument_SetsHeading + - Sarif_Report_UsesCustomHeading - id: SarifMark-Report-LineBreaks title: The tool shall format multiple results with proper line breaks. @@ -74,4 +68,4 @@ sections: children: - SarifMark-SarifResults-ToMarkdown tests: - - SarifResults_ToMarkdown_MultipleResults_EnforcesLineBreaks + - Sarif_GenerateReport_FormatsMultipleResultsWithLineBreaks diff --git a/docs/reqstream/sarifmark/sarif/sarif.yaml b/docs/reqstream/sarifmark/sarif/sarif.yaml index 43ef483..e303112 100644 --- a/docs/reqstream/sarifmark/sarif/sarif.yaml +++ b/docs/reqstream/sarifmark/sarif/sarif.yaml @@ -13,7 +13,6 @@ sections: children: - SarifMark-SarifResults-ParseResults tests: - - SarifMark_SarifReading - Sarif_ValidSarifFile_ProcessesSuccessfully - id: SarifMark-Sarif-Validation @@ -26,10 +25,8 @@ sections: - SarifMark-SarifResults-ValidatePath - SarifMark-SarifResults-ValidateStructure tests: - - SarifResults_Read_InvalidJson_ThrowsInvalidOperationException - - SarifResults_Read_MissingVersion_ThrowsInvalidOperationException - - SarifResults_Read_MissingRuns_ThrowsInvalidOperationException - - SarifResults_Read_EmptyRuns_ThrowsInvalidOperationException + - Sarif_NonExistentSarifFile_ShowsError + - Sarif_InvalidSarifFile_ShowsFormatError - id: SarifMark-Sarif-ToolInfo title: The tool shall extract tool information from SARIF files. @@ -41,10 +38,7 @@ sections: - SarifMark-SarifResults-ExtractTool - SarifMark-SarifResults-VersionPriority tests: - - SarifResults_Read_MissingTool_ThrowsInvalidOperationException - - SarifResults_Read_MissingDriver_ThrowsInvalidOperationException - - SarifResults_Read_MissingToolName_UsesUnknown - - SarifResults_Read_MissingToolVersion_UsesUnknown + - Sarif_ValidSarifFile_ProcessesSuccessfully - id: SarifMark-Sarif-Results title: The tool shall extract results from SARIF files. @@ -56,9 +50,7 @@ sections: - SarifMark-SarifResults-ParseResults - SarifMark-SarifResults-FilterSuppressions tests: - - SarifResults_Read_NoResults_ReturnsValidResults - - SarifResults_Read_EmptyResults_ReturnsValidResults - - SarifResults_Read_WithResults_ReturnsValidResults + - Sarif_ValidSarifFile_ProcessesSuccessfully - id: SarifMark-Sarif-Locations title: The tool shall extract location information from SARIF results. @@ -70,7 +62,7 @@ sections: - SarifMark-SarifResult-Uri - SarifMark-SarifResult-StartLine tests: - - SarifResults_Read_WithLocations_ReturnsResultsWithLocationData + - Sarif_Report_ContainsLocationInfo - id: SarifMark-Sarif-FilePaths title: The tool shall require valid file paths for SARIF input. @@ -81,10 +73,8 @@ sections: children: - SarifMark-SarifResults-ValidatePath tests: - - SarifResults_Read_NullPath_ThrowsArgumentException - - SarifResults_Read_EmptyPath_ThrowsArgumentException - - SarifResults_Read_WhitespacePath_ThrowsArgumentException - - SarifResults_Read_NonExistentFile_ThrowsFileNotFoundException + - Sarif_MissingSarifParameter_ShowsError + - Sarif_NonExistentSarifFile_ShowsError - id: SarifMark-Sarif-Required title: The tool shall require the --sarif parameter for analysis. diff --git a/docs/reqstream/sarifmark/self-test/self-test.yaml b/docs/reqstream/sarifmark/self-test/self-test.yaml index 64cf3ff..19dfac9 100644 --- a/docs/reqstream/sarifmark/self-test/self-test.yaml +++ b/docs/reqstream/sarifmark/self-test/self-test.yaml @@ -24,8 +24,8 @@ sections: children: - SarifMark-Validation-ResultsFile tests: - - Validation_Run_WithTrxResultsFile_WritesResultsFile - - Validation_Run_WithXmlResultsFile_WritesResultsFile + - SelfTest_ResultsFile_WritesTrxFile + - SelfTest_ResultsFile_WritesJUnitFile - id: SarifMark-Validate-TrxFormat title: The tool shall support TRX format for test results. @@ -36,7 +36,7 @@ sections: children: - SarifMark-Validation-ResultsFile tests: - - Validation_Run_WithTrxResultsFile_WritesResultsFile + - SelfTest_ResultsFile_WritesTrxFile - id: SarifMark-Validate-JUnitFormat title: The tool shall support JUnit format for test results. @@ -47,7 +47,7 @@ sections: children: - SarifMark-Validation-ResultsFile tests: - - Validation_Run_WithXmlResultsFile_WritesResultsFile + - SelfTest_ResultsFile_WritesJUnitFile - title: Quality Enforcement requirements: @@ -60,8 +60,7 @@ sections: children: - SarifMark-Validation-EnforcementTest tests: - - Cli_EnforceFlagWithIssues_ReturnsError - - SarifMark_Enforcement + - SelfTest_EnforceFlag_ReturnsNonZeroOnIssues - id: SarifMark-Enforce-ExitCode title: The tool shall return non-zero exit code when issues found in enforcement mode. @@ -72,5 +71,4 @@ sections: children: - SarifMark-Validation-EnforcementTest tests: - - Cli_EnforceFlagWithIssues_ReturnsError - - SarifMark_Enforcement + - SelfTest_EnforceFlag_ReturnsNonZeroOnIssues diff --git a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs index a54ab32..d55e948 100644 --- a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs @@ -185,4 +185,184 @@ public void Sarif_ReportDepth_IsConfigurable() } } } + + /// + /// Test that processing an invalid SARIF file shows a format error. + /// + [TestMethod] + public void Sarif_InvalidSarifFile_ShowsFormatError() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "invalid.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--sarif", sarifFile); + + // Assert + Assert.AreEqual(1, exitCode); + Assert.Contains("Error:", output); + } + + /// + /// Test that a generated report formats multiple results with proper line breaks. + /// + [TestMethod] + public void Sarif_GenerateReport_FormatsMultipleResultsWithLineBreaks() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "multi-result.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-multi-report-{Guid.NewGuid()}.md"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--report", reportFile); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("Found 2 issues", reportContent); + Assert.Contains("first.cs", reportContent); + Assert.Contains("second.cs", reportContent); + + // Verify results appear on separate lines with proper markdown line breaks + Assert.MatchesRegex(@"first\.cs.* \r?\nfile:///path/to/second\.cs", reportContent); + } + finally + { + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that a generated report contains result count information. + /// + [TestMethod] + public void Sarif_Report_ContainsResultCount() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-count-report-{Guid.NewGuid()}.md"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--report", reportFile); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("Found 1 issue", reportContent); + } + finally + { + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that a generated report contains location information for results. + /// + [TestMethod] + public void Sarif_Report_ContainsLocationInfo() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-location-report-{Guid.NewGuid()}.md"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--report", reportFile); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("file:///path/to/file.cs", reportContent); + } + finally + { + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } + + /// + /// Test that a generated report uses a custom heading when provided. + /// + [TestMethod] + public void Sarif_Report_UsesCustomHeading() + { + // Arrange + var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-heading-report-{Guid.NewGuid()}.md"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--report", reportFile, + "--heading", "Custom Analysis Heading"); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + + var reportContent = File.ReadAllText(reportFile); + Assert.Contains("Custom Analysis Heading", reportContent); + } + finally + { + if (File.Exists(reportFile)) + { + File.Delete(reportFile); + } + } + } } diff --git a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs index a3373b5..4b268fc 100644 --- a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs @@ -60,4 +60,100 @@ public void SelfTest_ValidateFlag_RunsSelfValidation() Assert.Contains("SarifMark version", output); Assert.Contains("Total Tests:", output); } + + /// + /// Test that validate flag with TRX results parameter writes a TRX file. + /// + [TestMethod] + public void SelfTest_ResultsFile_WritesTrxFile() + { + // Arrange + var resultsFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.trx"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--validate", + "--results", resultsFile); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(resultsFile), "TRX results file was not created"); + + var content = File.ReadAllText(resultsFile); + Assert.Contains(" + /// Test that enforce flag returns non-zero exit code when issues are found. + /// + [TestMethod] + public void SelfTest_EnforceFlag_ReturnsNonZeroOnIssues() + { + // Arrange + var baseDir = AppContext.BaseDirectory; + var sarifFile = PathHelpers.SafePathCombine( + PathHelpers.SafePathCombine(baseDir, "TestData"), + "sample.sarif"); + Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + + // Act + var exitCode = Runner.Run( + out var output, + "dotnet", + _dllPath, + "--sarif", sarifFile, + "--enforce"); + + // Assert + Assert.AreEqual(1, exitCode); + Assert.Contains("Issues found in SARIF file", output); + } + + /// + /// Test that validate flag with JUnit XML results parameter writes a JUnit XML file. + /// + [TestMethod] + public void SelfTest_ResultsFile_WritesJUnitFile() + { + // Arrange + var resultsFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.xml"); + + try + { + // Act + var exitCode = Runner.Run( + out _, + "dotnet", + _dllPath, + "--validate", + "--results", resultsFile); + + // Assert + Assert.AreEqual(0, exitCode); + Assert.IsTrue(File.Exists(resultsFile), "JUnit XML results file was not created"); + + var content = File.ReadAllText(resultsFile); + Assert.Contains("