diff --git a/.reviewmark.yaml b/.reviewmark.yaml index c026427..79c746b 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -24,6 +24,16 @@ evidence-source: # Each review-set groups requirements, source, and tests for a coherent software unit # so that an AI-assisted review can verify consistency across the full evidence chain. reviews: + # System-level review — requirements, design, integration tests, and platform support + - id: ReqStream-System + title: Review of ReqStream System-Level Design and Requirements + paths: + - "docs/reqstream/reqstream-system.yaml" + - "docs/reqstream/platform-requirements.yaml" + - "docs/design/introduction.md" + - "docs/design/system.md" + - "test/**/IntegrationTests.cs" + # Software unit reviews - one per class - id: ReqStream-Context title: Review of ReqStream Context Unit @@ -78,19 +88,8 @@ reviews: - "src/**/Linter.cs" - "test/**/LinterTests.cs" - # Platform and OTS dependency reviews - - id: Platform-Support - title: Review of Platform and Runtime Support Requirements - paths: - - "docs/reqstream/platform-requirements.yaml" - - - id: OTS-Dependencies - title: Review of Off-The-Shelf Software Dependencies + # Full requirements review — all requirements files reviewed together for completeness and consistency + - id: ReqStream-AllRequirements + title: Review of All ReqStream Requirements for Consistency paths: - - "docs/reqstream/ots-mstest.yaml" - - "docs/reqstream/ots-reqstream.yaml" - - "docs/reqstream/ots-buildmark.yaml" - - "docs/reqstream/ots-versionmark.yaml" - - "docs/reqstream/ots-sarifmark.yaml" - - "docs/reqstream/ots-sonarmark.yaml" - - "docs/reqstream/ots-reviewmark.yaml" + - "docs/reqstream/**/*.yaml" diff --git a/README.md b/README.md index f8e281c..1f115b4 100644 --- a/README.md +++ b/README.md @@ -166,19 +166,19 @@ with sections, requirements, test mappings, and file includes. sections: - title: System Security requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support credentials authentication. justification: | Authentication is critical to ensure only authorized users can access the system. This requirement establishes the foundation for our security posture. children: # Support linking to child requirements - - AUTH-001 + - Auth-CredentialValidation - title: Data Management sections: - title: User Authentication requirements: - - id: AUTH-001 + - id: Auth-CredentialValidation title: All requests shall have their credentials authenticated before being processed. justification: | Prevents unauthorized access to system resources and ensures compliance with @@ -190,7 +190,7 @@ sections: - title: Logging requirements: - - id: DATA-001 + - id: Logging-RequestLogging title: All requests shall be logged. # Include other requirement files - may contain requirements and/or test mappings @@ -200,7 +200,7 @@ includes: # Test mappings support defining tests separate from requirements mappings: - - id: DATA-001 + - id: Logging-RequestLogging tests: - Logging_ValidRequest_Logged - Logging_InvalidRequest_Logged @@ -239,17 +239,17 @@ Test source linking allows requirements to specify which test results should com ```yaml requirements: - - id: PLAT-001 + - id: Platform-Windows title: Shall support Windows tests: - windows@Test_PlatformFeature # Matches only from files containing "windows" - - id: PLAT-002 + - id: Platform-Linux title: Shall support Linux tests: - ubuntu@Test_PlatformFeature # Matches only from files containing "ubuntu" - - id: PLAT-003 + - id: Platform-CrossPlatform title: Shall support cross-platform features tests: - Test_CrossPlatformFeature # Aggregates from all test result files @@ -275,13 +275,13 @@ Tags are defined as an optional list in the requirement definition: sections: - title: System Security requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support credentials authentication. tags: - security - critical - - id: SYS-SEC-002 + - id: Security-AuditLogging title: The system shall log all authentication attempts. tags: - security @@ -289,7 +289,7 @@ sections: - title: Performance requirements: - - id: PERF-001 + - id: Performance-ResponseTime title: The system shall respond within 100ms. tags: - performance @@ -442,7 +442,7 @@ Given requirements with justifications: sections: - title: Security requirements: - - id: SEC-001 + - id: Security-DataEncryption title: The system shall encrypt all data at rest. justification: | Data encryption at rest protects sensitive information from unauthorized access @@ -454,7 +454,7 @@ The exported justifications document would look like: ```markdown # Security -## SEC-001 +## Security-DataEncryption **The system shall encrypt all data at rest.** diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml index c6a8c53..9f50865 100644 --- a/docs/design/definition.yaml +++ b/docs/design/definition.yaml @@ -6,6 +6,7 @@ resource-path: input-files: - docs/design/title.txt - docs/design/introduction.md + - docs/design/system.md - docs/design/program.md - docs/design/context.md - docs/design/validation.md diff --git a/docs/design/introduction.md b/docs/design/introduction.md index a4c7900..212fb97 100644 --- a/docs/design/introduction.md +++ b/docs/design/introduction.md @@ -28,6 +28,32 @@ The following topics are out of scope: - Build pipeline configuration - Deployment and packaging +## Software Structure + +The following tree shows how the ReqStream software items are organized across the system, +subsystem, and unit levels: + +```text +ReqStream (System) +├── Program Orchestration (Subsystem) +│ ├── Command-Line Interface (Subsystem) +│ │ └── Context (Unit) +│ └── Program (Unit) +├── Requirements (Subsystem) +│ ├── Requirements File Processing (Subsystem) +│ │ ├── Requirements (Unit) +│ │ ├── Section (Unit) +│ │ └── Requirement (Unit) +│ ├── Test Integration (Subsystem) +│ │ └── TraceMatrix (Unit) +│ └── Linting (Subsystem) +│ └── Linter (Unit) +└── Validation (Subsystem) + └── Validation (Unit) +``` + +Each unit is described in detail in its own chapter within this document. + ## Document Conventions Throughout this document: diff --git a/docs/design/system.md b/docs/design/system.md new file mode 100644 index 0000000..55bb0fd --- /dev/null +++ b/docs/design/system.md @@ -0,0 +1,92 @@ +# System Integration Design + +## Overview + +This chapter describes how the ReqStream software units work together as an integrated system. +Where the unit chapters (Program, Context, Validation, Requirements, TraceMatrix, Linter) each +describe one component in isolation, this chapter focuses on the end-to-end data flow, the +coordination points between units, and the integrated scenarios that the units collectively +enable. + +## System Data Flow + +The following table shows the direction of data between units during a standard requirements +processing invocation: + +| Source | Data | Destination | +| ------ | ---- | ----------- | +| CLI arguments | Parsed options | `Context` | +| `Context.RequirementsFiles` | Glob-expanded file paths | `Requirements.Read` | +| `Requirements.Read` | Requirement tree | `Program.ProcessRequirements` | +| `Context.TestFiles` | Glob-expanded file paths | `TraceMatrix` constructor | +| Requirement tree | Requirements | `TraceMatrix` constructor | +| `TraceMatrix` | Coverage data | `Program.EnforceRequirementsCoverage` | +| `TraceMatrix` / requirement tree | Export input | Report files | + +## Integrated Processing Pipeline + +The following sequence describes the full pipeline executed during a normal (non-version, non-help, +non-validate, non-lint) invocation: + +1. **Argument parsing** — `Context.Create(args)` parses all CLI flags and expands any glob + patterns in `--requirements` and `--tests` arguments. +2. **Requirements loading** — `Requirements.Read(context.RequirementsFiles)` reads and merges all + YAML requirements files into a single requirement tree. Files listed via `includes` are resolved + recursively. +3. **Report generation** — if `--report` is set, the requirements report is exported. If + `--justifications` is set, the justifications report is exported. +4. **Test result loading** — if `--tests` is set, a `TraceMatrix` is constructed. It reads each + test result file (TRX or JUnit), applies source-specific matching rules, and maps each test + result to the requirements that reference it. +5. **Trace matrix export** — if `--matrix` is set and a `TraceMatrix` was constructed, the trace + matrix report is exported. +6. **Enforcement** — if `--enforce` is set and a `TraceMatrix` was constructed, + `EnforceRequirementsCoverage` compares the satisfied-requirement count against the total count. + Any unsatisfied requirement causes an error to be written to `context`, which results in a + non-zero exit code. + +## Source-Specific Test Matching + +When test results are collected from multiple platforms or configurations, each result file +typically carries a platform identifier in its file name (for example `windows-latest.trx` or +`ubuntu-latest.trx`). The `TraceMatrix` unit supports source-specific matching through the +`filepart@testname` syntax in requirement test lists: + +```text +tests: + - windows-latest@Test_WindowsOnlyFeature + - ubuntu@Test_LinuxFeature + - Test_CrossPlatformFeature +``` + +A `filepart@testname` entry matches only test result files whose names contain `filepart`. A plain +`testname` entry aggregates results from all files. This mechanism is used in ReqStream's own +requirements to enforce that platform-specific requirements are satisfied by evidence from the +correct platform. + +## Self-Validation Flow + +When `--validate` is specified, `Program.Run` delegates entirely to `Validation.Run(context)`. +`Validation` is self-contained: it creates temporary directories, writes fixture files, and invokes +the same `Program` methods used in normal processing. The self-validation path exercises the +integrated pipeline internally and produces structured test-result output in TRX or JUnit format +so that the evidence can be fed back into ReqStream's own requirements enforcement. + +## Interactions Between Units + +| Calling unit | Called unit | Call site | Purpose | +| ------------ | ----------- | --------- | ------- | +| `Program` | `Context` | `Main` | Parses CLI arguments; owns output and exit code | +| `Program` | `Validation` | `Run` | Runs self-validation suite when `--validate` is set | +| `Program` | `Linter` | `Run` | Lints requirements files when `--lint` is set | +| `Program` | `Requirements` | `ProcessRequirements` | Reads and merges YAML requirement files | +| `Program` | `TraceMatrix` | `ProcessRequirements` | Loads test results and maps them to requirements | +| `Validation` | `Program` | test methods | Invokes `Program.Run` to exercise the full pipeline | + +## References + +- [ReqStream Architecture][arch] +- [ReqStream Repository][repo] + +[arch]: ../../ARCHITECTURE.md +[repo]: https://github.com/demaconsulting/ReqStream diff --git a/docs/guide/guide.md b/docs/guide/guide.md index 670fc4c..f6071f1 100644 --- a/docs/guide/guide.md +++ b/docs/guide/guide.md @@ -158,7 +158,7 @@ A requirements YAML file has a top-level `sections` array: sections: - title: My Section requirements: - - id: REQ-001 + - id: Core-BasicRequirement title: My first requirement ``` @@ -171,18 +171,18 @@ Sections provide hierarchical organization. Sections can contain requirements an sections: - title: System Requirements requirements: - - id: SYS-001 + - id: System-TopLevel title: Top-level system requirement sections: - title: Security requirements: - - id: SEC-001 + - id: Security-AuthRequired title: Security requirement - title: Performance requirements: - - id: PERF-001 + - id: Performance-ResponseTime title: Performance requirement ``` @@ -206,7 +206,7 @@ Example: ```yaml requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support credentials authentication. justification: | Authentication is critical to ensure only authorized users can access the system. @@ -215,8 +215,8 @@ requirements: - security - critical children: - - AUTH-001 - - AUTH-002 + - Auth-CredentialValidation + - Auth-FailedAttemptLogging ``` ## Test Mappings @@ -227,7 +227,7 @@ Tests can be mapped to requirements in two ways: ```yaml requirements: - - id: AUTH-001 + - id: Auth-CredentialValidation title: All requests shall have their credentials authenticated before being processed. tests: - Credentials_Valid_Allowed @@ -241,11 +241,11 @@ requirements: sections: - title: Logging requirements: - - id: DATA-001 + - id: Logging-RequestLogging title: All requests shall be logged. mappings: - - id: DATA-001 + - id: Logging-RequestLogging tests: - Logging_ValidRequest_Logged - Logging_InvalidRequest_Logged @@ -268,19 +268,19 @@ from different sources. This is particularly useful for matrix testing scenarios ```yaml requirements: - - id: PLAT-001 + - id: Platform-Windows title: Shall support Windows operating systems tests: - "windows-latest@Test_PlatformBasic" - "windows-latest@Test_FileSystem" - - id: PLAT-002 + - id: Platform-Linux title: Shall support Linux operating systems tests: - "ubuntu-latest@Test_PlatformBasic" - "ubuntu-latest@Test_FileSystem" - - id: PLAT-003 + - id: Platform-CrossPlatform title: Shall support cross-platform APIs tests: - Test_CrossPlatformAPI # Aggregates from all platforms @@ -313,16 +313,16 @@ themes (e.g., security, performance, compliance) and generating focused reports sections: - title: System Requirements requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support credentials authentication. tags: - security - critical - - id: SYS-PERF-001 + - id: Performance-ResponseTime title: The system shall respond within 100ms. tags: - performance - - id: SYS-SEC-002 + - id: Security-DataEncryption title: The system shall encrypt data at rest. tags: - security @@ -383,7 +383,7 @@ Large projects can be split across multiple YAML files using the `includes` sect sections: - title: Core Requirements requirements: - - id: CORE-001 + - id: Core-Functional title: Core requirement includes: @@ -414,7 +414,7 @@ sections: sections: - title: Security requirements: - - id: SEC-001 + - id: Security-AuthRequired title: Authentication required ``` @@ -427,11 +427,12 @@ sections: sections: - title: Security requirements: - - id: SEC-002 + - id: Security-AuthorizationRequired title: Authorization required ``` -When both files are loaded, the "Security" section will contain both SEC-001 and SEC-002. +When both files are loaded, the "Security" section will contain both Security-AuthRequired and +Security-AuthorizationRequired. ## Complete Example @@ -444,34 +445,34 @@ Here's a comprehensive example showing all features: sections: - title: System Security requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support credentials authentication. children: - - "AUTH-001" - - "AUTH-002" + - "Auth-CredentialValidation" + - "Auth-FailedAttemptLogging" - title: Data Management sections: - title: User Authentication requirements: - - id: AUTH-001 + - id: Auth-CredentialValidation title: All requests shall have their credentials authenticated before being processed. tests: - Credentials_Valid_Allowed - Credentials_Invalid_Refused - Credentials_Missing_Refused - - id: AUTH-002 + - id: Auth-FailedAttemptLogging title: Failed authentication attempts shall be logged. tests: - Authentication_Failed_Logged - title: Logging requirements: - - id: DATA-001 + - id: Logging-RequestLogging title: All requests shall be logged with timestamp and user information. - - id: DATA-002 + - id: Logging-RetentionPolicy title: Logs shall be retained for at least 90 days. # Include additional requirements from other files @@ -481,14 +482,14 @@ includes: # Test mappings separate from requirements mappings: - - id: DATA-001 + - id: Logging-RequestLogging tests: - Logging_ValidRequest_Logged - Logging_InvalidRequest_Logged - Logging_ContainsTimestamp - Logging_ContainsUserInfo - - id: DATA-002 + - id: Logging-RetentionPolicy tests: - LogRetention_OldLogs_Retained - LogRetention_VeryOldLogs_Deleted @@ -624,7 +625,7 @@ reqstream --requirements "docs/**/*.yaml" --lint ```text docs/requirements/unit.yaml(42,5): error: Unknown field 'tittle' in requirement -docs/requirements/unit.yaml(57,13): error: Duplicate requirement ID 'REQ-001' (first seen in docs/requirements/base.yaml) +docs/requirements/unit.yaml(57,13): error: Duplicate requirement ID 'Core-BasicRequirement' (first seen in docs/requirements/base.yaml) docs/requirements/other.yaml(10,1): error: Section missing required field 'title' ``` @@ -772,19 +773,19 @@ reqstream --requirements "docs/**/*.yaml" --report requirements_report.md ```markdown # System Security -## SYS-SEC-001 +## Security-CredentialAuthentication The system shall support credentials authentication. Children: -- AUTH-001 -- AUTH-002 +- Auth-CredentialValidation +- Auth-FailedAttemptLogging # Data Management ## User Authentication -### AUTH-001 +### Auth-CredentialValidation All requests shall have their credentials authenticated before being processed. @@ -819,13 +820,13 @@ The trace matrix includes: ```markdown # Trace Matrix -## SYS-SEC-001: The system shall support credentials authentication. +## Security-CredentialAuthentication: The system shall support credentials authentication. No direct tests (parent requirement) -Child requirements: AUTH-001, AUTH-002 +Child requirements: Auth-CredentialValidation, Auth-FailedAttemptLogging -## AUTH-001: All requests shall have their credentials authenticated before being processed. +## Auth-CredentialValidation: All requests shall have their credentials authenticated before being processed. Tests: - ✓ Credentials_Valid_Allowed (Passed) @@ -859,14 +860,14 @@ The justifications report includes: ```markdown # System Security -## SYS-SEC-001 +## Security-CredentialAuthentication **The system shall support credentials authentication.** Authentication is critical to ensure only authorized users can access the system. This requirement establishes the foundation for our security posture. -## AUTH-001 +## Auth-CredentialValidation **All requests shall have their credentials authenticated before being processed.** @@ -977,19 +978,19 @@ For a requirement to be satisfied: sections: - title: System Security requirements: - - id: SYS-SEC-001 + - id: Security-CredentialAuthentication title: The system shall support authentication. children: - - "AUTH-001" - - "AUTH-002" + - "Auth-CredentialValidation" + - "Auth-FailedAttemptLogging" - - id: AUTH-001 + - id: Auth-CredentialValidation title: Users shall authenticate with username and password. tests: - Test_UsernamePassword_Valid - Test_UsernamePassword_Invalid - - id: AUTH-002 + - id: Auth-FailedAttemptLogging title: Failed authentication attempts shall be logged. tests: - Test_FailedAuth_Logged @@ -997,9 +998,10 @@ sections: In this example: -- `AUTH-001` is satisfied if both its tests pass -- `AUTH-002` is satisfied if its test passes -- `SYS-SEC-001` is satisfied transitively through its children (if both `AUTH-001` and `AUTH-002` are satisfied) +- `Auth-CredentialValidation` is satisfied if both its tests pass +- `Auth-FailedAttemptLogging` is satisfied if its test passes +- `Security-CredentialAuthentication` is satisfied transitively through its children (if both + `Auth-CredentialValidation` and `Auth-FailedAttemptLogging` are satisfied) ## Enforcement Output @@ -1174,15 +1176,15 @@ tests if they're satisfied through child requirements: ```yaml requirements: - - id: HIGH-LEVEL-001 + - id: Security-Overall title: System shall be secure children: - - "SEC-001" - - "SEC-002" - - "SEC-003" + - "Security-AuthRequired" + - "Security-AuthorizationRequired" + - "Security-DataProtection" # Children have direct tests - - id: SEC-001 + - id: Security-AuthRequired title: Authentication required tests: - Test_Auth_Required @@ -1227,12 +1229,12 @@ Ensure parent requirements reference their children via the `children` field: ```yaml requirements: - - id: PARENT-001 + - id: System-ParentRequirement title: Parent requirement children: - - "CHILD-001" # Add child references + - "System-ChildRequirement" # Add child references - - id: CHILD-001 + - id: System-ChildRequirement title: Child requirement tests: - Test_Child @@ -1276,8 +1278,8 @@ generating reports. A: ReqStream doesn't enforce a specific ID format. You can use any format that makes sense for your project: -- `REQ-001`, `REQ-002`, ... -- `SYS-001`, `AUTH-001`, ... +- `Core-BasicRequirement`, `Core-AnotherRequirement`, ... +- `System-TopLevel`, `Auth-CredentialValidation`, ... - `FR-1.1`, `FR-1.2`, ... The only requirement is that IDs must be unique across all requirements files. @@ -1334,12 +1336,12 @@ matches the base filename (without extension) of the test result file. For examp ```yaml requirements: - - id: WIN-001 + - id: Platform-Windows title: Shall support Windows tests: - "windows-latest@Test_PlatformFeature" # Matches only from files containing "windows-latest" - - id: LIN-001 + - id: Platform-Linux title: Shall support Linux tests: - "ubuntu-latest@Test_PlatformFeature" # Matches only from files containing "ubuntu-latest" diff --git a/docs/reqstream/ots-reqstream.yaml b/docs/reqstream/ots-reqstream.yaml deleted file mode 100644 index 1d38613..0000000 --- a/docs/reqstream/ots-reqstream.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -sections: - - title: ReqStream Requirements - sections: - - title: OTS Software - requirements: - - id: ReqStream-OTS-ReqStream - title: ReqStream shall enforce that every requirement is linked to passing test evidence. - justification: | - DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to - produce a requirements report, justifications document, and traceability matrix. When - run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, - making unproven requirements a build-breaking condition. A successful pipeline run with - --enforce proves all requirements are covered and that ReqStream is functioning. - tags: [ots] - tests: - - ReqStream_EnforcementMode diff --git a/docs/reqstream/reqstream-system.yaml b/docs/reqstream/reqstream-system.yaml new file mode 100644 index 0000000..ada41ce --- /dev/null +++ b/docs/reqstream/reqstream-system.yaml @@ -0,0 +1,40 @@ +--- +sections: + - title: ReqStream Requirements + sections: + - title: System Integration + requirements: + - id: ReqStream-Sys-FullPipeline + title: >- + The system shall process requirements files, load test results, generate reports, and + enforce coverage in a single invocation. + justification: | + Combining requirements processing, test result loading, report generation, and + enforcement in a single invocation is the primary CI/CD use case. This system-level + requirement verifies that all subsystems — Requirements, TraceMatrix, and Program — + work correctly when integrated together. + tests: + - ReqStream_FullPipeline_GeneratesAllReportsAndEnforces + + - id: ReqStream-Sys-SourceFilter + title: >- + The system shall support source-specific test matching to restrict coverage evidence + to tests from named result files. + justification: | + When running tests across multiple platforms or configurations, each producing + separate test result files, source-specific matching ensures that platform-specific + requirements are satisfied only by test evidence from the appropriate source files. + tests: + - ReqStream_SourceFilter_MatchesTestsBySourceFile + + - id: ReqStream-Sys-EnforceMode + title: >- + The system shall exit with a non-zero code when enforcement mode is active and any + requirement lacks passing test evidence. + justification: | + Enforcement mode makes unproven requirements a build-breaking condition, ensuring + that the CI/CD pipeline fails if any requirement is not covered by passing tests. + This validates both the positive case (all requirements satisfied, exit code 0) and + the negative case (unsatisfied requirement, non-zero exit code). + tests: + - ReqStream_EnforcementMode diff --git a/requirements.yaml b/requirements.yaml index 5141d24..e719b69 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -1,14 +1,19 @@ --- includes: - - docs/reqstream/unit-context.yaml + # System-level requirements + - docs/reqstream/reqstream-system.yaml + # Program Orchestration subsystem (including Command-Line Interface) - docs/reqstream/unit-program.yaml - - docs/reqstream/unit-validation.yaml + - docs/reqstream/unit-context.yaml + # Requirements subsystem (Requirements File Processing, Test Integration, Linting) - docs/reqstream/unit-requirements.yaml - docs/reqstream/unit-trace-matrix.yaml - docs/reqstream/unit-linter.yaml + # Validation subsystem + - docs/reqstream/unit-validation.yaml + # Platform support and OTS software - docs/reqstream/platform-requirements.yaml - docs/reqstream/ots-mstest.yaml - - docs/reqstream/ots-reqstream.yaml - docs/reqstream/ots-buildmark.yaml - docs/reqstream/ots-versionmark.yaml - docs/reqstream/ots-sarifmark.yaml diff --git a/test/DemaConsulting.ReqStream.Tests/IntegrationTests.cs b/test/DemaConsulting.ReqStream.Tests/IntegrationTests.cs new file mode 100644 index 0000000..2f0f9dd --- /dev/null +++ b/test/DemaConsulting.ReqStream.Tests/IntegrationTests.cs @@ -0,0 +1,206 @@ +// Copyright (c) 2026 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +namespace DemaConsulting.ReqStream.Tests; + +/// +/// Integration tests for the ReqStream system, exercising the full pipeline across +/// multiple subsystems in end-to-end scenarios. +/// +[TestClass] +public class IntegrationTests +{ + private string _testDirectory = string.Empty; + + /// + /// Initialize test by creating a temporary test directory. + /// + [TestInitialize] + public void TestInitialize() + { + _testDirectory = Path.Combine(Path.GetTempPath(), $"reqstream_test_{Guid.NewGuid()}"); + Directory.CreateDirectory(_testDirectory); + } + + /// + /// Clean up test by deleting the temporary test directory. + /// + [TestCleanup] + public void TestCleanup() + { + if (Directory.Exists(_testDirectory)) + { + Directory.Delete(_testDirectory, recursive: true); + } + } + + /// + /// Integration test verifying that a single invocation can process requirements, load test + /// results, generate a requirements report, justifications, and trace matrix, and enforce + /// coverage — all subsystems working together correctly. + /// + [TestMethod] + public void ReqStream_FullPipeline_GeneratesAllReportsAndEnforces() + { + // Arrange: create requirements file with one covered requirement + var reqFile = Path.Combine(_testDirectory, "requirements.yaml"); + File.WriteAllText(reqFile, """ + sections: + - title: System Requirements + requirements: + - id: Integration-System-DoSomethingUseful + title: The system shall do something useful. + justification: | + This is a test justification. + tests: + - IntegrationTest1 + """); + + // Arrange: create TRX file with a passing test result + var testResults = new DemaConsulting.TestResults.TestResults { Name = "IntegrationRun" }; + testResults.Results.Add(new DemaConsulting.TestResults.TestResult + { + Name = "IntegrationTest1", + ClassName = "IntegrationTests", + CodeBase = "Tests.dll", + Outcome = DemaConsulting.TestResults.TestOutcome.Passed, + Duration = TimeSpan.FromSeconds(1) + }); + var trxFile = Path.Combine(_testDirectory, "results.trx"); + File.WriteAllText(trxFile, DemaConsulting.TestResults.IO.TrxSerializer.Serialize(testResults)); + + var reportFile = Path.Combine(_testDirectory, "requirements.md"); + var justificationsFile = Path.Combine(_testDirectory, "justifications.md"); + var matrixFile = Path.Combine(_testDirectory, "matrix.md"); + + // Act: run the full pipeline + var originalDir = Directory.GetCurrentDirectory(); + try + { + Directory.SetCurrentDirectory(_testDirectory); + + using var context = Context.Create([ + "--requirements", "requirements.yaml", + "--tests", "results.trx", + "--report", reportFile, + "--justifications", justificationsFile, + "--matrix", matrixFile, + "--enforce" + ]); + Program.Run(context); + + // Assert: enforcement passed (exit code 0) + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Directory.SetCurrentDirectory(originalDir); + } + + // Assert: all three output files were generated + Assert.IsTrue(File.Exists(reportFile), "Requirements report should be generated."); + Assert.IsTrue(File.Exists(justificationsFile), "Justifications report should be generated."); + Assert.IsTrue(File.Exists(matrixFile), "Trace matrix report should be generated."); + + // Assert: report contains the requirement ID and title + var reportContent = File.ReadAllText(reportFile); + Assert.IsTrue(reportContent.Contains("Integration-System-DoSomethingUseful"), "Requirements report should contain the requirement ID."); + Assert.IsTrue(reportContent.Contains("The system shall do something useful."), + "Requirements report should contain the requirement title."); + + // Assert: trace matrix contains the satisfied requirement + var matrixContent = File.ReadAllText(matrixFile); + Assert.IsTrue(matrixContent.Contains("Integration-System-DoSomethingUseful"), "Trace matrix should contain the requirement ID."); + Assert.IsTrue(matrixContent.Contains("satisfied with tests"), "Trace matrix should show requirements as satisfied."); + } + + /// + /// Integration test verifying that source-specific test matching restricts coverage evidence + /// to tests from the named result file, and that enforcement passes when the named source + /// provides the required passing test. + /// + [TestMethod] + public void ReqStream_SourceFilter_MatchesTestsBySourceFile() + { + // Arrange: create requirements file with source-specific test reference + var reqFile = Path.Combine(_testDirectory, "requirements.yaml"); + File.WriteAllText(reqFile, """ + sections: + - title: Platform Requirements + requirements: + - id: Integration-PlatformA-SourceFilter + title: The system shall pass the platform-specific test on platform-a. + justification: | + Platform-specific behavior must be validated on the target platform. + tests: + - platform-a@PlatformTest1 + """); + + // Arrange: create platform-a.trx with a passing test + var platformAResults = new DemaConsulting.TestResults.TestResults { Name = "PlatformARun" }; + platformAResults.Results.Add(new DemaConsulting.TestResults.TestResult + { + Name = "PlatformTest1", + ClassName = "PlatformTests", + CodeBase = "Tests.dll", + Outcome = DemaConsulting.TestResults.TestOutcome.Passed, + Duration = TimeSpan.FromSeconds(1) + }); + var platformAFile = Path.Combine(_testDirectory, "platform-a.trx"); + File.WriteAllText(platformAFile, DemaConsulting.TestResults.IO.TrxSerializer.Serialize(platformAResults)); + + // Arrange: create platform-b.trx without the platform-a test + var platformBResults = new DemaConsulting.TestResults.TestResults { Name = "PlatformBRun" }; + platformBResults.Results.Add(new DemaConsulting.TestResults.TestResult + { + // Include the same test name as in platform-a.trx, but with a failing outcome, + // so the test validates that the source-specific filter (platform-a@) is honored. + Name = "PlatformTest1", + ClassName = "PlatformTests", + CodeBase = "Tests.dll", + Outcome = DemaConsulting.TestResults.TestOutcome.Failed, + Duration = TimeSpan.FromSeconds(1) + }); + var platformBFile = Path.Combine(_testDirectory, "platform-b.trx"); + File.WriteAllText(platformBFile, DemaConsulting.TestResults.IO.TrxSerializer.Serialize(platformBResults)); + + // Act: run enforcement using both result files + var originalDir = Directory.GetCurrentDirectory(); + try + { + Directory.SetCurrentDirectory(_testDirectory); + + using var context = Context.Create([ + "--requirements", "requirements.yaml", + "--tests", "platform-a.trx", + "--tests", "platform-b.trx", + "--enforce" + ]); + Program.Run(context); + + // Assert: enforcement passed because platform-a.trx satisfies the source-filtered requirement + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Directory.SetCurrentDirectory(originalDir); + } + } +}