diff --git a/.reviewmark.yaml b/.reviewmark.yaml index d964e3c..a2c7b54 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -7,7 +7,9 @@ # Processed in order; prefix a pattern with '!' to exclude. needs-review: - "**/*.cs" # All C# source and test files + - "requirements.yaml" # Root requirements file - "docs/reqstream/*.yaml" # Per-software-item requirements files + - "docs/design/*.md" # Software design documents - "!**/obj/**" # Exclude build output - "!**/bin/**" # Exclude build output @@ -30,51 +32,104 @@ reviews: - id: ReviewMark-Context title: Review of Context software unit (command-line argument handling) paths: - - "docs/reqstream/cli-requirements.yaml" # requirements + - "docs/reqstream/unit-context.yaml" # requirements + - "docs/design/context.md" # design - "src/**/Context.cs" # implementation - "test/**/ContextTests.cs" # tests - id: ReviewMark-GlobMatcher title: Review of GlobMatcher software unit (file pattern matching) paths: - - "src/**/GlobMatcher.cs" # implementation - - "test/**/GlobMatcherTests.cs" # tests + - "docs/reqstream/unit-glob-matcher.yaml" # requirements + - "docs/design/glob-matcher.md" # design + - "src/**/GlobMatcher.cs" # implementation + - "test/**/GlobMatcherTests.cs" # tests - - id: ReviewMark-Index - title: Review of Index software unit (review evidence indexing) + - id: ReviewMark-ReviewIndex + title: Review of ReviewIndex software unit (review evidence indexing) paths: - - "docs/reqstream/index-requirements.yaml" # requirements - - "src/**/Index.cs" # implementation + - "docs/reqstream/unit-review-index.yaml" # requirements + - "docs/design/review-index.md" # design + - "src/**/ReviewIndex.cs" # implementation - "test/**/IndexTests.cs" # tests - id: ReviewMark-PathHelpers title: Review of PathHelpers software unit (file path utilities) paths: - - "src/**/PathHelpers.cs" # implementation - - "test/**/PathHelpersTests.cs" # tests + - "docs/reqstream/unit-path-helpers.yaml" # requirements + - "docs/design/path-helpers.md" # design + - "src/**/PathHelpers.cs" # implementation + - "test/**/PathHelpersTests.cs" # tests - id: ReviewMark-Program title: Review of Program software unit (main entry point and tool orchestration) paths: - - "docs/reqstream/cli-requirements.yaml" # requirements - - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/reqstream/unit-program.yaml" # requirements + - "docs/design/program.md" # design - "docs/guide/guide.md" # user guide - "src/**/Program.cs" # implementation - "test/**/ProgramTests.cs" # unit tests - - "test/**/IntegrationTests.cs" # integration tests - - "test/**/Runner.cs" # test infrastructure - "test/**/TestDirectory.cs" # test infrastructure - - "test/**/AssemblyInfo.cs" # test infrastructure - id: ReviewMark-Configuration title: Review of ReviewMarkConfiguration software unit (configuration parsing and processing) paths: - - "docs/reqstream/configuration-requirements.yaml" # requirements - - "src/**/ReviewMarkConfiguration.cs" # implementation - - "test/**/ReviewMarkConfigurationTests.cs" # tests + - "docs/reqstream/subsystem-configuration.yaml" # requirements + - "docs/design/review-mark-configuration.md" # design + - "src/**/ReviewMarkConfiguration.cs" # implementation + - "test/**/ReviewMarkConfigurationTests.cs" # tests - id: ReviewMark-Validation title: Review of Validation software unit (self-validation test execution) paths: - - "docs/reqstream/ots-requirements.yaml" # OTS requirements verified by self-validation + - "docs/reqstream/unit-validation.yaml" # requirements + - "docs/design/validation.md" # design - "src/**/Validation.cs" # implementation + - "test/**/ValidationTests.cs" # tests + + # Special review-sets + - id: ReviewMark-System + title: Review of ReviewMark system-level behavior, platform support, and integration + paths: + - "docs/reqstream/reviewmark-system.yaml" # system requirements + - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/system.md" # system design + - "test/**/IntegrationTests.cs" # integration tests + - "test/**/Runner.cs" # test infrastructure + - "test/**/AssemblyInfo.cs" # test infrastructure + + - id: ReviewMark-Design + title: Review of all ReviewMark design documentation + paths: + - "docs/reqstream/platform-requirements.yaml" # platform requirements + - "docs/design/introduction.md" # design introduction and architecture + - "docs/design/system.md" # system design + - "docs/design/context.md" # Context design + - "docs/design/glob-matcher.md" # GlobMatcher design + - "docs/design/review-index.md" # ReviewIndex design + - "docs/design/path-helpers.md" # PathHelpers design + - "docs/design/program.md" # Program design + - "docs/design/review-mark-configuration.md" # ReviewMarkConfiguration design + - "docs/design/validation.md" # Validation design + + - id: ReviewMark-AllRequirements + title: Review of all ReviewMark requirements files + paths: + - "requirements.yaml" # root requirements file + - "docs/reqstream/reviewmark-system.yaml" # system-level requirements + - "docs/reqstream/subsystem-cli.yaml" # CLI subsystem requirements + - "docs/reqstream/subsystem-configuration.yaml" # Configuration subsystem requirements + - "docs/reqstream/unit-context.yaml" # Context unit requirements + - "docs/reqstream/unit-program.yaml" # Program unit requirements + - "docs/reqstream/unit-review-index.yaml" # ReviewIndex unit requirements + - "docs/reqstream/unit-glob-matcher.yaml" # GlobMatcher unit requirements + - "docs/reqstream/unit-path-helpers.yaml" # PathHelpers unit requirements + - "docs/reqstream/unit-validation.yaml" # Validation unit requirements + - "docs/reqstream/platform-requirements.yaml" # Platform support requirements + - "docs/reqstream/ots-mstest.yaml" # MSTest OTS requirements + - "docs/reqstream/ots-reqstream.yaml" # ReqStream OTS requirements + - "docs/reqstream/ots-buildmark.yaml" # BuildMark OTS requirements + - "docs/reqstream/ots-versionmark.yaml" # VersionMark OTS requirements + - "docs/reqstream/ots-sarifmark.yaml" # SarifMark OTS requirements + - "docs/reqstream/ots-sonarmark.yaml" # SonarMark OTS requirements diff --git a/docs/design/context.md b/docs/design/context.md new file mode 100644 index 0000000..eed5e1e --- /dev/null +++ b/docs/design/context.md @@ -0,0 +1,59 @@ +# Context + +## Purpose + +The `Context` software unit is responsible for parsing command-line arguments and +providing a unified interface for output and logging throughout the tool. It acts as +the primary configuration carrier, passing parsed options from the CLI entry point +to all processing subsystems. + +## Properties + +The following properties are populated by `Context.Create()` from the command-line +arguments: + +| Property | Type | Description | +| -------- | ---- | ----------- | +| `Version` | bool | Requests version display | +| `Help` | bool | Requests help display | +| `Silent` | bool | Suppresses console output | +| `Validate` | bool | Requests self-validation run | +| `Lint` | bool | Requests configuration linting | +| `ResultsFile` | string? | Path for TRX/JUnit test results output | +| `DefinitionFile` | string | Path to the `.reviewmark.yaml` configuration | +| `PlanFile` | string? | Output path for the Review Plan document | +| `PlanDepth` | int | Heading depth for the Review Plan | +| `ReportFile` | string? | Output path for the Review Report document | +| `ReportDepth` | int | Heading depth for the Review Report | +| `IndexPaths` | string[]? | Paths to scan when building an evidence index | +| `WorkingDirectory` | string | Base directory for resolving relative paths | +| `Enforce` | bool | Fail if any review-set is not Current | +| `Elaborate` | bool | Expand file lists in generated documents | + +## Argument Parsing + +`Context.Create(string[] args)` is a factory method that processes the argument +array sequentially, recognizing both flag arguments (e.g., `--validate`) and +value arguments (e.g., `--plan `). Unrecognized or unsupported arguments +cause `Context.ParseArgument` to throw an `ArgumentException`, which callers of +`Context.Create` are expected to handle and surface as a CLI error. The resulting +`Context` instance holds the fully parsed state when argument parsing succeeds. + +## Output Methods + +| Method | Description | +| ------ | ----------- | +| `WriteLine(string)` | Writes a line to the console (unless `Silent` is set) and to the log file | +| `WriteError(string)` | Writes an error line to the console and to the log file | + +## Exit Code + +`Context.ExitCode` reflects the current error status of the tool run. It is set to +a non-zero value when an error is detected. The value of `ExitCode` is returned from +`Program.Main()` as the process exit code. + +## Logging + +When a log file path is provided via the relevant CLI argument, `Context` opens and +holds the log file handle for the duration of the tool run. All output written through +`WriteLine` and `WriteError` is duplicated to the log file. diff --git a/docs/design/definition.yaml b/docs/design/definition.yaml new file mode 100644 index 0000000..1b115fd --- /dev/null +++ b/docs/design/definition.yaml @@ -0,0 +1,18 @@ +--- +resource-path: + - docs/design + - docs/template +input-files: + - docs/design/title.txt + - docs/design/introduction.md + - docs/design/system.md + - docs/design/context.md + - docs/design/glob-matcher.md + - docs/design/review-index.md + - docs/design/path-helpers.md + - docs/design/program.md + - docs/design/review-mark-configuration.md + - docs/design/validation.md +template: template.html +table-of-contents: true +number-sections: true diff --git a/docs/design/glob-matcher.md b/docs/design/glob-matcher.md new file mode 100644 index 0000000..71c9a1a --- /dev/null +++ b/docs/design/glob-matcher.md @@ -0,0 +1,32 @@ +# GlobMatcher + +## Purpose + +The `GlobMatcher` software unit resolves an ordered list of glob patterns into a +concrete, sorted list of file paths relative to a base directory. It provides the +file enumeration primitive used by the Configuration subsystem to expand the +`needs-review` and `review-set` file lists defined in `.reviewmark.yaml`. + +## Algorithm + +`GlobMatcher.GetMatchingFiles(baseDirectory, patterns)` processes patterns in the +order they are declared. Patterns prefixed with `!` are exclusion patterns; all +others are inclusion patterns. Each inclusion pattern adds matching paths to the +result set; each exclusion pattern removes matching paths from the result set. +Because patterns are applied in declaration order, a later pattern can re-include +files excluded by an earlier one, or exclude files included by an earlier one. The +`**` wildcard matches any number of path segments, enabling recursive matching. +After all patterns are processed, the result set is sorted and returned. + +## Return Value + +The method returns a sorted list of relative file paths. Path separators are +normalized to forward slashes regardless of the host operating system, ensuring +consistent fingerprint computation across platforms. + +## Usage + +`GlobMatcher.GetMatchingFiles()` is called by `ReviewMarkConfiguration` to resolve: + +- The `needs-review` file list, which represents all files subject to review +- Each `review-set` file list, which represents the files covered by a specific review record diff --git a/docs/design/introduction.md b/docs/design/introduction.md new file mode 100644 index 0000000..648be94 --- /dev/null +++ b/docs/design/introduction.md @@ -0,0 +1,56 @@ +# Introduction + +This document describes the software design for the ReviewMark project. + +## Purpose + +ReviewMark is a .NET command-line tool for automated file-review evidence management +in regulated environments. It computes cryptographic fingerprints of defined file-sets, +queries a review evidence store for corresponding review records, and produces compliance +documents on each CI/CD run. + +This design document describes the internal architecture, subsystems, and software units +that together implement the ReviewMark tool. It is intended to support development, +review, and maintenance activities. + +## Scope + +This design document covers: + +- The software system decomposition into subsystems and software units +- The responsibilities and interfaces of each software unit +- The algorithms and data flows used for fingerprinting, evidence lookup, and document generation +- The self-validation framework + +This document does not cover: + +- External CI/CD pipeline configuration +- Evidence store setup or administration +- Requirements traceability (see the Requirements Specification) + +## Software Architecture + +The following diagram shows the decomposition of the ReviewMark software system into +subsystems and software units. + +```text +ReviewMark (Software System) +├── CLI Subsystem +│ ├── Program (Software Unit) +│ └── Context (Software Unit) +├── Configuration Subsystem +│ ├── ReviewMarkConfiguration (Software Unit) +│ └── GlobMatcher (Software Unit) +├── Index Subsystem +│ ├── ReviewIndex (Software Unit) +│ └── PathHelpers (Software Unit) +└── Validation (Software Unit) +``` + +## Audience + +This document is intended for: + +- Software developers working on ReviewMark +- Quality assurance teams performing design verification +- Project stakeholders reviewing architectural decisions diff --git a/docs/design/path-helpers.md b/docs/design/path-helpers.md new file mode 100644 index 0000000..942cae2 --- /dev/null +++ b/docs/design/path-helpers.md @@ -0,0 +1,35 @@ +# PathHelpers + +## Purpose + +The `PathHelpers` software unit provides safe path construction utilities that +prevent path traversal attacks. It is used by the Index subsystem when constructing +file system paths to evidence PDF files referenced in the evidence index. + +## SafePathCombine() + +`PathHelpers.SafePathCombine(basePath, relativePath)` combines a trusted base path +with an untrusted relative path from the evidence index, validating that the result +does not escape the base directory. + +The validation steps are: + +1. Reject any relative path that contains `..` segments (explicit traversal attempt). +2. Reject any relative path that is rooted (absolute path supplied where a relative one is required). +3. Combine the base path and relative path. +4. Verify that the combined path still begins with the base path (catches edge cases + such as platform-specific path normalization that might otherwise bypass the + earlier checks). +5. Return the combined path. + +The double-check strategy (pre-validation of segments plus post-combination +verification) defends against edge cases such as URL-encoded separators or +platform-specific path normalization that might otherwise bypass a single check. + +## Security Rationale + +Evidence index files may be loaded from external sources (file shares or URLs). +The `file` field in each index record is supplied by the evidence store and must +be treated as untrusted input. Without path validation, a maliciously crafted +index could direct the tool to read or reference files outside the intended +evidence directory. `SafePathCombine` eliminates this attack surface. diff --git a/docs/design/program.md b/docs/design/program.md new file mode 100644 index 0000000..6d260cc --- /dev/null +++ b/docs/design/program.md @@ -0,0 +1,47 @@ +# Program + +## Purpose + +The `Program` software unit is the main entry point of the ReviewMark tool. It is +responsible for constructing the execution context, dispatching to the appropriate +processing logic based on parsed flags, and returning a meaningful exit code to the +calling process. + +## Version Property + +`Program.Version` returns the tool version string. The version is embedded at build +time from the assembly metadata and follows semantic versioning conventions. + +## Main() Method + +`Program.Main(string[] args)` is the process entry point. It: + +1. Constructs a `Context` instance via `Context.Create(args)` inside a `using` block +2. Calls `Program.Run(Context)` to perform the requested operation +3. Returns `Context.ExitCode` as the process exit code + +Any unexpected exception that escapes `Run()` is logged to the standard error stream +via `Console.Error` and then rethrown. As a result, the process terminates due to the +unhandled exception and the final exit code is determined by the .NET runtime rather +than by `Program.Main` explicitly returning a non-zero value. + +## Run() Dispatch Logic + +`Program.Run(Context)` evaluates the parsed flags in the following priority order, +executing the first matching action and returning: + +1. If `--version` — print version and return +2. If `--help` — print banner and return +3. If `--validate` — run self-validation and return +4. If `--lint` — run configuration lint and return +5. If `--index` paths provided — scan and write evidence index, then return +6. Otherwise — generate Review Plan and/or Review Report and return + +Only one top-level action is performed per invocation. Actions later in the priority +order are not reached if an earlier flag is set. + +## PrintBanner() + +`Program.PrintBanner(Context)` writes the help text to the console via +`Context.WriteLine()`. The banner lists all supported flags and arguments with brief +descriptions. diff --git a/docs/design/review-index.md b/docs/design/review-index.md new file mode 100644 index 0000000..1a3ef95 --- /dev/null +++ b/docs/design/review-index.md @@ -0,0 +1,83 @@ +# ReviewIndex + +## Purpose + +The `ReviewIndex` software unit manages the loading, querying, and creation of the review +evidence index. It abstracts the evidence store behind a uniform interface so that +the rest of the tool does not need to know whether evidence is stored on a fileshare, +served over HTTP, or absent entirely. + +## ReviewEvidence Record + +`ReviewEvidence` is an immutable record that holds the in-memory representation of a +single review record once the index has been loaded or scanned. + +| Property | Type | Description | +| -------- | ---- | ----------- | +| `Id` | string | The review-set identifier | +| `Fingerprint` | string | The SHA-256 fingerprint of the reviewed files | +| `Date` | string | The date of the review (e.g. `2026-02-14`) | +| `Result` | string | The review outcome (`pass` or `fail`) | +| `File` | string | The file name of the review evidence PDF | + +The `ReviewIndex` holds these records in a two-level +`Dictionary>` keyed first by `Id` and +then by `Fingerprint`, which enables O(1) lookup by both fields simultaneously. + +## Evidence Index Format + +The evidence index is a JSON file (`index.json`) containing an array of review records. +Each record has the following fields: + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `id` | string | Unique identifier for the review record (matches the review-set `id` in `.reviewmark.yaml`) | +| `fingerprint` | string | SHA-256 fingerprint of the file-set at time of review | +| `date` | string | Date the review was conducted | +| `result` | string | Review outcome (`pass` or `fail`) | +| `file` | string | Relative path to the PDF evidence file | + +## ReviewIndex.Load() + +`ReviewIndex.Load(EvidenceSource)` selects a loading strategy based on the evidence +source type: + +| Source Type | Behavior | +| ----------- | -------- | +| `none` | Returns an empty index (equivalent to `ReviewIndex.Empty()`) | +| `fileshare` | Reads `index.json` from the specified file path | +| `url` | Downloads `index.json` from the specified HTTP or HTTPS URL | + +## ReviewIndex.Scan() + +`ReviewIndex.Scan(directory, patterns)` scans a directory for PDF files matching +the given glob patterns. For each PDF file found, it reads embedded metadata to +extract the review record fields and returns a populated in-memory `ReviewIndex`. +The caller (e.g., `Program`) is responsible for choosing an output path and calling +`Save(...)` on the returned index to produce `index.json` as part of the `--index` +workflow. + +## ReviewIndex.Empty() + +`ReviewIndex.Empty()` returns an index with no records. It is used when the evidence +source type is `none`, resulting in all review-sets being reported as Missing. + +## ReviewIndex.GetStatus() + +`ReviewIndex.GetStatus(id, fingerprint)` determines the review status of a +review-set by looking up the `id` in the loaded index: + +1. Look up `id` in the index + - If not found — return `Missing` +2. Check if there is a record whose `Fingerprint` matches the supplied `fingerprint` + - If no matching fingerprint exists — return `Stale` + - If a matching fingerprint exists: + - If the `Result` is `pass` — return `Current` + - If the `Result` is not `pass` — return `Failed` + +| Status | Meaning | +| ------ | ------- | +| `Current` | The review record matches the current fingerprint and has a passing result | +| `Failed` | The review record matches the current fingerprint but the result is not passing | +| `Stale` | A record exists for the id but the fingerprint does not match the current one | +| `Missing` | No review record exists for the id | diff --git a/docs/design/review-mark-configuration.md b/docs/design/review-mark-configuration.md new file mode 100644 index 0000000..9692eb1 --- /dev/null +++ b/docs/design/review-mark-configuration.md @@ -0,0 +1,75 @@ +# ReviewMarkConfiguration + +## Purpose + +The `ReviewMarkConfiguration` software unit is responsible for parsing the +`.reviewmark.yaml` configuration file and performing all review-set processing. +It coordinates file enumeration, fingerprint computation, evidence lookup, and +the generation of the Review Plan and Review Report compliance documents. + +## Configuration Model + +The `.reviewmark.yaml` file is deserialized into the following model: + +| Class | Description | +| ----- | ----------- | +| `ReviewMarkYaml` | Root configuration object containing the evidence source and review list | +| `EvidenceSourceYaml` | Describes how to locate the evidence index (`type`, `location`, optional `credentials`) | +| `ReviewYaml` | Describes a single review-set (`id`, `title`, file patterns) | + +## ReviewMarkConfiguration.Load() + +`ReviewMarkConfiguration.Load(definitionFile, workingDirectory)` reads and +deserializes the YAML file, resolves all glob patterns relative to the working +directory, computes fingerprints for each review-set, loads the evidence index, +and returns a fully initialized configuration object ready for plan/report generation. + +## Fingerprinting Algorithm + +The fingerprint for a review-set uniquely identifies the exact content of its file-set. +The algorithm is: + +1. For each file in the review-set, read its contents and compute a SHA-256 hash. +2. Collect all per-file hashes and sort them lexicographically. +3. Concatenate the sorted hashes and compute a SHA-256 hash of the result. +4. Return the final hash as a hex string — this is the review-set fingerprint. + +Sorting the per-file hashes before combining them ensures that the fingerprint is +sensitive to content changes but not to the order in which files happen to be +enumerated by the operating system. + +## Review Plan Generation + +The Review Plan is generated by `ReviewMarkConfiguration.PublishReviewPlan()`. It produces +a Markdown document that lists every file in the `needs-review` file-set and, for +each file, identifies which review-sets provide coverage. + +- The `--plan-depth` argument controls the heading level used for sections +- The `--elaborate` flag expands the file list for each review-set inline + +## Review Report Generation + +The Review Report is generated by `ReviewMarkConfiguration.PublishReviewReport()`. It +produces a Markdown document that lists every review-set with its current status. + +For each review-set the report includes: + +- The review-set `id` and `title` +- The current fingerprint of the file-set +- The review status: `Current`, `Stale`, `Missing`, or `Failed` + +Status is determined by looking up the current fingerprint in the loaded evidence +index to establish whether a passing, failing, stale, or missing review result exists. + +- The `--report-depth` argument controls the heading level used for sections +- The `--elaborate` flag expands the list of files covered by each review-set + +## Linting + +`ReviewMarkConfiguration.Lint(Context)` validates the loaded configuration for +correctness. Lint checks include: + +- All review-set `id` values are unique +- All glob patterns resolve to at least one file +- The `needs-review` file-set is non-empty +- All files in the `needs-review` set are covered by at least one review-set diff --git a/docs/design/system.md b/docs/design/system.md new file mode 100644 index 0000000..0f37a4f --- /dev/null +++ b/docs/design/system.md @@ -0,0 +1,73 @@ +# System Design + +This section describes the high-level behavior of the ReviewMark system and the workflow +that connects its subsystems. + +## Overview + +ReviewMark automates the evidence-gathering step of software review processes used in +regulated environments. On each CI/CD run, it determines which files are subject to +review, identifies the review evidence that covers them, and generates two compliance +documents: a Review Plan and a Review Report. + +## Main Workflow + +The following steps describe the end-to-end processing flow. + +1. Parse CLI arguments +2. Load `.reviewmark.yaml` +3. Resolve file lists via glob patterns +4. Compute SHA-256 fingerprints +5. Load evidence index + - `none` — use an empty index (no evidence store configured) + - `fileshare` — load `index.json` from a local or network file path + - `url` — download `index.json` from an HTTP or HTTPS URL +6. Generate Review Plan and/or Review Report +7. If `--enforce` flag is set: + - If all review-sets are Current — return success + - Otherwise — return a non-zero exit code + +## Evidence Source Types + +ReviewMark supports three evidence source types, configured in `.reviewmark.yaml`: + +| Source Type | Description | +| ----------- | ----------- | +| `none` | No evidence store; all review-sets are treated as missing | +| `fileshare` | Evidence index loaded from a local or network file path | +| `url` | Evidence index loaded from an HTTP or HTTPS URL | + +## Output Documents + +### Review Plan + +The Review Plan lists every file that is subject to review and identifies which +review-sets provide coverage for each file. It is generated by the `--plan` flag +and written to a configurable output path. + +### Review Report + +The Review Report lists every review-set defined in the configuration, the current +fingerprint of its file-set, and the review status (Current, Stale, Missing, or Failed). +It is generated by the `--report` flag and written to a configurable output path. + +The statuses have the following meanings: + +- **Current** — Evidence exists for the current fingerprint and the recorded result is `pass`. +- **Stale** — Evidence exists, but it corresponds to an older fingerprint than the current one. +- **Missing** — No evidence exists for this review-set. +- **Failed** — Evidence exists for the current fingerprint, but the recorded result is not `pass`. + +## Enforcement + +When the `--enforce` flag is set, ReviewMark returns a non-zero exit code if any +review-set does not have Current status (i.e., is Stale, Missing, or Failed). This allows +CI/CD pipelines to fail builds when review coverage is incomplete, out of date, or has +failed results for the current fingerprint. + +## Index Management + +The `--index` flag causes ReviewMark to scan a directory for PDF evidence files and +write an `index.json` file suitable for use as a fileshare evidence source. This +supports workflows where review PDFs are stored alongside source code or on a +shared network location. diff --git a/docs/design/title.txt b/docs/design/title.txt new file mode 100644 index 0000000..d140ba3 --- /dev/null +++ b/docs/design/title.txt @@ -0,0 +1,13 @@ +--- +title: ReviewMark Design +subtitle: Software Design Document for ReviewMark +author: DEMA Consulting +description: Software Design Document for ReviewMark +lang: en-US +keywords: + - ReviewMark + - Design + - Software Architecture + - .NET + - Command-Line Tool +--- diff --git a/docs/design/validation.md b/docs/design/validation.md new file mode 100644 index 0000000..04ff878 --- /dev/null +++ b/docs/design/validation.md @@ -0,0 +1,44 @@ +# Validation + +## Purpose + +The `Validation` software unit implements the self-validation framework for +ReviewMark. Self-validation allows the tool to verify its own correct operation +in a target environment, which is a requirement for regulated deployment contexts +where the tool itself is part of a qualified software chain. + +## Validation.Run() + +`Validation.Run(Context)` orchestrates all self-validation tests. It: + +1. Creates a test suite using the `DemaConsulting.TestResults` library +2. Executes each test case in sequence +3. Writes results to the configured output file (TRX or JUnit format) if `ResultsFile` is set +4. Writes a summary table and per-test results to the console via `Context.WriteLine()` +5. Sets `Context.ExitCode` to a non-zero value if any test fails + +## Test Output Format + +Results are written using the `DemaConsulting.TestResults` library, which supports +both TRX (Visual Studio Test Results) and JUnit XML output formats. The output format +is inferred from the file extension of `ResultsFile`. + +## Test Coverage + +The self-validation suite covers the following scenarios: + +- **Version display**: Tool correctly reports its version +- **Help display**: Tool correctly displays help text +- **Plan generation**: Review Plan is generated correctly for a known configuration +- **Report generation**: Review Report is generated correctly for a known configuration +- **Index scanning**: Evidence index is created correctly by scanning a directory +- **Enforce mode**: Tool returns non-zero exit code when enforce mode detects uncovered review sets +- **Working directory override**: Relative paths are resolved correctly when the working directory is overridden +- **Elaborate mode**: File lists are expanded in generated documents when elaborate mode is active +- **Lint mode**: Configuration errors are detected correctly + +## Console Output + +In addition to the structured results file, `Validation.Run()` writes a human-readable +summary to the console. The summary includes a table of all tests with their pass/fail +status, followed by detailed output for any failing tests to aid diagnosis. diff --git a/docs/reqstream/ots-buildmark.yaml b/docs/reqstream/ots-buildmark.yaml new file mode 100644 index 0000000..d59a4a7 --- /dev/null +++ b/docs/reqstream/ots-buildmark.yaml @@ -0,0 +1,20 @@ +--- +# BuildMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the BuildMark off-the-shelf documentation generation tool +# - BuildMark generates build-notes documentation from GitHub Actions metadata + +sections: + - title: BuildMark OTS Requirements + requirements: + - id: ReviewMark-OTS-BuildMark + title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. + justification: | + DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and + renders them as a markdown build-notes document included in the release artifacts. + It runs as part of the same CI pipeline that produces the TRX test results, so a + successful pipeline run is evidence that BuildMark executed without error. + tags: [ots] + tests: + - BuildMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-mstest.yaml b/docs/reqstream/ots-mstest.yaml new file mode 100644 index 0000000..98dd61a --- /dev/null +++ b/docs/reqstream/ots-mstest.yaml @@ -0,0 +1,28 @@ +--- +# MSTest OTS Requirements +# +# PURPOSE: +# - Define requirements for the MSTest off-the-shelf testing framework +# - MSTest is used to discover, execute, and report unit test results + +sections: + - title: MSTest OTS Requirements + requirements: + - id: ReviewMark-OTS-MSTest + title: MSTest shall execute unit tests and report results. + justification: | + MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used + by the project. It discovers and runs all test methods and writes TRX result files that + feed into coverage reporting and requirements traceability. Passing tests confirm the + framework is functioning correctly. + tags: [ots] + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_ShortVersionFlag_SetsVersionTrue diff --git a/docs/reqstream/ots-reqstream.yaml b/docs/reqstream/ots-reqstream.yaml new file mode 100644 index 0000000..908a75f --- /dev/null +++ b/docs/reqstream/ots-reqstream.yaml @@ -0,0 +1,21 @@ +--- +# ReqStream OTS Requirements +# +# PURPOSE: +# - Define requirements for the ReqStream off-the-shelf requirements traceability tool +# - ReqStream validates that every requirement is linked to passing test evidence + +sections: + - title: ReqStream OTS Requirements + requirements: + - id: ReviewMark-OTS-ReqStream + title: ReqStream shall enforce that every requirement is linked to passing test evidence. + justification: | + DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to + produce a requirements report, justifications document, and traceability matrix. When + run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, + making unproven requirements a build-breaking condition. A successful pipeline run with + --enforce proves all requirements are covered and that ReqStream is functioning. + tags: [ots] + tests: + - ReqStream_EnforcementMode diff --git a/docs/reqstream/ots-requirements.yaml b/docs/reqstream/ots-requirements.yaml deleted file mode 100644 index b763998..0000000 --- a/docs/reqstream/ots-requirements.yaml +++ /dev/null @@ -1,102 +0,0 @@ ---- -# OTS (Off-the-Shelf) Software Requirements -# -# PURPOSE: -# - Define requirements for third-party components used by ReviewMark -# - OTS requirements document which capabilities the project depends on -# - Tests verify the OTS component provides the required behavior in this environment - -sections: - - title: OTS Software Requirements - sections: - - title: MSTest - requirements: - - id: ReviewMark-OTS-MSTest - title: MSTest shall execute unit tests and report results. - justification: | - MSTest (MSTest.TestFramework and MSTest.TestAdapter) is the unit-testing framework used - by the project. It discovers and runs all test methods and writes TRX result files that - feed into coverage reporting and requirements traceability. Passing tests confirm the - framework is functioning correctly. - tags: [ots] - tests: - - Context_Create_NoArguments_ReturnsDefaultContext - - Context_Create_VersionFlag_SetsVersionTrue - - Context_Create_HelpFlag_SetsHelpTrue - - Context_Create_SilentFlag_SetsSilentTrue - - Context_Create_ValidateFlag_SetsValidateTrue - - Context_Create_ResultsFlag_SetsResultsFile - - Context_Create_LogFlag_OpensLogFile - - Context_Create_UnknownArgument_ThrowsArgumentException - - Context_Create_ShortVersionFlag_SetsVersionTrue - - - title: ReqStream - requirements: - - id: ReviewMark-OTS-ReqStream - title: ReqStream shall enforce that every requirement is linked to passing test evidence. - justification: | - DemaConsulting.ReqStream processes requirements.yaml and the TRX test-result files to - produce a requirements report, justifications document, and traceability matrix. When - run with --enforce, it exits with a non-zero code if any requirement lacks test evidence, - making unproven requirements a build-breaking condition. A successful pipeline run with - --enforce proves all requirements are covered and that ReqStream is functioning. - tags: [ots] - tests: - - ReqStream_EnforcementMode - - - title: BuildMark - requirements: - - id: ReviewMark-OTS-BuildMark - title: BuildMark shall generate build-notes documentation from GitHub Actions metadata. - justification: | - DemaConsulting.BuildMark queries the GitHub API to capture workflow run details and - renders them as a markdown build-notes document included in the release artifacts. - It runs as part of the same CI pipeline that produces the TRX test results, so a - successful pipeline run is evidence that BuildMark executed without error. - tags: [ots] - tests: - - BuildMark_MarkdownReportGeneration - - - title: VersionMark - requirements: - - id: ReviewMark-OTS-VersionMark - title: VersionMark shall publish captured tool-version information. - justification: | - DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the - pipeline and writes a versions markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that VersionMark executed without error. - tags: [ots] - tests: - - VersionMark_CapturesVersions - - VersionMark_GeneratesMarkdownReport - - - title: SarifMark - requirements: - - id: ReviewMark-OTS-SarifMark - title: SarifMark shall convert CodeQL SARIF results into a markdown report. - justification: | - DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and - renders it as a human-readable markdown document included in the release artifacts. - It runs in the same CI pipeline that produces the TRX test results, so a successful - pipeline run is evidence that SarifMark executed without error. - tags: [ots] - tests: - - SarifMark_SarifReading - - SarifMark_MarkdownReportGeneration - - - title: SonarMark - requirements: - - id: ReviewMark-OTS-SonarMark - title: SonarMark shall generate a SonarCloud quality report. - justification: | - DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and - renders it as a markdown document included in the release artifacts. It runs in the - same CI pipeline that produces the TRX test results, so a successful pipeline run is - evidence that SonarMark executed without error. - tags: [ots] - tests: - - SonarMark_QualityGateRetrieval - - SonarMark_IssuesRetrieval - - SonarMark_HotSpotsRetrieval - - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-sarifmark.yaml b/docs/reqstream/ots-sarifmark.yaml new file mode 100644 index 0000000..c49a525 --- /dev/null +++ b/docs/reqstream/ots-sarifmark.yaml @@ -0,0 +1,21 @@ +--- +# SarifMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the SarifMark off-the-shelf SARIF reporting tool +# - SarifMark converts CodeQL SARIF results into a human-readable markdown report + +sections: + - title: SarifMark OTS Requirements + requirements: + - id: ReviewMark-OTS-SarifMark + title: SarifMark shall convert CodeQL SARIF results into a markdown report. + justification: | + DemaConsulting.SarifMark reads the SARIF output produced by CodeQL code scanning and + renders it as a human-readable markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that SarifMark executed without error. + tags: [ots] + tests: + - SarifMark_SarifReading + - SarifMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-sonarmark.yaml b/docs/reqstream/ots-sonarmark.yaml new file mode 100644 index 0000000..791d57e --- /dev/null +++ b/docs/reqstream/ots-sonarmark.yaml @@ -0,0 +1,23 @@ +--- +# SonarMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the SonarMark off-the-shelf SonarCloud reporting tool +# - SonarMark generates a SonarCloud quality report as part of release artifacts + +sections: + - title: SonarMark OTS Requirements + requirements: + - id: ReviewMark-OTS-SonarMark + title: SonarMark shall generate a SonarCloud quality report. + justification: | + DemaConsulting.SonarMark retrieves quality-gate and metrics data from SonarCloud and + renders it as a markdown document included in the release artifacts. It runs in the + same CI pipeline that produces the TRX test results, so a successful pipeline run is + evidence that SonarMark executed without error. + tags: [ots] + tests: + - SonarMark_QualityGateRetrieval + - SonarMark_IssuesRetrieval + - SonarMark_HotSpotsRetrieval + - SonarMark_MarkdownReportGeneration diff --git a/docs/reqstream/ots-versionmark.yaml b/docs/reqstream/ots-versionmark.yaml new file mode 100644 index 0000000..58f0928 --- /dev/null +++ b/docs/reqstream/ots-versionmark.yaml @@ -0,0 +1,21 @@ +--- +# VersionMark OTS Requirements +# +# PURPOSE: +# - Define requirements for the VersionMark off-the-shelf tool-version documentation tool +# - VersionMark publishes captured tool-version information as part of release artifacts + +sections: + - title: VersionMark OTS Requirements + requirements: + - id: ReviewMark-OTS-VersionMark + title: VersionMark shall publish captured tool-version information. + justification: | + DemaConsulting.VersionMark reads version metadata for each dotnet tool used in the + pipeline and writes a versions markdown document included in the release artifacts. + It runs in the same CI pipeline that produces the TRX test results, so a successful + pipeline run is evidence that VersionMark executed without error. + tags: [ots] + tests: + - VersionMark_CapturesVersions + - VersionMark_GeneratesMarkdownReport diff --git a/docs/reqstream/reviewmark-system.yaml b/docs/reqstream/reviewmark-system.yaml new file mode 100644 index 0000000..5e51e6e --- /dev/null +++ b/docs/reqstream/reviewmark-system.yaml @@ -0,0 +1,67 @@ +--- +# ReviewMark System-Level Requirements +# +# PURPOSE: +# - Define system-level requirements describing what end-users need the ReviewMark tool to provide +# - These requirements capture the externally visible behavior of the complete ReviewMark system +# - Unit-level requirements (per-class behavior) are in the individual *-requirements.yaml files + +sections: + - title: System-Level Requirements + requirements: + - id: ReviewMark-System-ReviewPlan + title: >- + The tool shall generate a Review Plan document listing all files requiring review and their + review-set coverage. + justification: | + In regulated environments, auditors require evidence that every file subject to review + is covered by at least one named review-set. The Review Plan document provides this + evidence automatically on each CI/CD run, replacing manual tracking spreadsheets. + tests: + - ReviewMark_ReviewPlanGeneration + + - id: ReviewMark-System-ReviewReport + title: The tool shall generate a Review Report document listing every review-set and its current review status. + justification: | + Auditors need evidence that the review evidence for each review-set is current — + that the reviewed files have not changed since the review was conducted. The Review + Report provides this evidence automatically, showing Current, Stale, or Missing + status for each review-set. + tests: + - ReviewMark_ReviewReportGeneration + + - id: ReviewMark-System-Enforce + title: The tool shall return a non-zero exit code when enforcement is enabled and any review-set is not current. + justification: | + CI/CD pipelines must be able to gate releases on review coverage. The --enforce flag + enables this by causing the tool to exit with a non-zero code when any review-set has + Stale or Missing status, making incomplete review coverage a build-breaking condition. + tests: + - ReviewMark_Enforce + + - id: ReviewMark-System-IndexScan + title: The tool shall scan PDF evidence files and write an index.json when the --index flag is provided. + justification: | + Review evidence PDFs contain embedded metadata (id, fingerprint, date, result) in their + Keywords field. The --index command scans a directory of such PDFs and writes an + index.json, enabling the evidence store to be refreshed after new review PDFs are added + without manual maintenance of the index file. + tests: + - ReviewMark_IndexScan + + - id: ReviewMark-System-Validate + title: The tool shall execute self-validation tests when the --validate flag is provided. + justification: | + Regulated environments require tool qualification evidence to demonstrate that the tool + functions correctly in its specific deployment environment. The --validate flag triggers + a built-in test suite that exercises core tool behaviors and produces a pass/fail report. + tests: + - ReviewMark_VersionDisplay + - ReviewMark_HelpDisplay + - ReviewMark_ReviewPlanGeneration + - ReviewMark_ReviewReportGeneration + - ReviewMark_IndexScan + - ReviewMark_Enforce + - ReviewMark_WorkingDirectoryOverride + - ReviewMark_Elaborate + - ReviewMark_Lint diff --git a/docs/reqstream/cli-requirements.yaml b/docs/reqstream/subsystem-cli.yaml similarity index 100% rename from docs/reqstream/cli-requirements.yaml rename to docs/reqstream/subsystem-cli.yaml diff --git a/docs/reqstream/configuration-requirements.yaml b/docs/reqstream/subsystem-configuration.yaml similarity index 100% rename from docs/reqstream/configuration-requirements.yaml rename to docs/reqstream/subsystem-configuration.yaml diff --git a/docs/reqstream/unit-context.yaml b/docs/reqstream/unit-context.yaml new file mode 100644 index 0000000..2c880de --- /dev/null +++ b/docs/reqstream/unit-context.yaml @@ -0,0 +1,41 @@ +--- +# Context Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Context software unit +# - This unit parses command-line arguments into an in-memory context object +# - It also provides unified output and logging across the tool + +sections: + - title: Context Unit Requirements + requirements: + - id: ReviewMark-Context-Parsing + title: The Context unit shall parse command-line arguments into a strongly-typed Context object. + justification: | + All downstream processing reads options from the Context object rather than + directly from the raw argument array. The Context.Create factory method processes + arguments sequentially, recognizing flag and value arguments, and returns a fully + initialized Context. Unknown arguments must raise an ArgumentException so the + caller can report a clear error message. + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_VersionFlag_SetsVersionTrue + - Context_Create_HelpFlag_SetsHelpTrue + - Context_Create_SilentFlag_SetsSilentTrue + - Context_Create_ValidateFlag_SetsValidateTrue + - Context_Create_ResultsFlag_SetsResultsFile + - Context_Create_LogFlag_OpensLogFile + - Context_Create_UnknownArgument_ThrowsArgumentException + - Context_Create_ShortVersionFlag_SetsVersionTrue + + - id: ReviewMark-Context-Output + title: The Context unit shall provide WriteLine and WriteError methods for unified output and logging. + justification: | + All output goes through Context so that the --silent flag is honoured and + optionally duplicated to a log file opened by the --log flag. WriteError must + additionally set the error exit code so that the process exits with a non-zero + status when any error is reported. + tests: + - Context_WriteError_NotSilent_WritesToConsole + - Context_WriteError_SetsErrorExitCode + - Context_WriteLine_Silent_DoesNotWriteToConsole diff --git a/docs/reqstream/unit-glob-matcher.yaml b/docs/reqstream/unit-glob-matcher.yaml new file mode 100644 index 0000000..2529257 --- /dev/null +++ b/docs/reqstream/unit-glob-matcher.yaml @@ -0,0 +1,29 @@ +--- +# GlobMatcher Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the GlobMatcher software unit +# - This unit resolves ordered include/exclude glob patterns to a list of files +# - It is used by ReviewMarkConfiguration to resolve needs-review and review-set file lists + +sections: + - title: GlobMatcher Unit Requirements + requirements: + - id: ReviewMark-GlobMatcher-IncludeExclude + title: >- + The GlobMatcher shall resolve ordered include and exclude glob patterns to a sorted list of + relative file paths. + justification: | + Review-set and needs-review configurations specify files using ordered glob patterns, + where patterns prefixed with '!' are exclusions. The GlobMatcher must apply these + patterns in declaration order so that a later include can re-add files removed by an + earlier exclude, and vice versa. The result must be sorted to ensure deterministic + fingerprinting regardless of filesystem iteration order. + tests: + - GlobMatcher_GetMatchingFiles_SingleIncludePattern_ReturnsMatchingFiles + - GlobMatcher_GetMatchingFiles_ExcludePattern_ExcludesMatchingFiles + - GlobMatcher_GetMatchingFiles_ReIncludeAfterExclude_ReturnsReIncludedFiles + - GlobMatcher_GetMatchingFiles_IncludeAndExclude_ReturnsFilteredFiles + - GlobMatcher_GetMatchingFiles_NullBaseDirectory_ThrowsArgumentNullException + - GlobMatcher_GetMatchingFiles_NullPatterns_ThrowsArgumentNullException + - GlobMatcher_GetMatchingFiles_NoMatchingFiles_ReturnsEmptyList diff --git a/docs/reqstream/unit-path-helpers.yaml b/docs/reqstream/unit-path-helpers.yaml new file mode 100644 index 0000000..f9295bc --- /dev/null +++ b/docs/reqstream/unit-path-helpers.yaml @@ -0,0 +1,24 @@ +--- +# PathHelpers Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the PathHelpers software unit +# - This unit provides safe path operations that prevent path traversal attacks +# - It is used by ReviewIndex.cs and Validation.cs when constructing file paths + +sections: + - title: PathHelpers Unit Requirements + requirements: + - id: ReviewMark-PathHelpers-SafeCombine + title: The PathHelpers shall safely combine a base path and a relative path, rejecting path traversal attempts. + justification: | + When constructing file paths from user-supplied or externally-sourced components + (such as relative paths read from an evidence index), the tool must prevent path + traversal attacks. SafePathCombine validates that the relative path does not + contain '..' sequences or absolute path components, and performs a defense-in-depth + check that the resolved combined path remains under the base directory. + tests: + - PathHelpers_SafePathCombine_ValidPaths_CombinesCorrectly + - PathHelpers_SafePathCombine_PathTraversalWithDoubleDots_ThrowsArgumentException + - PathHelpers_SafePathCombine_AbsolutePath_ThrowsArgumentException + - PathHelpers_SafePathCombine_NestedPaths_CombinesCorrectly diff --git a/docs/reqstream/unit-program.yaml b/docs/reqstream/unit-program.yaml new file mode 100644 index 0000000..356f383 --- /dev/null +++ b/docs/reqstream/unit-program.yaml @@ -0,0 +1,45 @@ +--- +# Program Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Program software unit +# - This unit is the main entry point and top-level orchestrator of the tool +# - It dispatches to processing logic based on parsed CLI flags + +sections: + - title: Program Unit Requirements + requirements: + - id: ReviewMark-Program-EntryPoint + title: >- + The Program unit shall construct a Context, dispatch to the appropriate operation, + and return the Context exit code as the process exit code. + justification: | + Program.Main is the process entry point. It must create the execution context, + call Program.Run to perform the requested operation, and return the exit code + from the context so that callers can detect success or failure programmatically. + Unexpected exceptions are written to error output and then rethrown, so callers + may observe either a normal exit code or a process termination due to an + unhandled exception. + tests: + - Program_Run_WithVersionFlag_DisplaysVersionOnly + - Program_Version_ReturnsNonEmptyString + - Program_Run_WithHelpFlag_DisplaysUsageInformation + + - id: ReviewMark-Program-Dispatch + title: >- + The Program unit shall dispatch to exactly one operation per invocation based on + the priority order of CLI flags. + justification: | + --version, --help, --validate, --lint, --index, and plan/report operations must + be evaluated in a fixed priority order so that the behavior is predictable and + documented. Only the first matching flag action is executed; later flags are + not reached. + tests: + - Program_Run_WithVersionFlag_DisplaysVersionOnly + - Program_Run_WithHelpFlag_DisplaysUsageInformation + - Program_Run_WithValidateFlag_RunsValidation + - Program_Run_WithLintFlag_ValidConfig_ReportsSuccess + - Program_Run_WithHelpFlag_IncludesElaborateOption + - Program_Run_WithHelpFlag_IncludesLintOption + - Program_Run_WithElaborateFlag_OutputsElaboration + - Program_Run_WithElaborateFlag_UnknownId_ReportsError diff --git a/docs/reqstream/index-requirements.yaml b/docs/reqstream/unit-review-index.yaml similarity index 99% rename from docs/reqstream/index-requirements.yaml rename to docs/reqstream/unit-review-index.yaml index 0abf7b8..ad05a88 100644 --- a/docs/reqstream/index-requirements.yaml +++ b/docs/reqstream/unit-review-index.yaml @@ -1,5 +1,5 @@ --- -# Index Software Unit Requirements +# ReviewIndex Software Unit Requirements # # PURPOSE: # - Define requirements for the ReviewIndex software unit diff --git a/docs/reqstream/unit-validation.yaml b/docs/reqstream/unit-validation.yaml new file mode 100644 index 0000000..622ccd1 --- /dev/null +++ b/docs/reqstream/unit-validation.yaml @@ -0,0 +1,34 @@ +--- +# Validation Software Unit Requirements +# +# PURPOSE: +# - Define requirements for the Validation software unit +# - This unit provides self-validation test execution for regulated environments +# - Self-validation proves the tool is functioning correctly in its deployment environment + +sections: + - title: Validation Unit Requirements + requirements: + - id: ReviewMark-Validation-Run + title: The tool shall execute self-validation tests and report results when the --validate flag is provided. + justification: | + In regulated environments, tool qualification evidence is required to demonstrate + that the tool functions correctly in its deployment environment. Self-validation + runs a suite of functional tests covering core behaviors and reports pass/fail + results with a summary count, giving quality assurance teams the evidence they need. + tests: + - Validation_Run_NullContext_ThrowsArgumentNullException + - Validation_Run_WritesValidationHeader + - Validation_Run_WritesSummaryWithTotalTests + - Validation_Run_AllTestsPass_ExitCodeIsZero + + - id: ReviewMark-Validation-ResultsFile + title: The tool shall write self-validation results to a TRX or JUnit XML file when --results is provided. + justification: | + CI/CD pipelines and requirements traceability tools (such as ReqStream) consume + test result files in standard formats. By supporting both TRX (MSTest) and JUnit + XML output, the self-validation results can be fed directly into pipeline tooling + without additional conversion steps. + tests: + - Validation_Run_WithTrxResultsFile_WritesFile + - Validation_Run_WithXmlResultsFile_WritesFile diff --git a/requirements.yaml b/requirements.yaml index 3654661..341b6db 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -24,8 +24,19 @@ # --- includes: - - docs/reqstream/cli-requirements.yaml - - docs/reqstream/configuration-requirements.yaml - - docs/reqstream/index-requirements.yaml + - docs/reqstream/reviewmark-system.yaml + - docs/reqstream/subsystem-cli.yaml + - docs/reqstream/subsystem-configuration.yaml + - docs/reqstream/unit-context.yaml + - docs/reqstream/unit-program.yaml + - docs/reqstream/unit-review-index.yaml + - docs/reqstream/unit-glob-matcher.yaml + - docs/reqstream/unit-path-helpers.yaml + - docs/reqstream/unit-validation.yaml - docs/reqstream/platform-requirements.yaml - - docs/reqstream/ots-requirements.yaml + - docs/reqstream/ots-mstest.yaml + - docs/reqstream/ots-reqstream.yaml + - docs/reqstream/ots-buildmark.yaml + - docs/reqstream/ots-versionmark.yaml + - docs/reqstream/ots-sarifmark.yaml + - docs/reqstream/ots-sonarmark.yaml diff --git a/src/DemaConsulting.ReviewMark/Index.cs b/src/DemaConsulting.ReviewMark/ReviewIndex.cs similarity index 100% rename from src/DemaConsulting.ReviewMark/Index.cs rename to src/DemaConsulting.ReviewMark/ReviewIndex.cs diff --git a/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs b/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs new file mode 100644 index 0000000..d7d9e03 --- /dev/null +++ b/test/DemaConsulting.ReviewMark.Tests/ValidationTests.cs @@ -0,0 +1,202 @@ +// Copyright (c) DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +namespace DemaConsulting.ReviewMark.Tests; + +/// +/// Unit tests for the class. +/// +[TestClass] +public class ValidationTests +{ + /// + /// Test that Run throws ArgumentNullException when context is null. + /// + [TestMethod] + public void Validation_Run_NullContext_ThrowsArgumentNullException() + { + // Act & Assert + Assert.Throws(() => Validation.Run(null!)); + } + + /// + /// Test that Run writes a validation header containing system information. + /// + [TestMethod] + public void Validation_Run_WritesValidationHeader() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — output contains the markdown header and table headings + var output = outWriter.ToString(); + Assert.Contains("DEMA Consulting ReviewMark", output); + Assert.Contains("Tool Version", output); + Assert.Contains("Machine Name", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run writes a summary with a total test count. + /// + [TestMethod] + public void Validation_Run_WritesSummaryWithTotalTests() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — output contains the summary section + var output = outWriter.ToString(); + Assert.Contains("Total Tests:", output); + Assert.Contains("Passed:", output); + Assert.Contains("Failed:", output); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run returns a zero exit code when all tests pass. + /// + [TestMethod] + public void Validation_Run_AllTestsPass_ExitCodeIsZero() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate"]); + + // Act + Validation.Run(context); + + // Assert — exit code is zero (no errors) + Assert.AreEqual(0, context.ExitCode); + } + finally + { + Console.SetOut(originalOut); + } + } + + /// + /// Test that Run writes results to a TRX file when --results is provided with a .trx extension. + /// + [TestMethod] + public void Validation_Run_WithTrxResultsFile_WritesFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"reviewmark-validation-{Guid.NewGuid()}.trx"); + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert — results file exists and has content + Assert.IsTrue(File.Exists(resultsFile), "TRX results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.IsFalse(string.IsNullOrWhiteSpace(content), "TRX results file is empty"); + Assert.Contains("TestRun", content); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } + } + + /// + /// Test that Run writes results to a JUnit XML file when --results is provided with a .xml extension. + /// + [TestMethod] + public void Validation_Run_WithXmlResultsFile_WritesFile() + { + // Arrange + var resultsFile = Path.Combine(Path.GetTempPath(), $"reviewmark-validation-{Guid.NewGuid()}.xml"); + try + { + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + using var context = Context.Create(["--validate", "--results", resultsFile]); + + // Act + Validation.Run(context); + + // Assert — results file exists and has content + Assert.IsTrue(File.Exists(resultsFile), "XML results file was not created"); + var content = File.ReadAllText(resultsFile); + Assert.IsFalse(string.IsNullOrWhiteSpace(content), "XML results file is empty"); + Assert.Contains("testsuites", content); + } + finally + { + Console.SetOut(originalOut); + } + } + finally + { + if (File.Exists(resultsFile)) + { + File.Delete(resultsFile); + } + } + } +}