diff --git a/.reviewmark.yaml b/.reviewmark.yaml index ad6a153..ddb246e 100644 --- a/.reviewmark.yaml +++ b/.reviewmark.yaml @@ -7,6 +7,9 @@ # Processed in order; prefix a pattern with '!' to exclude. needs-review: - "**/*.cs" # All C# source and test files + - "requirements.yaml" # Root requirements file + - "docs/reqstream/**/*.yaml" # All requirements files + - "docs/design/**/*.md" # All design documents - "!**/obj/**" # Exclude build output - "!**/bin/**" # Exclude build output @@ -22,53 +25,246 @@ evidence-source: # Each review-set groups requirements, source, and tests for a coherent software unit # so that an AI-assisted review can verify consistency across the full evidence chain. reviews: - # Software unit reviews - one per class + # System-level review + - id: VHDLTest-System + title: Review of VHDLTest System + paths: + - "docs/reqstream/vhdltest-system.yaml" + - "docs/reqstream/platform-requirements.yaml" + - "docs/design/introduction.md" + - "docs/design/system.md" + - "src/**/Program.cs" + - "test/**/IntegrationTests.cs" + - "test/**/Runner.cs" + - "test/**/GlobalUsings.cs" + + # Design review + - id: VHDLTest-Design + title: Review of VHDLTest Design Documentation + paths: + - "docs/reqstream/vhdltest-system.yaml" + - "docs/reqstream/platform-requirements.yaml" + - "docs/design/**/*.md" + + # All requirements review + - id: VHDLTest-AllRequirements + title: Review of All VHDLTest Requirements + paths: + - "requirements.yaml" + - "docs/reqstream/**/*.yaml" + + # OTS review + - id: VHDLTest-OTS + title: Review of VHDLTest OTS Dependencies + paths: + - "docs/reqstream/ots-mstest.yaml" + - "docs/reqstream/ots-reqstream.yaml" + - "docs/reqstream/ots-buildmark.yaml" + - "docs/reqstream/ots-versionmark.yaml" + - "docs/reqstream/ots-sarifmark.yaml" + - "docs/reqstream/ots-sonarmark.yaml" + + # Cli subsystem review + - id: VHDLTest-Cli + title: Review of VHDLTest Cli Subsystem + paths: + - "docs/reqstream/cli/subsystem-cli.yaml" # subsystem requirements + - "docs/design/cli/cli.md" # subsystem design + - "test/**/Cli/CliSubsystemTests.cs" # subsystem integration tests + + # Context unit review - id: VHDLTest-Context title: Review of VHDLTest Context Unit paths: - - "src/**/Context.cs" - - "test/**/ContextTests.cs" + - "docs/reqstream/cli/unit-context.yaml" + - "docs/design/cli/context.md" + - "src/**/Cli/Context.cs" + - "test/**/Cli/ContextTests.cs" + # ConfigDocument unit review + - id: VHDLTest-ConfigDocument + title: Review of VHDLTest ConfigDocument Unit + paths: + - "docs/reqstream/cli/unit-config-document.yaml" + - "docs/design/cli/config-document.md" + - "src/**/Cli/ConfigDocument.cs" + - "test/**/Cli/ConfigDocumentTests.cs" + + # Options unit review + - id: VHDLTest-Options + title: Review of VHDLTest Options Unit + paths: + - "docs/reqstream/cli/unit-options.yaml" + - "docs/design/cli/options.md" + - "src/**/Cli/Options.cs" + - "test/**/Cli/OptionsTests.cs" + + # Program unit review - id: VHDLTest-Program title: Review of VHDLTest Program Unit paths: - - "src/**/Program.cs" - - "test/**/UsageTests.cs" - - "test/**/VersionTests.cs" - - "test/**/ExitCodeTests.cs" - - "test/**/Runner.cs" - - "test/**/GlobalUsings.cs" + - "docs/reqstream/unit-program.yaml" # requirements + - "docs/design/program.md" # design + - "src/**/Program.cs" # implementation + - "test/**/ProgramTests.cs" # unit tests - - id: VHDLTest-Validation - title: Review of VHDLTest Validation Unit + # Run subsystem review + - id: VHDLTest-Run + title: Review of VHDLTest Run Subsystem paths: - - "src/**/Validation.cs" - - "test/**/ValidationTests.cs" + - "docs/reqstream/run/subsystem-run.yaml" # subsystem requirements + - "docs/design/run/run.md" # subsystem design + - "test/**/Run/RunSubsystemTests.cs" # subsystem integration tests - - id: VHDLTest-Config - title: Review of VHDLTest Configuration Units + # RunProcessor unit review + - id: VHDLTest-RunProcessor + title: Review of VHDLTest RunProcessor Unit paths: - - "src/**/ConfigDocument.cs" - - "src/**/Options.cs" - - "test/**/ConfigDocumentTests.cs" - - "test/**/OptionsTests.cs" + - "docs/reqstream/run/unit-run-processor.yaml" # requirements + - "docs/design/run/run-processor.md" # design + - "src/**/Run/RunProcessor.cs" # implementation + - "test/**/Run/RunProcessorTests.cs" # unit tests - - id: VHDLTest-Run - title: Review of VHDLTest Run Processing Units + # RunProgram unit review + - id: VHDLTest-RunProgram + title: Review of VHDLTest RunProgram Unit + paths: + - "docs/reqstream/run/unit-run-program.yaml" # requirements + - "docs/design/run/run-program.md" # design + - "src/**/Run/RunProgram.cs" # implementation + + # RunResults unit review + - id: VHDLTest-RunResults + title: Review of VHDLTest RunResults Unit paths: - - "src/**/Run/*.cs" - - "test/**/RunProcessorTests.cs" + - "docs/reqstream/run/unit-run-results.yaml" # requirements + - "docs/design/run/run-results.md" # design + - "src/**/Run/RunResults.cs" # implementation + # Run types unit review (RunLine, RunLineRule, RunLineType) + - id: VHDLTest-RunTypes + title: Review of VHDLTest Run Types Unit + paths: + - "docs/reqstream/run/unit-run-types.yaml" # requirements + - "docs/design/run/run-line.md" # design + - "docs/design/run/run-line-rule.md" # design + - "docs/design/run/run-line-type.md" # design + - "src/**/Run/RunLine.cs" # implementation + - "src/**/Run/RunLineRule.cs" # implementation + - "src/**/Run/RunLineType.cs" # implementation + + # Results subsystem review - id: VHDLTest-Results - title: Review of VHDLTest Test Results Units + title: Review of VHDLTest Results Subsystem + paths: + - "docs/reqstream/results/subsystem-results.yaml" # subsystem requirements + - "docs/design/results/results.md" # subsystem design + - "test/**/Results/ResultsSubsystemTests.cs" # subsystem integration tests + + # TestResult unit review + - id: VHDLTest-TestResult + title: Review of VHDLTest TestResult Unit paths: - - "src/**/Results/*.cs" - - "test/**/TestResultTests.cs" - - "test/**/TestResultsTests.cs" + - "docs/reqstream/results/unit-test-result.yaml" # requirements + - "docs/design/results/test-result.md" # design + - "src/**/Results/TestResult.cs" # implementation + - "test/**/Results/TestResultTests.cs" # unit tests + # TestResults unit review + - id: VHDLTest-TestResults + title: Review of VHDLTest TestResults Unit + paths: + - "docs/reqstream/results/unit-test-results.yaml" # requirements + - "docs/design/results/test-results.md" # design + - "src/**/Results/TestResults.cs" # implementation + - "test/**/Results/TestResultsTests.cs" # unit tests + + # Simulators subsystem review - id: VHDLTest-Simulators - title: Review of VHDLTest Simulator Units + title: Review of VHDLTest Simulators Subsystem + paths: + - "docs/reqstream/simulators/subsystem-simulators.yaml" # subsystem requirements + - "docs/design/simulators/simulators.md" # subsystem design + - "docs/design/simulators/simulator.md" # Simulator base class design + - "src/**/Simulators/Simulator.cs" # base class (no dedicated unit review) + - "test/**/Simulators/SimulatorsSubsystemTests.cs" # subsystem integration tests + + # SimulatorFactory unit review + - id: VHDLTest-SimulatorFactory + title: Review of VHDLTest SimulatorFactory Unit + paths: + - "docs/reqstream/simulators/unit-simulator-factory.yaml" # requirements + - "docs/design/simulators/simulator-factory.md" # design + - "src/**/Simulators/SimulatorFactory.cs" # implementation + - "test/**/Simulators/SimulatorFactoryTests.cs" # unit tests + + # GhdlSimulator unit review + - id: VHDLTest-GhdlSimulator + title: Review of VHDLTest GhdlSimulator Unit + paths: + - "docs/reqstream/simulators/unit-ghdl-simulator.yaml" # requirements + - "docs/design/simulators/ghdl-simulator.md" # design + - "src/**/Simulators/GhdlSimulator.cs" # implementation + - "test/**/Simulators/GhdlSimulatorTests.cs" # unit tests + + # NvcSimulator unit review + - id: VHDLTest-NvcSimulator + title: Review of VHDLTest NvcSimulator Unit + paths: + - "docs/reqstream/simulators/unit-nvc-simulator.yaml" # requirements + - "docs/design/simulators/nvc-simulator.md" # design + - "src/**/Simulators/NvcSimulator.cs" # implementation + - "test/**/Simulators/NvcSimulatorTests.cs" # unit tests + + # ModelSimSimulator unit review + - id: VHDLTest-ModelSimSimulator + title: Review of VHDLTest ModelSimSimulator Unit + paths: + - "docs/reqstream/simulators/unit-modelsim-simulator.yaml" # requirements + - "docs/design/simulators/modelsim-simulator.md" # design + - "src/**/Simulators/ModelSimSimulator.cs" # implementation + - "test/**/Simulators/ModelSimSimulatorTests.cs" # unit tests + + # QuestaSimSimulator unit review + - id: VHDLTest-QuestaSimSimulator + title: Review of VHDLTest QuestaSimSimulator Unit + paths: + - "docs/reqstream/simulators/unit-questasim-simulator.yaml" # requirements + - "docs/design/simulators/questasim-simulator.md" # design + - "src/**/Simulators/QuestaSimSimulator.cs" # implementation + - "test/**/Simulators/QuestaSimSimulatorTests.cs" # unit tests + + # VivadoSimulator unit review + - id: VHDLTest-VivadoSimulator + title: Review of VHDLTest VivadoSimulator Unit + paths: + - "docs/reqstream/simulators/unit-vivado-simulator.yaml" # requirements + - "docs/design/simulators/vivado-simulator.md" # design + - "src/**/Simulators/VivadoSimulator.cs" # implementation + - "test/**/Simulators/VivadoSimulatorTests.cs" # unit tests + + # ActiveHdlSimulator unit review + - id: VHDLTest-ActiveHdlSimulator + title: Review of VHDLTest ActiveHdlSimulator Unit + paths: + - "docs/reqstream/simulators/unit-active-hdl-simulator.yaml" # requirements + - "docs/design/simulators/active-hdl-simulator.md" # design + - "src/**/Simulators/ActiveHdlSimulator.cs" # implementation + - "test/**/Simulators/ActiveHdlSimulatorTests.cs" # unit tests + + # SelfTest subsystem review + - id: VHDLTest-SelfTest + title: Review of VHDLTest SelfTest Subsystem + paths: + - "docs/reqstream/self-test/subsystem-self-test.yaml" # subsystem requirements + - "docs/design/self-test/self-test.md" # subsystem design + - "test/**/SelfTest/ValidationTests.cs" # subsystem integration tests + + # Validation unit review + - id: VHDLTest-Validation + title: Review of VHDLTest Validation Unit paths: - - "src/**/Simulators/*.cs" - - "test/**/*SimulatorTests.cs" - - "test/**/SimulatorFactoryTests.cs" + - "docs/reqstream/self-test/unit-validation.yaml" # requirements + - "docs/design/self-test/validation.md" # design + - "src/**/SelfTest/Validation.cs" # implementation + - "test/**/SelfTest/ValidationTests.cs" # unit/integration tests diff --git a/docs/design/cli/cli.md b/docs/design/cli/cli.md new file mode 100644 index 0000000..681eb92 --- /dev/null +++ b/docs/design/cli/cli.md @@ -0,0 +1,21 @@ +# Cli Subsystem Design + +## Overview + +The Cli subsystem implements command-line interface handling for VHDLTest. It encompasses +all logic for parsing command-line arguments, reading configuration files, and managing +output channels. + +## Units + +| Unit | File | Responsibility | +| -------------- | ----------------------- | --------------------------------------------------- | +| Context | `Cli/Context.cs` | Parses command-line arguments and owns I/O channels | +| ConfigDocument | `Cli/ConfigDocument.cs` | Deserializes the YAML test configuration | +| Options | `Cli/Options.cs` | Holds parsed configuration options | + +## Interactions + +The Cli subsystem is consumed primarily by `Program.cs`, which creates a `Context` +from the raw argument array, reads the config file path from it, then constructs +`Options` from a parsed `ConfigDocument`. diff --git a/docs/design/cli/config-document.md b/docs/design/cli/config-document.md new file mode 100644 index 0000000..877ddbc --- /dev/null +++ b/docs/design/cli/config-document.md @@ -0,0 +1,20 @@ +# ConfigDocument Unit Design + +## Overview + +`ConfigDocument.cs` implements the `ConfigDocument` class, which deserializes the +YAML configuration file that specifies the VHDL test suite to run. + +## Responsibilities + +- Deserialize YAML configuration using YamlDotNet +- Expose test file lists and simulator name +- Provide a static `Parse` method for loading from a file path + +## Data Model + +| Property | Type | Description | +| ----------- | ---------- | --------------------------------- | +| `Simulator` | `string?` | Name of simulator to use | +| `Results` | `string?` | Default results file path | +| `Tests` | `string[]` | Array of test bench file patterns | diff --git a/docs/design/cli/context.md b/docs/design/cli/context.md new file mode 100644 index 0000000..a25e951 --- /dev/null +++ b/docs/design/cli/context.md @@ -0,0 +1,30 @@ +# Context Unit Design + +## Overview + +`Context.cs` implements the `Context` class, which is responsible for parsing the +raw command-line argument array and exposing the parsed flags and output channels +to the rest of the application. + +## Responsibilities + +- Parse `-v`/`--version`, `-?`/`-h`/`--help`, `--silent`, `--verbose`, + `--validate`, `--depth`, `--results`, `--log`, and positional arguments +- Open an optional log-file writer when `--log` is specified +- Expose typed properties for each flag +- Write output to both console and log-file through `WriteLine`/`WriteError` +- Set an error exit code when `WriteError` is called + +## Data Model + +| Property | Type | Description | +| ---------- | --------- | -------------------------------------------- | +| `Version` | `bool` | True when `--version` or `-v` was passed | +| `Help` | `bool` | True when `--help`, `-h`, or `-?` was passed | +| `Silent` | `bool` | True when `--silent` was passed | +| `Verbose` | `bool` | True when `--verbose` was passed | +| `Validate` | `bool` | True when `--validate` was passed | +| `Depth` | `int` | Heading depth for validation reports | +| `Results` | `string?` | Path to results output file | +| `Config` | `string?` | Path to the YAML configuration file | +| `ExitCode` | `int` | Process exit code (0 = success, 1 = failure) | diff --git a/docs/design/cli/options.md b/docs/design/cli/options.md new file mode 100644 index 0000000..af2a84a --- /dev/null +++ b/docs/design/cli/options.md @@ -0,0 +1,18 @@ +# Options Unit Design + +## Overview + +`Options.cs` implements the `Options` record, which holds the resolved options +for a test run after combining command-line arguments with the configuration file. + +## Responsibilities + +- Hold the working directory and parsed `ConfigDocument` +- Provide a static `Parse` method that combines `Context` and file system state + +## Data Model + +| Property | Type | Description | +| ------------------ | ---------------- | --------------------------------------------- | +| `WorkingDirectory` | `string` | Resolved working directory for test execution | +| `Config` | `ConfigDocument` | Deserialized configuration document | diff --git a/docs/design/introduction.md b/docs/design/introduction.md new file mode 100644 index 0000000..c0aba46 --- /dev/null +++ b/docs/design/introduction.md @@ -0,0 +1,123 @@ +# Introduction + +This document provides the detailed design for VHDLTest, a .NET command-line application for +running VHDL unit tests and generating test reports. + +## Purpose + +The purpose of this document is to describe the internal design of each software unit that +comprises VHDLTest. It captures data models, algorithms, key methods, and inter-unit +interactions at a level of detail sufficient for formal code review, compliance verification, +and future maintenance. The document does not restate requirements; it explains how they +are realized. + +## Scope + +This document covers the detailed design of the following software units: + +- **Program** — entry point and execution orchestrator (`Program.cs`) +- **Context** — command-line argument parser and I/O owner (`Cli/Context.cs`) +- **ConfigDocument** — YAML configuration document parser (`Cli/ConfigDocument.cs`) +- **Options** — parsed options holder (`Cli/Options.cs`) +- **Simulators** — VHDL simulator integrations (`Simulators/`) +- **Run** — VHDL simulation run processing (`Run/`) +- **Results** — test result data model (`Results/`) +- **Validation** — self-validation test runner (`SelfTest/Validation.cs`) + +The following topics are out of scope: + +- External library internals +- Build pipeline configuration +- Deployment and packaging + +## Software Structure + +The following tree shows how the VHDLTest software items are organized across the +system, subsystem, and unit levels: + +```text +VHDLTest (System) +├── Program (Unit) +├── Cli (Subsystem) +│ ├── Context (Unit) +│ ├── ConfigDocument (Unit) +│ └── Options (Unit) +├── Simulators (Subsystem) +│ ├── Simulator (Unit) +│ ├── SimulatorFactory (Unit) +│ ├── GhdlSimulator (Unit) +│ ├── NvcSimulator (Unit) +│ ├── ModelSimSimulator (Unit) +│ ├── QuestaSimSimulator (Unit) +│ ├── VivadoSimulator (Unit) +│ └── ActiveHdlSimulator (Unit) +├── Run (Subsystem) +│ ├── RunProcessor (Unit) +│ ├── RunProgram (Unit) +│ ├── RunResults (Unit) +│ ├── RunLine (Unit) +│ ├── RunLineRule (Unit) +│ └── RunLineType (Unit) +├── Results (Subsystem) +│ ├── TestResult (Unit) +│ └── TestResults (Unit) +└── SelfTest (Subsystem) + └── Validation (Unit) +``` + +Each unit is described in detail in its own chapter within this document. + +## Folder Layout + +The source code folder structure mirrors the top-level subsystem breakdown above, giving +reviewers an explicit navigation aid from design to code: + +```text +src/DEMAConsulting.VHDLTest/ +├── Program.cs — entry point and execution orchestrator +├── Cli/ +│ ├── Context.cs — command-line argument parser and I/O owner +│ ├── ConfigDocument.cs — YAML configuration document parser +│ └── Options.cs — parsed options holder +├── Simulators/ +│ ├── Simulator.cs — base class for all simulators +│ ├── SimulatorFactory.cs — creates simulator instances +│ ├── GhdlSimulator.cs — GHDL simulator integration +│ ├── NvcSimulator.cs — NVC simulator integration +│ ├── ModelSimSimulator.cs — ModelSim simulator integration +│ ├── QuestaSimSimulator.cs — QuestaSim simulator integration +│ ├── VivadoSimulator.cs — Vivado simulator integration +│ └── ActiveHdlSimulator.cs — Active-HDL simulator integration +├── Run/ +│ ├── RunProcessor.cs — processes simulation run output +│ ├── RunProgram.cs — executes simulation programs +│ ├── RunResults.cs — simulation run results +│ ├── RunLine.cs — individual output line +│ ├── RunLineRule.cs — output line matching rule +│ └── RunLineType.cs — output line type classification +├── Results/ +│ ├── TestResult.cs — individual test result +│ └── TestResults.cs — collection of test results +└── SelfTest/ + └── Validation.cs — self-validation test runner +``` + +The test project mirrors the same layout under `test/DEMAConsulting.VHDLTest.Tests/`. + +## Document Conventions + +Throughout this document: + +- Class names, method names, property names, and file names appear in `monospace` font. +- The word **shall** denotes a design constraint that the implementation must satisfy. +- Section headings within each unit chapter follow a consistent structure: overview, data model, + methods/algorithms, and interactions with other units. +- Text tables are used in preference to diagrams, which may not render in all PDF viewers. + +## References + +- [VHDLTest User Guide][user-guide] +- [VHDLTest Repository][repo] + +[user-guide]: ../user_guide/introduction.md +[repo]: https://github.com/demaconsulting/VHDLTest diff --git a/docs/design/program.md b/docs/design/program.md new file mode 100644 index 0000000..3fa5faf --- /dev/null +++ b/docs/design/program.md @@ -0,0 +1,23 @@ +# Program Unit Design + +## Overview + +`Program.cs` is the entry point of VHDLTest. It parses the command-line context, +dispatches to the appropriate handler (version, help, validate, or test execution), +and returns a process exit code. + +## Responsibilities + +- Initialize the `Context` from command-line arguments +- Display version information when requested +- Display help/usage information when requested +- Invoke self-validation when requested +- Load configuration and execute VHDL tests when a configuration file is specified +- Return appropriate process exit codes + +## Interactions + +- Depends on `Cli/Context.cs` for command-line argument parsing +- Depends on `Cli/Options.cs` and `Cli/ConfigDocument.cs` for test configuration +- Depends on `Simulators/` for VHDL simulation +- Depends on `SelfTest/Validation.cs` for self-validation diff --git a/docs/design/results/results.md b/docs/design/results/results.md new file mode 100644 index 0000000..6f41fca --- /dev/null +++ b/docs/design/results/results.md @@ -0,0 +1,18 @@ +# Results Subsystem Design + +## Overview + +The Results subsystem provides the data model for storing and serializing VHDL test +results in standard formats (TRX and JUnit XML). + +## Units + +| Unit | File | Responsibility | +| ----------- | ------------------------ | --------------------------------------------- | +| TestResult | `Results/TestResult.cs` | Represents a single test result | +| TestResults | `Results/TestResults.cs` | Collection of test results with serialization | + +## Interactions + +- Populated by the Run subsystem with pass/fail outcomes +- Serialized to TRX or JUnit XML output files when `--results` is specified diff --git a/docs/design/results/test-result.md b/docs/design/results/test-result.md new file mode 100644 index 0000000..35ef0c5 --- /dev/null +++ b/docs/design/results/test-result.md @@ -0,0 +1,15 @@ +# TestResult Unit Design + +## Overview + +`TestResult.cs` represents the outcome of a single VHDL test bench execution. + +## Data Model + +| Property | Type | Description | +| ------------ | ------------ | ---------------------------------------------------------------- | +| `ClassName` | `string` | Fully qualified test class name | +| `TestName` | `string` | Test bench name (logical test identifier) | +| `RunResults` | `RunResults` | Execution results, including `Duration` as a `double` (seconds) | +| `Passed` | `bool` | Derived: true if the test passed based on `RunResults` | +| `Failed` | `bool` | Derived: logical negation of `Passed` | diff --git a/docs/design/results/test-results.md b/docs/design/results/test-results.md new file mode 100644 index 0000000..ca81be2 --- /dev/null +++ b/docs/design/results/test-results.md @@ -0,0 +1,13 @@ +# TestResults Unit Design + +## Overview + +`TestResults.cs` is a collection of `TestResult` instances with serialization +support for TRX and JUnit XML formats. + +## Responsibilities + +- Hold a list of `TestResult` instances +- Serialize results to TRX format for MSTest-compatible reporting +- Serialize results to JUnit XML format for CI/CD system integration +- Compute aggregate pass/fail counts diff --git a/docs/design/run/run-line-rule.md b/docs/design/run/run-line-rule.md new file mode 100644 index 0000000..45d8fe3 --- /dev/null +++ b/docs/design/run/run-line-rule.md @@ -0,0 +1,11 @@ +# RunLineRule Unit Design + +## Overview + +`RunLineRule.cs` defines a pattern rule for classifying simulator output lines +into `RunLineType` categories. + +## Responsibilities + +- Match output lines against a regular expression pattern +- Return the associated `RunLineType` when a match is found diff --git a/docs/design/run/run-line-type.md b/docs/design/run/run-line-type.md new file mode 100644 index 0000000..b60b0f3 --- /dev/null +++ b/docs/design/run/run-line-type.md @@ -0,0 +1,15 @@ +# RunLineType Unit Design + +## Overview + +`RunLineType.cs` is an enumeration of output line classification types used +to categorize simulator output lines. + +## Values + +| Value | Description | +| --------- | ----------------------------------------------- | +| `Text` | Standard textual output | +| `Info` | Informational message | +| `Warning` | Warning or non-fatal issue | +| `Error` | Output indicating an error or failure condition | diff --git a/docs/design/run/run-line.md b/docs/design/run/run-line.md new file mode 100644 index 0000000..8bc835b --- /dev/null +++ b/docs/design/run/run-line.md @@ -0,0 +1,12 @@ +# RunLine Unit Design + +## Overview + +`RunLine.cs` represents a single line of output captured from a simulator execution. + +## Data Model + +| Property | Type | Description | +| ---------- | ------------- | ----------------------------------- | +| `Text` | `string` | The text content of the output line | +| `Type` | `RunLineType` | The classification of this line | diff --git a/docs/design/run/run-processor.md b/docs/design/run/run-processor.md new file mode 100644 index 0000000..b7d93a9 --- /dev/null +++ b/docs/design/run/run-processor.md @@ -0,0 +1,12 @@ +# RunProcessor Unit Design + +## Overview + +`RunProcessor.cs` coordinates the execution of simulator programs and processes +their output to determine test outcomes. + +## Responsibilities + +- Apply `RunLineRule` patterns to classify output lines +- Determine pass/fail status for each test bench execution +- Aggregate results across multiple test benches diff --git a/docs/design/run/run-program.md b/docs/design/run/run-program.md new file mode 100644 index 0000000..cdb3739 --- /dev/null +++ b/docs/design/run/run-program.md @@ -0,0 +1,11 @@ +# RunProgram Unit Design + +## Overview + +`RunProgram.cs` executes an external simulator program and captures its output. + +## Responsibilities + +- Launch an external process with specified arguments +- Capture stdout and stderr output +- Return a `RunResults` instance with exit code and output lines diff --git a/docs/design/run/run-results.md b/docs/design/run/run-results.md new file mode 100644 index 0000000..49e3165 --- /dev/null +++ b/docs/design/run/run-results.md @@ -0,0 +1,16 @@ +# RunResults Unit Design + +## Overview + +`RunResults.cs` holds the results of executing an external simulator program. + +## Data Model + +| Property | Type | Description | +| ------------ | ----------------------------- | ------------------------------------------- | +| `Summary` | `string` | High-level summary of the run | +| `Start` | `DateTime` | Timestamp when the run started | +| `Duration` | `double` | Duration of the run in seconds | +| `ExitCode` | `int` | Process exit code | +| `Output` | `string` | Full captured stdout/stderr text | +| `Lines` | `ReadOnlyCollection` | Captured output lines, split into run lines | diff --git a/docs/design/run/run.md b/docs/design/run/run.md new file mode 100644 index 0000000..f34534f --- /dev/null +++ b/docs/design/run/run.md @@ -0,0 +1,24 @@ +# Run Subsystem Design + +## Overview + +The Run subsystem handles the execution of VHDL simulation programs and the processing +of their output to determine test outcomes. + +## Units + +| Unit | File | Responsibility | +| ------------ | --------------------- | ------------------------------------------------- | +| RunProcessor | `Run/RunProcessor.cs` | Coordinates execution and output processing | +| RunProgram | `Run/RunProgram.cs` | Executes an external program and captures output | +| RunResults | `Run/RunResults.cs` | Holds execution results (exit code, output lines) | +| RunLine | `Run/RunLine.cs` | Represents a single line of program output | +| RunLineRule | `Run/RunLineRule.cs` | Rule for classifying output lines | +| RunLineType | `Run/RunLineType.cs` | Enumeration of output line type classifications | + +## Design Flow + +1. `RunProgram` executes the simulator executable and captures stdout/stderr +2. `RunResults` holds the captured output as a list of `RunLine` instances +3. `RunLineRule` patterns classify each `RunLine` into a `RunLineType` +4. `RunProcessor` aggregates classified lines to determine overall pass/fail diff --git a/docs/design/self-test/self-test.md b/docs/design/self-test/self-test.md new file mode 100644 index 0000000..13c074f --- /dev/null +++ b/docs/design/self-test/self-test.md @@ -0,0 +1,19 @@ +# SelfTest Subsystem Design + +## Overview + +The SelfTest subsystem implements VHDLTest's self-validation capability, allowing +users to verify that the tool is correctly installed and functioning in their environment. + +## Units + +| Unit | File | Responsibility | +| ---------- | ------------------------ | ------------------------------------------ | +| Validation | `SelfTest/Validation.cs` | Executes and reports self-validation tests | + +## Interactions + +- Invoked by `Program.cs` when `--validate` is specified +- Uses embedded VHDL test files from `ValidationFiles/` for test execution +- Produces output via the `Context` I/O channels +- Optionally writes a TRX or JUnit results file when `--results` is specified diff --git a/docs/design/self-test/validation.md b/docs/design/self-test/validation.md new file mode 100644 index 0000000..e549906 --- /dev/null +++ b/docs/design/self-test/validation.md @@ -0,0 +1,20 @@ +# Validation Unit Design + +## Overview + +`SelfTest/Validation.cs` implements the self-validation test runner for VHDLTest. +It uses a set of embedded VHDL files to verify the tool is correctly installed +and that the configured simulator is functioning correctly. + +## Responsibilities + +- Execute a set of built-in VHDL tests using the available simulator +- Report validation results via the `Context` output channels +- Optionally save results to a TRX or JUnit file +- Include system information (OS, .NET runtime) in the validation report + +## Interactions + +- Called by `Program.cs` when `--validate` is present in the `Context` +- Uses `SimulatorFactory` to obtain the active simulator +- Uses `Results/TestResults.cs` to build and serialize the results report diff --git a/docs/design/simulators/active-hdl-simulator.md b/docs/design/simulators/active-hdl-simulator.md new file mode 100644 index 0000000..87f3f63 --- /dev/null +++ b/docs/design/simulators/active-hdl-simulator.md @@ -0,0 +1,12 @@ +# ActiveHdlSimulator Unit Design + +## Overview + +`ActiveHdlSimulator.cs` implements VHDL simulation using the Active-HDL simulator +from Aldec. + +## Responsibilities + +- Invoke Active-HDL compilation commands for each test file +- Invoke Active-HDL simulation for each test bench +- Parse Active-HDL output to determine pass/fail status diff --git a/docs/design/simulators/ghdl-simulator.md b/docs/design/simulators/ghdl-simulator.md new file mode 100644 index 0000000..74e9d7a --- /dev/null +++ b/docs/design/simulators/ghdl-simulator.md @@ -0,0 +1,11 @@ +# GhdlSimulator Unit Design + +## Overview + +`GhdlSimulator.cs` implements VHDL simulation using the GHDL open-source simulator. + +## Responsibilities + +- Invoke GHDL analysis (compilation) for each test file +- Invoke GHDL elaboration and run for each test bench +- Parse GHDL output to determine pass/fail status diff --git a/docs/design/simulators/modelsim-simulator.md b/docs/design/simulators/modelsim-simulator.md new file mode 100644 index 0000000..456db99 --- /dev/null +++ b/docs/design/simulators/modelsim-simulator.md @@ -0,0 +1,11 @@ +# ModelSimSimulator Unit Design + +## Overview + +`ModelSimSimulator.cs` implements VHDL simulation using the ModelSim commercial simulator. + +## Responsibilities + +- Invoke ModelSim vcom (compilation) for each test file +- Invoke ModelSim vsim (simulation) for each test bench +- Parse ModelSim output to determine pass/fail status diff --git a/docs/design/simulators/nvc-simulator.md b/docs/design/simulators/nvc-simulator.md new file mode 100644 index 0000000..6e524c0 --- /dev/null +++ b/docs/design/simulators/nvc-simulator.md @@ -0,0 +1,11 @@ +# NvcSimulator Unit Design + +## Overview + +`NvcSimulator.cs` implements VHDL simulation using the NVC open-source simulator. + +## Responsibilities + +- Invoke NVC analysis and elaboration for each test bench +- Invoke NVC run for each test bench +- Parse NVC output to determine pass/fail status diff --git a/docs/design/simulators/questasim-simulator.md b/docs/design/simulators/questasim-simulator.md new file mode 100644 index 0000000..5539b6d --- /dev/null +++ b/docs/design/simulators/questasim-simulator.md @@ -0,0 +1,11 @@ +# QuestaSimSimulator Unit Design + +## Overview + +`QuestaSimSimulator.cs` implements VHDL simulation using the QuestaSim commercial simulator. + +## Responsibilities + +- Invoke QuestaSim vcom (compilation) for each test file +- Invoke QuestaSim vsim (simulation) for each test bench +- Parse QuestaSim output to determine pass/fail status diff --git a/docs/design/simulators/simulator-factory.md b/docs/design/simulators/simulator-factory.md new file mode 100644 index 0000000..757e75f --- /dev/null +++ b/docs/design/simulators/simulator-factory.md @@ -0,0 +1,13 @@ +# SimulatorFactory Unit Design + +## Overview + +`SimulatorFactory.cs` implements the factory method for creating `Simulator` instances +by name. + +## Responsibilities + +- Map simulator name strings to concrete `Simulator` implementations +- Return the appropriate simulator for a given configuration +- Return `null` for unknown simulator names +- Callers are responsible for handling this case (for example, by throwing a descriptive exception) diff --git a/docs/design/simulators/simulator.md b/docs/design/simulators/simulator.md new file mode 100644 index 0000000..aac6d0b --- /dev/null +++ b/docs/design/simulators/simulator.md @@ -0,0 +1,12 @@ +# Simulator Unit Design + +## Overview + +`Simulator.cs` defines the abstract `Simulator` base class that all VHDL simulator +integrations must inherit from. + +## Responsibilities + +- Define the `SimulatorName` property contract +- Define the `Compile` and `Simulate` method contracts +- Provide shared utilities used by concrete simulator implementations diff --git a/docs/design/simulators/simulators.md b/docs/design/simulators/simulators.md new file mode 100644 index 0000000..942feac --- /dev/null +++ b/docs/design/simulators/simulators.md @@ -0,0 +1,32 @@ +# Simulators Subsystem Design + +## Overview + +The Simulators subsystem provides integration with multiple VHDL simulators. It implements +a factory pattern for creating simulator instances and an abstract base class defining +the interface all simulators must implement. + +## Units + +| Unit | File | Responsibility | +| ------------------ | ---------------------------------- | ------------------------------------------------ | +| Simulator | `Simulators/Simulator.cs` | Abstract base class defining simulator interface | +| SimulatorFactory | `Simulators/SimulatorFactory.cs` | Creates simulator instances by name | +| GhdlSimulator | `Simulators/GhdlSimulator.cs` | GHDL open-source simulator integration | +| NvcSimulator | `Simulators/NvcSimulator.cs` | NVC open-source simulator integration | +| ModelSimSimulator | `Simulators/ModelSimSimulator.cs` | ModelSim commercial simulator integration | +| QuestaSimSimulator | `Simulators/QuestaSimSimulator.cs` | QuestaSim commercial simulator integration | +| VivadoSimulator | `Simulators/VivadoSimulator.cs` | Vivado FPGA simulator integration | +| ActiveHdlSimulator | `Simulators/ActiveHdlSimulator.cs` | Active-HDL simulator integration | +| MockSimulator | `Simulators/MockSimulator.cs` | Test double for simulator testing | + +## Design Pattern + +The subsystem uses the Factory pattern. `SimulatorFactory.Get(name)` returns the +appropriate `Simulator`-derived instance based on the simulator name string found +in the configuration file. + +## Interactions + +- Consumed by `Program.cs` via `SimulatorFactory` to obtain the active simulator +- Each simulator receives `RunProgram` callbacks to execute compilation and simulation steps diff --git a/docs/design/simulators/vivado-simulator.md b/docs/design/simulators/vivado-simulator.md new file mode 100644 index 0000000..33fbb89 --- /dev/null +++ b/docs/design/simulators/vivado-simulator.md @@ -0,0 +1,12 @@ +# VivadoSimulator Unit Design + +## Overview + +`VivadoSimulator.cs` implements VHDL simulation using the Vivado simulator +from Xilinx/AMD. + +## Responsibilities + +- Invoke `xvhdl` analysis for each VHDL source file +- Invoke `xelab` elaboration and simulation for each test bench +- Parse Vivado output to determine pass/fail status diff --git a/docs/design/system.md b/docs/design/system.md new file mode 100644 index 0000000..03551fe --- /dev/null +++ b/docs/design/system.md @@ -0,0 +1,54 @@ +# VHDLTest System Design + +## Overview + +VHDLTest is a .NET command-line tool for running VHDL unit tests and generating test +reports. It accepts command-line arguments, loads a YAML configuration file, invokes a +VHDL simulator, processes the simulation output, and reports test pass/fail results. + +## Responsibilities + +- Provide a command-line interface for VHDL test execution +- Support multiple VHDL simulator back-ends +- Parse simulation output to determine test outcomes +- Report test results in standard formats (TRX, JUnit) +- Provide self-validation tests + +## Top-Level Unit + +| Unit | File | Responsibility | +| ------- | ------------ | ----------------------------------------------------- | +| Program | `Program.cs` | Entry point, command dispatch, and process exit code | + +## Subsystems + +| Subsystem | Folder | Responsibility | +| ---------- | ------------- | ---------------------------------------------------- | +| Cli | `Cli/` | Command-line argument parsing and I/O channels | +| Simulators | `Simulators/` | VHDL simulator integrations | +| Run | `Run/` | Simulation program execution and output processing | +| Results | `Results/` | Test result data model and report generation | +| SelfTest | `SelfTest/` | Self-validation test runner | + +## Execution Flow + +1. `Program.Main` creates a `Context` from the raw command-line argument array + (Cli subsystem) +2. `Program.Run` dispatches based on the context flags: + - **Version flag** — prints the version string and returns + - **Help flag** — prints version banner and usage information, then returns + - **Validate flag** — delegates to `Validation.Run` (SelfTest subsystem) and returns + - **Config file specified** — executes VHDL tests: + 1. `SimulatorFactory.Get` selects the named simulator (Simulators subsystem) + 2. `Options.Parse` reads the YAML configuration file (Cli subsystem) + 3. `TestResults.Execute` drives the simulator through the Run subsystem and + collects test outcomes (Results subsystem) + 4. `TestResults.PrintSummary` reports the summary to the context + 5. If a results file is specified, `TestResults.SaveResults` writes the TRX/JUnit output + - **No config file** — writes an error and prints usage, then returns +3. `Program.Main` sets `Environment.ExitCode` from `context.ExitCode` + +## Interactions + +The `Program` unit is the sole entry point and acts as the orchestrator. It depends on +all subsystems but has no reverse dependencies — no subsystem calls back into `Program`. diff --git a/docs/reqstream/cli/subsystem-cli.yaml b/docs/reqstream/cli/subsystem-cli.yaml new file mode 100644 index 0000000..9abc26d --- /dev/null +++ b/docs/reqstream/cli/subsystem-cli.yaml @@ -0,0 +1,17 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Cli Subsystem + requirements: + - id: VHDLTest-Cli-Pipeline + title: >- + VHDLTest shall parse command-line arguments, load the configuration file, + and produce a combined Options object. + justification: | + The end-to-end CLI pipeline — argument parsing, configuration loading, and option + combination — must function correctly as an integrated whole. Verifying this + integration catches issues that cannot be found by testing each unit in isolation. + tests: + - CliSubsystem_ParseArgsAndLoadConfig_WithValidConfig_ProducesCorrectOptions + - CliSubsystem_ParseArgsAndLoadConfig_WithMissingConfig_ThrowsFileNotFoundException diff --git a/docs/reqstream/cli/unit-config-document.yaml b/docs/reqstream/cli/unit-config-document.yaml new file mode 100644 index 0000000..4740ab8 --- /dev/null +++ b/docs/reqstream/cli/unit-config-document.yaml @@ -0,0 +1,23 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: ConfigDocument Unit + requirements: + - id: VHDLTest-ConfigDoc-Read + title: The ConfigDocument class shall deserialize a YAML configuration file. + justification: | + Deserializing the YAML configuration into a typed object ensures configuration + values are validated at load time and provides a clear interface for downstream + code to access test parameters. + tests: + - ConfigDocument_ReadFile_ValidFile_ReadsSuccessfully + + - id: VHDLTest-ConfigDoc-Missing + title: The ConfigDocument class shall throw FileNotFoundException for missing files. + justification: | + Throwing a descriptive exception for missing configuration files allows the + application to surface a clear error message to the user rather than a + cryptic null-reference error. + tests: + - ConfigDocument_ReadFile_MissingFile_ThrowsFileNotFoundException diff --git a/docs/reqstream/cli/unit-context.yaml b/docs/reqstream/cli/unit-context.yaml new file mode 100644 index 0000000..f34bf5a --- /dev/null +++ b/docs/reqstream/cli/unit-context.yaml @@ -0,0 +1,33 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Context Unit + requirements: + - id: VHDLTest-Context-Parse + title: The Context class shall parse all supported command-line flags and arguments. + justification: | + Comprehensive flag parsing ensures all command-line options are correctly interpreted + and stored as typed properties, providing a reliable interface between the CLI and + application logic. + tests: + - Context_Create_NoArguments_ReturnsDefaultContext + - Context_Create_WithConfigFile_SetsConfigFile + - Context_Create_WithResultsFile_SetsResultsFile + - Context_Create_WithSimulator_SetsSimulator + - Context_Create_WithVerbose_SetsVerboseFlag + - Context_Create_WithExitZero_SetsExitZeroFlag + - Context_Create_WithValidate_SetsValidateFlag + - Context_Create_WithCustomTest_SetsCustomTest + - Context_Create_WithCustomTests_SetsCustomTests + + - id: VHDLTest-Context-Error + title: The Context class shall throw InvalidOperationException for unrecognized arguments. + justification: | + Throwing a typed exception for unrecognized arguments allows the application entry + point to provide a clear error message and prevents silent misconfiguration. + tests: + - Context_Create_UnknownArgument_ThrowsInvalidOperationException + - Context_Create_MissingConfigValue_ThrowsInvalidOperationException + - Context_Create_MissingResultsValue_ThrowsInvalidOperationException + - Context_Create_MissingSimulatorValue_ThrowsInvalidOperationException diff --git a/docs/reqstream/cli/unit-options.yaml b/docs/reqstream/cli/unit-options.yaml new file mode 100644 index 0000000..02d05a8 --- /dev/null +++ b/docs/reqstream/cli/unit-options.yaml @@ -0,0 +1,32 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Options Unit + requirements: + - id: VHDLTest-Options-Parse + title: The Options class shall combine Context and ConfigDocument into test run options. + justification: | + Combining the command-line context and configuration document into a single Options + record provides a single, validated source of truth for test execution, simplifying + downstream logic. + tests: + - Options_Parse_ValidConfigFile_ParsesSuccessfully + - Options_Parse_WithVerboseFlag_ParsesSuccessfully + - Options_Parse_WithCustomTest_ParsesSuccessfully + + - id: VHDLTest-Options-NoConfig + title: The Options class shall throw InvalidOperationException when no config file is provided. + justification: | + Validating that a configuration file is specified before attempting to parse options + provides an early and clear error message, improving user experience. + tests: + - Options_Parse_NoConfigProvided_ThrowsInvalidOperationException + + - id: VHDLTest-Options-MissingConfig + title: The Options class shall throw FileNotFoundException for a missing configuration file. + justification: | + Propagating the FileNotFoundException from ConfigDocument.ReadFile ensures the caller + receives a descriptive error when the specified configuration file cannot be found. + tests: + - Options_Parse_MissingConfigFile_ThrowsFileNotFoundException diff --git a/docs/reqstream/results/subsystem-results.yaml b/docs/reqstream/results/subsystem-results.yaml new file mode 100644 index 0000000..3f394b6 --- /dev/null +++ b/docs/reqstream/results/subsystem-results.yaml @@ -0,0 +1,22 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Results Subsystem + requirements: + - id: VHDLTest-Results-Collect + title: VHDLTest shall collect individual test results and report pass and fail counts. + justification: | + Aggregating individual test outcomes into a summary with pass/fail counts lets + users and CI/CD systems quickly determine overall test health without inspecting + every individual test result. + tests: + - ResultsSubsystem_CollectAndSummarize_WithMixedResults_ReportsCorrectPassFailCounts + + - id: VHDLTest-Results-Save + title: VHDLTest shall save a collection of test results to a results file. + justification: | + Persisting the collected test results to a file enables integration with CI/CD + platforms and test reporting tools that consume standard test result formats. + tests: + - ResultsSubsystem_SaveMixedResults_ToTrxFile_CreatesTrxFileWithCorrectCounts diff --git a/docs/reqstream/results/unit-test-result.yaml b/docs/reqstream/results/unit-test-result.yaml new file mode 100644 index 0000000..e966f2f --- /dev/null +++ b/docs/reqstream/results/unit-test-result.yaml @@ -0,0 +1,23 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: TestResult Unit + requirements: + - id: VHDLTest-TestResult-Pass + title: TestResult shall record a passed test when the run result indicates success. + justification: | + Accurately recording pass results ensures that the test report correctly + reflects the outcome of each test bench execution and provides traceability + from simulator output to the final test report. + tests: + - TestResult_Constructor_WithInfoResult_CreatesPassedTest + + - id: VHDLTest-TestResult-Fail + title: TestResult shall record a failed test when the run result indicates an error. + justification: | + Accurately recording failure results is essential for CI/CD systems to detect + and report test failures, enabling rapid identification and resolution of + VHDL design defects. + tests: + - TestResult_Constructor_WithErrorResult_CreatesFailedTest diff --git a/docs/reqstream/results/unit-test-results.yaml b/docs/reqstream/results/unit-test-results.yaml new file mode 100644 index 0000000..1e32b8c --- /dev/null +++ b/docs/reqstream/results/unit-test-results.yaml @@ -0,0 +1,34 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: TestResults Unit + requirements: + - id: VHDLTest-TestResults-Trx + title: TestResults shall serialize test results to TRX format. + justification: | + TRX is the standard test results format for .NET and Microsoft tooling. Saving + results in TRX format enables integration with Visual Studio, Azure DevOps, and + other .NET-ecosystem CI/CD tools. + tests: + - TestResults_SaveResults_WithTrxExtension_CreatesTrxFile + - TestResults_SaveToTrx_WithTestResults_CreatesTrxFile + - TestResults_SaveResults_WithUnknownExtension_CreatesTrxFile + + - id: VHDLTest-TestResults-JUnit + title: TestResults shall serialize test results to JUnit XML format. + justification: | + JUnit XML is the de-facto standard test results format for CI/CD systems such as + Jenkins, GitLab CI, and GitHub Actions. Saving results in JUnit XML format enables + broad CI/CD integration. + tests: + - TestResults_SaveResults_WithXmlExtension_CreatesJUnitFile + - TestResults_SaveResults_WithFailedTest_CreatesJUnitFileWithFailure + + - id: VHDLTest-TestResults-NullFile + title: TestResults shall raise an exception when a null file path is provided. + justification: | + Raising a clear exception for a null path prevents silent failures and provides + callers with an actionable error message. + tests: + - TestResults_SaveResults_WithNullFileName_ThrowsArgumentException diff --git a/docs/reqstream/run/subsystem-run.yaml b/docs/reqstream/run/subsystem-run.yaml new file mode 100644 index 0000000..cec006d --- /dev/null +++ b/docs/reqstream/run/subsystem-run.yaml @@ -0,0 +1,15 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Run Subsystem + requirements: + - id: VHDLTest-Run-Execute + title: VHDLTest shall execute simulator programs and capture their classified output. + justification: | + Executing the simulator as an external process and classifying its output allows + VHDLTest to work with any command-line VHDL simulator without requiring + programmatic integration. + tests: + - RunSubsystem_ExecuteRealProgram_WithClassificationRules_ProducesClassifiedRunResults + - RunSubsystem_ExecuteRealProgram_WithErrorExitCode_ProducesErrorRunResults diff --git a/docs/reqstream/run/unit-run-processor.yaml b/docs/reqstream/run/unit-run-processor.yaml new file mode 100644 index 0000000..1f92a9d --- /dev/null +++ b/docs/reqstream/run/unit-run-processor.yaml @@ -0,0 +1,29 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: RunProcessor Unit + requirements: + - id: VHDLTest-RunProcessor-Pass + title: RunProcessor shall classify simulator output as a pass when pass patterns match. + justification: | + Correct classification of pass output is essential for accurate test reporting and + for distinguishing successful test bench executions from failures. + tests: + - RunProcessor_Execute_ProgramWithSuccess_ReturnsInfoResult + + - id: VHDLTest-RunProcessor-Fail + title: RunProcessor shall classify simulator output as a fail when fail patterns match. + justification: | + Correct classification of fail output enables VHDLTest to accurately identify + test failures and report them to the user and CI/CD system. + tests: + - RunProcessor_Execute_ProgramWithError_ReturnsErrorResult + + - id: VHDLTest-RunProcessor-MissingProgram + title: RunProcessor shall raise an exception when the simulator program cannot be found. + justification: | + Raising a clear exception for missing programs helps users quickly diagnose + configuration errors, such as an incorrectly specified simulator path. + tests: + - RunProcessor_Execute_MissingProgram_ThrowsException diff --git a/docs/reqstream/run/unit-run-program.yaml b/docs/reqstream/run/unit-run-program.yaml new file mode 100644 index 0000000..7c92228 --- /dev/null +++ b/docs/reqstream/run/unit-run-program.yaml @@ -0,0 +1,17 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: RunProgram Unit + requirements: + - id: VHDLTest-RunProgram-Execute + title: RunProgram shall execute an external program and return its combined output. + justification: | + Executing simulator tools as external processes decouples VHDLTest from any + specific simulator API, allowing it to integrate with any command-line VHDL + simulator. Capturing both stdout and stderr ensures no output is lost when + diagnosing failures. + tests: + - RunProcessor_Execute_ProgramWithSuccess_ReturnsInfoResult + - RunProcessor_Execute_ProgramWithError_ReturnsErrorResult + - RunProcessor_Execute_MissingProgram_ThrowsException diff --git a/docs/reqstream/run/unit-run-results.yaml b/docs/reqstream/run/unit-run-results.yaml new file mode 100644 index 0000000..94187de --- /dev/null +++ b/docs/reqstream/run/unit-run-results.yaml @@ -0,0 +1,15 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: RunResults Unit + requirements: + - id: VHDLTest-RunResults-Print + title: RunResults shall print classified output lines with color-coded severity. + justification: | + Color-coded output helps users quickly distinguish between informational, + warning, and error output lines when reviewing simulator output in the console, + reducing the time needed to identify failures. + tests: + - RunProcessor_Execute_ProgramWithSuccess_ReturnsInfoResult + - RunProcessor_Execute_ProgramWithError_ReturnsErrorResult diff --git a/docs/reqstream/run/unit-run-types.yaml b/docs/reqstream/run/unit-run-types.yaml new file mode 100644 index 0000000..f3656a3 --- /dev/null +++ b/docs/reqstream/run/unit-run-types.yaml @@ -0,0 +1,16 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Run Types Unit + requirements: + - id: VHDLTest-RunTypes-Classify + title: RunLineRule shall classify simulator output lines into RunLineType categories. + justification: | + Pattern-based line classification allows each simulator integration to define + its own output patterns without changing the core execution logic, making it + easy to add support for new simulators or update patterns when simulator + output formats change. + tests: + - RunProcessor_Execute_ProgramWithSuccess_ReturnsInfoResult + - RunProcessor_Execute_ProgramWithError_ReturnsErrorResult diff --git a/docs/reqstream/self-test/subsystem-self-test.yaml b/docs/reqstream/self-test/subsystem-self-test.yaml new file mode 100644 index 0000000..f2eae3e --- /dev/null +++ b/docs/reqstream/self-test/subsystem-self-test.yaml @@ -0,0 +1,41 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: SelfTest Subsystem + requirements: + - id: VHDLTest-Val-Validate + title: VHDLTest shall support self-validation mode. + justification: | + Enables users to verify that VHDLTest is correctly installed and configured in + their environment. This diagnostic capability helps identify configuration issues + before running actual test suites, reducing troubleshooting time. + tests: + - IntegrationTest_ValidateFlag_PerformsValidationAndReturnsSuccess + + - id: VHDLTest-Val-Depth + title: VHDLTest shall support configurable validation report depth. + justification: | + Enables control of markdown header depth in validation reports when embedding the + validation output in other documents. This ensures proper document hierarchy and + formatting when validation reports are included as sections in larger documents. + tests: + - IntegrationTest_ValidateFlagWithDepth_PerformsValidationWithDepth + + - id: VHDLTest-Val-Results + title: VHDLTest shall save validation results to file. + justification: | + Enables persistent storage of validation results for documentation, compliance, + and historical comparison purposes. Saved reports can be shared with team members + or included in bug reports for support. + tests: + - IntegrationTest_ValidateFlagWithResultsFile_SavesValidationResults + + - id: VHDLTest-Val-SystemInfo + title: VHDLTest shall include system information in validation reports. + justification: | + Provides essential context for validation results by capturing the execution + environment (OS version, runtime, etc.). This information is crucial for + reproducing issues and diagnosing environment-specific problems. + tests: + - IntegrationTest_ValidateFlag_IncludesOSVersionInReport diff --git a/docs/reqstream/self-test/unit-validation.yaml b/docs/reqstream/self-test/unit-validation.yaml new file mode 100644 index 0000000..f66bf23 --- /dev/null +++ b/docs/reqstream/self-test/unit-validation.yaml @@ -0,0 +1,30 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Validation Unit + requirements: + - id: VHDLTest-Validation-Run + title: The Validation class shall execute built-in VHDL test benches to verify correct operation. + justification: | + Running built-in test benches during self-validation provides concrete evidence that + the simulator integration is working correctly in the user's environment, rather than + just checking that the tool starts up. + tests: + - IntegrationTest_ValidateFlag_PerformsValidationAndReturnsSuccess + + - id: VHDLTest-Validation-SystemInfo + title: The Validation class shall include OS and runtime information in the validation report. + justification: | + Including system information in the validation report provides essential context for + diagnosing environment-specific issues and for compliance documentation. + tests: + - IntegrationTest_ValidateFlag_IncludesOSVersionInReport + + - id: VHDLTest-Validation-Results + title: The Validation class shall save results to a file when a results path is specified. + justification: | + Saving validation results to a file enables integration with CI/CD pipelines and + provides a persistent record of validation outcomes for compliance documentation. + tests: + - IntegrationTest_ValidateFlagWithResultsFile_SavesValidationResults diff --git a/docs/reqstream/simulators/subsystem-simulators.yaml b/docs/reqstream/simulators/subsystem-simulators.yaml new file mode 100644 index 0000000..6592d1b --- /dev/null +++ b/docs/reqstream/simulators/subsystem-simulators.yaml @@ -0,0 +1,15 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Simulators Subsystem + requirements: + - id: VHDLTest-Simulators-Factory + title: VHDLTest shall select the simulator implementation by name and use it to classify output. + justification: | + A factory pattern for simulator selection decouples test execution from the + concrete simulator implementation. Verifying that the factory and simulator + processors work together end-to-end ensures the subsystem interfaces are correct. + tests: + - SimulatorsSubsystem_GetSimulatorAndProcessCompileOutput_WithErrorOutput_ClassifiesAsError + - SimulatorsSubsystem_GetUnknownSimulatorAndProcessCleanOutput_ReturnsNullAndClassifiesText diff --git a/docs/reqstream/simulators/unit-active-hdl-simulator.yaml b/docs/reqstream/simulators/unit-active-hdl-simulator.yaml new file mode 100644 index 0000000..e26f78c --- /dev/null +++ b/docs/reqstream/simulators/unit-active-hdl-simulator.yaml @@ -0,0 +1,38 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: ActiveHdlSimulator Unit + requirements: + - id: VHDLTest-Sim-ActiveHDL + title: VHDLTest shall support the ActiveHDL simulator. + justification: | + Active-HDL from Aldec is a popular VHDL simulator particularly in the aerospace + and defense industries. Supporting Active-HDL ensures VHDLTest can be used in + regulated environments and specialized application domains. + tests: + - SimulatorFactory_Get_ActiveHDLSimulator_ReturnsActiveHDLSimulator + - ActiveHdlSimulator_SimulatorName_ReturnsActiveHDL + + - id: VHDLTest-Sim-ActiveHDL-Compile + title: ActiveHdlSimulator shall classify Active-HDL compilation output lines by severity. + justification: | + Accurate classification of compilation output enables VHDLTest to detect + warnings and errors during the analysis phase and report them clearly to + the user. + tests: + - ActiveHdlSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - ActiveHdlSimulator_CompileProcessor_WarningOutput_ReturnsWarningResult + - ActiveHdlSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-ActiveHDL-Test + title: ActiveHdlSimulator shall classify Active-HDL simulation output lines by severity. + justification: | + Accurate classification of simulation output enables VHDLTest to detect pass, + warning, and error conditions during test bench execution and reflect them + in the test report. + tests: + - ActiveHdlSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - ActiveHdlSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - ActiveHdlSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - ActiveHdlSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult diff --git a/docs/reqstream/simulators/unit-ghdl-simulator.yaml b/docs/reqstream/simulators/unit-ghdl-simulator.yaml new file mode 100644 index 0000000..252ab93 --- /dev/null +++ b/docs/reqstream/simulators/unit-ghdl-simulator.yaml @@ -0,0 +1,40 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: GhdlSimulator Unit + requirements: + - id: VHDLTest-Sim-GHDL + title: VHDLTest shall support the GHDL simulator. + justification: | + GHDL is the most widely-used open-source VHDL simulator and is essential for + teams working with free and open-source toolchains. Supporting GHDL enables + cost-effective VHDL development and testing without commercial licenses. + tests: + - SimulatorFactory_Get_GhdlSimulator_ReturnsGhdlSimulator + - GhdlSimulator_SimulatorName_ReturnsGHDL + - ghdl@VHDLTest_TestPasses + - ghdl@VHDLTest_TestFails + + - id: VHDLTest-Sim-GHDL-Compile + title: GhdlSimulator shall classify GHDL compilation output lines by severity. + justification: | + Accurate classification of compilation output enables VHDLTest to detect + warnings and errors during the analysis phase and report them clearly to + the user. + tests: + - GhdlSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - GhdlSimulator_CompileProcessor_WarningOutput_ReturnsWarningResult + - GhdlSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-GHDL-Test + title: GhdlSimulator shall classify GHDL simulation output lines by severity. + justification: | + Accurate classification of simulation output enables VHDLTest to detect pass, + warning, and error conditions during test bench execution and reflect them + in the test report. + tests: + - GhdlSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - GhdlSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - GhdlSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - GhdlSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult diff --git a/docs/reqstream/simulators/unit-modelsim-simulator.yaml b/docs/reqstream/simulators/unit-modelsim-simulator.yaml new file mode 100644 index 0000000..c658f30 --- /dev/null +++ b/docs/reqstream/simulators/unit-modelsim-simulator.yaml @@ -0,0 +1,36 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: ModelSimSimulator Unit + requirements: + - id: VHDLTest-Sim-ModelSim + title: VHDLTest shall support the ModelSim simulator. + justification: | + ModelSim is an industry-standard commercial VHDL simulator widely used in + professional FPGA and ASIC development. Support for ModelSim is critical for + enterprise users and teams with existing ModelSim-based workflows. + tests: + - SimulatorFactory_Get_ModelSimSimulator_ReturnsModelSimSimulator + - ModelSimSimulator_SimulatorName_ReturnsModelSim + + - id: VHDLTest-Sim-ModelSim-Compile + title: ModelSimSimulator shall classify ModelSim compilation output lines by severity. + justification: | + Accurate classification of compilation output enables VHDLTest to detect + errors during the vcom analysis phase and report them clearly to the user. + tests: + - ModelSimSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - ModelSimSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-ModelSim-Test + title: ModelSimSimulator shall classify ModelSim simulation output lines by severity. + justification: | + Accurate classification of simulation output enables VHDLTest to detect pass, + warning, and error conditions during vsim execution and reflect them in the + test report. + tests: + - ModelSimSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - ModelSimSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - ModelSimSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - ModelSimSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult diff --git a/docs/reqstream/simulators/unit-nvc-simulator.yaml b/docs/reqstream/simulators/unit-nvc-simulator.yaml new file mode 100644 index 0000000..bc90303 --- /dev/null +++ b/docs/reqstream/simulators/unit-nvc-simulator.yaml @@ -0,0 +1,41 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: NvcSimulator Unit + requirements: + - id: VHDLTest-Sim-NVC + title: VHDLTest shall support the NVC simulator. + justification: | + NVC is a modern, fast open-source VHDL simulator with good standards compliance. + Supporting NVC provides users with an alternative open-source option that offers + better performance than GHDL in certain scenarios and supports newer VHDL standards. + tests: + - SimulatorFactory_Get_NVCSimulator_ReturnsNVCSimulator + - NvcSimulator_SimulatorName_ReturnsNVC + - nvc@VHDLTest_TestPasses + - nvc@VHDLTest_TestFails + + - id: VHDLTest-Sim-NVC-Compile + title: NvcSimulator shall classify NVC compilation output lines by severity. + justification: | + Accurate classification of compilation output enables VHDLTest to detect + warnings and errors during the analysis phase and report them clearly to + the user. + tests: + - NvcSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - NvcSimulator_CompileProcessor_InfoOutput_ReturnsInfoResult + - NvcSimulator_CompileProcessor_WarningOutput_ReturnsWarningResult + - NvcSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-NVC-Test + title: NvcSimulator shall classify NVC simulation output lines by severity. + justification: | + Accurate classification of simulation output enables VHDLTest to detect pass, + warning, and error conditions during test bench execution and reflect them + in the test report. + tests: + - NvcSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - NvcSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - NvcSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - NvcSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult diff --git a/docs/reqstream/simulators/unit-questasim-simulator.yaml b/docs/reqstream/simulators/unit-questasim-simulator.yaml new file mode 100644 index 0000000..51174b4 --- /dev/null +++ b/docs/reqstream/simulators/unit-questasim-simulator.yaml @@ -0,0 +1,37 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: QuestaSimSimulator Unit + requirements: + - id: VHDLTest-Sim-QuestaSim + title: VHDLTest shall support the QuestaSim simulator. + justification: | + QuestaSim is the advanced verification tool from Siemens/Mentor Graphics, offering + enhanced debugging and coverage features. Supporting QuestaSim enables VHDLTest + to integrate with high-end verification environments. + tests: + - SimulatorFactory_Get_QuestaSimSimulator_ReturnsQuestaSimSimulator + - QuestaSimSimulator_SimulatorName_ReturnsQuestaSim + + - id: VHDLTest-Sim-QuestaSim-Compile + title: QuestaSimSimulator shall classify QuestaSim compilation output lines by severity. + justification: | + Accurate classification of compilation output enables VHDLTest to detect + errors during the vcom analysis phase and report them clearly to the user. + tests: + - QuestaSimSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - QuestaSimSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-QuestaSim-Test + title: QuestaSimSimulator shall classify QuestaSim simulation output lines by severity. + justification: | + Accurate classification of simulation output enables VHDLTest to detect pass, + warning, failure, and error conditions during vsim execution and reflect them + in the test report. + tests: + - QuestaSimSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - QuestaSimSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - QuestaSimSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - QuestaSimSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult + - QuestaSimSimulator_TestProcessor_FailureOutput_ReturnsErrorResult diff --git a/docs/reqstream/simulators/unit-simulator-factory.yaml b/docs/reqstream/simulators/unit-simulator-factory.yaml new file mode 100644 index 0000000..715ce27 --- /dev/null +++ b/docs/reqstream/simulators/unit-simulator-factory.yaml @@ -0,0 +1,28 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: SimulatorFactory Unit + requirements: + - id: VHDLTest-SimulatorFactory-Get + title: SimulatorFactory shall return the correct simulator instance for a given name. + justification: | + The factory pattern centralizes simulator selection, allowing the rest of the + application to remain independent of the specific simulator being used and + enabling easy extension with new simulators. + tests: + - SimulatorFactory_Get_GhdlSimulator_ReturnsGhdlSimulator + - SimulatorFactory_Get_ModelSimSimulator_ReturnsModelSimSimulator + - SimulatorFactory_Get_VivadoSimulator_ReturnsVivadoSimulator + - SimulatorFactory_Get_ActiveHDLSimulator_ReturnsActiveHDLSimulator + - SimulatorFactory_Get_NVCSimulator_ReturnsNVCSimulator + - SimulatorFactory_Get_QuestaSimSimulator_ReturnsQuestaSimSimulator + + - id: VHDLTest-SimulatorFactory-Unknown + title: SimulatorFactory shall return null for unknown simulator names. + justification: | + Returning null for unknown simulator names allows the calling code to provide + a descriptive error message, rather than surfacing an opaque exception from + within the factory. + tests: + - SimulatorFactory_Get_UnknownSimulator_ReturnsNull diff --git a/docs/reqstream/simulators/unit-vivado-simulator.yaml b/docs/reqstream/simulators/unit-vivado-simulator.yaml new file mode 100644 index 0000000..4a10584 --- /dev/null +++ b/docs/reqstream/simulators/unit-vivado-simulator.yaml @@ -0,0 +1,36 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: VivadoSimulator Unit + requirements: + - id: VHDLTest-Sim-Vivado + title: VHDLTest shall support the Vivado simulator. + justification: | + Vivado is Xilinx's primary design suite and simulator for their FPGA products. + Supporting Vivado simulator is essential for teams developing designs targeting + Xilinx/AMD FPGAs and enables seamless integration with the Vivado toolchain. + tests: + - SimulatorFactory_Get_VivadoSimulator_ReturnsVivadoSimulator + - VivadoSimulator_SimulatorName_ReturnsVivado + + - id: VHDLTest-Sim-Vivado-Compile + title: VivadoSimulator shall classify Vivado compilation output lines by severity. + justification: | + Accurate classification of xvhdl/xelab output enables VHDLTest to detect + errors during the elaboration phase and report them clearly to the user. + tests: + - VivadoSimulator_CompileProcessor_CleanOutput_ReturnsTextResult + - VivadoSimulator_CompileProcessor_ErrorOutput_ReturnsErrorResult + + - id: VHDLTest-Sim-Vivado-Test + title: VivadoSimulator shall classify Vivado simulation output lines by severity. + justification: | + Accurate classification of Vivado simulation output enables VHDLTest to + detect pass, warning, and error conditions and reflect them in the test + report. + tests: + - VivadoSimulator_TestProcessor_CleanOutput_ReturnsTextResult + - VivadoSimulator_TestProcessor_InfoOutput_ReturnsInfoResult + - VivadoSimulator_TestProcessor_WarningOutput_ReturnsWarningResult + - VivadoSimulator_TestProcessor_ErrorOutput_ReturnsErrorResult diff --git a/docs/reqstream/subsystem-simulators.yaml b/docs/reqstream/subsystem-simulators.yaml deleted file mode 100644 index f788088..0000000 --- a/docs/reqstream/subsystem-simulators.yaml +++ /dev/null @@ -1,69 +0,0 @@ ---- -sections: - - title: VHDLTest Requirements - sections: - - title: Simulator Support - requirements: - - id: VHDLTest-Sim-GHDL - title: VHDLTest shall support the GHDL simulator. - justification: | - GHDL is the most widely-used open-source VHDL simulator and is essential for - teams working with free and open-source toolchains. Supporting GHDL enables - cost-effective VHDL development and testing without commercial licenses. - tests: - - SimulatorFactory_Get_GhdlSimulator_ReturnsGhdlSimulator - - GhdlSimulator_SimulatorName_ReturnsGHDL - - ghdl@VHDLTest_TestPasses - - ghdl@VHDLTest_TestFails - - - id: VHDLTest-Sim-ModelSim - title: VHDLTest shall support the ModelSim simulator. - justification: | - ModelSim is an industry-standard commercial VHDL simulator widely used in - professional FPGA and ASIC development. Support for ModelSim is critical for - enterprise users and teams with existing ModelSim-based workflows. - tests: - - SimulatorFactory_Get_ModelSimSimulator_ReturnsModelSimSimulator - - ModelSimSimulator_SimulatorName_ReturnsModelSim - - - id: VHDLTest-Sim-QuestaSim - title: VHDLTest shall support the QuestaSim simulator. - justification: | - QuestaSim is the advanced verification tool from Siemens/Mentor Graphics, offering - enhanced debugging and coverage features. Supporting QuestaSim enables VHDLTest - to integrate with high-end verification environments. - tests: - - SimulatorFactory_Get_QuestaSimSimulator_ReturnsQuestaSimSimulator - - QuestaSimSimulator_SimulatorName_ReturnsQuestaSim - - - id: VHDLTest-Sim-Vivado - title: VHDLTest shall support the Vivado simulator. - justification: | - Vivado is Xilinx's primary design suite and simulator for their FPGA products. - Supporting Vivado simulator is essential for teams developing designs targeting - Xilinx/AMD FPGAs and enables seamless integration with the Vivado toolchain. - tests: - - SimulatorFactory_Get_VivadoSimulator_ReturnsVivadoSimulator - - VivadoSimulator_SimulatorName_ReturnsVivado - - - id: VHDLTest-Sim-ActiveHDL - title: VHDLTest shall support the ActiveHDL simulator. - justification: | - Active-HDL from Aldec is a popular VHDL simulator particularly in the aerospace - and defense industries. Supporting Active-HDL ensures VHDLTest can be used in - regulated environments and specialized application domains. - tests: - - SimulatorFactory_Get_ActiveHDLSimulator_ReturnsActiveHDLSimulator - - ActiveHdlSimulator_SimulatorName_ReturnsActiveHDL - - - id: VHDLTest-Sim-NVC - title: VHDLTest shall support the NVC simulator. - justification: | - NVC is a modern, fast open-source VHDL simulator with good standards compliance. - Supporting NVC provides users with an alternative open-source option that offers - better performance than GHDL in certain scenarios and supports newer VHDL standards. - tests: - - SimulatorFactory_Get_NVCSimulator_ReturnsNVCSimulator - - NvcSimulator_SimulatorName_ReturnsNVC - - nvc@VHDLTest_TestPasses - - nvc@VHDLTest_TestFails diff --git a/docs/reqstream/unit-program.yaml b/docs/reqstream/unit-program.yaml new file mode 100644 index 0000000..26857a5 --- /dev/null +++ b/docs/reqstream/unit-program.yaml @@ -0,0 +1,31 @@ +--- +sections: + - title: VHDLTest Requirements + sections: + - title: Program Unit + requirements: + - id: VHDLTest-Program-Usage + title: Program shall return a non-zero exit code when no configuration file is found. + justification: | + When the tool is invoked without a valid configuration file, Program.Run must + signal failure via a non-zero exit code so that callers (CI pipelines, scripts) + can detect the error condition reliably. + tests: + - Program_Run_WithNoConfig_ReturnsNonZeroExitCode + + - id: VHDLTest-Program-Version + title: Program shall display version information when the version flag is given. + justification: | + Enables users to confirm the installed version for troubleshooting and + compatibility verification. + tests: + - Program_Version_IsNotEmpty + - Program_Run_WithVersionFlag_ReturnsZeroExitCode + + - id: VHDLTest-Program-Help + title: Program shall display help information when a help flag is given. + justification: | + Built-in help reduces the need for users to consult external documentation + and supports multiple conventional help flag forms. + tests: + - Program_Run_WithHelpFlag_ReturnsZeroExitCode diff --git a/docs/reqstream/vhdltest-system.yaml b/docs/reqstream/vhdltest-system.yaml index 10ea3de..4d48dc6 100644 --- a/docs/reqstream/vhdltest-system.yaml +++ b/docs/reqstream/vhdltest-system.yaml @@ -71,41 +71,3 @@ sections: in pipelines where failures are handled by separate post-processing steps. tests: - IntegrationTest_TestExecutionErrorWithExit0_ReturnsZeroExitCode - - - title: Validation - requirements: - - id: VHDLTest-Val-Validate - title: VHDLTest shall support self-validation mode. - justification: | - Enables users to verify that VHDLTest is correctly installed and configured in - their environment. This diagnostic capability helps identify configuration issues - before running actual test suites, reducing troubleshooting time. - tests: - - IntegrationTest_ValidateFlag_PerformsValidationAndReturnsSuccess - - - id: VHDLTest-Val-Depth - title: VHDLTest shall support configurable validation report depth. - justification: | - Enables control of markdown header depth in validation reports when embedding the - validation output in other documents. This ensures proper document hierarchy and - formatting when validation reports are included as sections in larger documents. - tests: - - IntegrationTest_ValidateFlagWithDepth_PerformsValidationWithDepth - - - id: VHDLTest-Val-Results - title: VHDLTest shall save validation results to file. - justification: | - Enables persistent storage of validation results for documentation, compliance, - and historical comparison purposes. Saved reports can be shared with team members - or included in bug reports for support. - tests: - - IntegrationTest_ValidateFlagWithResultsFile_SavesValidationResults - - - id: VHDLTest-Val-SystemInfo - title: VHDLTest shall include system information in validation reports. - justification: | - Provides essential context for validation results by capturing the execution - environment (OS version, runtime, etc.). This information is crucial for - reproducing issues and diagnosing environment-specific problems. - tests: - - IntegrationTest_ValidateFlag_IncludesOSVersionInReport diff --git a/requirements.yaml b/requirements.yaml index aaed323..3dd1e74 100644 --- a/requirements.yaml +++ b/requirements.yaml @@ -29,8 +29,30 @@ --- includes: - docs/reqstream/vhdltest-system.yaml + - docs/reqstream/unit-program.yaml - docs/reqstream/platform-requirements.yaml - - docs/reqstream/subsystem-simulators.yaml + - docs/reqstream/cli/subsystem-cli.yaml + - docs/reqstream/cli/unit-context.yaml + - docs/reqstream/cli/unit-config-document.yaml + - docs/reqstream/cli/unit-options.yaml + - docs/reqstream/simulators/subsystem-simulators.yaml + - docs/reqstream/simulators/unit-simulator-factory.yaml + - docs/reqstream/simulators/unit-ghdl-simulator.yaml + - docs/reqstream/simulators/unit-nvc-simulator.yaml + - docs/reqstream/simulators/unit-modelsim-simulator.yaml + - docs/reqstream/simulators/unit-questasim-simulator.yaml + - docs/reqstream/simulators/unit-vivado-simulator.yaml + - docs/reqstream/simulators/unit-active-hdl-simulator.yaml + - docs/reqstream/run/subsystem-run.yaml + - docs/reqstream/run/unit-run-processor.yaml + - docs/reqstream/run/unit-run-program.yaml + - docs/reqstream/run/unit-run-results.yaml + - docs/reqstream/run/unit-run-types.yaml + - docs/reqstream/results/subsystem-results.yaml + - docs/reqstream/results/unit-test-result.yaml + - docs/reqstream/results/unit-test-results.yaml + - docs/reqstream/self-test/subsystem-self-test.yaml + - docs/reqstream/self-test/unit-validation.yaml - docs/reqstream/ots-mstest.yaml - docs/reqstream/ots-reqstream.yaml - docs/reqstream/ots-buildmark.yaml diff --git a/src/DEMAConsulting.VHDLTest/ConfigDocument.cs b/src/DEMAConsulting.VHDLTest/Cli/ConfigDocument.cs similarity index 98% rename from src/DEMAConsulting.VHDLTest/ConfigDocument.cs rename to src/DEMAConsulting.VHDLTest/Cli/ConfigDocument.cs index cf12bae..849f3b2 100644 --- a/src/DEMAConsulting.VHDLTest/ConfigDocument.cs +++ b/src/DEMAConsulting.VHDLTest/Cli/ConfigDocument.cs @@ -21,7 +21,7 @@ using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -namespace DEMAConsulting.VHDLTest; +namespace DEMAConsulting.VHDLTest.Cli; /// /// Configuration document class diff --git a/src/DEMAConsulting.VHDLTest/Context.cs b/src/DEMAConsulting.VHDLTest/Cli/Context.cs similarity index 99% rename from src/DEMAConsulting.VHDLTest/Context.cs rename to src/DEMAConsulting.VHDLTest/Cli/Context.cs index 015be8d..7526658 100644 --- a/src/DEMAConsulting.VHDLTest/Context.cs +++ b/src/DEMAConsulting.VHDLTest/Cli/Context.cs @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest; +namespace DEMAConsulting.VHDLTest.Cli; /// /// Program Arguments Class diff --git a/src/DEMAConsulting.VHDLTest/Options.cs b/src/DEMAConsulting.VHDLTest/Cli/Options.cs similarity index 98% rename from src/DEMAConsulting.VHDLTest/Options.cs rename to src/DEMAConsulting.VHDLTest/Cli/Options.cs index fc93161..ff2f119 100644 --- a/src/DEMAConsulting.VHDLTest/Options.cs +++ b/src/DEMAConsulting.VHDLTest/Cli/Options.cs @@ -18,7 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest; +namespace DEMAConsulting.VHDLTest.Cli; /// /// Program Options Class diff --git a/src/DEMAConsulting.VHDLTest/Program.cs b/src/DEMAConsulting.VHDLTest/Program.cs index 7656bfc..a2278bf 100644 --- a/src/DEMAConsulting.VHDLTest/Program.cs +++ b/src/DEMAConsulting.VHDLTest/Program.cs @@ -19,7 +19,9 @@ // SOFTWARE. using System.Reflection; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; +using DEMAConsulting.VHDLTest.SelfTest; using DEMAConsulting.VHDLTest.Simulators; namespace DEMAConsulting.VHDLTest; diff --git a/src/DEMAConsulting.VHDLTest/Results/TestResult.cs b/src/DEMAConsulting.VHDLTest/Results/TestResult.cs index 335a0cb..3de1004 100644 --- a/src/DEMAConsulting.VHDLTest/Results/TestResult.cs +++ b/src/DEMAConsulting.VHDLTest/Results/TestResult.cs @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Run; namespace DEMAConsulting.VHDLTest.Results; diff --git a/src/DEMAConsulting.VHDLTest/Results/TestResults.cs b/src/DEMAConsulting.VHDLTest/Results/TestResults.cs index d568b22..5fd9a4a 100644 --- a/src/DEMAConsulting.VHDLTest/Results/TestResults.cs +++ b/src/DEMAConsulting.VHDLTest/Results/TestResults.cs @@ -20,6 +20,7 @@ using DemaConsulting.TestResults; using DemaConsulting.TestResults.IO; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; diff --git a/src/DEMAConsulting.VHDLTest/Run/RunResults.cs b/src/DEMAConsulting.VHDLTest/Run/RunResults.cs index ea261cf..5cfe401 100644 --- a/src/DEMAConsulting.VHDLTest/Run/RunResults.cs +++ b/src/DEMAConsulting.VHDLTest/Run/RunResults.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Collections.ObjectModel; +using DEMAConsulting.VHDLTest.Cli; namespace DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Validation.cs b/src/DEMAConsulting.VHDLTest/SelfTest/Validation.cs similarity index 99% rename from src/DEMAConsulting.VHDLTest/Validation.cs rename to src/DEMAConsulting.VHDLTest/SelfTest/Validation.cs index c4bbf3f..742b6f6 100644 --- a/src/DEMAConsulting.VHDLTest/Validation.cs +++ b/src/DEMAConsulting.VHDLTest/SelfTest/Validation.cs @@ -20,10 +20,11 @@ using System.Collections.ObjectModel; using System.Runtime.InteropServices; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; -namespace DEMAConsulting.VHDLTest; +namespace DEMAConsulting.VHDLTest.SelfTest; /// /// Validation runner diff --git a/src/DEMAConsulting.VHDLTest/Simulators/ActiveHdlSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/ActiveHdlSimulator.cs index 8a462e3..e85e4e8 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/ActiveHdlSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/ActiveHdlSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/GhdlSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/GhdlSimulator.cs index 5f85063..d35f639 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/GhdlSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/GhdlSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/MockSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/MockSimulator.cs index 1635fa4..b5b521c 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/MockSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/MockSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/ModelSimSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/ModelSimSimulator.cs index 6cc65bd..640a08d 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/ModelSimSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/ModelSimSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/NvcSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/NvcSimulator.cs index 47db228..82ae08e 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/NvcSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/NvcSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/QuestaSimSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/QuestaSimSimulator.cs index 0387563..281941e 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/QuestaSimSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/QuestaSimSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/Simulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/Simulator.cs index 28d16a0..d8dec20 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/Simulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/Simulator.cs @@ -18,6 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/src/DEMAConsulting.VHDLTest/Simulators/VivadoSimulator.cs b/src/DEMAConsulting.VHDLTest/Simulators/VivadoSimulator.cs index ee8112c..a259dfc 100644 --- a/src/DEMAConsulting.VHDLTest/Simulators/VivadoSimulator.cs +++ b/src/DEMAConsulting.VHDLTest/Simulators/VivadoSimulator.cs @@ -19,6 +19,7 @@ // SOFTWARE. using System.Text; +using DEMAConsulting.VHDLTest.Cli; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; diff --git a/test/DEMAConsulting.VHDLTest.Tests/Cli/CliSubsystemTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Cli/CliSubsystemTests.cs new file mode 100644 index 0000000..7cda631 --- /dev/null +++ b/test/DEMAConsulting.VHDLTest.Tests/Cli/CliSubsystemTests.cs @@ -0,0 +1,97 @@ +// Copyright (c) 2023 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DEMAConsulting.VHDLTest.Cli; + +namespace DEMAConsulting.VHDLTest.Tests.Cli; + +/// +/// Subsystem integration tests for the Cli subsystem. +/// These tests verify that , , and +/// work together as a complete command-line configuration pipeline. +/// +[TestClass] +public class CliSubsystemTests +{ + /// + /// Configuration file name + /// + private const string ConfigFile = "cli-subsystem-test.yaml"; + + /// + /// Configuration file contents + /// + private const string ConfigContent = + """ + files: + - src/design.vhd + - src/testbench.vhd + + tests: + - test_entity + """; + + /// + /// Test that command-line arguments, config file loading, and options parsing + /// work together to produce a correctly configured Options object. + /// + [TestMethod] + public void CliSubsystem_ParseArgsAndLoadConfig_WithValidConfig_ProducesCorrectOptions() + { + try + { + // Arrange - write a valid config file + File.WriteAllText(ConfigFile, ConfigContent); + + // Act - run the full Cli pipeline: args -> Context -> Options + var context = Context.Create(["-c", ConfigFile, "--verbose"]); + var options = Options.Parse(context); + + // Assert - verify the subsystem produced the correct options + Assert.IsNotNull(context); + Assert.IsTrue(context.Verbose); + Assert.AreEqual(ConfigFile, context.ConfigFile); + Assert.IsNotNull(options); + Assert.HasCount(2, options.Config.Files); + Assert.AreEqual("src/design.vhd", options.Config.Files[0]); + Assert.AreEqual("src/testbench.vhd", options.Config.Files[1]); + Assert.HasCount(1, options.Config.Tests); + Assert.AreEqual("test_entity", options.Config.Tests[0]); + } + finally + { + File.Delete(ConfigFile); + } + } + + /// + /// Test that the Cli subsystem correctly propagates a missing config file error + /// through the Context and Options pipeline. + /// + [TestMethod] + public void CliSubsystem_ParseArgsAndLoadConfig_WithMissingConfig_ThrowsFileNotFoundException() + { + // Arrange - create context specifying a non-existent config file + var context = Context.Create(["-c", "missing-config.yaml"]); + + // Act & Assert - verify the subsystem surfaces the missing file error from Options.Parse + Assert.ThrowsExactly(() => Options.Parse(context)); + } +} diff --git a/test/DEMAConsulting.VHDLTest.Tests/ConfigDocumentTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Cli/ConfigDocumentTests.cs similarity index 97% rename from test/DEMAConsulting.VHDLTest.Tests/ConfigDocumentTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Cli/ConfigDocumentTests.cs index b772276..f2d200e 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ConfigDocumentTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Cli/ConfigDocumentTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest.Tests; +using DEMAConsulting.VHDLTest.Cli; + +namespace DEMAConsulting.VHDLTest.Tests.Cli; /// /// Tests for configuration documents diff --git a/test/DEMAConsulting.VHDLTest.Tests/ContextTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Cli/ContextTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/ContextTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Cli/ContextTests.cs index 35e79f7..051126d 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ContextTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Cli/ContextTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest.Tests; +using DEMAConsulting.VHDLTest.Cli; + +namespace DEMAConsulting.VHDLTest.Tests.Cli; /// /// Tests for argument parsing diff --git a/test/DEMAConsulting.VHDLTest.Tests/OptionsTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Cli/OptionsTests.cs similarity index 98% rename from test/DEMAConsulting.VHDLTest.Tests/OptionsTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Cli/OptionsTests.cs index 63ba346..f0d7915 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/OptionsTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Cli/OptionsTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest.Tests; +using DEMAConsulting.VHDLTest.Cli; + +namespace DEMAConsulting.VHDLTest.Tests.Cli; /// /// Tests for parsing options diff --git a/test/DEMAConsulting.VHDLTest.Tests/ExitCodeTests.cs b/test/DEMAConsulting.VHDLTest.Tests/IntegrationTests.cs similarity index 55% rename from test/DEMAConsulting.VHDLTest.Tests/ExitCodeTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/IntegrationTests.cs index 99cc4db..d2b0e55 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ExitCodeTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/IntegrationTests.cs @@ -18,14 +18,144 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +using System.Text.RegularExpressions; + namespace DEMAConsulting.VHDLTest.Tests; /// -/// Tests for exit code +/// System-level integration tests for VHDLTest. +/// These tests run the VHDLTest tool as a whole and verify end-to-end behavior. /// [TestClass] -public class ExitCodeTests +public partial class IntegrationTests { + /// + /// Regular expression to check for version + /// + /// Version regex + [GeneratedRegex(@"\d+\.\d+\.\d+.*")] + private static partial Regex VersionRegex(); + + /// + /// Test usage information is reported when no arguments are specified + /// + [TestMethod] + public void IntegrationTest_NoArguments_DisplaysUsageAndReturnsError() + { + // Run the application + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll"); + + // Verify error + Assert.AreNotEqual(0, exitCode); + + // Verify usage reported + Assert.Contains("Error: Missing arguments", output); + Assert.Contains("Usage: VHDLTest", output); + } + + /// + /// Test usage information is reported when the '-h' parameter is specified + /// + [TestMethod] + public void IntegrationTest_HelpShortFlag_DisplaysUsageAndReturnsSuccess() + { + // Run the application + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll", + "-h"); + + // Verify no error + Assert.AreEqual(0, exitCode); + + // Verify usage reported + Assert.Contains("Usage: VHDLTest", output); + } + + /// + /// Test usage information is reported when the '-?' parameter is specified + /// + [TestMethod] + public void IntegrationTest_HelpQuestionFlag_DisplaysUsageAndReturnsSuccess() + { + // Run the application + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll", + "-?"); + + // Verify no error + Assert.AreEqual(0, exitCode); + + // Verify usage reported + Assert.Contains("Usage: VHDLTest", output); + } + + /// + /// Test usage information is reported when the '--help' parameter is specified + /// + [TestMethod] + public void IntegrationTest_HelpLongFlag_DisplaysUsageAndReturnsSuccess() + { + // Run the application + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll", + "--help"); + + // Verify no error + Assert.AreEqual(0, exitCode); + + // Verify usage reported + Assert.Contains("Usage: VHDLTest", output); + } + + /// + /// Test version information is reported when the '-v' parameter is specified + /// + [TestMethod] + public void IntegrationTest_VersionShortFlag_DisplaysVersionAndReturnsSuccess() + { + // Query version + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll", + "-v"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify version reported + Assert.MatchesRegex(VersionRegex(), output); + } + + /// + /// Test version information is reported when the '--version' parameter is specified + /// + [TestMethod] + public void IntegrationTest_VersionLongFlag_DisplaysVersionAndReturnsSuccess() + { + // Query version + var exitCode = Runner.Run( + out var output, + "dotnet", + "DEMAConsulting.VHDLTest.dll", + "--version"); + + // Verify success + Assert.AreEqual(0, exitCode); + + // Verify version reported + Assert.MatchesRegex(VersionRegex(), output); + } + /// /// Test non-zero exit code with compile errors /// @@ -99,7 +229,7 @@ public void IntegrationTest_TestExecutionError_ReturnsNonZeroExitCode() } /// - /// Test non-zero exit code with test-execution errors + /// Test zero exit code is returned when exit-0 is specified and tests fail /// [TestMethod] public void IntegrationTest_TestExecutionErrorWithExit0_ReturnsZeroExitCode() @@ -136,7 +266,7 @@ public void IntegrationTest_TestExecutionErrorWithExit0_ReturnsZeroExitCode() } /// - /// Test non-zero exit code with test-execution errors + /// Test zero exit code is returned when all tests pass /// [TestMethod] public void IntegrationTest_TestsPassed_ReturnsZeroExitCode() diff --git a/test/DEMAConsulting.VHDLTest.Tests/ProgramTests.cs b/test/DEMAConsulting.VHDLTest.Tests/ProgramTests.cs new file mode 100644 index 0000000..118ecfa --- /dev/null +++ b/test/DEMAConsulting.VHDLTest.Tests/ProgramTests.cs @@ -0,0 +1,88 @@ +// Copyright (c) 2023 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DEMAConsulting.VHDLTest.Cli; + +namespace DEMAConsulting.VHDLTest.Tests; + +/// +/// Unit tests for class. +/// +[TestClass] +public class ProgramTests +{ + /// + /// Test that the version string is not empty + /// + [TestMethod] + public void Program_Version_IsNotEmpty() + { + // Assert - verify version is populated + Assert.IsFalse(string.IsNullOrEmpty(Program.Version)); + } + + /// + /// Test that Run returns zero exit code when version flag is given + /// + [TestMethod] + public void Program_Run_WithVersionFlag_ReturnsZeroExitCode() + { + // Arrange - create a silent context with version flag + using var context = Context.Create(["--version", "--silent"]); + + // Act - run the program + Program.Run(context); + + // Assert - verify zero exit code + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that Run returns zero exit code when help flag is given + /// + [TestMethod] + public void Program_Run_WithHelpFlag_ReturnsZeroExitCode() + { + // Arrange - create a silent context with help flag + using var context = Context.Create(["--help", "--silent"]); + + // Act - run the program + Program.Run(context); + + // Assert - verify zero exit code + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that Run returns non-zero exit code when no config is given + /// + [TestMethod] + public void Program_Run_WithNoConfig_ReturnsNonZeroExitCode() + { + // Arrange - create a silent context with no config file + using var context = Context.Create(["--silent"]); + + // Act - run the program + Program.Run(context); + + // Assert - verify non-zero exit code + Assert.AreNotEqual(0, context.ExitCode); + } +} diff --git a/test/DEMAConsulting.VHDLTest.Tests/Results/ResultsSubsystemTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Results/ResultsSubsystemTests.cs new file mode 100644 index 0000000..f9169db --- /dev/null +++ b/test/DEMAConsulting.VHDLTest.Tests/Results/ResultsSubsystemTests.cs @@ -0,0 +1,119 @@ +// Copyright (c) 2023 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using System.Collections.ObjectModel; +using DEMAConsulting.VHDLTest.Run; +using VHDLTestResult = DEMAConsulting.VHDLTest.Results.TestResult; +using VHDLTestResults = DEMAConsulting.VHDLTest.Results.TestResults; + +namespace DEMAConsulting.VHDLTest.Tests.Results; + +/// +/// Subsystem integration tests for the Results subsystem. +/// These tests verify that and +/// work together to collect, summarize, and persist test results. +/// +[TestClass] +public class ResultsSubsystemTests +{ + /// + /// Test that TestResult and TestResults work together to correctly track + /// pass and fail counts across a collection of test outcomes. + /// + [TestMethod] + public void ResultsSubsystem_CollectAndSummarize_WithMixedResults_ReportsCorrectPassFailCounts() + { + // Arrange - create passing and failing RunResults objects + var passRunResults = new RunResults( + RunLineType.Info, + new DateTime(2024, 8, 10, 0, 0, 0, DateTimeKind.Utc), + 1.0, + 0, + "Passed", + new ReadOnlyCollection([new RunLine(RunLineType.Info, "Passed")])); + + var failRunResults = new RunResults( + RunLineType.Error, + new DateTime(2024, 8, 10, 0, 0, 1, DateTimeKind.Utc), + 1.0, + 1, + "Error: assertion failed", + new ReadOnlyCollection([new RunLine(RunLineType.Error, "Error: assertion failed")])); + + // Act - wrap each RunResults in a TestResult and add to TestResults + var testResults = new VHDLTestResults("SubsystemTestRun", "VHDLTest"); + testResults.Tests.Add(new VHDLTestResult("Suite", "Test1", passRunResults)); + testResults.Tests.Add(new VHDLTestResult("Suite", "Test2", passRunResults)); + testResults.Tests.Add(new VHDLTestResult("Suite", "Test3", failRunResults)); + + // Assert - TestResults correctly aggregates pass and fail counts + Assert.AreEqual(3, testResults.Tests.Count); + Assert.AreEqual(2, testResults.Passes.Count()); + Assert.AreEqual(1, testResults.Fails.Count()); + } + + /// + /// Test that TestResults correctly saves a combined pass/fail result set to + /// a TRX file, verifying that TestResult and TestResults integrate through + /// the serialization pipeline. + /// + [TestMethod] + public void ResultsSubsystem_SaveMixedResults_ToTrxFile_CreatesTrxFileWithCorrectCounts() + { + const string resultsFile = "results-subsystem-test.trx"; + + try + { + // Arrange - build a mixed result set + var passRunResults = new RunResults( + RunLineType.Info, + new DateTime(2024, 8, 10, 0, 0, 0, DateTimeKind.Utc), + 1.0, + 0, + "Passed", + new ReadOnlyCollection([new RunLine(RunLineType.Info, "Passed")])); + + var failRunResults = new RunResults( + RunLineType.Error, + new DateTime(2024, 8, 10, 0, 0, 1, DateTimeKind.Utc), + 1.0, + 1, + "Error: assertion failed", + new ReadOnlyCollection([new RunLine(RunLineType.Error, "Error: assertion failed")])); + + var testResults = new VHDLTestResults("SubsystemTestRun", "VHDLTest"); + testResults.Tests.Add(new VHDLTestResult("Suite", "Test1", passRunResults)); + testResults.Tests.Add(new VHDLTestResult("Suite", "Test2", failRunResults)); + + // Act - save the combined results to a TRX file + testResults.SaveResults(resultsFile); + + // Assert - file was created and contains both test results + Assert.IsTrue(File.Exists(resultsFile)); + var content = File.ReadAllText(resultsFile); + Assert.IsTrue(content.Contains("Test1")); + Assert.IsTrue(content.Contains("Test2")); + } + finally + { + File.Delete(resultsFile); + } + } +} diff --git a/test/DEMAConsulting.VHDLTest.Tests/TestResultTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Results/TestResultTests.cs similarity index 92% rename from test/DEMAConsulting.VHDLTest.Tests/TestResultTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Results/TestResultTests.cs index be31590..acdf643 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/TestResultTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Results/TestResultTests.cs @@ -20,11 +20,12 @@ using System.Collections.ObjectModel; using DEMAConsulting.VHDLTest.Run; +using VHDLTestResult = DEMAConsulting.VHDLTest.Results.TestResult; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Results; /// -/// Tests for class. +/// Tests for class. /// [TestClass] public class TestResultTests @@ -36,7 +37,7 @@ public class TestResultTests public void TestResult_Constructor_WithInfoResult_CreatesPassedTest() { // Construct the result - var result = new Results.TestResult( + var result = new VHDLTestResult( "test", "test", new RunResults( @@ -65,7 +66,7 @@ public void TestResult_Constructor_WithInfoResult_CreatesPassedTest() public void TestResult_Constructor_WithErrorResult_CreatesFailedTest() { // Construct the result - var result = new Results.TestResult( + var result = new VHDLTestResult( "test", "test", new RunResults( diff --git a/test/DEMAConsulting.VHDLTest.Tests/TestResultsTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Results/TestResultsTests.cs similarity index 96% rename from test/DEMAConsulting.VHDLTest.Tests/TestResultsTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Results/TestResultsTests.cs index b74c1d8..695ed09 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/TestResultsTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Results/TestResultsTests.cs @@ -21,11 +21,12 @@ using System.Collections.ObjectModel; using DEMAConsulting.VHDLTest.Results; using DEMAConsulting.VHDLTest.Run; +using VHDLTestResult = DEMAConsulting.VHDLTest.Results.TestResult; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Results; /// -/// Tests for class. +/// Tests for class. /// [TestClass] public class TestResultsTests @@ -38,7 +39,7 @@ public void TestResults_SaveResults_WithTrxExtension_CreatesTrxFile() { var results = new TestResults("TestRun", "TestCodeBase"); results.Tests.Add( - new Results.TestResult( + new VHDLTestResult( "TestClass", "TestName", new RunResults( RunLineType.Info, @@ -81,7 +82,7 @@ public void TestResults_SaveResults_WithXmlExtension_CreatesJUnitFile() { var results = new TestResults("TestRun", "TestCodeBase"); results.Tests.Add( - new Results.TestResult( + new VHDLTestResult( "TestClass", "TestName", new RunResults( RunLineType.Info, @@ -124,7 +125,7 @@ public void TestResults_SaveResults_WithFailedTest_CreatesJUnitFileWithFailure() { var results = new TestResults("TestRun", "TestCodeBase"); results.Tests.Add( - new Results.TestResult( + new VHDLTestResult( "TestClass", "FailedTest", new RunResults( RunLineType.Error, @@ -167,7 +168,7 @@ public void TestResults_SaveToTrx_WithTestResults_CreatesTrxFile() { var results = new TestResults("TestRun", "TestCodeBase"); results.Tests.Add( - new Results.TestResult( + new VHDLTestResult( "TestClass", "TestName", new RunResults( RunLineType.Info, @@ -218,7 +219,7 @@ public void TestResults_SaveResults_WithUnknownExtension_CreatesTrxFile() { var results = new TestResults("TestRun", "TestCodeBase"); results.Tests.Add( - new Results.TestResult( + new VHDLTestResult( "TestClass", "TestName", new RunResults( RunLineType.Info, diff --git a/test/DEMAConsulting.VHDLTest.Tests/RunProcessorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Run/RunProcessorTests.cs similarity index 98% rename from test/DEMAConsulting.VHDLTest.Tests/RunProcessorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Run/RunProcessorTests.cs index f114962..8ce3d9a 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/RunProcessorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Run/RunProcessorTests.cs @@ -20,7 +20,7 @@ using DEMAConsulting.VHDLTest.Run; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Run; /// /// Tests for class. diff --git a/test/DEMAConsulting.VHDLTest.Tests/Run/RunSubsystemTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Run/RunSubsystemTests.cs new file mode 100644 index 0000000..ac1ca0e --- /dev/null +++ b/test/DEMAConsulting.VHDLTest.Tests/Run/RunSubsystemTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) 2023 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DEMAConsulting.VHDLTest.Run; + +namespace DEMAConsulting.VHDLTest.Tests.Run; + +/// +/// Subsystem integration tests for the Run subsystem. +/// These tests verify that , , and +/// work together to execute programs and classify their output. +/// +[TestClass] +public class RunSubsystemTests +{ + /// + /// Test that RunProcessor executes a real program via RunProgram, and produces a + /// RunResults object with correctly classified output lines. + /// + [TestMethod] + public void RunSubsystem_ExecuteRealProgram_WithClassificationRules_ProducesClassifiedRunResults() + { + // Arrange - create a processor with an Info classification rule + var processor = new RunProcessor( + [ + RunLineRule.Create(RunLineType.Info, "Usage") + ]); + + // Act - execute a real program through the full Run pipeline + var results = processor.Execute("dotnet", "", "help"); + + // Assert - RunProgram ran the program and RunProcessor classified the output into RunResults + Assert.IsNotNull(results); + Assert.AreEqual(0, results.ExitCode); + Assert.IsTrue(results.Output.Length > 0); + Assert.IsTrue(results.Lines.Count > 0); + Assert.IsTrue(results.Lines.Any(l => l.Type == RunLineType.Info)); + Assert.IsTrue(results.Duration >= 0.0); + } + + /// + /// Test that RunProcessor correctly surfaces a non-zero exit code from RunProgram + /// as an Error summary in the RunResults. + /// + [TestMethod] + public void RunSubsystem_ExecuteRealProgram_WithErrorExitCode_ProducesErrorRunResults() + { + // Arrange - create a processor with no special classification rules + var processor = new RunProcessor([]); + + // Act - execute dotnet with an unknown command to produce a non-zero exit code + var results = processor.Execute("dotnet", "", "unknown-command"); + + // Assert - RunResults reflects the non-zero exit code as an error summary + Assert.IsNotNull(results); + Assert.AreNotEqual(0, results.ExitCode); + Assert.AreEqual(RunLineType.Error, results.Summary); + } +} diff --git a/test/DEMAConsulting.VHDLTest.Tests/ValidationTests.cs b/test/DEMAConsulting.VHDLTest.Tests/SelfTest/ValidationTests.cs similarity index 98% rename from test/DEMAConsulting.VHDLTest.Tests/ValidationTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/SelfTest/ValidationTests.cs index 997e31e..5110ddd 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ValidationTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/SelfTest/ValidationTests.cs @@ -18,7 +18,9 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -namespace DEMAConsulting.VHDLTest.Tests; +using DEMAConsulting.VHDLTest.Tests; + +namespace DEMAConsulting.VHDLTest.Tests.SelfTest; /// /// Tests for validation diff --git a/test/DEMAConsulting.VHDLTest.Tests/ActiveHdlSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/ActiveHdlSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/ActiveHdlSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/ActiveHdlSimulatorTests.cs index 5c44dce..7e086fb 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ActiveHdlSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/ActiveHdlSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for the ActiveHDL simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/GhdlSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/GhdlSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/GhdlSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/GhdlSimulatorTests.cs index e9c568e..80c9c61 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/GhdlSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/GhdlSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for GHDL simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/ModelSimSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/ModelSimSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/ModelSimSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/ModelSimSimulatorTests.cs index 7cb5c76..52d491b 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/ModelSimSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/ModelSimSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for ModelSim simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/NvcSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/NvcSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/NvcSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/NvcSimulatorTests.cs index 22a0849..36eb384 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/NvcSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/NvcSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for NVC simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/QuestaSimSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/QuestaSimSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/QuestaSimSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/QuestaSimSimulatorTests.cs index 615606f..ed33212 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/QuestaSimSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/QuestaSimSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for QuestaSim simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/SimulatorFactoryTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorFactoryTests.cs similarity index 98% rename from test/DEMAConsulting.VHDLTest.Tests/SimulatorFactoryTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorFactoryTests.cs index 1115ec0..abe1ac8 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/SimulatorFactoryTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorFactoryTests.cs @@ -20,7 +20,7 @@ using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for class. diff --git a/test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorsSubsystemTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorsSubsystemTests.cs new file mode 100644 index 0000000..78d27aa --- /dev/null +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/SimulatorsSubsystemTests.cs @@ -0,0 +1,79 @@ +// Copyright (c) 2023 DEMA Consulting +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +using DEMAConsulting.VHDLTest.Run; +using DEMAConsulting.VHDLTest.Simulators; + +namespace DEMAConsulting.VHDLTest.Tests.Simulators; + +/// +/// Subsystem integration tests for the Simulators subsystem. +/// These tests verify that and the simulator +/// processors work together to select and classify simulator output. +/// +[TestClass] +public class SimulatorsSubsystemTests +{ + /// + /// Test that the factory returns the GHDL simulator by name, and that the + /// simulator's compile processor correctly classifies error output. + /// + [TestMethod] + public void SimulatorsSubsystem_GetSimulatorAndProcessCompileOutput_WithErrorOutput_ClassifiesAsError() + { + // Arrange - obtain GHDL simulator via the factory + var simulator = SimulatorFactory.Get("GHDL"); + + // Act - use the simulator's compile processor to classify error output + var results = GhdlSimulator.CompileProcessor.Parse( + new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2024, 1, 1, 0, 0, 1, DateTimeKind.Utc), + "design.vhd:10:5: error: undefined identifier 'x'", + 1); + + // Assert - factory returned GHDL and processor classified the line as an error + Assert.IsNotNull(simulator); + Assert.AreEqual("GHDL", simulator.SimulatorName); + Assert.AreEqual(RunLineType.Error, results.Summary); + Assert.IsTrue(results.Lines.Any(l => l.Type == RunLineType.Error)); + } + + /// + /// Test that the factory returns null for an unrecognized simulator name, and + /// that the NVC simulator's test processor correctly classifies clean output. + /// + [TestMethod] + public void SimulatorsSubsystem_GetUnknownSimulatorAndProcessCleanOutput_ReturnsNullAndClassifiesText() + { + // Arrange - attempt to obtain an unknown simulator via the factory + var simulator = SimulatorFactory.Get("unknown-simulator"); + + // Act - use the NVC test processor independently to classify clean output + var results = NvcSimulator.TestProcessor.Parse( + new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2024, 1, 1, 0, 0, 1, DateTimeKind.Utc), + "Simulation complete", + 0); + + // Assert - unknown simulator yields null; processor classified clean output as text + Assert.IsNull(simulator); + Assert.AreEqual(RunLineType.Text, results.Summary); + } +} diff --git a/test/DEMAConsulting.VHDLTest.Tests/VivadoSimulatorTests.cs b/test/DEMAConsulting.VHDLTest.Tests/Simulators/VivadoSimulatorTests.cs similarity index 99% rename from test/DEMAConsulting.VHDLTest.Tests/VivadoSimulatorTests.cs rename to test/DEMAConsulting.VHDLTest.Tests/Simulators/VivadoSimulatorTests.cs index 500b57d..85218ac 100644 --- a/test/DEMAConsulting.VHDLTest.Tests/VivadoSimulatorTests.cs +++ b/test/DEMAConsulting.VHDLTest.Tests/Simulators/VivadoSimulatorTests.cs @@ -21,7 +21,7 @@ using DEMAConsulting.VHDLTest.Run; using DEMAConsulting.VHDLTest.Simulators; -namespace DEMAConsulting.VHDLTest.Tests; +namespace DEMAConsulting.VHDLTest.Tests.Simulators; /// /// Tests for Vivado simulator diff --git a/test/DEMAConsulting.VHDLTest.Tests/UsageTests.cs b/test/DEMAConsulting.VHDLTest.Tests/UsageTests.cs deleted file mode 100644 index 3125eb9..0000000 --- a/test/DEMAConsulting.VHDLTest.Tests/UsageTests.cs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2023 DEMA Consulting -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -namespace DEMAConsulting.VHDLTest.Tests; - -/// -/// Tests for program usage -/// -[TestClass] -public class UsageTests -{ - /// - /// Test usage information is reported when no arguments are specified - /// - [TestMethod] - public void IntegrationTest_NoArguments_DisplaysUsageAndReturnsError() - { - // Run the application - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll"); - - // Verify error - Assert.AreNotEqual(0, exitCode); - - // Verify usage reported - Assert.Contains("Error: Missing arguments", output); - Assert.Contains("Usage: VHDLTest", output); - } - - /// - /// Test usage information is reported when the '-h' parameter is specified - /// - [TestMethod] - public void IntegrationTest_HelpShortFlag_DisplaysUsageAndReturnsSuccess() - { - // Run the application - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll", - "-h"); - - // Verify no error - Assert.AreEqual(0, exitCode); - - // Verify usage reported - Assert.Contains("Usage: VHDLTest", output); - } - - /// - /// Test usage information is reported when the '-?' parameter is specified - /// - [TestMethod] - public void IntegrationTest_HelpQuestionFlag_DisplaysUsageAndReturnsSuccess() - { - // Run the application - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll", - "-?"); - - // Verify no error - Assert.AreEqual(0, exitCode); - - // Verify usage reported - Assert.Contains("Usage: VHDLTest", output); - } - - /// - /// Test usage information is reported when the '--help' parameter is specified - /// - [TestMethod] - public void IntegrationTest_HelpLongFlag_DisplaysUsageAndReturnsSuccess() - { - // Run the application - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll", - "--help"); - - // Verify no error - Assert.AreEqual(0, exitCode); - - // Verify usage reported - Assert.Contains("Usage: VHDLTest", output); - } -} diff --git a/test/DEMAConsulting.VHDLTest.Tests/VersionTests.cs b/test/DEMAConsulting.VHDLTest.Tests/VersionTests.cs deleted file mode 100644 index ccc5f58..0000000 --- a/test/DEMAConsulting.VHDLTest.Tests/VersionTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) 2023 DEMA Consulting -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -using System.Text.RegularExpressions; - -namespace DEMAConsulting.VHDLTest.Tests; - -/// -/// Tests for program version -/// -[TestClass] -public partial class VersionTests -{ - /// - /// Regular expression to check for version - /// - /// - [GeneratedRegex(@"\d+\.\d+\.\d+.*")] - private static partial Regex VersionRegex(); - - /// - /// Test version information is reported when the '-v' parameter is specified - /// - [TestMethod] - public void IntegrationTest_VersionShortFlag_DisplaysVersionAndReturnsSuccess() - { - // Query version - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll", - "-v"); - - // Verify success - Assert.AreEqual(0, exitCode); - - // Verify version reported - Assert.MatchesRegex(VersionRegex(), output); - } - - /// - /// Test version information is reported when the '--version' parameter is specified - /// - [TestMethod] - public void IntegrationTest_VersionLongFlag_DisplaysVersionAndReturnsSuccess() - { - // Query version - var exitCode = Runner.Run( - out var output, - "dotnet", - "DEMAConsulting.VHDLTest.dll", - "--version"); - - // Verify success - Assert.AreEqual(0, exitCode); - - // Verify version reported - Assert.MatchesRegex(VersionRegex(), output); - } -}