diff --git a/docs/design/sarifmark/cli/cli.md b/docs/design/sarifmark/cli/cli.md index 8bfa19c..ae94a9e 100644 --- a/docs/design/sarifmark/cli/cli.md +++ b/docs/design/sarifmark/cli/cli.md @@ -23,12 +23,29 @@ The `Cli` subsystem contains the following software unit: The `Cli` subsystem exposes the following interface to the rest of the tool: +**Methods:** + | Interface | Direction | Description | |----------------------|-----------|---------------------------------------------------------------| | `Context.Create` | Outbound | Factory method constructing a `Context` from `string[] args`. | | `Context.WriteLine` | Outbound | Writes a message to console and optional log file. | | `Context.WriteError` | Outbound | Writes an error to stderr and sets the error exit code. | -| `Context.ExitCode` | Outbound | Returns 0 for success or 1 when errors have been reported. | + +**Parsed flags and parameters** (set by `Create`, read by the application layer): + +| Property | Type | CLI flag(s) | Description | +|---------------|-----------|--------------------------|-----------------------------------------------| +| `Version` | `bool` | `-v`, `--version` | Version query flag | +| `Help` | `bool` | `-?`, `-h`, `--help` | Help flag | +| `Silent` | `bool` | `--silent` | Suppress console output flag | +| `Validate` | `bool` | `--validate` | Self-validation mode flag | +| `Enforce` | `bool` | `--enforce` | Enforcement mode flag | +| `SarifFile` | `string?` | `--sarif ` | Path to the SARIF input file | +| `ReportFile` | `string?` | `--report ` | Path for the markdown report output file | +| `ReportDepth` | `int` | `--report-depth ` | Markdown heading depth for the report | +| `Heading` | `string?` | `--heading ` | Custom heading text for the report | +| `ResultsFile` | `string?` | `--results ` | Path for the self-validation results file | +| `ExitCode` | `int` | *(derived)* | 0 until `WriteError` is called, then 1 | ## Interactions diff --git a/docs/design/sarifmark/sarif/sarif.md b/docs/design/sarifmark/sarif/sarif.md index 9f0517e..23f3909 100644 --- a/docs/design/sarifmark/sarif/sarif.md +++ b/docs/design/sarifmark/sarif/sarif.md @@ -8,7 +8,7 @@ markdown reports from the extracted results. It consists of two records: `SarifResults` (the full results collection with reading and reporting logic). This layer satisfies requirements `SarifMark-Sarif-Reading`, `SarifMark-Sarif-Validation`, `SarifMark-Sarif-ToolInfo`, `SarifMark-Sarif-Results`, -`SarifMark-Sarif-Locations`, `SarifMark-Sarif-FilePaths`, `SarifMark-Sarif-Required`, +`SarifMark-Sarif-Locations`, `SarifMark-Sarif-FilePaths`, `SarifMark-Sarif-Processing`, `SarifMark-Report-Markdown`, `SarifMark-Report-Depth`, `SarifMark-Report-Counts`, `SarifMark-Report-Locations`, `SarifMark-Report-Headings`, and `SarifMark-Report-LineBreaks`. @@ -51,10 +51,10 @@ formats each result with location information and result counts. This satisfies ## CLI Integration -The requirement `SarifMark-Sarif-Required` (the tool shall require the `--sarif` parameter -for analysis) is enforced at the command-line layer rather than within this library. The +The requirement `SarifMark-System-SarifRequired` (the tool shall require the `--sarif` parameter +for analysis) is enforced at the application layer rather than within this library. The `ProcessSarifAnalysis` method in `Program.cs` validates that `--sarif` is provided before -invoking the SARIF reading layer. See the Command Line document for full details. +invoking the SARIF reading layer. See the Program Class document for full details. ## Class Details diff --git a/docs/reqstream/sarifmark/cli/cli.yaml b/docs/reqstream/sarifmark/cli/cli.yaml index 36cbd4c..467795c 100644 --- a/docs/reqstream/sarifmark/cli/cli.yaml +++ b/docs/reqstream/sarifmark/cli/cli.yaml @@ -2,10 +2,10 @@ sections: - title: SarifMark Requirements sections: - - title: Command-Line Interface + - title: CLI Subsystem requirements: - id: SarifMark-Cli-Interface - title: The tool shall provide a command-line interface. + title: The CLI subsystem shall accept command-line arguments and route output. justification: >- A command-line interface enables automation, scripting, and integration into CI/CD pipelines, making the tool suitable for DevOps workflows and batch processing scenarios. @@ -13,33 +13,33 @@ sections: children: - SarifMark-Context-Create tests: - - Cli_VersionFlag_OutputsVersion - - Cli_HelpFlag_OutputsUsageInformation + - Cli_VersionFlag_SetsVersionFlag + - Cli_HelpFlag_SetsHelpFlag - id: SarifMark-Cli-Version - title: The tool shall display version information when requested. + title: The CLI shall recognize the --version flag. justification: >- - Version information allows users to identify the tool version for troubleshooting, compatibility - verification, and ensuring consistent behavior across different environments. + Recognizing the version flag allows the application layer to identify the requested operation + and display the tool version, supporting troubleshooting and compatibility verification. tags: [public] children: - SarifMark-Context-VersionFlag tests: - - Cli_VersionFlag_OutputsVersion + - Cli_VersionFlag_SetsVersionFlag - id: SarifMark-Cli-Help - title: The tool shall display help information when requested. + title: The CLI shall recognize the --help flag. justification: >- - Help information improves discoverability and usability by providing users with guidance on - available options, parameters, and usage patterns without consulting external documentation. + Recognizing the help flag allows the application layer to display usage information, + improving discoverability without consulting external documentation. tags: [public] children: - SarifMark-Context-HelpFlag tests: - - Cli_HelpFlag_OutputsUsageInformation + - Cli_HelpFlag_SetsHelpFlag - id: SarifMark-Cli-Silent - title: The tool shall support silent mode to suppress console output. + title: The CLI shall suppress console output when the silent flag is set. justification: >- Silent mode enables cleaner integration into automated workflows and CI/CD pipelines where console noise needs to be minimized, while still allowing log files to capture detailed information. @@ -50,7 +50,7 @@ sections: - Cli_SilentFlag_SuppressesOutput - id: SarifMark-Cli-Log - title: The tool shall support writing output to a log file. + title: The CLI shall write output to a log file when the log parameter is set. justification: >- Log file support enables persistent record-keeping, debugging, and audit trails, especially valuable in automated environments where console output may not be readily available. @@ -61,18 +61,30 @@ sections: - Cli_LogFile_WritesOutputToFile - id: SarifMark-Cli-Enforce - title: The tool shall support enforcing quality gate checks. + title: The CLI shall recognize the --enforce flag. justification: >- - Quality gate enforcement enables the tool to act as a quality gate in CI/CD pipelines, failing - builds when issues are detected and ensuring code quality standards are maintained. + Recognizing the enforce flag allows the application layer to determine whether to apply + quality gate behavior, making it suitable for integration into CI/CD pipelines. tags: [public] children: - SarifMark-Context-EnforceFlag tests: - - Cli_EnforceFlagWithIssues_ReturnsError + - Cli_EnforceFlag_SetsEnforceFlag + + - id: SarifMark-Cli-WriteError + title: The CLI shall write errors to stderr and set a non-zero exit code. + justification: >- + Writing errors to stderr separates diagnostic output from the regular output stream, and + automatically setting a non-zero exit code ensures that callers receive a failure signal + without requiring explicit exit code management throughout the codebase. + tags: [public] + children: + - SarifMark-Context-WriteError + tests: + - Cli_WriteError_SetsExitCodeToOne - id: SarifMark-Cli-InvalidArgs - title: The tool shall reject unknown command-line arguments. + title: The CLI shall reject unknown command-line arguments. justification: >- Rejecting unknown arguments prevents silent failures from typos or incorrect parameters, ensuring users receive immediate feedback about misconfigured commands. @@ -80,4 +92,70 @@ sections: children: - SarifMark-Context-UnknownArgs tests: - - Cli_UnknownArgument_ShowsError + - Cli_UnknownArgument_ThrowsArgumentException + + - id: SarifMark-Cli-Validate + title: The CLI shall recognize the --validate flag. + justification: >- + Recognizing the validate flag allows the application layer to activate self-validation mode, + enabling verification that the tool operates correctly in the deployment environment. + tags: [public] + children: + - SarifMark-Context-ValidateFlag + tests: + - Cli_ValidateFlag_SetsValidateFlag + + - id: SarifMark-Cli-Sarif + title: The CLI shall accept a SARIF file path via the --sarif parameter. + justification: >- + Exposing a dedicated SARIF file path parameter gives the application layer a clean, named value + that decouples argument parsing from file processing. + tags: [public] + children: + - SarifMark-Context-SarifParam + tests: + - Cli_SarifParameter_SetsSarifFilePath + + - id: SarifMark-Cli-Report + title: The CLI shall accept a report file path via the --report parameter. + justification: >- + A dedicated report file path parameter gives the application layer a clear output target, + keeping argument parsing separate from report generation logic. + tags: [public] + children: + - SarifMark-Context-ReportParam + tests: + - Cli_ReportParameter_SetsReportFilePath + + - id: SarifMark-Cli-ReportDepth + title: The CLI shall accept a heading depth via the --report-depth parameter. + justification: >- + Exposing a validated heading depth parameter allows the application layer to control markdown + report heading levels without reimplementing argument parsing or range checking. + tags: [public] + children: + - SarifMark-Context-ReportDepthParam + tests: + - Cli_ReportDepthParameter_SetsReportDepth + + - id: SarifMark-Cli-Heading + title: The CLI shall accept a custom heading via the --heading parameter. + justification: >- + Exposing a custom heading parameter lets the application layer pass user-specified titles + directly to the report formatter, keeping argument handling separate from output logic. + tags: [public] + children: + - SarifMark-Context-HeadingParam + tests: + - Cli_HeadingParameter_SetsCustomHeading + + - id: SarifMark-Cli-Results + title: The CLI shall accept a results file path via the --results parameter. + justification: >- + A dedicated results file path parameter gives the application layer a clear destination for + test result output during self-validation, keeping argument parsing separate from output logic. + tags: [public] + children: + - SarifMark-Context-ResultsParam + tests: + - Cli_ResultsParameter_SetsResultsFilePath diff --git a/docs/reqstream/sarifmark/sarif/report.yaml b/docs/reqstream/sarifmark/sarif/report.yaml index bcb61f9..a47431d 100644 --- a/docs/reqstream/sarifmark/sarif/report.yaml +++ b/docs/reqstream/sarifmark/sarif/report.yaml @@ -13,7 +13,7 @@ sections: children: - SarifMark-SarifResults-ToMarkdown tests: - - Sarif_GenerateReport_CreatesReportFile + - Sarif_GenerateReport_ProducesMarkdownContent - id: SarifMark-Report-Depth title: The tool shall support configurable markdown heading depth. diff --git a/docs/reqstream/sarifmark/sarif/sarif.yaml b/docs/reqstream/sarifmark/sarif/sarif.yaml index e303112..f013d76 100644 --- a/docs/reqstream/sarifmark/sarif/sarif.yaml +++ b/docs/reqstream/sarifmark/sarif/sarif.yaml @@ -25,8 +25,8 @@ sections: - SarifMark-SarifResults-ValidatePath - SarifMark-SarifResults-ValidateStructure tests: - - Sarif_NonExistentSarifFile_ShowsError - - Sarif_InvalidSarifFile_ShowsFormatError + - Sarif_NonExistentSarifFile_ThrowsFileNotFoundException + - Sarif_InvalidSarifFile_ThrowsInvalidOperationException - id: SarifMark-Sarif-ToolInfo title: The tool shall extract tool information from SARIF files. @@ -65,7 +65,7 @@ sections: - Sarif_Report_ContainsLocationInfo - id: SarifMark-Sarif-FilePaths - title: The tool shall require valid file paths for SARIF input. + title: The SARIF subsystem shall require valid file paths for SARIF input. justification: >- Path validation ensures clear error messages when files are missing or incorrectly specified, improving user experience and preventing confusing runtime failures. @@ -73,19 +73,7 @@ sections: children: - SarifMark-SarifResults-ValidatePath tests: - - Sarif_MissingSarifParameter_ShowsError - - Sarif_NonExistentSarifFile_ShowsError - - - id: SarifMark-Sarif-Required - title: The tool shall require the --sarif parameter for analysis. - justification: >- - Requiring explicit SARIF file specification ensures intentional operation and prevents accidental - execution without proper input, improving tool safety and predictability. - tags: [public] - children: - - SarifMark-SarifResults-ValidatePath - tests: - - Sarif_MissingSarifParameter_ShowsError + - Sarif_NonExistentSarifFile_ThrowsFileNotFoundException - id: SarifMark-Sarif-Processing title: The tool shall process valid SARIF files successfully. diff --git a/docs/reqstream/sarifmark/sarifmark.yaml b/docs/reqstream/sarifmark/sarifmark.yaml index eaa9fc5..32f7819 100644 --- a/docs/reqstream/sarifmark/sarifmark.yaml +++ b/docs/reqstream/sarifmark/sarifmark.yaml @@ -37,6 +37,17 @@ sections: tests: - IntegrationTest_ValidateFlag_RunsSelfValidation + - id: SarifMark-System-SarifRequired + title: The tool shall require the --sarif parameter for SARIF analysis. + tags: [public] + justification: >- + Requiring explicit SARIF file specification ensures intentional operation and prevents + accidental execution without proper input, improving tool safety and predictability. + children: + - SarifMark-Sarif-FilePaths + tests: + - IntegrationTest_MissingSarifParameter_ShowsError + - id: SarifMark-System-SarifAnalysis title: The tool shall read and analyze SARIF files. tags: [public] diff --git a/docs/reqstream/sarifmark/self-test/self-test.yaml b/docs/reqstream/sarifmark/self-test/self-test.yaml index 19dfac9..a8f4c2a 100644 --- a/docs/reqstream/sarifmark/self-test/self-test.yaml +++ b/docs/reqstream/sarifmark/self-test/self-test.yaml @@ -60,7 +60,7 @@ sections: children: - SarifMark-Validation-EnforcementTest tests: - - SelfTest_EnforceFlag_ReturnsNonZeroOnIssues + - SelfTest_EnforcementTest_RunsWithinValidation - id: SarifMark-Enforce-ExitCode title: The tool shall return non-zero exit code when issues found in enforcement mode. @@ -71,4 +71,4 @@ sections: children: - SarifMark-Validation-EnforcementTest tests: - - SelfTest_EnforceFlag_ReturnsNonZeroOnIssues + - SelfTest_EnforcementTest_RunsWithinValidation diff --git a/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs b/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs index 58f374f..c234c54 100644 --- a/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Cli/CliTests.cs @@ -26,67 +26,32 @@ namespace DemaConsulting.SarifMark.Tests; [TestClass] public class CliTests { - private string _dllPath = string.Empty; - private string _testDataPath = string.Empty; - - /// - /// Initialize test by locating the SarifMark DLL and test data. - /// - [TestInitialize] - public void TestInitialize() - { - var baseDir = AppContext.BaseDirectory; - _dllPath = PathHelpers.SafePathCombine(baseDir, "DemaConsulting.SarifMark.dll"); - _testDataPath = PathHelpers.SafePathCombine(baseDir, "TestData"); - - Assert.IsTrue(File.Exists(_dllPath), $"Could not find SarifMark DLL at {_dllPath}"); - } - /// - /// Test that version flag outputs version information. + /// Test that version flag sets the version flag in context. /// [TestMethod] - public void Cli_VersionFlag_OutputsVersion() + public void Cli_VersionFlag_SetsVersionFlag() { - // Arrange - No special setup needed - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--version"); + using var context = Context.Create(["--version"]); // Assert - Assert.AreEqual(0, exitCode); - Assert.IsFalse(string.IsNullOrWhiteSpace(output)); - Assert.DoesNotContain("Error", output); - Assert.DoesNotContain("Copyright", output); + Assert.IsTrue(context.Version); + Assert.AreEqual(0, context.ExitCode); } /// - /// Test that help flag outputs usage information. + /// Test that help flag sets the help flag in context. /// [TestMethod] - public void Cli_HelpFlag_OutputsUsageInformation() + public void Cli_HelpFlag_SetsHelpFlag() { - // Arrange - No special setup needed - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--help"); + using var context = Context.Create(["--help"]); // Assert - Assert.AreEqual(0, exitCode); - Assert.Contains("Usage: sarifmark", output); - Assert.Contains("Options:", output); - Assert.Contains("--version", output); - Assert.Contains("--help", output); - Assert.Contains("--sarif", output); - Assert.MatchesRegex(@"--report(?!-)", output); + Assert.IsTrue(context.Help); + Assert.AreEqual(0, context.ExitCode); } /// @@ -96,21 +61,26 @@ public void Cli_HelpFlag_OutputsUsageInformation() public void Cli_SilentFlag_SuppressesOutput() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); - Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--silent", - "--sarif", sarifFile); + // Act + using var context = Context.Create(["--silent"]); + context.WriteLine("SarifMark version 1.0"); + context.WriteLine("Copyright"); + var output = outWriter.ToString(); - // Assert - Assert.AreEqual(0, exitCode); - Assert.DoesNotContain("SarifMark version", output); - Assert.DoesNotContain("Copyright", output); + // Assert + Assert.AreEqual(0, context.ExitCode); + Assert.AreEqual(string.Empty, output); + } + finally + { + Console.SetOut(originalOut); + } } /// @@ -120,23 +90,19 @@ public void Cli_SilentFlag_SuppressesOutput() public void Cli_LogFile_WritesOutputToFile() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); - Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - - var logFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-log-{Guid.NewGuid()}.log"); + var logFile = Path.Combine(Path.GetTempPath(), $"test-log-{Guid.NewGuid()}.log"); try { // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--log", logFile, - "--sarif", sarifFile); + using (var context = Context.Create(["--silent", "--log", logFile])) + { + context.WriteLine("SarifMark version 1.0"); + context.WriteLine("SARIF File: test.sarif"); + context.WriteLine("Tool: TestTool"); + } // Assert - Assert.AreEqual(0, exitCode); Assert.IsTrue(File.Exists(logFile), "Log file was not created"); var logContent = File.ReadAllText(logFile); @@ -154,46 +120,143 @@ public void Cli_LogFile_WritesOutputToFile() } /// - /// Test that enforce flag with issues returns error exit code. + /// Test that enforce flag sets the enforce flag in context. /// [TestMethod] - public void Cli_EnforceFlagWithIssues_ReturnsError() + public void Cli_EnforceFlag_SetsEnforceFlag() + { + // Act + using var context = Context.Create(["--enforce"]); + + // Assert + Assert.IsTrue(context.Enforce); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that WriteError writes to stderr and sets exit code to one. + /// + [TestMethod] + public void Cli_WriteError_SetsExitCodeToOne() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); - Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); + var originalError = Console.Error; + try + { + using var errWriter = new StringWriter(); + Console.SetError(errWriter); + + // Act + using var context = Context.Create([]); + context.WriteError("Test error message"); + var output = errWriter.ToString(); + + // Assert + Assert.AreEqual(1, context.ExitCode); + Assert.Contains("Test error message", output); + } + finally + { + Console.SetError(originalError); + } + } + + /// + /// Test that unknown arguments are rejected by throwing ArgumentException. + /// + [TestMethod] + public void Cli_UnknownArgument_ThrowsArgumentException() + { + // Arrange - No special setup needed // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--enforce"); + var ex = Assert.ThrowsExactly(() => Context.Create(["--unknown-flag"])); // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("Issues found in SARIF file", output); + Assert.Contains("unknown-flag", ex.Message); } /// - /// Test that unknown arguments are rejected with error. + /// Test that validate flag sets the validate flag in context. /// [TestMethod] - public void Cli_UnknownArgument_ShowsError() + public void Cli_ValidateFlag_SetsValidateFlag() { - // Arrange - No special setup needed + // Act + using var context = Context.Create(["--validate"]); + + // Assert + Assert.IsTrue(context.Validate); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that sarif parameter sets the SARIF file path in context. + /// + [TestMethod] + public void Cli_SarifParameter_SetsSarifFilePath() + { + // Act + using var context = Context.Create(["--sarif", "analysis.sarif"]); + + // Assert + Assert.AreEqual("analysis.sarif", context.SarifFile); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that report parameter sets the report file path in context. + /// + [TestMethod] + public void Cli_ReportParameter_SetsReportFilePath() + { + // Act + using var context = Context.Create(["--report", "report.md"]); + // Assert + Assert.AreEqual("report.md", context.ReportFile); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that report-depth parameter sets the report depth in context. + /// + [TestMethod] + public void Cli_ReportDepthParameter_SetsReportDepth() + { + // Act + using var context = Context.Create(["--report-depth", "3"]); + + // Assert + Assert.AreEqual(3, context.ReportDepth); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that heading parameter sets the custom heading in context. + /// + [TestMethod] + public void Cli_HeadingParameter_SetsCustomHeading() + { + // Act + using var context = Context.Create(["--heading", "My Analysis"]); + + // Assert + Assert.AreEqual("My Analysis", context.Heading); + Assert.AreEqual(0, context.ExitCode); + } + + /// + /// Test that results parameter sets the results file path in context. + /// + [TestMethod] + public void Cli_ResultsParameter_SetsResultsFilePath() + { // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--unknown-flag"); + using var context = Context.Create(["--results", "results.trx"]); // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("Error:", output); - Assert.Contains("unknown-flag", output); + Assert.AreEqual("results.trx", context.ResultsFile); + Assert.AreEqual(0, context.ExitCode); } } diff --git a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs index d55e948..e79fa58 100644 --- a/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/Sarif/SarifTests.cs @@ -26,39 +26,15 @@ namespace DemaConsulting.SarifMark.Tests; [TestClass] public class SarifTests { - private string _dllPath = string.Empty; private string _testDataPath = string.Empty; /// - /// Initialize test by locating the SarifMark DLL and test data. + /// Initialize test by locating test data. /// [TestInitialize] public void TestInitialize() { - var baseDir = AppContext.BaseDirectory; - _dllPath = PathHelpers.SafePathCombine(baseDir, "DemaConsulting.SarifMark.dll"); - _testDataPath = PathHelpers.SafePathCombine(baseDir, "TestData"); - - Assert.IsTrue(File.Exists(_dllPath), $"Could not find SarifMark DLL at {_dllPath}"); - } - - /// - /// Test that missing sarif parameter shows error. - /// - [TestMethod] - public void Sarif_MissingSarifParameter_ShowsError() - { - // Arrange - No special setup needed - - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath); - - // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("--sarif parameter is required", output); + _testDataPath = Path.Combine(AppContext.BaseDirectory, "TestData"); } /// @@ -68,83 +44,46 @@ public void Sarif_MissingSarifParameter_ShowsError() public void Sarif_ValidSarifFile_ProcessesSuccessfully() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", sarifFile); + var results = SarifResults.Read(sarifFile); // Assert - Assert.AreEqual(0, exitCode); - Assert.Contains("SarifMark version", output); - Assert.Contains("SARIF File:", output); - Assert.Contains("Reading SARIF file...", output); - Assert.Contains("Tool: TestTool", output); - Assert.Contains("Results: 1", output); + Assert.AreEqual("TestTool", results.ToolName); + Assert.AreEqual(1, results.ResultCount); } /// - /// Test that processing a non-existent SARIF file shows error. + /// Test that reading a non-existent SARIF file throws FileNotFoundException. /// [TestMethod] - public void Sarif_NonExistentSarifFile_ShowsError() + public void Sarif_NonExistentSarifFile_ThrowsFileNotFoundException() { - // Arrange - No special setup needed - - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", "nonexistent.sarif"); + // Arrange + var nonExistentPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.sarif"); - // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("Error:", output); + // Act / Assert + Assert.ThrowsExactly(() => SarifResults.Read(nonExistentPath)); } /// - /// Test that generating a report file succeeds. + /// Test that generating a report produces the expected markdown content. /// [TestMethod] - public void Sarif_GenerateReport_CreatesReportFile() + public void Sarif_GenerateReport_ProducesMarkdownContent() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-report-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.Contains("Writing report to", output); - Assert.Contains("Report generated successfully", output); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(1); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("# TestTool Analysis", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Assert + Assert.Contains("# TestTool Analysis", reportContent); } /// @@ -154,58 +93,29 @@ public void Sarif_GenerateReport_CreatesReportFile() public void Sarif_ReportDepth_IsConfigurable() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-report-depth-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile, - "--report-depth", "3"); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(3); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("### TestTool Analysis", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Assert + Assert.Contains("### TestTool Analysis", reportContent); } /// - /// Test that processing an invalid SARIF file shows a format error. + /// Test that reading an invalid SARIF file throws InvalidOperationException. /// [TestMethod] - public void Sarif_InvalidSarifFile_ShowsFormatError() + public void Sarif_InvalidSarifFile_ThrowsInvalidOperationException() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "invalid.sarif"); + var sarifFile = Path.Combine(_testDataPath, "invalid.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", sarifFile); - - // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("Error:", output); + // Act / Assert + Assert.ThrowsExactly(() => SarifResults.Read(sarifFile)); } /// @@ -215,40 +125,20 @@ public void Sarif_InvalidSarifFile_ShowsFormatError() public void Sarif_GenerateReport_FormatsMultipleResultsWithLineBreaks() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "multi-result.sarif"); + var sarifFile = Path.Combine(_testDataPath, "multi-result.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-multi-report-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(1); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("Found 2 issues", reportContent); - Assert.Contains("first.cs", reportContent); - Assert.Contains("second.cs", reportContent); + // Assert + Assert.Contains("Found 2 issues", reportContent); + Assert.Contains("first.cs", reportContent); + Assert.Contains("second.cs", reportContent); - // Verify results appear on separate lines with proper markdown line breaks - Assert.MatchesRegex(@"first\.cs.* \r?\nfile:///path/to/second\.cs", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Verify results appear on separate lines with proper markdown line breaks + Assert.MatchesRegex(@"first\.cs.* \r?\nfile:///path/to/second\.cs", reportContent); } /// @@ -258,35 +148,15 @@ public void Sarif_GenerateReport_FormatsMultipleResultsWithLineBreaks() public void Sarif_Report_ContainsResultCount() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-count-report-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(1); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("Found 1 issue", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Assert + Assert.Contains("Found 1 issue", reportContent); } /// @@ -296,35 +166,15 @@ public void Sarif_Report_ContainsResultCount() public void Sarif_Report_ContainsLocationInfo() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-location-report-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(1); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("file:///path/to/file.cs", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Assert + Assert.Contains("file:///path/to/file.cs", reportContent); } /// @@ -334,35 +184,14 @@ public void Sarif_Report_ContainsLocationInfo() public void Sarif_Report_UsesCustomHeading() { // Arrange - var sarifFile = PathHelpers.SafePathCombine(_testDataPath, "sample.sarif"); + var sarifFile = Path.Combine(_testDataPath, "sample.sarif"); Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - var reportFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-heading-report-{Guid.NewGuid()}.md"); - - try - { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--report", reportFile, - "--heading", "Custom Analysis Heading"); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(reportFile), "Report file was not created"); + // Act + var results = SarifResults.Read(sarifFile); + var reportContent = results.ToMarkdown(1, "Custom Analysis Heading"); - var reportContent = File.ReadAllText(reportFile); - Assert.Contains("Custom Analysis Heading", reportContent); - } - finally - { - if (File.Exists(reportFile)) - { - File.Delete(reportFile); - } - } + // Assert + Assert.Contains("Custom Analysis Heading", reportContent); } } diff --git a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs index 4b268fc..fb68363 100644 --- a/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs +++ b/test/DemaConsulting.SarifMark.Tests/SelfTest/SelfTestTests.cs @@ -26,39 +26,32 @@ namespace DemaConsulting.SarifMark.Tests; [TestClass] public class SelfTestTests { - private string _dllPath = string.Empty; - - /// - /// Initialize test by locating the SarifMark DLL. - /// - [TestInitialize] - public void TestInitialize() - { - var baseDir = AppContext.BaseDirectory; - _dllPath = PathHelpers.SafePathCombine(baseDir, "DemaConsulting.SarifMark.dll"); - - Assert.IsTrue(File.Exists(_dllPath), $"Could not find SarifMark DLL at {_dllPath}"); - } - /// /// Test that validate flag runs self-validation. /// [TestMethod] public void SelfTest_ValidateFlag_RunsSelfValidation() { - // Arrange - No special setup needed - - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--validate"); - - // Assert - Assert.AreEqual(0, exitCode); - Assert.Contains("SarifMark version", output); - Assert.Contains("Total Tests:", output); + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + + // Act + using var context = Context.Create(["--validate"]); + Validation.Run(context); + var output = outWriter.ToString(); + + // Assert + Assert.AreEqual(0, context.ExitCode); + Assert.Contains("Total Tests:", output); + } + finally + { + Console.SetOut(originalOut); + } } /// @@ -68,24 +61,31 @@ public void SelfTest_ValidateFlag_RunsSelfValidation() public void SelfTest_ResultsFile_WritesTrxFile() { // Arrange - var resultsFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.trx"); + var resultsFile = Path.Combine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.trx"); try { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--validate", - "--results", resultsFile); + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(resultsFile), "TRX results file was not created"); + // Act + using var context = Context.Create(["--validate", "--results", resultsFile]); + Validation.Run(context); - var content = File.ReadAllText(resultsFile); - Assert.Contains(" - /// Test that enforce flag returns non-zero exit code when issues are found. - /// - [TestMethod] - public void SelfTest_EnforceFlag_ReturnsNonZeroOnIssues() - { - // Arrange - var baseDir = AppContext.BaseDirectory; - var sarifFile = PathHelpers.SafePathCombine( - PathHelpers.SafePathCombine(baseDir, "TestData"), - "sample.sarif"); - Assert.IsTrue(File.Exists(sarifFile), $"Test SARIF file not found at {sarifFile}"); - - // Act - var exitCode = Runner.Run( - out var output, - "dotnet", - _dllPath, - "--sarif", sarifFile, - "--enforce"); - - // Assert - Assert.AreEqual(1, exitCode); - Assert.Contains("Issues found in SARIF file", output); - } - /// /// Test that validate flag with JUnit XML results parameter writes a JUnit XML file. /// @@ -129,24 +103,31 @@ public void SelfTest_EnforceFlag_ReturnsNonZeroOnIssues() public void SelfTest_ResultsFile_WritesJUnitFile() { // Arrange - var resultsFile = PathHelpers.SafePathCombine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.xml"); + var resultsFile = Path.Combine(Path.GetTempPath(), $"test-results-{Guid.NewGuid()}.xml"); try { - // Act - var exitCode = Runner.Run( - out _, - "dotnet", - _dllPath, - "--validate", - "--results", resultsFile); + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); - // Assert - Assert.AreEqual(0, exitCode); - Assert.IsTrue(File.Exists(resultsFile), "JUnit XML results file was not created"); + // Act + using var context = Context.Create(["--validate", "--results", resultsFile]); + Validation.Run(context); + + // Assert + Assert.AreEqual(0, context.ExitCode); + Assert.IsTrue(File.Exists(resultsFile), "JUnit XML results file was not created"); - var content = File.ReadAllText(resultsFile); - Assert.Contains(" + /// Test that enforcement mode behavior is verified by the self-validation suite. + /// + [TestMethod] + public void SelfTest_EnforcementTest_RunsWithinValidation() + { + // Arrange + var originalOut = Console.Out; + try + { + using var outWriter = new StringWriter(); + Console.SetOut(outWriter); + + // Act - Run validation which internally exercises enforcement mode + using var context = Context.Create(["--validate"]); + Validation.Run(context); + var output = outWriter.ToString(); + + // Assert - enforcement test within validation runs and passes + Assert.AreEqual(0, context.ExitCode); + Assert.Contains("SarifMark_Enforcement - Passed", output); + } + finally + { + Console.SetOut(originalOut); + } + } }