Skip to content

Replace WIN32 with .NET tool for nanoCLR in test executor #202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 3, 2023
Merged

Replace WIN32 with .NET tool for nanoCLR in test executor #202

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0fbde56
Replace WIN32 version with .NET tool for nanoCLR in test executor
josesimoes Mar 31, 2023
02a7203
Improvements following review
josesimoes Apr 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions poc/TestOfTestFramework/NFUnitTest.nfproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,4 @@
<ProjectConfigurationsDeclaredAsItems />
</ProjectCapabilities>
</ProjectExtensions>
<Import Project="..\packages\nanoFramework.TestFramework.2.1.60\build\nanoFramework.TestFramework.targets" Condition="Exists('..\packages\nanoFramework.TestFramework.2.1.60\build\nanoFramework.TestFramework.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105.The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\nanoFramework.TestFramework.2.1.60\build\nanoFramework.TestFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\nanoFramework.TestFramework.2.1.60\build\nanoFramework.TestFramework.targets'))" />
</Target>
</Project>
204 changes: 88 additions & 116 deletions source/TestAdapter/Executor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
// See LICENSE file in the project root for full license information.
//

using CliWrap;
using CliWrap.Buffered;
using ICSharpCode.Decompiler;
using ICSharpCode.Decompiler.CSharp;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
Expand Down Expand Up @@ -78,6 +80,7 @@ public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrame
try
{
InitializeLogger(runContext, frameworkHandle);

foreach (var source in sources)
{
var testsCases = TestDiscoverer.ComposeTestCases(source);
Expand Down Expand Up @@ -120,8 +123,10 @@ public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrame
}
else
{
// we are connecting to WIN32 nanoCLR
results = RunTestOnEmulator(groups.ToList());
// we are connecting to nanoCLR CLI
results = RunTestOnEmulatorAsync(
groups.ToList(),
_logger).GetAwaiter().GetResult();
}

foreach (var result in results)
Expand Down Expand Up @@ -536,7 +541,7 @@ await Task.Run(async delegate
}

_logger.LogMessage($"Tests finished.", Settings.LoggingLevel.Verbose);
CheckAllTests(output.ToString(), results);
ParseTestResults(output.ToString(), results);
}
else
{
Expand Down Expand Up @@ -566,17 +571,29 @@ private List<TestResult> PrepareListResult(List<TestCase> tests)
return results;
}

private List<TestResult> RunTestOnEmulator(List<TestCase> tests)
private async Task<List<TestResult>> RunTestOnEmulatorAsync(
List<TestCase> tests,
LogMessenger _logger)
{
List<TestResult> results = PrepareListResult(tests);

_logger.LogMessage(
"Setting up test runner in *** nanoCLR WIN32***",
"Setting up test runner in *** nanoCLR CLI ***",
Settings.LoggingLevel.Detailed);

_logger.LogMessage(
$"Timeout set to {_testSessionTimeout}ms",
Settings.LoggingLevel.Verbose);

List<TestResult> results = PrepareListResult(tests);
// check if nanoCLR needs to be installed/updated
if (!NanoCLRHelper.NanoClrIsInstalled
&& !NanoCLRHelper.InstallNanoClr(_logger))
{
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = "Failed to install/update nanoCLR CLI. Check log for details.";

return results;
}

_logger.LogMessage(
"Processing assemblies to load into test runner...",
Expand All @@ -586,143 +603,98 @@ private List<TestResult> RunTestOnEmulator(List<TestCase> tests)
var workingDirectory = Path.GetDirectoryName(source);
var allPeFiles = Directory.GetFiles(workingDirectory, "*.pe");

// prepare the process start of the WIN32 nanoCLR
_nanoClr = new Process();
// prepare launch of nanoCLR CLI
StringBuilder arguments = new StringBuilder();

AutoResetEvent outputWaitHandle = new AutoResetEvent(false);
AutoResetEvent errorWaitHandle = new AutoResetEvent(false);
StringBuilder output = new StringBuilder();
StringBuilder error = new StringBuilder();
// assemblies to load
arguments.Append("run --assemblies ");

try
foreach (var pe in allPeFiles)
{
// prepare parameters to load nanoCLR, include:
// 1. unit test launcher
// 2. mscorlib
// 3. test framework
// 4. test application
StringBuilder str = new StringBuilder();
foreach (var pe in allPeFiles)
{
str.Append($" -load \"{Path.Combine(workingDirectory, pe)}\"");
}

string parameter = str.ToString();
arguments.Append($" \"{Path.Combine(workingDirectory, pe)}\"");
}

_logger.LogMessage(
$"Parameters to pass to nanoCLR: <{parameter}>",
Settings.LoggingLevel.Verbose);
// should we use a local nanoCLR instance?
if (!string.IsNullOrEmpty(_settings.PathToLocalCLRInstance))
{
arguments.Append($" --localinstance \"{_settings.PathToLocalCLRInstance}\"");
}

var nanoClrLocation = TestObjectHelper.GetNanoClrLocation();
if (string.IsNullOrEmpty(nanoClrLocation))
{
_logger.LogPanicMessage("Can't find nanoCLR Win32 in any of the directories!");
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = "Can't find nanoCLR Win32 in any of the directories!";
return results;
}
// if requested, set diagnostic output
if(_settings.Logging > Settings.LoggingLevel.None)
{
arguments.Append(" -v diag");
}

_logger.LogMessage($"Found nanoCLR Win32: {nanoClrLocation}", Settings.LoggingLevel.Verbose);
_nanoClr.StartInfo = new ProcessStartInfo(nanoClrLocation, parameter)
{
WorkingDirectory = workingDirectory,
UseShellExecute = false,
RedirectStandardError = true,
RedirectStandardOutput = true
};
_logger.LogMessage(
$"Launching nanoCLR with these arguments: '{arguments}'",
Settings.LoggingLevel.Verbose);

_logger.LogMessage(
$"Launching process with nanoCLR (from {Path.GetFullPath(TestObjectHelper.GetNanoClrLocation())})",
Settings.LoggingLevel.Verbose);
// launch nanoCLR
var cmd = Cli.Wrap("nanoclr")
.WithArguments(arguments.ToString())
.WithValidation(CommandResultValidation.None);

// launch nanoCLR
if (!_nanoClr.Start())
{
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = "Failed to start nanoCLR";
// setup cancellation token with a timeout of 5 seconds
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(TimeSpan.FromSeconds(5));

_logger.LogPanicMessage(
"Failed to start nanoCLR!");
}
var cliResult = await cmd.ExecuteBufferedAsync(cts.Token);
var exitCode = cliResult.ExitCode;

// read standard output
var output = cliResult.StandardOutput;

_nanoClr.OutputDataReceived += (sender, e) =>
if (exitCode == 0)
{
if (e.Data == null)
{
outputWaitHandle.Set();
}
else
try
{
output.AppendLine(e.Data);
}
};
// process output to gather tests results
ParseTestResults(output, results);

_nanoClr.ErrorDataReceived += (sender, e) =>
{
if (e.Data == null)
{
errorWaitHandle.Set();
}
else
{
error.AppendLine(e.Data);
}
};
_logger.LogMessage(output, Settings.LoggingLevel.Verbose);

_nanoClr.Start();

_nanoClr.BeginOutputReadLine();
_nanoClr.BeginErrorReadLine();

_logger.LogMessage(
$"nanoCLR started @ process ID: {_nanoClr.Id}",
Settings.LoggingLevel.Detailed);
if (!output.Contains(Done))
{
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = output;
}

var notPassedOrFailed = results.Where(m => m.Outcome != TestOutcome.Failed
&& m.Outcome != TestOutcome.Passed
&& m.Outcome != TestOutcome.Skipped);

// wait for exit, no worries about the outcome
_nanoClr.WaitForExit(_testSessionTimeout);
if (notPassedOrFailed.Any())
{
notPassedOrFailed.First().ErrorMessage = output;
}

CheckAllTests(output.ToString(), results);
_logger.LogMessage(output.ToString(), Settings.LoggingLevel.Verbose);
if (!output.ToString().Contains(Done))
{
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = output.ToString();
}
}
catch (Exception ex)
{
_logger.LogMessage(
$"Fatal exception when processing test results: >>>{ex.Message}\r\n{output}",
Settings.LoggingLevel.Detailed);

var notPassedOrFailed = results.Where(m => m.Outcome != TestOutcome.Failed && m.Outcome != TestOutcome.Passed && m.Outcome != TestOutcome.Skipped);
if (notPassedOrFailed.Any())
{
notPassedOrFailed.First().ErrorMessage = output.ToString();
results.First().Outcome = TestOutcome.Failed;
}
}

}
catch (Exception ex)
{
_logger.LogMessage(
$"Fatal exception when processing test results: >>>{ex.Message}\r\n{output}\r\n{error}",
Settings.LoggingLevel.Detailed);

results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = $"Fatal exception when processing test results. Set logging to 'Detailed' for details.";
}
finally
{
if (!_nanoClr.HasExited)
else
{
_logger.LogMessage(
"Attempting to kill nanoCLR process...",
Settings.LoggingLevel.Verbose);
_logger.LogPanicMessage($"nanoCLR ended with '{exitCode}' exit code.\r\n>>>>>>>>>>>>>\r\n{output}\r\n>>>>>>>>>>>>>");

_nanoClr.Kill();
_nanoClr.WaitForExit(2000);
results.First().Outcome = TestOutcome.Failed;
results.First().ErrorMessage = $"nanoCLR execution ended with exit code: {exitCode}. Check log for details.";

return results;
}
}

return results;
}

private void CheckAllTests(string rawOutput, List<TestResult> results)
private void ParseTestResults(string rawOutput, List<TestResult> results)
{
var outputStrings = Regex.Replace(
rawOutput,
Expand Down
77 changes: 77 additions & 0 deletions source/TestAdapter/NanoCLRHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
//
// Copyright (c) .NET Foundation and Contributors
// See LICENSE file in the project root for full license information.
//

using CliWrap;
using CliWrap.Buffered;
using nanoFramework.TestPlatform.TestAdapter;
using System;
using System.Text.RegularExpressions;
using System.Threading;

namespace nanoFramework.TestAdapter
{
internal class NanoCLRHelper
{
/// <summary>
/// Flag to report if nanoCLR CLI .NET tool is installed.
/// </summary>
public static bool NanoClrIsInstalled { get; private set; } = false;

public static bool InstallNanoClr(LogMessenger logger)
{
logger.LogMessage(
"Install/upate nanoclr tool",
Settings.LoggingLevel.Verbose);

var cmd = Cli.Wrap("dotnet")
.WithArguments("tool update -g nanoclr")
.WithValidation(CommandResultValidation.None);

// setup cancellation token with a timeout of 1 minute
using (var cts = new CancellationTokenSource())
{
cts.CancelAfter(TimeSpan.FromMinutes(1));

var cliResult = cmd.ExecuteBufferedAsync(cts.Token).Task.Result;

if (cliResult.ExitCode == 0)
{
var regexResult = Regex.Match(cliResult.StandardOutput, @"((?>version ')(?'version'\d+\.\d+\.\d+)(?>'))");

if (regexResult.Success)
{
logger.LogMessage($"Install/update successful. Running v{regexResult.Groups["version"].Value}",
Settings.LoggingLevel.Verbose);

NanoClrIsInstalled = true;
}
else
{
logger.LogPanicMessage($"*** Failed to install/update nanoclr. {cliResult.StandardOutput}.");

NanoClrIsInstalled = false;
}
}
else
{
logger.LogPanicMessage(
$"Failed to install/update nanoclr. Exit code {cliResult.ExitCode}."
+ Environment.NewLine
+ Environment.NewLine
+ "****************************************"
+ Environment.NewLine
+ "*** WON'T BE ABLE TO RUN UNITS TESTS ***"
+ Environment.NewLine
+ "****************************************");

NanoClrIsInstalled = false;
}
}

// report outcome
return NanoClrIsInstalled;
}
}
}
7 changes: 6 additions & 1 deletion source/TestAdapter/Settings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
namespace nanoFramework.TestPlatform.TestAdapter
{
/// <summary>
/// Settings for the nanoFramework tests
/// Settings for .NET nanoFramework Test Adapter.
/// </summary>
public class Settings
{
Expand All @@ -24,6 +24,11 @@ public class Settings
/// </summary>
public string RealHardwarePort { get; set; } = string.Empty;

/// <summary>
/// Path to a local nanoCLR instance to use to run Unit Tests.
/// </summary>
public string PathToLocalCLRInstance { get; set; } = string.Empty;

/// <summary>
/// Level of logging for test execution.
/// </summary>
Expand Down
Loading