diff --git a/poc/TestOfTestFramework/NFUnitTest.nfproj b/poc/TestOfTestFramework/NFUnitTest.nfproj index 9c96dfc..a58f7a9 100644 --- a/poc/TestOfTestFramework/NFUnitTest.nfproj +++ b/poc/TestOfTestFramework/NFUnitTest.nfproj @@ -52,11 +52,4 @@ - - - - This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105.The missing file is {0}. - - - \ No newline at end of file diff --git a/source/TestAdapter/Executor.cs b/source/TestAdapter/Executor.cs index 97675f0..8153728 100644 --- a/source/TestAdapter/Executor.cs +++ b/source/TestAdapter/Executor.cs @@ -4,6 +4,8 @@ // See LICENSE file in the project root for full license information. // +using CliWrap; +using CliWrap.Buffered; using ICSharpCode.Decompiler; using ICSharpCode.Decompiler.CSharp; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -78,6 +80,7 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame try { InitializeLogger(runContext, frameworkHandle); + foreach (var source in sources) { var testsCases = TestDiscoverer.ComposeTestCases(source); @@ -120,8 +123,10 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame } else { - // we are connecting to WIN32 nanoCLR - results = RunTestOnEmulator(groups.ToList()); + // we are connecting to nanoCLR CLI + results = RunTestOnEmulatorAsync( + groups.ToList(), + _logger).GetAwaiter().GetResult(); } foreach (var result in results) @@ -536,7 +541,7 @@ await Task.Run(async delegate } _logger.LogMessage($"Tests finished.", Settings.LoggingLevel.Verbose); - CheckAllTests(output.ToString(), results); + ParseTestResults(output.ToString(), results); } else { @@ -566,17 +571,29 @@ private List PrepareListResult(List tests) return results; } - private List RunTestOnEmulator(List tests) + private async Task> RunTestOnEmulatorAsync( + List tests, + LogMessenger _logger) { + List results = PrepareListResult(tests); + _logger.LogMessage( - "Setting up test runner in *** nanoCLR WIN32***", + "Setting up test runner in *** nanoCLR CLI ***", Settings.LoggingLevel.Detailed); _logger.LogMessage( $"Timeout set to {_testSessionTimeout}ms", Settings.LoggingLevel.Verbose); - List results = PrepareListResult(tests); + // check if nanoCLR needs to be installed/updated + if (!NanoCLRHelper.NanoClrIsInstalled + && !NanoCLRHelper.InstallNanoClr(_logger)) + { + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = "Failed to install/update nanoCLR CLI. Check log for details."; + + return results; + } _logger.LogMessage( "Processing assemblies to load into test runner...", @@ -586,143 +603,96 @@ private List RunTestOnEmulator(List tests) var workingDirectory = Path.GetDirectoryName(source); var allPeFiles = Directory.GetFiles(workingDirectory, "*.pe"); - // prepare the process start of the WIN32 nanoCLR - _nanoClr = new Process(); + // prepare launch of nanoCLR CLI + StringBuilder arguments = new StringBuilder(); - AutoResetEvent outputWaitHandle = new AutoResetEvent(false); - AutoResetEvent errorWaitHandle = new AutoResetEvent(false); - StringBuilder output = new StringBuilder(); - StringBuilder error = new StringBuilder(); + // assemblies to load + arguments.Append("run --assemblies "); - try + foreach (var pe in allPeFiles) { - // prepare parameters to load nanoCLR, include: - // 1. unit test launcher - // 2. mscorlib - // 3. test framework - // 4. test application - StringBuilder str = new StringBuilder(); - foreach (var pe in allPeFiles) - { - str.Append($" -load \"{Path.Combine(workingDirectory, pe)}\""); - } - - string parameter = str.ToString(); - - _logger.LogMessage( - $"Parameters to pass to nanoCLR: <{parameter}>", - Settings.LoggingLevel.Verbose); - - var nanoClrLocation = TestObjectHelper.GetNanoClrLocation(); - if (string.IsNullOrEmpty(nanoClrLocation)) - { - _logger.LogPanicMessage("Can't find nanoCLR Win32 in any of the directories!"); - results.First().Outcome = TestOutcome.Failed; - results.First().ErrorMessage = "Can't find nanoCLR Win32 in any of the directories!"; - return results; - } + arguments.Append($" \"{Path.Combine(workingDirectory, pe)}\""); + } - _logger.LogMessage($"Found nanoCLR Win32: {nanoClrLocation}", Settings.LoggingLevel.Verbose); - _nanoClr.StartInfo = new ProcessStartInfo(nanoClrLocation, parameter) - { - WorkingDirectory = workingDirectory, - UseShellExecute = false, - RedirectStandardError = true, - RedirectStandardOutput = true - }; + // should we use a local nanoCLR instance? + if (!string.IsNullOrEmpty(_settings.PathToLocalCLRInstance)) + { + arguments.Append($" --localinstance \"{_settings.PathToLocalCLRInstance}\""); + } - _logger.LogMessage( - $"Launching process with nanoCLR (from {Path.GetFullPath(TestObjectHelper.GetNanoClrLocation())})", - Settings.LoggingLevel.Verbose); + // if requested, set diagnostic output + if(_settings.Logging > Settings.LoggingLevel.None) + { + arguments.Append(" -v diag"); + } - // launch nanoCLR - if (!_nanoClr.Start()) - { - results.First().Outcome = TestOutcome.Failed; - results.First().ErrorMessage = "Failed to start nanoCLR"; + _logger.LogMessage( + $"Launching nanoCLR with these arguments: '{arguments}'", + Settings.LoggingLevel.Verbose); - _logger.LogPanicMessage( - "Failed to start nanoCLR!"); - } + // launch nanoCLR + var cmd = Cli.Wrap("nanoclr") + .WithArguments(arguments.ToString()) + .WithValidation(CommandResultValidation.None); - _nanoClr.OutputDataReceived += (sender, e) => - { - if (e.Data == null) - { - outputWaitHandle.Set(); - } - else - { - output.AppendLine(e.Data); - } - }; + // setup cancellation token with a timeout of 5 seconds + using (var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5))) + { + var cliResult = await cmd.ExecuteBufferedAsync(cts.Token); + var exitCode = cliResult.ExitCode; + + // read standard output + var output = cliResult.StandardOutput; - _nanoClr.ErrorDataReceived += (sender, e) => + if (exitCode == 0) { - if (e.Data == null) - { - errorWaitHandle.Set(); - } - else + try { - error.AppendLine(e.Data); - } - }; - - _nanoClr.Start(); + // process output to gather tests results + ParseTestResults(output, results); - _nanoClr.BeginOutputReadLine(); - _nanoClr.BeginErrorReadLine(); + _logger.LogMessage(output, Settings.LoggingLevel.Verbose); - _logger.LogMessage( - $"nanoCLR started @ process ID: {_nanoClr.Id}", - Settings.LoggingLevel.Detailed); + if (!output.Contains(Done)) + { + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = output; + } + var notPassedOrFailed = results.Where(m => m.Outcome != TestOutcome.Failed + && m.Outcome != TestOutcome.Passed + && m.Outcome != TestOutcome.Skipped); - // wait for exit, no worries about the outcome - _nanoClr.WaitForExit(_testSessionTimeout); + if (notPassedOrFailed.Any()) + { + notPassedOrFailed.First().ErrorMessage = output; + } - CheckAllTests(output.ToString(), results); - _logger.LogMessage(output.ToString(), Settings.LoggingLevel.Verbose); - if (!output.ToString().Contains(Done)) - { - results.First().Outcome = TestOutcome.Failed; - results.First().ErrorMessage = output.ToString(); - } + } + catch (Exception ex) + { + _logger.LogMessage( + $"Fatal exception when processing test results: >>>{ex.Message}\r\n{output}", + Settings.LoggingLevel.Detailed); - var notPassedOrFailed = results.Where(m => m.Outcome != TestOutcome.Failed && m.Outcome != TestOutcome.Passed && m.Outcome != TestOutcome.Skipped); - if (notPassedOrFailed.Any()) - { - notPassedOrFailed.First().ErrorMessage = output.ToString(); + results.First().Outcome = TestOutcome.Failed; + } } - - } - catch (Exception ex) - { - _logger.LogMessage( - $"Fatal exception when processing test results: >>>{ex.Message}\r\n{output}\r\n{error}", - Settings.LoggingLevel.Detailed); - - results.First().Outcome = TestOutcome.Failed; - results.First().ErrorMessage = $"Fatal exception when processing test results. Set logging to 'Detailed' for details."; - } - finally - { - if (!_nanoClr.HasExited) + else { - _logger.LogMessage( - "Attempting to kill nanoCLR process...", - Settings.LoggingLevel.Verbose); + _logger.LogPanicMessage($"nanoCLR ended with '{exitCode}' exit code.\r\n>>>>>>>>>>>>>\r\n{output}\r\n>>>>>>>>>>>>>"); - _nanoClr.Kill(); - _nanoClr.WaitForExit(2000); + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = $"nanoCLR execution ended with exit code: {exitCode}. Check log for details."; + + return results; } } return results; } - private void CheckAllTests(string rawOutput, List results) + private void ParseTestResults(string rawOutput, List results) { var outputStrings = Regex.Replace( rawOutput, diff --git a/source/TestAdapter/NanoCLRHelper.cs b/source/TestAdapter/NanoCLRHelper.cs new file mode 100644 index 0000000..8ac0b91 --- /dev/null +++ b/source/TestAdapter/NanoCLRHelper.cs @@ -0,0 +1,79 @@ +// +// Copyright (c) .NET Foundation and Contributors +// See LICENSE file in the project root for full license information. +// + +using CliWrap; +using CliWrap.Buffered; +using nanoFramework.TestPlatform.TestAdapter; +using System; +using System.Text.RegularExpressions; +using System.Threading; + +namespace nanoFramework.TestAdapter +{ + internal class NanoCLRHelper + { + /// + /// Flag to report if nanoCLR CLI .NET tool is installed. + /// + public static bool NanoClrIsInstalled { get; private set; } = false; + + public static bool InstallNanoClr(LogMessenger logger) + { + logger.LogMessage( + "Install/upate nanoclr tool", + Settings.LoggingLevel.Verbose); + + var cmd = Cli.Wrap("dotnet") + .WithArguments("tool update -g nanoclr") + .WithValidation(CommandResultValidation.None); + + // setup cancellation token with a timeout of 1 minute + using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1))) + { + var cliResult = cmd.ExecuteBufferedAsync(cts.Token).Task.Result; + + if (cliResult.ExitCode == 0) + { + // this will be either (on update): + // Tool 'nanoclr' was successfully updated from version '1.0.205' to version '1.0.208'. + // or (update becoming reinstall with same version, if there is no new version): + // Tool 'nanoclr' was reinstalled with the latest stable version (version '1.0.208'). + var regexResult = Regex.Match(cliResult.StandardOutput, @"((?>version ')(?'version'\d+\.\d+\.\d+)(?>'))"); + + if (regexResult.Success) + { + logger.LogMessage($"Install/update successful. Running v{regexResult.Groups["version"].Value}", + Settings.LoggingLevel.Verbose); + + NanoClrIsInstalled = true; + } + else + { + logger.LogPanicMessage($"*** Failed to install/update nanoclr. {cliResult.StandardOutput}."); + + NanoClrIsInstalled = false; + } + } + else + { + logger.LogPanicMessage( + $"Failed to install/update nanoclr. Exit code {cliResult.ExitCode}." + + Environment.NewLine + + Environment.NewLine + + "****************************************" + + Environment.NewLine + + "*** WON'T BE ABLE TO RUN UNITS TESTS ***" + + Environment.NewLine + + "****************************************"); + + NanoClrIsInstalled = false; + } + } + + // report outcome + return NanoClrIsInstalled; + } + } +} diff --git a/source/TestAdapter/Settings.cs b/source/TestAdapter/Settings.cs index 5c87073..fd1b17b 100644 --- a/source/TestAdapter/Settings.cs +++ b/source/TestAdapter/Settings.cs @@ -10,7 +10,7 @@ namespace nanoFramework.TestPlatform.TestAdapter { /// - /// Settings for the nanoFramework tests + /// Settings for .NET nanoFramework Test Adapter. /// public class Settings { @@ -24,6 +24,11 @@ public class Settings /// public string RealHardwarePort { get; set; } = string.Empty; + /// + /// Path to a local nanoCLR instance to use to run Unit Tests. + /// + public string PathToLocalCLRInstance { get; set; } = string.Empty; + /// /// Level of logging for test execution. /// diff --git a/source/TestAdapter/nanoFramework.TestAdapter.csproj b/source/TestAdapter/nanoFramework.TestAdapter.csproj index 57a4a39..10b4b10 100644 --- a/source/TestAdapter/nanoFramework.TestAdapter.csproj +++ b/source/TestAdapter/nanoFramework.TestAdapter.csproj @@ -10,39 +10,15 @@ + - - 1.8.0.751 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - https://dl.cloudsmith.io/public/net-nanoframework/nanoframework-images-dev/raw/names/WIN32_nanoCLR/versions/latest/nanoFramework.nanoCLR.exe - - - - - - - - - - - - - diff --git a/source/TestAdapter/packages.lock.json b/source/TestAdapter/packages.lock.json index 8bc9720..0bc5ab0 100644 --- a/source/TestAdapter/packages.lock.json +++ b/source/TestAdapter/packages.lock.json @@ -2,6 +2,20 @@ "version": 1, "dependencies": { ".NETFramework,Version=v4.8": { + "CliWrap": { + "type": "Direct", + "requested": "[3.6.0, )", + "resolved": "3.6.0", + "contentHash": "AY6LvRZOEYuAiuaWPLnIDddJUnpiPpiSvfoPwweEXI1orRNnsAwf6sOv9Tt0J4GFrlwejFF/INuR57iEKIh7bw==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "7.0.0", + "System.Buffers": "4.5.1", + "System.Memory": "4.5.5", + "System.Runtime.InteropServices.RuntimeInformation": "4.3.0", + "System.Threading.Tasks.Extensions": "4.5.4", + "System.ValueTuple": "4.5.0" + } + }, "ICSharpCode.Decompiler": { "type": "Direct", "requested": "[7.2.1.6856, )", @@ -23,12 +37,6 @@ "System.Reflection.Metadata": "1.6.0" } }, - "nanoFramework.nanoCLR.Win32": { - "type": "Direct", - "requested": "[1.8.0.751, )", - "resolved": "1.8.0.751", - "contentHash": "fyT81Ee1KUQiqpFtotFt0s+X5Zh5HKpWKM+BokoBU7lbLG+kt0/8b4oetlyafrs97Vt+uF2LXb6bLwZvmhV7SA==" - }, "nanoFramework.Tools.Debugger.Net": { "type": "Direct", "requested": "[2.4.11, )", @@ -169,6 +177,11 @@ "resolved": "6.0.0", "contentHash": "/iUeP3tq1S0XdNNoMz5C9twLSrM/TH+qElHkXWaPvuNOt+99G75NrV0OS2EqHx5wMN7popYjpc8oTjC1y16DLg==" }, + "System.Runtime.InteropServices.RuntimeInformation": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "cbz4YJMqRDR7oLeMRbdYv7mYzc++17lNhScCX0goO2XpGWdvAt60CGN+FHdePUEHCe/Jy9jUlvNAiNdM+7jsOw==" + }, "System.Security.AccessControl": { "type": "Transitive", "resolved": "5.0.0", @@ -189,6 +202,11 @@ "dependencies": { "System.Runtime.CompilerServices.Unsafe": "4.5.3" } + }, + "System.ValueTuple": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "okurQJO6NRE/apDIP23ajJ0hpiNmJ+f0BwOlB/cSqTLQlw5upkf+5+96+iG2Jw40G1fCVCyPz/FhIABUjMR+RQ==" } } } diff --git a/source/nanoFramework.TestFramework.targets b/source/nanoFramework.TestFramework.targets deleted file mode 100644 index 25f4c36..0000000 --- a/source/nanoFramework.TestFramework.targets +++ /dev/null @@ -1,15 +0,0 @@ - - - https://dl.cloudsmith.io/public/net-nanoframework/nanoframework-images/raw/names/WIN32_nanoCLR/versions/latest/nanoFramework.nanoCLR.exe - - - - - - - - \ No newline at end of file diff --git a/source/package.nuspec b/source/package.nuspec index aff031c..bbd1a41 100644 --- a/source/package.nuspec +++ b/source/package.nuspec @@ -23,11 +23,9 @@ - -