From f35456102688f67e4cfd9b67f0b462a3accd9ae7 Mon Sep 17 00:00:00 2001 From: josesimoes Date: Fri, 26 Feb 2021 17:44:30 +0000 Subject: [PATCH 1/7] Start work on test adapter for hardware --- poc/TestOfTestFramework/nano.runsettings | 2 + source/TestAdapter/DeploymentAssembly.cs | 35 ++ source/TestAdapter/Executor.cs | 312 +++++++++++++++++- source/TestAdapter/Settings.cs | 6 +- .../nanoFramework - Backup.TestAdapter.csproj | 27 ++ .../nanoFramework.TestAdapter.csproj | 3 +- 6 files changed, 380 insertions(+), 5 deletions(-) create mode 100644 source/TestAdapter/DeploymentAssembly.cs create mode 100644 source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj diff --git a/poc/TestOfTestFramework/nano.runsettings b/poc/TestOfTestFramework/nano.runsettings index 62f0a00..521afd3 100644 --- a/poc/TestOfTestFramework/nano.runsettings +++ b/poc/TestOfTestFramework/nano.runsettings @@ -9,5 +9,7 @@ None + True + \ No newline at end of file diff --git a/source/TestAdapter/DeploymentAssembly.cs b/source/TestAdapter/DeploymentAssembly.cs new file mode 100644 index 0000000..7ba32ad --- /dev/null +++ b/source/TestAdapter/DeploymentAssembly.cs @@ -0,0 +1,35 @@ +// +// Copyright (c) .NET Foundation and Contributors +// See LICENSE file in the project root for full license information. +// + +using System.Collections.Generic; + +namespace nanoFramework.TestPlatform.TestAdapter +{ + public class DeploymentAssembly + { + /// + /// Path to the EXE or DLL file. + /// + public string Path { get; set; } + + /// + /// Assembly version of the EXE or DLL. + /// + public string Version { get; set; } + + /// + /// Required version of the native implementation of the class library. + /// Only used in class libraries. Can be empty on the core library and user EXE and DLLs. + /// + public string NativeVersion { get; set; } + + public DeploymentAssembly(string path, string version, string nativeVersion) + { + Path = path; + Version = version; + NativeVersion = nativeVersion; + } + } +} \ No newline at end of file diff --git a/source/TestAdapter/Executor.cs b/source/TestAdapter/Executor.cs index 042fed6..9bf0171 100644 --- a/source/TestAdapter/Executor.cs +++ b/source/TestAdapter/Executor.cs @@ -7,6 +7,9 @@ using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using nanoFramework.TestAdapter; +using nanoFramework.Tools.Debugger; +using nanoFramework.Tools.Debugger.Extensions; +using nanoFramework.Tools.Debugger.WireProtocol; using System; using System.Collections.Generic; using System.Diagnostics; @@ -15,6 +18,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading; +using System.Threading.Tasks; namespace nanoFramework.TestPlatform.TestAdapter { @@ -32,6 +36,12 @@ class Executor : ITestExecutor private LogMessenger _logger; private Process _nanoClr; + // number of retries when performing a deploy operation + private const int _numberOfRetries = 5; + + // timeout when performing a deploy operation + private const int _timeoutMiliseconds = 1000; + private IFrameworkHandle _frameworkHandle = null; /// @@ -95,7 +105,19 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame $"Test group is '{source}'", Settings.LoggingLevel.Detailed); - var results = RunTest(groups.ToList()); + // + List results; + + if(_settings.IsRealHardware) + { + // we are connecting to a real device + results = RunTestOnHardwareAsync(groups.ToList()); + } + else + { + // we are connecting to WIN32 nanoCLR + results = RunTest(groups.ToList()); + } foreach (var result in results) { @@ -104,6 +126,294 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame } } + private async System.Threading.Tasks.Task> RunTestOnHardwareAsync(List lists) + { + var serialDebugClient = PortBase.CreateInstanceForSerial("", new System.Collections.Generic.List() { "COM16" }); + + var device = serialDebugClient.NanoFrameworkDevices[0]; + + // check if debugger engine exists + if (device.DebugEngine == null) + { + device.CreateDebugEngine(); + } + + bool deviceIsInInitializeState = false; + int retryCount = 0; + + bool connectResult = await device.DebugEngine.ConnectAsync(5000, true); + + if (connectResult) + { + // erase the device + var eraseResult = await Task.Run(async delegate + { + //MessageCentre.InternalErrorMessage("Erase deployment block storage."); + + return await device.EraseAsync( + EraseOptions.Deployment, + CancellationToken.None, + null, + null); + }); + + if (eraseResult) + { + + // initial check + if (device.DebugEngine.IsDeviceInInitializeState()) + { + //MessageCentre.InternalErrorMessage("Device status verified as being in initialized state. Requesting to resume execution."); + + // set flag + deviceIsInInitializeState = true; + + // device is still in initialization state, try resume execution + device.DebugEngine.ResumeExecution(); + } + + // handle the workflow required to try resuming the execution on the device + // only required if device is not already there + // retry 5 times with a 500ms interval between retries + while (retryCount++ < _numberOfRetries && deviceIsInInitializeState) + { + if (!device.DebugEngine.IsDeviceInInitializeState()) + { + //MessageCentre.InternalErrorMessage("Device has completed initialization."); + + // done here + deviceIsInInitializeState = false; + break; + } + + //MessageCentre.InternalErrorMessage($"Waiting for device to report initialization completed ({retryCount}/{_numberOfRetries})."); + + // provide feedback to user on the 1st pass + if (retryCount == 0) + { + //await outputPaneWriter.WriteLineAsync(ResourceStrings.WaitingDeviceInitialization); + } + + if (device.DebugEngine.IsConnectedTonanoBooter) + { + // MessageCentre.InternalErrorMessage("Device reported running nanoBooter. Requesting to load nanoCLR."); + + // request nanoBooter to load CLR + device.DebugEngine.ExecuteMemory(0); + } + else if (device.DebugEngine.IsConnectedTonanoCLR) + { + //MessageCentre.InternalErrorMessage("Device reported running nanoCLR. Requesting to reboot nanoCLR."); + + await Task.Run(delegate + { + // already running nanoCLR try rebooting the CLR + device.DebugEngine.RebootDevice(RebootOptions.ClrOnly); + }); + } + + // wait before next pass + // use a back-off strategy of increasing the wait time to accommodate slower or less responsive targets (such as networked ones) + await Task.Delay(TimeSpan.FromMilliseconds(_timeoutMiliseconds * (retryCount + 1))); + + await Task.Yield(); + } + + // check if device is still in initialized state + if (!deviceIsInInitializeState) + { + // device has left initialization state + //await outputPaneWriter.WriteLineAsync(ResourceStrings.DeviceInitialized); + + await Task.Yield(); + + + + ////////////////////////////////////////////////////////// + // sanity check for devices without native assemblies ?!?! + if (device.DeviceInfo.NativeAssemblies.Count == 0) + { + // MessageCentre.InternalErrorMessage("Device reporting no assemblies loaded. This can not happen. Sanity check failed."); + + // there are no assemblies deployed?! + //throw new DeploymentException($"Couldn't find any native assemblies deployed in {_viewModelLocator.DeviceExplorer.SelectedDevice.Description}! If the situation persists reboot the device."); + } + + //MessageCentre.InternalErrorMessage("Computing deployment blob."); + + + + // build a list with the full path for each DLL, referenced DLL and EXE + List assemblyList = new List(); + + + //var source = tests.First().Source; + //var nfUnitTestLauncherLocation = source.Replace(Path.GetFileName(source), "nanoFramework.UnitTestLauncher.pe"); + //var workingDirectory = Path.GetDirectoryName(nfUnitTestLauncherLocation); + + // load tests + assemblyList.Add( + new DeploymentAssembly(source, "", "")); + + // TODO + + + //var mscorlibLocation = source.Replace(Path.GetFileName(source), "mscorlib.pe"); + //var nfTestFrameworkLocation = source.Replace(Path.GetFileName(source), "nanoFramework.TestFramework.pe"); + //var nfAssemblyUnderTestLocation = source.Replace(".dll", ".pe"); + + // TODO do we need to check vbersions? + + //foreach (string assemblyPath in assemblyPathsToDeploy) + //{ + // // load assembly in order to get the versions + // var decompiler = new CSharpDecompiler(assemblyPath, decompilerSettings); + // var assemblyProperties = decompiler.DecompileModuleAndAssemblyAttributesToString(); + + // // read attributes using a Regex + + // // AssemblyVersion + // string pattern = @"(?<=AssemblyVersion\("")(.*)(?=\""\)])"; + // var match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); + // string assemblyVersion = match[0].Value; + + // // AssemblyNativeVersion + // pattern = @"(?<=AssemblyNativeVersion\("")(.*)(?=\""\)])"; + // match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); + + // // only class libs have this attribute, therefore sanity check is required + // string nativeVersion = ""; + // if (match.Count == 1) + // { + // nativeVersion = match[0].Value; + // } + + // assemblyList.Add(new DeploymentAssembly(assemblyPath, assemblyVersion, nativeVersion)); + //} + + //// if there are referenced project, the assembly list contains repeated assemblies so need to use Linq Distinct() + //// an IEqualityComparer is required implementing the proper comparison + //List distinctAssemblyList = assemblyList.Distinct(new DeploymentAssemblyDistinctEquality()).ToList(); + + //// build a list with the PE files corresponding to each DLL and EXE + //List peCollection = distinctAssemblyList.Select(a => new DeploymentAssembly(a.Path.Replace(".dll", ".pe").Replace(".exe", ".pe"), a.Version, a.NativeVersion)).ToList(); + + //// build a list with the PE files corresponding to a DLL for native support checking + //// only need to check libraries because EXEs don't have native counterpart + //List peCollectionToCheck = distinctAssemblyList.Where(i => i.Path.EndsWith(".dll")).Select(a => new DeploymentAssembly(a.Path.Replace(".dll", ".pe"), a.Version, a.NativeVersion)).ToList(); + + //await Task.Yield(); + + //var checkAssembliesResult = await CheckNativeAssembliesAvailabilityAsync(device.DeviceInfo.NativeAssemblies, peCollectionToCheck); + //if (checkAssembliesResult != "") + //{ + // MessageCentre.InternalErrorMessage("Found assemblies mismatches when checking for deployment pre-check."); + + // // can't deploy + // throw new DeploymentException(checkAssembliesResult); + //} + + //await Task.Yield(); + + // Keep track of total assembly size + long totalSizeOfAssemblies = 0; + + // TODO use this code to load the PE files + + //// now we will re-deploy all system assemblies + //foreach (DeploymentAssembly peItem in peCollection) + //{ + // // append to the deploy blob the assembly + // using (FileStream fs = File.Open(peItem.Path, FileMode.Open, FileAccess.Read)) + // { + // long length = (fs.Length + 3) / 4 * 4; + // await outputPaneWriter.WriteLineAsync($"Adding {Path.GetFileNameWithoutExtension(peItem.Path)} v{peItem.Version} ({length.ToString()} bytes) to deployment bundle"); + // byte[] buffer = new byte[length]; + + // await Task.Yield(); + + // await fs.ReadAsync(buffer, 0, (int)fs.Length); + // assemblies.Add(buffer); + + // // Increment totalizer + // totalSizeOfAssemblies += length; + // } + //} + + //await outputPaneWriter.WriteLineAsync($"Deploying {peCollection.Count:N0} assemblies to device... Total size in bytes is {totalSizeOfAssemblies.ToString()}."); + //MessageCentre.InternalErrorMessage("Deploying assemblies."); + + //// need to keep a copy of the deployment blob for the second attempt (if needed) + //var assemblyCopy = new List(assemblies); + + await Task.Yield(); + + await Task.Run(async delegate + { + // OK to skip erase as we just did that + // no need to reboot device + if (!device.DebugEngine.DeploymentExecute( + assemblyCopy, + false, + true, + null, + null)) + { + // if the first attempt fails, give it another try + + // wait before next pass + await Task.Delay(TimeSpan.FromSeconds(1)); + + await Task.Yield(); + + //MessageCentre.InternalErrorMessage("Deploying assemblies. Second attempt."); + + //// !! need to use the deployment blob copy + //assemblyCopy = new List(assemblies); + + //// can't skip erase as we just did that + //// no need to reboot device + //if (!device.DebugEngine.DeploymentExecute( + // assemblyCopy, + // false, + // false, + // progressIndicator, + // logProgressIndicator)) + //{ + // MessageCentre.InternalErrorMessage("Deployment failed."); + + // // throw exception to signal deployment failure + // throw new DeploymentException("Deploy failed."); + //} + } + }); + + await Task.Yield(); + + // attach listner for messages + device.DebugEngine.OnMessage -= new MessageEventHandler(OnMessage); + + + device.DebugEngine.RebootDevice(RebootOptions.ClrOnly); + + + } + else + { + // after retry policy applied seems that we couldn't resume execution on the device... + +// MessageCentre.InternalErrorMessage("Failed to initialize device."); + + } + } + } + } + + private void OnMessage(IncomingMessage message, string text) + { + throw new NotImplementedException(); + } + /// public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { diff --git a/source/TestAdapter/Settings.cs b/source/TestAdapter/Settings.cs index af46540..7da63d2 100644 --- a/source/TestAdapter/Settings.cs +++ b/source/TestAdapter/Settings.cs @@ -10,7 +10,7 @@ namespace nanoFramework.TestPlatform.TestAdapter { /// - /// Settings for the nanoFramweork tests + /// Settings for the nanoFramework tests /// public class Settings { @@ -23,12 +23,12 @@ public class Settings /// /// True to run the tests on real hardware /// - public bool IsRealHarware { get; set; } = false; + public bool IsRealHardware { get; set; } = false; /// /// The serial port number to run the tests on a real hardware /// - public string RealHarwarePort { get; set; } = string.Empty; + public string RealHardwarePort { get; set; } = string.Empty; /// /// Level of logging for test execution. diff --git a/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj b/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj new file mode 100644 index 0000000..9189d48 --- /dev/null +++ b/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj @@ -0,0 +1,27 @@ + + + + net4.8 + true + key.snk + + + + + + 1.6.1-preview.223 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + 3.3.37 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/source/TestAdapter/nanoFramework.TestAdapter.csproj b/source/TestAdapter/nanoFramework.TestAdapter.csproj index 50841dd..9596dee 100644 --- a/source/TestAdapter/nanoFramework.TestAdapter.csproj +++ b/source/TestAdapter/nanoFramework.TestAdapter.csproj @@ -1,7 +1,7 @@  - net4.6 + net4.8 true key.snk @@ -13,6 +13,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all + 3.3.37 runtime; build; native; contentfiles; analyzers; buildtransitive From 20aa086a7c2e86a678e1293a26e3074d6ed5df8b Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 14:55:02 +0300 Subject: [PATCH 2/7] adding minimal hardare loading --- source/TestAdapter/Executor.cs | 257 +++++++++--------- source/TestAdapter/Settings.cs | 8 +- .../nanoFramework.TestAdapter.csproj | 1 + 3 files changed, 136 insertions(+), 130 deletions(-) diff --git a/source/TestAdapter/Executor.cs b/source/TestAdapter/Executor.cs index f6b2e49..2797702 100644 --- a/source/TestAdapter/Executor.cs +++ b/source/TestAdapter/Executor.cs @@ -4,6 +4,8 @@ // See LICENSE file in the project root for full license information. // +using ICSharpCode.Decompiler; +using ICSharpCode.Decompiler.CSharp; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using nanoFramework.TestAdapter; @@ -108,10 +110,10 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame // List results; - if(_settings.IsRealHardware) + if (_settings.IsRealHardware) { // we are connecting to a real device - results = RunTestOnHardwareAsync(groups.ToList()); + results = RunTestOnHardwareAsync(groups.ToList()).GetAwaiter().GetResult(); } else { @@ -126,9 +128,15 @@ public void RunTests(IEnumerable tests, IRunContext runContext, IFrame } } - private async System.Threading.Tasks.Task> RunTestOnHardwareAsync(List lists) + private async Task> RunTestOnHardwareAsync(List tests) { - var serialDebugClient = PortBase.CreateInstanceForSerial("", new System.Collections.Generic.List() { "COM16" }); + List results = PrepareListResult(tests); + List assemblies = new List(); + string port = _settings.RealHardwarePort == string.Empty ? "COM4" : _settings.RealHardwarePort; + var serialDebugClient = PortBase.CreateInstanceForSerial("", null, true, new List() { port }); + + _logger.LogMessage($"Checking device on port {port}.", Settings.LoggingLevel.Verbose); + _logger.LogMessage($"{serialDebugClient.NanoFrameworkDevices.Count}", Settings.LoggingLevel.Verbose); var device = serialDebugClient.NanoFrameworkDevices[0]; @@ -136,20 +144,21 @@ private async System.Threading.Tasks.Task> RunTestOnHardwareAsy if (device.DebugEngine == null) { device.CreateDebugEngine(); + _logger.LogMessage($"Debug engine created.", Settings.LoggingLevel.Verbose); } bool deviceIsInInitializeState = false; int retryCount = 0; bool connectResult = await device.DebugEngine.ConnectAsync(5000, true); + _logger.LogMessage($"Device connect result is {connectResult}.", Settings.LoggingLevel.Verbose); if (connectResult) { // erase the device var eraseResult = await Task.Run(async delegate { - //MessageCentre.InternalErrorMessage("Erase deployment block storage."); - + _logger.LogMessage($"Erase deployment block storage.", Settings.LoggingLevel.Error); return await device.EraseAsync( EraseOptions.Deployment, CancellationToken.None, @@ -157,14 +166,14 @@ private async System.Threading.Tasks.Task> RunTestOnHardwareAsy null); }); + _logger.LogMessage($"Erase result is {eraseResult}.", Settings.LoggingLevel.Verbose); if (eraseResult) { // initial check if (device.DebugEngine.IsDeviceInInitializeState()) { - //MessageCentre.InternalErrorMessage("Device status verified as being in initialized state. Requesting to resume execution."); - + _logger.LogMessage($"Device status verified as being in initialized state. Requesting to resume execution.", Settings.LoggingLevel.Error); // set flag deviceIsInInitializeState = true; @@ -179,36 +188,32 @@ private async System.Threading.Tasks.Task> RunTestOnHardwareAsy { if (!device.DebugEngine.IsDeviceInInitializeState()) { - //MessageCentre.InternalErrorMessage("Device has completed initialization."); - + _logger.LogMessage($"Device has completed initialization.", Settings.LoggingLevel.Verbose); // done here deviceIsInInitializeState = false; break; } - //MessageCentre.InternalErrorMessage($"Waiting for device to report initialization completed ({retryCount}/{_numberOfRetries})."); - + _logger.LogMessage($"Waiting for device to report initialization completed ({retryCount}/{_numberOfRetries}).", Settings.LoggingLevel.Verbose); // provide feedback to user on the 1st pass if (retryCount == 0) { - //await outputPaneWriter.WriteLineAsync(ResourceStrings.WaitingDeviceInitialization); + _logger.LogMessage($"Waiting for device to initialize.", Settings.LoggingLevel.Verbose); } if (device.DebugEngine.IsConnectedTonanoBooter) { - // MessageCentre.InternalErrorMessage("Device reported running nanoBooter. Requesting to load nanoCLR."); - + _logger.LogMessage($"Device reported running nanoBooter. Requesting to load nanoCLR.", Settings.LoggingLevel.Verbose); // request nanoBooter to load CLR device.DebugEngine.ExecuteMemory(0); } else if (device.DebugEngine.IsConnectedTonanoCLR) { - //MessageCentre.InternalErrorMessage("Device reported running nanoCLR. Requesting to reboot nanoCLR."); - + _logger.LogMessage($"Device reported running nanoCLR. Requesting to reboot nanoCLR.", Settings.LoggingLevel.Error); await Task.Run(delegate { - // already running nanoCLR try rebooting the CLR - device.DebugEngine.RebootDevice(RebootOptions.ClrOnly); + // already running nanoCLR try rebooting the CLR + device.DebugEngine.RebootDevice(RebootOptions.ClrOnly); }); } @@ -223,87 +228,74 @@ await Task.Run(delegate if (!deviceIsInInitializeState) { // device has left initialization state - //await outputPaneWriter.WriteLineAsync(ResourceStrings.DeviceInitialized); - + _logger.LogMessage($"Device is initialized and ready!", Settings.LoggingLevel.Verbose); await Task.Yield(); - ////////////////////////////////////////////////////////// // sanity check for devices without native assemblies ?!?! if (device.DeviceInfo.NativeAssemblies.Count == 0) { - // MessageCentre.InternalErrorMessage("Device reporting no assemblies loaded. This can not happen. Sanity check failed."); - + _logger.LogMessage($"Device reporting no assemblies loaded. This can not happen. Sanity check failed.", Settings.LoggingLevel.Error); // there are no assemblies deployed?! - //throw new DeploymentException($"Couldn't find any native assemblies deployed in {_viewModelLocator.DeviceExplorer.SelectedDevice.Description}! If the situation persists reboot the device."); + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = $"Couldn't find any native assemblies deployed in {device.Description}, {device.TargetName} on {device.SerialNumber}! If the situation persists reboot the device."; + return results; } - //MessageCentre.InternalErrorMessage("Computing deployment blob."); - - - + _logger.LogMessage($"Computing deployment blob.", Settings.LoggingLevel.Verbose); // build a list with the full path for each DLL, referenced DLL and EXE List assemblyList = new List(); + var source = tests.First().Source; + var workingDirectory = Path.GetDirectoryName(source); + var allPeFiles = Directory.GetFiles(workingDirectory, "*.pe"); - //var source = tests.First().Source; - //var nfUnitTestLauncherLocation = source.Replace(Path.GetFileName(source), "nanoFramework.UnitTestLauncher.pe"); - //var workingDirectory = Path.GetDirectoryName(nfUnitTestLauncherLocation); - - // load tests - assemblyList.Add( - new DeploymentAssembly(source, "", "")); - - // TODO - - - //var mscorlibLocation = source.Replace(Path.GetFileName(source), "mscorlib.pe"); - //var nfTestFrameworkLocation = source.Replace(Path.GetFileName(source), "nanoFramework.TestFramework.pe"); - //var nfAssemblyUnderTestLocation = source.Replace(".dll", ".pe"); - - // TODO do we need to check vbersions? - - //foreach (string assemblyPath in assemblyPathsToDeploy) + // load tests in case we don't need to check the version: + //foreach (var pe in allPeFiles) //{ - // // load assembly in order to get the versions - // var decompiler = new CSharpDecompiler(assemblyPath, decompilerSettings); - // var assemblyProperties = decompiler.DecompileModuleAndAssemblyAttributesToString(); - - // // read attributes using a Regex + // assemblyList.Add( + // new DeploymentAssembly(Path.Combine(workingDirectory, pe), "", "")); + //} - // // AssemblyVersion - // string pattern = @"(?<=AssemblyVersion\("")(.*)(?=\""\)])"; - // var match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); - // string assemblyVersion = match[0].Value; + // TODO do we need to check versions? + var decompilerSettings = new DecompilerSettings + { + LoadInMemory = false, + ThrowOnAssemblyResolveErrors = false + }; - // // AssemblyNativeVersion - // pattern = @"(?<=AssemblyNativeVersion\("")(.*)(?=\""\)])"; - // match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); + foreach (string assemblyPath in allPeFiles) + { + // load assembly in order to get the versions + var decompiler = new CSharpDecompiler(Path.Combine(workingDirectory, assemblyPath), decompilerSettings); + var assemblyProperties = decompiler.DecompileModuleAndAssemblyAttributesToString(); - // // only class libs have this attribute, therefore sanity check is required - // string nativeVersion = ""; - // if (match.Count == 1) - // { - // nativeVersion = match[0].Value; - // } + // read attributes using a Regex - // assemblyList.Add(new DeploymentAssembly(assemblyPath, assemblyVersion, nativeVersion)); - //} + // AssemblyVersion + string pattern = @"(?<=AssemblyVersion\("")(.*)(?=\""\)])"; + var match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); + string assemblyVersion = match[0].Value; - //// if there are referenced project, the assembly list contains repeated assemblies so need to use Linq Distinct() - //// an IEqualityComparer is required implementing the proper comparison - //List distinctAssemblyList = assemblyList.Distinct(new DeploymentAssemblyDistinctEquality()).ToList(); + // AssemblyNativeVersion + pattern = @"(?<=AssemblyNativeVersion\("")(.*)(?=\""\)])"; + match = Regex.Matches(assemblyProperties, pattern, RegexOptions.IgnoreCase); - //// build a list with the PE files corresponding to each DLL and EXE - //List peCollection = distinctAssemblyList.Select(a => new DeploymentAssembly(a.Path.Replace(".dll", ".pe").Replace(".exe", ".pe"), a.Version, a.NativeVersion)).ToList(); + // only class libs have this attribute, therefore sanity check is required + string nativeVersion = ""; + if (match.Count == 1) + { + nativeVersion = match[0].Value; + } - //// build a list with the PE files corresponding to a DLL for native support checking - //// only need to check libraries because EXEs don't have native counterpart - //List peCollectionToCheck = distinctAssemblyList.Where(i => i.Path.EndsWith(".dll")).Select(a => new DeploymentAssembly(a.Path.Replace(".dll", ".pe"), a.Version, a.NativeVersion)).ToList(); + assemblyList.Add(new DeploymentAssembly(Path.Combine(workingDirectory, assemblyPath), assemblyVersion, nativeVersion)); + } - //await Task.Yield(); + _logger.LogMessage($"Added {assemblyList.Count} assemblies to deploy.", Settings.LoggingLevel.Verbose); + await Task.Yield(); + //TODO: shall we chack the assembly availability? //var checkAssembliesResult = await CheckNativeAssembliesAvailabilityAsync(device.DeviceInfo.NativeAssemblies, peCollectionToCheck); //if (checkAssembliesResult != "") //{ @@ -320,31 +312,29 @@ await Task.Run(delegate // TODO use this code to load the PE files - //// now we will re-deploy all system assemblies - //foreach (DeploymentAssembly peItem in peCollection) - //{ - // // append to the deploy blob the assembly - // using (FileStream fs = File.Open(peItem.Path, FileMode.Open, FileAccess.Read)) - // { - // long length = (fs.Length + 3) / 4 * 4; - // await outputPaneWriter.WriteLineAsync($"Adding {Path.GetFileNameWithoutExtension(peItem.Path)} v{peItem.Version} ({length.ToString()} bytes) to deployment bundle"); - // byte[] buffer = new byte[length]; - - // await Task.Yield(); + // now we will re-deploy all system assemblies + foreach (DeploymentAssembly peItem in assemblyList) + { + // append to the deploy blob the assembly + using (FileStream fs = File.Open(peItem.Path, FileMode.Open, FileAccess.Read)) + { + long length = (fs.Length + 3) / 4 * 4; + _logger.LogMessage($"Adding {Path.GetFileNameWithoutExtension(peItem.Path)} v{peItem.Version} ({length} bytes) to deployment bundle", Settings.LoggingLevel.Verbose); + byte[] buffer = new byte[length]; - // await fs.ReadAsync(buffer, 0, (int)fs.Length); - // assemblies.Add(buffer); + await Task.Yield(); - // // Increment totalizer - // totalSizeOfAssemblies += length; - // } - //} + await fs.ReadAsync(buffer, 0, (int)fs.Length); + assemblies.Add(buffer); - //await outputPaneWriter.WriteLineAsync($"Deploying {peCollection.Count:N0} assemblies to device... Total size in bytes is {totalSizeOfAssemblies.ToString()}."); - //MessageCentre.InternalErrorMessage("Deploying assemblies."); + // Increment totalizer + totalSizeOfAssemblies += length; + } + } - //// need to keep a copy of the deployment blob for the second attempt (if needed) - //var assemblyCopy = new List(assemblies); + _logger.LogMessage($"Deploying {assemblyList.Count:N0} assemblies to device... Total size in bytes is {totalSizeOfAssemblies}.", Settings.LoggingLevel.Verbose); + // need to keep a copy of the deployment blob for the second attempt (if needed) + var assemblyCopy = new List(assemblies); await Task.Yield(); @@ -366,29 +356,35 @@ await Task.Run(async delegate await Task.Yield(); - //MessageCentre.InternalErrorMessage("Deploying assemblies. Second attempt."); - - //// !! need to use the deployment blob copy - //assemblyCopy = new List(assemblies); - - //// can't skip erase as we just did that - //// no need to reboot device - //if (!device.DebugEngine.DeploymentExecute( - // assemblyCopy, - // false, - // false, - // progressIndicator, - // logProgressIndicator)) - //{ - // MessageCentre.InternalErrorMessage("Deployment failed."); - - // // throw exception to signal deployment failure - // throw new DeploymentException("Deploy failed."); - //} + _logger.LogMessage("Deploying assemblies. Second attempt.", Settings.LoggingLevel.Verbose); + + // !! need to use the deployment blob copy + assemblyCopy = new List(assemblies); + + // can't skip erase as we just did that + // no need to reboot device + if (!device.DebugEngine.DeploymentExecute( + assemblyCopy, + false, + false, + null, + null)) + { + _logger.LogMessage("Deployment failed.", Settings.LoggingLevel.Error); + + // throw exception to signal deployment failure + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = $"Deployment failed in {device.Description}, {device.TargetName} on {device.SerialNumber}! If the situation persists reboot the device."; + } } }); await Task.Yield(); + // If there has been an issue before, the first test is marked as failed + if (results.First().Outcome == TestOutcome.Failed) + { + return results; + } // attach listner for messages device.DebugEngine.OnMessage -= new MessageEventHandler(OnMessage); @@ -402,11 +398,13 @@ await Task.Run(async delegate { // after retry policy applied seems that we couldn't resume execution on the device... -// MessageCentre.InternalErrorMessage("Failed to initialize device."); + // MessageCentre.InternalErrorMessage("Failed to initialize device."); } } } + + return results; } private void OnMessage(IncomingMessage message, string text) @@ -429,6 +427,19 @@ public void RunTests(IEnumerable sources, IRunContext runContext, IFrame } } + private List PrepareListResult(List tests) + { + List results = new List(); + + foreach (var test in tests) + { + TestResult result = new TestResult(test) { Outcome = TestOutcome.None }; + results.Add(result); + } + + return results; + } + private List RunTest(List tests) { _logger.LogMessage( @@ -445,13 +456,7 @@ private List RunTest(List tests) $"Timeout set to {runTimeout}ms", Settings.LoggingLevel.Verbose); - List results = new List(); - - foreach (var test in tests) - { - TestResult result = new TestResult(test) { Outcome = TestOutcome.None }; - results.Add(result); - } + List results = PrepareListResult(tests); _logger.LogMessage( "Processing assemblies to load into test runner...", @@ -459,7 +464,7 @@ private List RunTest(List tests) var source = tests.First().Source; var workingDirectory = Path.GetDirectoryName(source); - var allPeFiles = Directory.GetFiles(workingDirectory, "*.pe"); + var allPeFiles = Directory.GetFiles(workingDirectory, "*.pe"); // prepare the process start of the WIN32 nanoCLR _nanoClr = new Process(); @@ -477,7 +482,7 @@ private List RunTest(List tests) // 3. test framework // 4. test application StringBuilder str = new StringBuilder(); - foreach(var pe in allPeFiles) + foreach (var pe in allPeFiles) { str.Append($" -load {Path.Combine(workingDirectory, pe)}"); } diff --git a/source/TestAdapter/Settings.cs b/source/TestAdapter/Settings.cs index 7da63d2..8c5bace 100644 --- a/source/TestAdapter/Settings.cs +++ b/source/TestAdapter/Settings.cs @@ -55,16 +55,16 @@ public static Settings Extract(XmlNode node) } } - var isrealhard = node.SelectSingleNode(nameof(IsRealHarware))?.FirstChild; + var isrealhard = node.SelectSingleNode(nameof(IsRealHardware))?.FirstChild; if (isrealhard != null && isrealhard.NodeType == XmlNodeType.Text) { - settings.IsRealHarware = isrealhard.Value.ToLower() == "true" ? true : false; + settings.IsRealHardware = isrealhard.Value.ToLower() == "true" ? true : false; } - var realhardport = node.SelectSingleNode(nameof(RealHarwarePort))?.FirstChild; + var realhardport = node.SelectSingleNode(nameof(RealHardwarePort))?.FirstChild; if (realhardport != null && realhardport.NodeType == XmlNodeType.Text) { - settings.RealHarwarePort = realhardport.Value; + settings.RealHardwarePort = realhardport.Value; } var loggingLevel = node.SelectSingleNode(nameof(Logging))?.FirstChild; diff --git a/source/TestAdapter/nanoFramework.TestAdapter.csproj b/source/TestAdapter/nanoFramework.TestAdapter.csproj index 9596dee..e21e3ab 100644 --- a/source/TestAdapter/nanoFramework.TestAdapter.csproj +++ b/source/TestAdapter/nanoFramework.TestAdapter.csproj @@ -7,6 +7,7 @@ + 1.6.1-preview.223 From 208379abdec8b105909f8e97beb9076ce9ea34a8 Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 16:35:10 +0300 Subject: [PATCH 3/7] Adding hardware support --- source/TestAdapter/Executor.cs | 156 ++++++++++++++++++--------------- 1 file changed, 86 insertions(+), 70 deletions(-) diff --git a/source/TestAdapter/Executor.cs b/source/TestAdapter/Executor.cs index 2797702..dad9495 100644 --- a/source/TestAdapter/Executor.cs +++ b/source/TestAdapter/Executor.cs @@ -133,10 +133,16 @@ private async Task> RunTestOnHardwareAsync(List tests List results = PrepareListResult(tests); List assemblies = new List(); string port = _settings.RealHardwarePort == string.Empty ? "COM4" : _settings.RealHardwarePort; - var serialDebugClient = PortBase.CreateInstanceForSerial("", null, true, new List() { port }); + //var serialDebugClient = PortBase.CreateInstanceForSerial("", null, true, new List() { port }); + var serialDebugClient = PortBase.CreateInstanceForSerial("", null, true, null); _logger.LogMessage($"Checking device on port {port}.", Settings.LoggingLevel.Verbose); - _logger.LogMessage($"{serialDebugClient.NanoFrameworkDevices.Count}", Settings.LoggingLevel.Verbose); + while (!serialDebugClient.IsDevicesEnumerationComplete) + { + Thread.Sleep(1); + } + + _logger.LogMessage($"Found: {serialDebugClient.NanoFrameworkDevices.Count} devices", Settings.LoggingLevel.Verbose); var device = serialDebugClient.NanoFrameworkDevices[0]; @@ -268,7 +274,14 @@ await Task.Run(delegate foreach (string assemblyPath in allPeFiles) { // load assembly in order to get the versions - var decompiler = new CSharpDecompiler(Path.Combine(workingDirectory, assemblyPath), decompilerSettings); + var file = Path.Combine(workingDirectory, assemblyPath.Replace(".pe", ".dll")); + if (!File.Exists(file)) + { + // Check with an exe + file = Path.Combine(workingDirectory, assemblyPath.Replace(".pe", ".exe")); + } + + var decompiler = new CSharpDecompiler(file, decompilerSettings); ; var assemblyProperties = decompiler.DecompileModuleAndAssemblyAttributesToString(); // read attributes using a Regex @@ -295,18 +308,6 @@ await Task.Run(delegate _logger.LogMessage($"Added {assemblyList.Count} assemblies to deploy.", Settings.LoggingLevel.Verbose); await Task.Yield(); - //TODO: shall we chack the assembly availability? - //var checkAssembliesResult = await CheckNativeAssembliesAvailabilityAsync(device.DeviceInfo.NativeAssemblies, peCollectionToCheck); - //if (checkAssembliesResult != "") - //{ - // MessageCentre.InternalErrorMessage("Found assemblies mismatches when checking for deployment pre-check."); - - // // can't deploy - // throw new DeploymentException(checkAssembliesResult); - //} - - //await Task.Yield(); - // Keep track of total assembly size long totalSizeOfAssemblies = 0; @@ -386,13 +387,28 @@ await Task.Run(async delegate return results; } + StringBuilder output = new StringBuilder(); + bool isFinished = false; // attach listner for messages - device.DebugEngine.OnMessage -= new MessageEventHandler(OnMessage); - + device.DebugEngine.OnMessage += (message, text) => + { + _logger.LogMessage(text, Settings.LoggingLevel.Verbose); + output.AppendLine(text); + if (text.Contains(Done)) + { + isFinished = true; + } + }; device.DebugEngine.RebootDevice(RebootOptions.ClrOnly); + while (!isFinished) + { + Thread.Sleep(1); + } + _logger.LogMessage($"Tests finished.", Settings.LoggingLevel.Verbose); + CheckAllTests(output.ToString(), results); } else { @@ -407,11 +423,6 @@ await Task.Run(async delegate return results; } - private void OnMessage(IncomingMessage message, string text) - { - throw new NotImplementedException(); - } - /// public void RunTests(IEnumerable sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { @@ -548,54 +559,7 @@ private List RunTest(List tests) // wait for exit, no worries about the outcome _nanoClr.WaitForExit(runTimeout); - var outputStrings = Regex.Split(output.ToString(), @"((\r)+)?(\n)+((\r)+)?").Where(m => !string.IsNullOrEmpty(m)); - - _logger.LogMessage( - "Parsing test results...", - Settings.LoggingLevel.Verbose); - - foreach (var line in outputStrings) - { - if (line.Contains(TestPassed)) - { - // Format is "Test passed: MethodName, ticks"; - // We do get split with space if the coma is missing, happens time to time - string method = line.Substring(line.IndexOf(TestPassed) + TestPassed.Length).Split(',')[0].Split(' ')[0]; - string ticks = line.Substring(line.IndexOf(TestPassed) + TestPassed.Length + method.Length + 2); - long ticksNum = 0; - - try - { - ticksNum = Convert.ToInt64(ticks); - } - catch (Exception) - { - // We won't do anything - } - - // Find the test - var res = results.Where(m => m.TestCase.DisplayName == method); - if (res.Any()) - { - res.First().Duration = TimeSpan.FromTicks(ticksNum); - res.First().Outcome = TestOutcome.Passed; - } - } - else if (line.Contains(TestFailed)) - { - // Format is "Test passed: MethodName, Exception message"; - string method = line.Substring(line.IndexOf(TestFailed) + TestFailed.Length).Split(',')[0].Split(' ')[0]; - string exception = line.Substring(line.IndexOf(TestFailed) + TestPassed.Length + method.Length + 2); - - // Find the test - var res = results.Where(m => m.TestCase.DisplayName == method); - if (res.Any()) - { - res.First().ErrorMessage = exception; - res.First().Outcome = TestOutcome.Failed; - } - } - } + CheckAllTests(output.ToString(), results); if (!output.ToString().Contains(Done)) { @@ -634,5 +598,57 @@ private List RunTest(List tests) return results; } + + private void CheckAllTests(string toCheck, List results) + { + var outputStrings = Regex.Split(toCheck, @"((\r)+)?(\n)+((\r)+)?").Where(m => !string.IsNullOrEmpty(m)); + + _logger.LogMessage( + "Parsing test results...", + Settings.LoggingLevel.Verbose); + + foreach (var line in outputStrings) + { + if (line.Contains(TestPassed)) + { + // Format is "Test passed: MethodName, ticks"; + // We do get split with space if the coma is missing, happens time to time + string method = line.Substring(line.IndexOf(TestPassed) + TestPassed.Length).Split(',')[0].Split(' ')[0]; + string ticks = line.Substring(line.IndexOf(TestPassed) + TestPassed.Length + method.Length + 2); + long ticksNum = 0; + + try + { + ticksNum = Convert.ToInt64(ticks); + } + catch (Exception) + { + // We won't do anything + } + + // Find the test + var res = results.Where(m => m.TestCase.DisplayName == method); + if (res.Any()) + { + res.First().Duration = TimeSpan.FromTicks(ticksNum); + res.First().Outcome = TestOutcome.Passed; + } + } + else if (line.Contains(TestFailed)) + { + // Format is "Test passed: MethodName, Exception message"; + string method = line.Substring(line.IndexOf(TestFailed) + TestFailed.Length).Split(',')[0].Split(' ')[0]; + string exception = line.Substring(line.IndexOf(TestFailed) + TestPassed.Length + method.Length + 2); + + // Find the test + var res = results.Where(m => m.TestCase.DisplayName == method); + if (res.Any()) + { + res.First().ErrorMessage = exception; + res.First().Outcome = TestOutcome.Failed; + } + } + } + } } } From 3a44143100802db7ec3e90e9a27b4ffd92c0b794 Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 16:38:32 +0300 Subject: [PATCH 4/7] adding error message in case of initializaion error --- source/TestAdapter/Executor.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/TestAdapter/Executor.cs b/source/TestAdapter/Executor.cs index dad9495..9892446 100644 --- a/source/TestAdapter/Executor.cs +++ b/source/TestAdapter/Executor.cs @@ -144,6 +144,13 @@ private async Task> RunTestOnHardwareAsync(List tests _logger.LogMessage($"Found: {serialDebugClient.NanoFrameworkDevices.Count} devices", Settings.LoggingLevel.Verbose); + if(serialDebugClient.NanoFrameworkDevices.Count == 0) + { + results.First().Outcome = TestOutcome.Failed; + results.First().ErrorMessage = $"Couldn't find any device, please try to disable the device scanning in the Visual Studio Extension! If the situation persists reboot the device as well."; + return results; + } + var device = serialDebugClient.NanoFrameworkDevices[0]; // check if debugger engine exists @@ -412,10 +419,7 @@ await Task.Run(async delegate } else { - // after retry policy applied seems that we couldn't resume execution on the device... - - // MessageCentre.InternalErrorMessage("Failed to initialize device."); - + _logger.LogMessage("Failed to initialize device.", Settings.LoggingLevel.Error); } } } From 33d822081f192eeb8eeae764e565683c8c25562c Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 17:29:27 +0300 Subject: [PATCH 5/7] Fixing nuget creation --- source/package.nuspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/package.nuspec b/source/package.nuspec index dae81c8..e9df111 100644 --- a/source/package.nuspec +++ b/source/package.nuspec @@ -22,8 +22,8 @@ - - + + From b0989fe398d331304fd0ebf33fee353790380ca1 Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 17:31:02 +0300 Subject: [PATCH 6/7] Ading entry for hardware support in runsettings --- source/runsettings/nano.runsettings | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/runsettings/nano.runsettings b/source/runsettings/nano.runsettings index 62f0a00..fa881e3 100644 --- a/source/runsettings/nano.runsettings +++ b/source/runsettings/nano.runsettings @@ -8,6 +8,7 @@ Framework40 - None + None + False \ No newline at end of file From 61bc9ac1a533d95638cfb06aab9cdb1359ae3034 Mon Sep 17 00:00:00 2001 From: Laurent Ellerbach Date: Sat, 27 Feb 2021 22:07:19 +0300 Subject: [PATCH 7/7] removing backup csproj --- .../nanoFramework - Backup.TestAdapter.csproj | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj diff --git a/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj b/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj deleted file mode 100644 index 9189d48..0000000 --- a/source/TestAdapter/nanoFramework - Backup.TestAdapter.csproj +++ /dev/null @@ -1,27 +0,0 @@ - - - - net4.8 - true - key.snk - - - - - - 1.6.1-preview.223 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - 3.3.37 - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - -