From f0a95a3581efbd0cb670e3ffd644f4b146de0d87 Mon Sep 17 00:00:00 2001 From: Sarabjot Singh Date: Tue, 17 Apr 2018 16:25:51 +0530 Subject: [PATCH 1/2] Adding global test timeout configurable via runsettings --- .../MSTest.CoreAdapter/Execution/TypeCache.cs | 7 ++- .../MSTest.CoreAdapter/MSTestSettings.cs | 18 +++++++ .../Execution/TypeCacheTests.cs | 52 +++++++++++++++++++ .../MSTestSettingsTests.cs | 31 ++++++++++- 4 files changed, 106 insertions(+), 2 deletions(-) diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs index 2e2e2a2f5b..673bcac514 100644 --- a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs +++ b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs @@ -286,7 +286,7 @@ private TestClassInfo CreateClassInfo(Type classType, TestMethod testMethod) foreach (var methodInfo in classType.GetTypeInfo().DeclaredMethods) { // Update test initialize/cleanup method - this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods, testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType); + this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods, testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType); if (this.IsAssemblyOrClassInitializeMethod(methodInfo, classInitializeAttributeType)) { @@ -634,6 +634,7 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod) { Debug.Assert(methodInfo != null, "TestMethod should be non-null"); var timeoutAttribute = this.reflectionHelper.GetAttribute(methodInfo); + var globalTimeout = MSTestSettings.CurrentSettings.TestTimeout; if (timeoutAttribute != null) { @@ -645,6 +646,10 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod) return timeoutAttribute.Timeout; } + else if (globalTimeout > 0) + { + return globalTimeout; + } return TestMethodInfo.TimeoutWhenNotSet; } diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs index ef5a534b02..c2e60e92e0 100644 --- a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs +++ b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs @@ -55,6 +55,7 @@ public MSTestSettings() this.ForcedLegacyMode = false; this.TestSettingsFile = null; this.DisableParallelization = false; + this.TestTimeout = 0; } /// @@ -143,6 +144,11 @@ private set /// public bool DisableParallelization { get; private set; } + /// + /// Gets specified global test case timeout + /// + public int TestTimeout { get; private set; } + /// /// Populate settings based on existing settings object. /// @@ -157,6 +163,7 @@ public static void PopulateSettings(MSTestSettings settings) CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers; CurrentSettings.ParallelizationScope = settings.ParallelizationScope; CurrentSettings.DisableParallelization = settings.DisableParallelization; + CurrentSettings.TestTimeout = settings.TestTimeout; } /// @@ -274,6 +281,7 @@ private static MSTestSettings ToSettings(XmlReader reader) // true // false // false + // 5000 // // 4 // TestClass @@ -362,6 +370,16 @@ private static MSTestSettings ToSettings(XmlReader reader) break; } + case "TESTTIMEOUT": + { + if (int.TryParse(reader.ReadInnerXml(), out int testTimeout) && testTimeout > 0) + { + settings.TestTimeout = testTimeout; + } + + break; + } + default: { PlatformServiceProvider.Instance.SettingsProvider.Load(reader.ReadSubtree()); diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs index 60c4779c04..48caad252b 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs @@ -54,6 +54,7 @@ public void TestInit() public void Cleanup() { PlatformServiceProvider.Instance = null; + MSTestSettings.Reset(); } #region GetTestMethodInfo tests @@ -815,6 +816,57 @@ public void GetTestMethodInfoShouldThrowWhenTimeoutIsIncorrect() Assert.AreEqual(expectedMessage, exception.Message); } + [TestMethodV1] + public void GetTestMethodInfoWhenTimeoutAttributeNotSetShouldReturnTestMethodInfoWithGlobalTimeout() + { + string runSettingxml = + @" + + 4000 + + "; + + MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + var type = typeof(DummyTestClassWithTestMethods); + var methodInfo = type.GetMethod("TestMethod"); + var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false); + + var testMethodInfo = this.typeCache.GetTestMethodInfo( + testMethod, + new TestContextImplementation(testMethod, null, new Dictionary()), + false); + + Assert.AreEqual(4000, testMethodInfo.TestMethodOptions.Timeout); + } + + [TestMethodV1] + public void GetTestMethodInfoWhenTimeoutAttributeSetShouldReturnTimeoutBasedOnAtrributeEvenIfGlobalTimeoutSet() + { + string runSettingxml = + @" + + 4000 + + "; + + MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + var type = typeof(DummyTestClassWithTestMethods); + var methodInfo = type.GetMethod("TestMethodWithTimeout"); + var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false); + + this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(methodInfo, typeof(UTF.TimeoutAttribute), false)) + .Returns(true); + + var testMethodInfo = this.typeCache.GetTestMethodInfo( + testMethod, + new TestContextImplementation(testMethod, null, new Dictionary()), + false); + + Assert.AreEqual(10, testMethodInfo.TestMethodOptions.Timeout); + } + [TestMethodV1] public void GetTestMethodInfoShouldReturnTestMethodInfoForMethodsAdornedWithADerivedTestMethodAttribute() { diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs index b4f21f53ee..4b8caf333d 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs @@ -195,6 +195,35 @@ public void CaptureDebugTracesShouldBeConsumedFromRunSettingsWhenSpecified() Assert.AreEqual(adapterSettings.CaptureDebugTraces, false); } + [TestMethod] + public void TestTimeoutShouldBeConsumedFromRunSettingsWhenSpecified() + { + string runSettingxml = + @" + + 4000 + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.TestTimeout, 4000); + } + + [TestMethod] + public void TestTimeoutShouldBeSetToZeroIfNotSpecifiedInRunSettings() + { + string runSettingxml = + @" + + + "; + + MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias); + + Assert.AreEqual(adapterSettings.TestTimeout, 0); + } + [TestMethod] public void ParallelizationSettingsShouldNotBeSetByDefault() { @@ -524,7 +553,7 @@ public void GetSettingsShouldOnlyPassTheElementSubTreeToPlatformService() actualReader.Read(); observedxml = actualReader.ReadOuterXml(); } - }); + }); MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsName); Assert.AreEqual(expectedrunSettingxml, observedxml); From 1f96284fb754b4e9855be7b9d3e01e8572b38beb Mon Sep 17 00:00:00 2001 From: Sarabjot Singh Date: Wed, 18 Apr 2018 15:23:58 +0530 Subject: [PATCH 2/2] Adding negative test as per comments. --- .../Execution/TypeCacheTests.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs index 48caad252b..3943a84714 100644 --- a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs +++ b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs @@ -867,6 +867,30 @@ public void GetTestMethodInfoWhenTimeoutAttributeSetShouldReturnTimeoutBasedOnAt Assert.AreEqual(10, testMethodInfo.TestMethodOptions.Timeout); } + [TestMethodV1] + public void GetTestMethodInfoForInvalidGLobalTimeoutShouldReturnTestMethodInfoWithTimeoutZero() + { + string runSettingxml = + @" + + 30.5 + + "; + + MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias)); + + var type = typeof(DummyTestClassWithTestMethods); + var methodInfo = type.GetMethod("TestMethod"); + var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false); + + var testMethodInfo = this.typeCache.GetTestMethodInfo( + testMethod, + new TestContextImplementation(testMethod, null, new Dictionary()), + false); + + Assert.AreEqual(0, testMethodInfo.TestMethodOptions.Timeout); + } + [TestMethodV1] public void GetTestMethodInfoShouldReturnTestMethodInfoForMethodsAdornedWithADerivedTestMethodAttribute() {