[ci] Improve TimeZoneInfo Emulator test reliability; Emulator 30.3.5 (d…

…otnet#5566)

The TimeZoneInfo Emulator tests have been very unreliable recently.
We speculate that this is down to the emulator not running as fast as
it used to on the CI system, possibly due to commit 75317db.

Improve the reliability of the TimeZoneInfo Emulator tests:

Add `[NonParallelizable]` to tests to ensure that they are not run in
parallel.  These tests are run on an emulator, and For Great Sanity™
we don't want these tests run in parallel *anyway*: the tests involve
changing the `persist.sys.timezone` system property, which is
effectively a global variable.  We can't sanely run these in parallel
(unless we spin up multiple emulators…?).

Add a `[Retry]` attribute to at least re-run a test once if it fails.
This might help if the emulator is running slowly.

Add the following commands both before and after each test:

	RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
	RunAdbCommand ($"shell am kill --user all {proj.PackageName}");

These commands will force the application to stop, and force any
background tasks the application has to stop as well.  This is the
best way we could find to completely quit an application.  While we
were expecting the `TimeZoneChanged` notification to fire if the app
was already running, it turns out that when setting the date time via
`adb`, the broadcast does *not* get sent.  The broadcast is only sent
when the timezone is changed via the Settings screen.  Thus exiting
the application completely and forcing a restart is the best option
for this particular test scenario.

We should investigate to see if we can script the timezone change via
the UI so we can test the broadcast, but that should be done later.

Run the TimeZoneInfo tests across *4* CI nodes, not 3 nodes, as the
previous changes increased execution time enough to cause frequent
timeouts.  Running across more CI nodes helps reduce the overall time.

Finally, bump to Android Emulator v30.3.5:

  * https://androidstudio.googleblog.com/2020/05/emulator-30013-canary.html
  * https://androidstudio.googleblog.com/2020/05/emulator-30014-canary.html
  * https://androidstudio.googleblog.com/2020/05/emulator-30015-canary.html
  * https://androidstudio.googleblog.com/2020/06/emulator-30016-canary.html
  * https://androidstudio.googleblog.com/2020/06/emulator-30017-canary.html
  * https://androidstudio.googleblog.com/2020/06/emulator-30018-canary.html
  * https://androidstudio.googleblog.com/2020/06/emulator-30019-canary.html
  * https://androidstudio.googleblog.com/2020/07/emulator-30020-canary.html
  * https://androidstudio.googleblog.com/2020/07/emulator-30021-canary.html
  * https://androidstudio.googleblog.com/2020/07/emulator-30022-canary.html
  * https://androidstudio.googleblog.com/2020/07/emulator-30023-canary.html
  * https://androidstudio.googleblog.com/2020/08/emulator-30024-canary.html
  * https://androidstudio.googleblog.com/2020/08/emulator-30025-canary.html
  * https://androidstudio.googleblog.com/2020/08/emulator-30026-canary.html
  * https://androidstudio.googleblog.com/2020/08/emulator-3010-canary.html
  * https://androidstudio.googleblog.com/2020/08/emulator-3011-canary.html
  * https://androidstudio.googleblog.com/2020/09/emulator-3013-canary.html

Configuration.props

@@ -161,8 +161,8 @@
     <CommandLineToolsFolder Condition=" '$(CommandLineToolsFolder)' == '' ">1.0</CommandLineToolsFolder>
     <CommandLineToolsVersion Condition=" '$(CommandLineToolsVersion)' == '' ">6200805_latest</CommandLineToolsVersion>
     <CommandLineToolsBinPath Condition=" '$(CommandLineToolsBinPath)' == '' ">$(AndroidSdkFullPath)\cmdline-tools\$(CommandLineToolsFolder)\bin</CommandLineToolsBinPath>
-    <EmulatorVersion Condition=" '$(EmulatorVersion)' == '' ">6466327</EmulatorVersion>
-    <EmulatorPkgRevision Condition=" '$(EmulatorPkgRevision)' == '' ">30.0.12</EmulatorPkgRevision>
+    <EmulatorVersion Condition=" '$(EmulatorVersion)' == '' ">7033400</EmulatorVersion>
+    <EmulatorPkgRevision Condition=" '$(EmulatorPkgRevision)' == '' ">30.3.5</EmulatorPkgRevision>
     <EmulatorToolPath Condition=" '$(EmulatorToolPath)' == '' ">$(AndroidSdkFullPath)\emulator</EmulatorToolPath>
     <EmulatorToolExe Condition=" '$(EmulatorToolExe)' == '' ">emulator</EmulatorToolExe>
     <NdkBuildPath Condition=" '$(NdkBuildPath)' == '' And '$(HostOS)' != 'Windows' ">$(AndroidNdkDirectory)\ndk-build</NdkBuildPath>

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/StartAndroidEmulator.cs

@@ -62,7 +62,7 @@ void Run (string emulator)
 				return;
 
 			var port = string.IsNullOrEmpty (Port) ? "" : $" -port {Port}";
-			var arguments = $"-verbose -avd {ImageName}{port} -cache-size 512";
+			var arguments = $"-verbose -no-boot-anim -no-audio -no-snapshot -cache-size 512 -timezone \"Etc/UTC\" -avd {ImageName}{port}";
 			Log.LogMessage (MessageImportance.Low, $"Tool {emulator} execution started with arguments: {arguments}");
 			var psi = new ProcessStartInfo () {
 				FileName                = emulator,

build-tools/automation/azure-pipelines.yaml

@@ -918,6 +918,10 @@ stages:
     parameters:
       node_id: 3
 
+  - template: yaml-templates\run-timezoneinfo-tests.yaml
+    parameters:
+      node_id: 4
+
 - stage: bcl_tests
   displayName: BCL Emulator Tests
   dependsOn: mac_build

build-tools/automation/yaml-templates/run-timezoneinfo-tests.yaml

@@ -8,7 +8,7 @@ jobs:
     displayName: macOS-${{ parameters.node_id }}
     pool:
       vmImage: $(HostedMacImage)
-    timeoutInMinutes: 90
+    timeoutInMinutes: 120
     cancelTimeoutInMinutes: 5
     workspace:
       clean: all

src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs

@@ -125,8 +125,8 @@ protected static bool MonitorAdbLogcat (Func<string, bool> action, string logcat
 					proc.BeginOutputReadLine ();
 					TimeSpan time = TimeSpan.FromSeconds (timeout);
 					while (!stdout_done.IsSet && !didActionSucceed && time.TotalMilliseconds > 0) {
-						proc.WaitForExit (100);
-						time -= TimeSpan.FromMilliseconds (100);
+						proc.WaitForExit (10);
+						time -= TimeSpan.FromMilliseconds (10);
 					}
 					proc.Kill ();
 					proc.WaitForExit ();

tests/MSBuildDeviceIntegration/Tests/DeploymentTest.cs

@@ -35,12 +35,18 @@ public void BeforeDeploymentTests ()
 			if (debuggable != "1") {
 				Assert.Ignore ("TimeZone tests need to use `su root` and this device does not support that feature. Try using an emulator.");
 			}
+			// Disable auto timezone
+			RunAdbCommand ("shell settings put global auto_time_zone 0");
 
 			proj = new XamarinFormsAndroidApplicationProject ();
 			proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86");
 			var mainPage = proj.Sources.First (x => x.Include () == "MainPage.xaml.cs");
 			var source = mainPage.TextContent ().Replace ("InitializeComponent ();", @"InitializeComponent ();
+			Console.WriteLine ($""TimeZoneInfoNative={Java.Util.TimeZone.Default.ID}"");
 			Console.WriteLine ($""TimeZoneInfo={TimeZoneInfo.Local.DisplayName}"");
+");
+			source = source.Replace ("Console.WriteLine (\"Button was Clicked!\");", @"Console.WriteLine (""Button was Clicked!"");
+			Console.WriteLine ($""TimeZoneInfoClick={TimeZoneInfo.Local.DisplayName}"");
 ");
 			mainPage.TextContent = () => source;
 			builder = CreateApkBuilder (Path.Combine ("temp", "DeploymentTests"));
@@ -183,7 +189,7 @@ public void CheckXamarinFormsAppDeploysAndAButtonWorks ()
 			}, Path.Combine (Root, builder.ProjectDirectory, "button-logcat.log")), "Button Should have been Clicked.");
 		}
 
-		private const int NODE_COUNT = 3;
+		private const int NODE_COUNT = 4;
 
 		static object [] GetTimeZoneTestCases (int node)
 		{
@@ -203,38 +209,72 @@ static object [] GetTimeZoneTestCases (int node)
 			return tests.Where (p => tests.IndexOf (p) % NODE_COUNT == node).ToArray ();
 		}
 
-		[Test]
+		[Test, NonParallelizable]
 		[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 0 })]
 		[Category ("TimeZoneInfo")]
+		[Retry (2)]
 		public void CheckTimeZoneInfoIsCorrectNode1 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
 
-		[Test]
+		[Test, NonParallelizable]
 		[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 1 })]
 		[Category ("TimeZoneInfo")]
+		[Retry (2)]
 		public void CheckTimeZoneInfoIsCorrectNode2 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
 
-		[Test]
+		[Test, NonParallelizable]
 		[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 2 })]
 		[Category ("TimeZoneInfo")]
+		[Retry (2)]
 		public void CheckTimeZoneInfoIsCorrectNode3 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
 
+		[Test, NonParallelizable]
+		[TestCaseSource (nameof (GetTimeZoneTestCases), new object [] { 3 })]
+		[Category ("TimeZoneInfo")]
+		[Retry (2)]
+		public void CheckTimeZoneInfoIsCorrectNode4 (string timeZone) => CheckTimeZoneInfoIsCorrect (timeZone);
+
 		public void CheckTimeZoneInfoIsCorrect (string timeZone)
 		{
 			AssertHasDevices ();
 
 			string currentTimeZone = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim ();
+			string deviceTz = string.Empty;
+			string logFile = Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log");
 			try {
-				RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\"");
+				for (int attempt = 0; attempt < 5; attempt++) {
+					RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{timeZone}\"");
+					deviceTz = RunAdbCommand ("shell getprop persist.sys.timezone")?.Trim ();
+					if (deviceTz == timeZone) {
+						break;
+					}
+				}
+				Assert.AreEqual (timeZone, deviceTz, $"The command to set the device timezone to {timeZone} failed. Current device timezone is {deviceTz}");
+				ClearAdbLogcat ();
+				RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
+				RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
+				WaitFor ((int)TimeSpan.FromSeconds (2).TotalMilliseconds);
 				ClearAdbLogcat ();
 				AdbStartActivity ($"{proj.PackageName}/{proj.JavaPackageName}.MainActivity");
-				Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity",
-					Path.Combine (Root, builder.ProjectDirectory, $"startup-logcat-{timeZone.Replace ("/", "-")}.log")), "Activity should have started");
+				Assert.IsTrue (WaitForActivityToStart (proj.PackageName, "MainActivity", logFile), "Activity should have started");
+				string line = "";
+				string logCatFile = Path.Combine (Root, builder.ProjectDirectory, $"timezone-logcat-{timeZone.Replace ("/", "-")}.log");
+				ClickButton (proj.PackageName, "myXFButton", "CLICK ME");
 				Assert.IsTrue (MonitorAdbLogcat ((l) => {
-					return l.Contains ($"TimeZoneInfo={timeZone}");
-				}, Path.Combine (Root, builder.ProjectDirectory, $"timezone-logcat-{timeZone.Replace ("/", "-")}.log")), $"TimeZone should have been {timeZone}");
+					if (l.Contains ("TimeZoneInfoClick=")) {
+						line = l;
+						return l.Contains ($"{timeZone}");
+					}
+					return false;
+				}, logCatFile, timeout:30), $"TimeZone should have been {timeZone}. We found : {line}");
 			} finally {
-				if (!string.IsNullOrEmpty (currentTimeZone))
+				RunAdbCommand ($"shell am force-stop --user all {proj.PackageName}");
+				RunAdbCommand ($"shell am kill --user all {proj.PackageName}");
+				if (!string.IsNullOrEmpty (currentTimeZone)) {
 					RunAdbCommand ($"shell su root setprop persist.sys.timezone \"{currentTimeZone}\"");
+				}
+				if (File.Exists (logFile)) {
+					TestContext.AddTestAttachment (logFile);
+				}
 			}
 		}
 	}