[Pytest] Sort unit tests before running. (apache#9188)

* [Pytest] Sort unit tests before running.

By default, pytest will sort tests to maximize the re-use of fixtures.
However, this assumes that all fixtures have an equal cost to
generate, and no caches outside of those managed by pytest.  A fixture
for a `tvm.testing.parameter` is effectively free, while a fixture
maintaining a cache of reference data
`tvm.testing.utils._fixture_cache` be quite large.

Since most of the TVM fixtures are specific to a python function, sort
the test ordering by python function, so that
tvm.testing.utils._fixture_cache can be cleared sooner rather than
later.

* Updated TestTargetAutoParametrization

When sorting the tests, the order of parametrizations may change.
Therefore, the tests checking for automatic target parametrization
shouldn't depend on order.

python/tvm/testing/plugin.py

@@ -74,6 +74,7 @@ def pytest_collection_modifyitems(config, items):
     # pylint: disable=unused-argument
     _count_num_fixture_uses(items)
     _remove_global_fixture_definitions(items)
+    _sort_tests(items)
 
 
 @pytest.fixture
@@ -236,6 +237,25 @@ def _remove_global_fixture_definitions(items):
                 delattr(module, name)
 
 
+def _sort_tests(items):
+    """Sort tests by file/function.
+
+    By default, pytest will sort tests to maximize the re-use of
+    fixtures.  However, this assumes that all fixtures have an equal
+    cost to generate, and no caches outside of those managed by
+    pytest.  A tvm.testing.parameter is effectively free, while
+    reference data for testing may be quite large.  Since most of the
+    TVM fixtures are specific to a python function, sort the test
+    ordering by python function, so that
+    tvm.testing.utils._fixture_cache can be cleared sooner rather than
+    later.
+
+    Should be called from pytest_collection_modifyitems.
+
+    """
+    items.sort(key=lambda item: item.location)
+
+
 def _target_to_requirement(target):
     if isinstance(target, str):
         target = tvm.target.Target(target)

tests/python/unittest/test_tvm_testing_features.py

@@ -46,8 +46,11 @@ def test_device_parametrization(self, dev):
         self.devices_used.append(dev)
 
     def test_all_targets_used(self):
-        assert self.targets_used == self.enabled_targets
-        assert self.devices_used == self.enabled_devices
+        assert sorted(self.targets_used) == sorted(self.enabled_targets)
+
+    def test_all_devices_used(self):
+        sort_key = lambda dev: (dev.device_type, dev.device_id)
+        assert sorted(self.devices_used, key=sort_key) == sorted(self.enabled_devices, key=sort_key)
 
     targets_with_explicit_list = []
 
@@ -70,9 +73,9 @@ def test_exclude_target(self, target):
         self.targets_with_exclusion.append(target)
 
     def test_all_nonexcluded_targets_ran(self):
-        assert self.targets_with_exclusion == [
-            target for target in self.enabled_targets if not target.startswith("llvm")
-        ]
+        assert sorted(self.targets_with_exclusion) == sorted(
+            [target for target in self.enabled_targets if not target.startswith("llvm")]
+        )
 
     run_targets_with_known_failure = []
 
@@ -85,7 +88,7 @@ def test_known_failing_target(self, target):
         assert "llvm" not in target
 
     def test_all_targets_ran(self):
-        assert self.run_targets_with_known_failure == self.enabled_targets
+        assert sorted(self.run_targets_with_known_failure) == sorted(self.enabled_targets)
 
     @tvm.testing.known_failing_targets("llvm")
     @tvm.testing.parametrize_targets("llvm")