Add top level track filter

README.md

@@ -155,7 +155,7 @@ Logging shows that the plugin first listed all tracks and then generated test fu
 
 #### Filter tests based on tracks
 
-Marking classes or functions with @pytest.mark.track("track_name") like the following test class member 
+Marking classes, functions or modules with @pytest.mark.track("track_name") like the following test class member 
 
 ``` python
 class TestCustomParameters:
@@ -202,6 +202,11 @@ INFO     pytest_rally.rally:rally.py:91 Removing Rally config from [/Users/nikos
 INFO     pytest_rally.elasticsearch:elasticsearch.py:104 Stopping Elasticsearch: [esrally stop --installation-id=1ee3c852-b86c-4126-ad1e-ac2088e30335]
 ==================================================== 1 passed, 4 skipped in 40.22s =====================================================
 ```
+
+In the case the --track-filter contains a top-level track name e.g. elastic, or elastic/logs, all tests marked with this top-level e.g. elastic, elastic/logs, elastic/security will be executed.
+
+Note that in the case of multi-level marking (We mark both a test class and a test function within that class), the closest marker (e.g. marker to function) will take precedence over the farther marker (e.g. module) and will be matched against the --track-filter items.
+
 #### Test execution
 
 Because our `test_autogenerated` function uses the [`es_cluster` fixture](#es_cluster), `pytest-rally` will install and start an Elasticsearch cluster during setup and stop it during teardown. All of our autogenerated tests will run their races with this cluster as their benchmark candidate.

pytest_rally/plugin.py

@@ -119,7 +119,7 @@ def default_params(track, challenge):
 
 @pytest.hookimpl
 def pytest_generate_tests(metafunc):
-    tfilter = metafunc.config.getoption('track_filter')
+    track_filter = metafunc.config.getoption('track_filter')
     current_class = metafunc.cls
     desired_class = metafunc.config.option.test_class
 
@@ -129,7 +129,7 @@ def pytest_generate_tests(metafunc):
             params = []
             tracks_and_challenges = r.all_tracks_and_challenges()
             for track, challenges in tracks_and_challenges:
-                if not tfilter or track in tfilter:
+                if not track_filter or track in track_filter or track.split("/")[0] in track_filter:
                     params += [(default_params(track, challenge)) for challenge in challenges]
             metafunc.parametrize("track,challenge,rally_options", params)
             metafunc.definition.parent.add_marker("autogenerated")
@@ -150,5 +150,5 @@ def pytest_collection_modifyitems(session,config,items):
                 marker_tracks = track_marker.args[0]
                 if isinstance(marker_tracks, str):
                     marker_tracks = marker_tracks.split(",")
-                if not any(track in track_filter for track in marker_tracks):
+                if not any(track in track_filter or track.split("/")[0] in track_filter for track in marker_tracks):
                     item.add_marker(pytest.mark.skip(reason=f"Skipping test for tracks {marker_tracks} not in track_filter {track_filter}"))

tests/examples/marked_tracks.py

@@ -17,30 +17,48 @@
 
 import pytest
 
-# Marks at module level for tests with track filters that doesn't include the listed track names 
-pytestmark = pytest.mark.track("test-track", "test-track2", "test-track3")
+# Marks at module level 
+pytestmark = pytest.mark.track("test-track")
 
-class TestMarkedFunctions:
-    @pytest.mark.track("test-track","test-track2")
+class TestMarkedModule:
     def test_mark_track(self, es_cluster, rally):
         rally.race(track="test-track",challenge="index-only")
-        rally.race(track="test-track2",challenge="force-merge")
-
+
+class TestMarkedFunctions:
     @pytest.mark.track("test-track2")
     def test_mark_track2(self, es_cluster, rally):
         rally.race(track="test-track2",challenge="cluster-health")
 
-    @pytest.mark.track("test-track3")
-    def test_mark_track3(self, es_cluster, rally):
-        rally.race(track="test-track3",challenge="index-only")
-
 @pytest.mark.track("test-track")
 class TestMarkedClass:
-    @pytest.mark.track("test-track","test-track2")
     def test_mark_track(self, es_cluster, rally):
-        rally.race(track="test-track",challenge="index-only")
-        rally.race(track="test-track2",challenge="force-merge")
+        rally.race(track="test-track",challenge="force-merge")
 
-    @pytest.mark.track("test-track3")
+    @pytest.mark.track("test-track2")
     def test_mark_track3(self, es_cluster, rally):
-        rally.race(track="test-track3",challenge="index-only")
+        rally.race(track="test-track2",challenge="index-only")
+
+class TestMarkedTrackList:
+    @pytest.mark.track(["many-tracks/sub-track", "many-tracks/sub-track2"])
+    def test_mark_track_list(self, es_cluster, rally):
+        rally.race(track="many-tracks/sub-track",challenge="index-only")
+        rally.race(track="many-tracks/sub-track2",challenge="index-only")
+
+    @pytest.mark.track("many-tracks/sub-track,many-tracks/sub-track2")
+    def test_mark_track_list_comma_separated(self, es_cluster, rally):
+        rally.race(track="many-tracks/sub-track",challenge="index-only")
+        rally.race(track="many-tracks/sub-track2",challenge="index-only")
+
+class TestMarkedSubTrack:
+    @pytest.mark.track("many-tracks/sub-track")
+    def test_mark_sub_track(self, es_cluster, rally):
+        rally.race(track="many-tracks/sub-track",challenge="index-only")
+
+    @pytest.mark.track("many-tracks/sub-track2")
+    def test_mark_sub_track2(self, es_cluster, rally):
+        rally.race(track="many-tracks/sub-track2",challenge="index-only")
+
+    @pytest.mark.track("many-tracks/sub-track3")
+    def test_mark_sub_track3(self, es_cluster, rally):
+        rally.race(track="many-tracks/sub-track3",challenge="index-only")
+

tests/resources/track-repo/test-track3/index.json → tests/resources/track-repo/many-tracks/sub-track/index.json

@@ -21,7 +21,7 @@
         "type": "text"
       },
       "population": {
-        "type": "long"
+        "type": "int"
       }
     }
   }

tests/resources/track-repo/many-tracks/sub-track/track.json

@@ -0,0 +1,104 @@
+{
+  "version": 2,
+  "description": "Track 1 for testing pytest-rally",
+  "indices": [
+    {
+      "name": "test",
+      "body": "index.json"
+    }
+  ],
+  "corpora": [
+    {
+      "name": "sub-track",
+      "documents": [
+        {
+          "source-file": "documents.json",
+          "document-count": 100,
+          "uncompressed-bytes": 100
+        }
+      ]
+    }
+  ],
+  "challenges": [
+    {
+      "name": "index-and-query",
+      "default": true,
+      "schedule": [
+        {
+          "operation": {
+            "operation-type": "delete-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "create-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "cluster-health",
+            "request-params": {
+              "wait_for_status": "green"
+            },
+            "retry-until-success": true
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "bulk",
+            "bulk-size": 5000
+          },
+          "warmup-time-period": 120,
+          "clients": 8
+        },
+        {
+          "operation": {
+            "name": "query-match-all",
+            "operation-type": "search",
+            "body": {
+              "query": {
+                "match_all": {}
+              }
+            }
+          },
+          "clients": 8,
+          "warmup-iterations": 1000,
+          "iterations": 1000,
+          "target-throughput": 100
+        }
+      ]
+    },
+    {
+      "name": "index-only",
+      "schedule": [
+        {
+          "operation": {
+            "operation-type": "delete-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "create-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "cluster-health",
+            "request-params": {
+              "wait_for_status": "green"
+            },
+            "retry-until-success": true
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "bulk",
+            "bulk-size": 5000
+          },
+          "warmup-time-period": 120,
+          "clients": 8
+        }
+      ]
+    }
+  ]
+}

tests/resources/track-repo/many-tracks/sub-track2/index.json

@@ -0,0 +1,28 @@
+{
+  "settings": {
+    "index.number_of_replicas": 0
+  },
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "geonameid": {
+        "type": "long"
+      },
+      "name": {
+        "type": "text"
+      },
+      "latitude": {
+        "type": "double"
+      },
+      "longitude": {
+        "type": "double"
+      },
+      "country_code": {
+        "type": "double"
+      },
+      "population": {
+        "type": "long"
+      }
+    }
+  }
+}

tests/resources/track-repo/many-tracks/sub-track2/track.json

@@ -0,0 +1,109 @@
+{
+  "version": 2,
+  "description": "Track 2 for testing pytest-rally",
+  "indices": [
+    {
+      "name": "test",
+      "body": "index.json"
+    }
+  ],
+  "corpora": [
+    {
+      "name": "sub-track2",
+      "documents": [
+        {
+          "source-file": "documents.json",
+          "document-count": 100,
+          "uncompressed-bytes": 100
+        }
+      ]
+    }
+  ],
+  "challenges": [
+    {
+      "name": "index-and-query",
+      "default": true,
+      "schedule": [
+        {
+          "operation": {
+            "operation-type": "delete-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "create-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "cluster-health",
+            "request-params": {
+              "wait_for_status": "green"
+            },
+            "retry-until-success": true
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "bulk",
+            "bulk-size": 5000
+          },
+          "warmup-time-period": 120,
+          "clients": 8
+        },
+        {
+          "operation": {
+            "operation-type": "force-merge"
+          }
+        },
+        {
+          "operation": {
+            "name": "query-match-all",
+            "operation-type": "search",
+            "body": {
+              "query": {
+                "match_all": {}
+              }
+            }
+          },
+          "clients": 8,
+          "warmup-iterations": 1000,
+          "iterations": 1000,
+          "target-throughput": 100
+        }
+      ]
+    },
+    {
+      "name": "index-only",
+      "schedule": [
+        {
+          "operation": {
+            "operation-type": "delete-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "create-index"
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "cluster-health",
+            "request-params": {
+              "wait_for_status": "green"
+            },
+            "retry-until-success": true
+          }
+        },
+        {
+          "operation": {
+            "operation-type": "bulk",
+            "bulk-size": 5000
+          },
+          "warmup-time-period": 120,
+          "clients": 8
+        }
+      ]
+    }
+  ]
+}

tests/resources/track-repo/many-tracks/sub-track3/index.json

@@ -0,0 +1,28 @@
+{
+  "settings": {
+    "index.number_of_replicas": 0
+  },
+  "mappings": {
+    "dynamic": "strict",
+    "properties": {
+      "geonameid": {
+        "type": "long"
+      },
+      "name": {
+        "type": "text"
+      },
+      "latitude": {
+        "type": "double"
+      },
+      "longitude": {
+        "type": "double"
+      },
+      "country_code": {
+        "type": "double"
+      },
+      "population": {
+        "type": "long"
+      }
+    }
+  }
+}