diff --git a/README.md b/README.md index fd5afb0..80e46a8 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Logging shows that the plugin first listed all tracks and then generated test fu #### Filter tests based on tracks -Marking classes or functions with @pytest.mark.track("track_name") like the following test class member +Marking classes, functions or modules with @pytest.mark.track("track_name") like the following test class member ``` python class TestCustomParameters: @@ -202,6 +202,11 @@ INFO pytest_rally.rally:rally.py:91 Removing Rally config from [/Users/nikos INFO pytest_rally.elasticsearch:elasticsearch.py:104 Stopping Elasticsearch: [esrally stop --installation-id=1ee3c852-b86c-4126-ad1e-ac2088e30335] ==================================================== 1 passed, 4 skipped in 40.22s ===================================================== ``` + +In the case the --track-filter contains a top-level track name e.g. elastic, or elastic/logs, all tests marked with this top-level e.g. elastic, elastic/logs, elastic/security will be executed. + +Note that in the case of multi-level marking (We mark both a test class and a test function within that class), the closest marker (e.g. marker to function) will take precedence over the farther marker (e.g. module) and will be matched against the --track-filter items. + #### Test execution Because our `test_autogenerated` function uses the [`es_cluster` fixture](#es_cluster), `pytest-rally` will install and start an Elasticsearch cluster during setup and stop it during teardown. All of our autogenerated tests will run their races with this cluster as their benchmark candidate. diff --git a/pytest_rally/plugin.py b/pytest_rally/plugin.py index 5290661..91172cc 100644 --- a/pytest_rally/plugin.py +++ b/pytest_rally/plugin.py @@ -119,7 +119,7 @@ def default_params(track, challenge): @pytest.hookimpl def pytest_generate_tests(metafunc): - tfilter = metafunc.config.getoption('track_filter') + track_filter = metafunc.config.getoption('track_filter') current_class = metafunc.cls desired_class = metafunc.config.option.test_class @@ -129,7 +129,7 @@ def pytest_generate_tests(metafunc): params = [] tracks_and_challenges = r.all_tracks_and_challenges() for track, challenges in tracks_and_challenges: - if not tfilter or track in tfilter: + if not track_filter or track in track_filter or track.split("/")[0] in track_filter: params += [(default_params(track, challenge)) for challenge in challenges] metafunc.parametrize("track,challenge,rally_options", params) metafunc.definition.parent.add_marker("autogenerated") @@ -150,5 +150,5 @@ def pytest_collection_modifyitems(session,config,items): marker_tracks = track_marker.args[0] if isinstance(marker_tracks, str): marker_tracks = marker_tracks.split(",") - if not any(track in track_filter for track in marker_tracks): + if not any(track in track_filter or track.split("/")[0] in track_filter for track in marker_tracks): item.add_marker(pytest.mark.skip(reason=f"Skipping test for tracks {marker_tracks} not in track_filter {track_filter}")) diff --git a/tests/examples/marked_tracks.py b/tests/examples/marked_tracks.py index 50f8643..101921a 100644 --- a/tests/examples/marked_tracks.py +++ b/tests/examples/marked_tracks.py @@ -17,30 +17,48 @@ import pytest -# Marks at module level for tests with track filters that doesn't include the listed track names -pytestmark = pytest.mark.track("test-track", "test-track2", "test-track3") +# Marks at module level +pytestmark = pytest.mark.track("test-track") -class TestMarkedFunctions: - @pytest.mark.track("test-track","test-track2") +class TestMarkedModule: def test_mark_track(self, es_cluster, rally): rally.race(track="test-track",challenge="index-only") - rally.race(track="test-track2",challenge="force-merge") - + +class TestMarkedFunctions: @pytest.mark.track("test-track2") def test_mark_track2(self, es_cluster, rally): rally.race(track="test-track2",challenge="cluster-health") - @pytest.mark.track("test-track3") - def test_mark_track3(self, es_cluster, rally): - rally.race(track="test-track3",challenge="index-only") - @pytest.mark.track("test-track") class TestMarkedClass: - @pytest.mark.track("test-track","test-track2") def test_mark_track(self, es_cluster, rally): - rally.race(track="test-track",challenge="index-only") - rally.race(track="test-track2",challenge="force-merge") + rally.race(track="test-track",challenge="force-merge") - @pytest.mark.track("test-track3") + @pytest.mark.track("test-track2") def test_mark_track3(self, es_cluster, rally): - rally.race(track="test-track3",challenge="index-only") + rally.race(track="test-track2",challenge="index-only") + +class TestMarkedTrackList: + @pytest.mark.track(["many-tracks/sub-track", "many-tracks/sub-track2"]) + def test_mark_track_list(self, es_cluster, rally): + rally.race(track="many-tracks/sub-track",challenge="index-only") + rally.race(track="many-tracks/sub-track2",challenge="index-only") + + @pytest.mark.track("many-tracks/sub-track,many-tracks/sub-track2") + def test_mark_track_list_comma_separated(self, es_cluster, rally): + rally.race(track="many-tracks/sub-track",challenge="index-only") + rally.race(track="many-tracks/sub-track2",challenge="index-only") + +class TestMarkedSubTrack: + @pytest.mark.track("many-tracks/sub-track") + def test_mark_sub_track(self, es_cluster, rally): + rally.race(track="many-tracks/sub-track",challenge="index-only") + + @pytest.mark.track("many-tracks/sub-track2") + def test_mark_sub_track2(self, es_cluster, rally): + rally.race(track="many-tracks/sub-track2",challenge="index-only") + + @pytest.mark.track("many-tracks/sub-track3") + def test_mark_sub_track3(self, es_cluster, rally): + rally.race(track="many-tracks/sub-track3",challenge="index-only") + diff --git a/tests/resources/track-repo/test-track3/index.json b/tests/resources/track-repo/many-tracks/sub-track/index.json similarity index 94% rename from tests/resources/track-repo/test-track3/index.json rename to tests/resources/track-repo/many-tracks/sub-track/index.json index c36b03e..a008839 100644 --- a/tests/resources/track-repo/test-track3/index.json +++ b/tests/resources/track-repo/many-tracks/sub-track/index.json @@ -21,7 +21,7 @@ "type": "text" }, "population": { - "type": "long" + "type": "int" } } } diff --git a/tests/resources/track-repo/many-tracks/sub-track/track.json b/tests/resources/track-repo/many-tracks/sub-track/track.json new file mode 100644 index 0000000..6950ab7 --- /dev/null +++ b/tests/resources/track-repo/many-tracks/sub-track/track.json @@ -0,0 +1,104 @@ +{ + "version": 2, + "description": "Track 1 for testing pytest-rally", + "indices": [ + { + "name": "test", + "body": "index.json" + } + ], + "corpora": [ + { + "name": "sub-track", + "documents": [ + { + "source-file": "documents.json", + "document-count": 100, + "uncompressed-bytes": 100 + } + ] + } + ], + "challenges": [ + { + "name": "index-and-query", + "default": true, + "schedule": [ + { + "operation": { + "operation-type": "delete-index" + } + }, + { + "operation": { + "operation-type": "create-index" + } + }, + { + "operation": { + "operation-type": "cluster-health", + "request-params": { + "wait_for_status": "green" + }, + "retry-until-success": true + } + }, + { + "operation": { + "operation-type": "bulk", + "bulk-size": 5000 + }, + "warmup-time-period": 120, + "clients": 8 + }, + { + "operation": { + "name": "query-match-all", + "operation-type": "search", + "body": { + "query": { + "match_all": {} + } + } + }, + "clients": 8, + "warmup-iterations": 1000, + "iterations": 1000, + "target-throughput": 100 + } + ] + }, + { + "name": "index-only", + "schedule": [ + { + "operation": { + "operation-type": "delete-index" + } + }, + { + "operation": { + "operation-type": "create-index" + } + }, + { + "operation": { + "operation-type": "cluster-health", + "request-params": { + "wait_for_status": "green" + }, + "retry-until-success": true + } + }, + { + "operation": { + "operation-type": "bulk", + "bulk-size": 5000 + }, + "warmup-time-period": 120, + "clients": 8 + } + ] + } + ] +} diff --git a/tests/resources/track-repo/many-tracks/sub-track2/index.json b/tests/resources/track-repo/many-tracks/sub-track2/index.json new file mode 100644 index 0000000..35c1cf3 --- /dev/null +++ b/tests/resources/track-repo/many-tracks/sub-track2/index.json @@ -0,0 +1,28 @@ +{ + "settings": { + "index.number_of_replicas": 0 + }, + "mappings": { + "dynamic": "strict", + "properties": { + "geonameid": { + "type": "long" + }, + "name": { + "type": "text" + }, + "latitude": { + "type": "double" + }, + "longitude": { + "type": "double" + }, + "country_code": { + "type": "double" + }, + "population": { + "type": "long" + } + } + } +} diff --git a/tests/resources/track-repo/many-tracks/sub-track2/track.json b/tests/resources/track-repo/many-tracks/sub-track2/track.json new file mode 100644 index 0000000..5e77e15 --- /dev/null +++ b/tests/resources/track-repo/many-tracks/sub-track2/track.json @@ -0,0 +1,109 @@ +{ + "version": 2, + "description": "Track 2 for testing pytest-rally", + "indices": [ + { + "name": "test", + "body": "index.json" + } + ], + "corpora": [ + { + "name": "sub-track2", + "documents": [ + { + "source-file": "documents.json", + "document-count": 100, + "uncompressed-bytes": 100 + } + ] + } + ], + "challenges": [ + { + "name": "index-and-query", + "default": true, + "schedule": [ + { + "operation": { + "operation-type": "delete-index" + } + }, + { + "operation": { + "operation-type": "create-index" + } + }, + { + "operation": { + "operation-type": "cluster-health", + "request-params": { + "wait_for_status": "green" + }, + "retry-until-success": true + } + }, + { + "operation": { + "operation-type": "bulk", + "bulk-size": 5000 + }, + "warmup-time-period": 120, + "clients": 8 + }, + { + "operation": { + "operation-type": "force-merge" + } + }, + { + "operation": { + "name": "query-match-all", + "operation-type": "search", + "body": { + "query": { + "match_all": {} + } + } + }, + "clients": 8, + "warmup-iterations": 1000, + "iterations": 1000, + "target-throughput": 100 + } + ] + }, + { + "name": "index-only", + "schedule": [ + { + "operation": { + "operation-type": "delete-index" + } + }, + { + "operation": { + "operation-type": "create-index" + } + }, + { + "operation": { + "operation-type": "cluster-health", + "request-params": { + "wait_for_status": "green" + }, + "retry-until-success": true + } + }, + { + "operation": { + "operation-type": "bulk", + "bulk-size": 5000 + }, + "warmup-time-period": 120, + "clients": 8 + } + ] + } + ] +} diff --git a/tests/resources/track-repo/many-tracks/sub-track3/index.json b/tests/resources/track-repo/many-tracks/sub-track3/index.json new file mode 100644 index 0000000..35c1cf3 --- /dev/null +++ b/tests/resources/track-repo/many-tracks/sub-track3/index.json @@ -0,0 +1,28 @@ +{ + "settings": { + "index.number_of_replicas": 0 + }, + "mappings": { + "dynamic": "strict", + "properties": { + "geonameid": { + "type": "long" + }, + "name": { + "type": "text" + }, + "latitude": { + "type": "double" + }, + "longitude": { + "type": "double" + }, + "country_code": { + "type": "double" + }, + "population": { + "type": "long" + } + } + } +} diff --git a/tests/resources/track-repo/test-track3/track.json b/tests/resources/track-repo/many-tracks/sub-track3/track.json similarity index 98% rename from tests/resources/track-repo/test-track3/track.json rename to tests/resources/track-repo/many-tracks/sub-track3/track.json index 5beb2f7..e08de5a 100644 --- a/tests/resources/track-repo/test-track3/track.json +++ b/tests/resources/track-repo/many-tracks/sub-track3/track.json @@ -9,7 +9,7 @@ ], "corpora": [ { - "name": "test-track3", + "name": "sub-track3", "documents": [ { "source-file": "documents.json", diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 605baaf..2019bf6 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -25,7 +25,7 @@ class TestPlugin: # this should be sorted as per Rally list tracks output - tracks = ["test-track", "test-track2", "test-track3"] + tracks = [ "many-tracks/sub-track", "many-tracks/sub-track2", "many-tracks/sub-track3", "test-track", "test-track2"] challenges = ["index-and-query", "index-only"] def test_generates_tests_from_list_tracks(self, pytester, example, temp_repo): @@ -69,21 +69,32 @@ def test_runs_correct_install_command(self, caplog, temp_repo, distribution_vers assert actual == expected def test_track_filter_limits_autogenerated_tracks(self, pytester, example, temp_repo): - def expected_test_names(track_filter): - filter_items = None if track_filter == "" else [t.strip() for t in track_filter.split(",")] - if filter_items and all(f not in self.tracks for f in filter_items): - result = [f"test_track_challenge[{DEFAULT_TRACK_AND_CHALLENGE}]"] - else: - result = [ - f"test_track_challenge[{track}-{challenge}]" - for track in self.tracks if not filter_items or track in filter_items - for challenge in self.challenges - ] - return result - - test_track_filters = ["","test-track2", "test-track,test-track2", "test-track500"] - for track_filter in test_track_filters: - expected = expected_test_names(track_filter) + all_expected = [ + f"test_track_challenge[{track}-{challenge}]" + for track in self.tracks + for challenge in self.challenges + ] + expected_map = { + "": all_expected, + "test-track2": [ + f"test_track_challenge[test-track2-{challenge}]" for challenge in self.challenges + ], + "test-track,test-track2": [ + *[f"test_track_challenge[test-track-{challenge}]" for challenge in self.challenges], + *[f"test_track_challenge[test-track2-{challenge}]" for challenge in self.challenges], + ], + "test-track500": [f"test_track_challenge[{DEFAULT_TRACK_AND_CHALLENGE}]"] , + "many-tracks": [ + *[f"test_track_challenge[many-tracks/sub-track-{challenge}]" for challenge in self.challenges], + *[f"test_track_challenge[many-tracks/sub-track2-{challenge}]" for challenge in self.challenges], + *[f"test_track_challenge[many-tracks/sub-track3-{challenge}]" for challenge in self.challenges], + ], + "many-tracks/sub-track": [ + f"test_track_challenge[many-tracks/sub-track-{challenge}]" for challenge in self.challenges + ], + } + + for track_filter, expected in expected_map.items(): generated, _ = pytester.inline_genitems( example["all_tracks_and_challenges"], f"--track-repository={temp_repo}", @@ -92,23 +103,27 @@ def expected_test_names(track_filter): assert [func.name for func in generated] == expected def test_track_filter_skips_tracks(self, caplog, temp_repo, example, run): - def expected_tracks_filtered(track_filter): - filter_items = None if track_filter == "" else [t.strip() for t in track_filter.split(",")] - return set([track for track in self.tracks if not filter_items or track in filter_items]) - - test_track_filters = ["", "test-track2", "test-track2,test-track", "test-track500"] - for track_filter in test_track_filters: + expected_map = { + "": { *self.tracks }, + "test-track2": { "test-track2" }, + "test-track2,test-track": { "test-track", "test-track2" }, + "test-track500": set(), + "many-tracks/sub-track": { "many-tracks/sub-track", "many-tracks/sub-track2" }, + "many-tracks/sub-track3": { "many-tracks/sub-track3" }, + "many-tracks": { "many-tracks/sub-track", "many-tracks/sub-track2", "many-tracks/sub-track3" }, + } + + for track_filter, expected_tracks in expected_map.items(): caplog.clear() - run_function = run(example["marked_tracks"], f"--track-filter={track_filter}") + run(example["marked_tracks"], f"--track-filter={track_filter}") races = [r for r in caplog.records if "esrally race" in r.message] - raced_tracks = [] + raced_tracks = set() for r in races: - track_match = re.search(r'--track="([^"]+)"', r.message) + track_match = re.search(r'--track="([^\"]+)"', r.message) if track_match: - raced_tracks.append(track_match.group(1)) - expected_tracks = expected_tracks_filtered(track_filter) - actual_tracks = set(raced_tracks) - assert actual_tracks == expected_tracks + raced_tracks.add(track_match.group(1)) + print(f"track_filter='{track_filter}' => raced_tracks={raced_tracks}, expected_tracks={expected_tracks}") + assert raced_tracks == expected_tracks def test_skip_autogenerated_tests_option(self, pytester, example, temp_repo): expected_all = [