diff --git a/tests/e2e/singlecard/models/conftest.py b/tests/e2e/singlecard/models/conftest.py index 2b25c1a9294..6c98f7d8465 100644 --- a/tests/e2e/singlecard/models/conftest.py +++ b/tests/e2e/singlecard/models/conftest.py @@ -30,6 +30,12 @@ def pytest_addoption(parser): default="./benchmarks/accuracy/Qwen3-8B-Base.md", help="Path to the report output file", ) + parser.addoption( + "--report-dir", + action="store", + default="./benchmarks/accuracy", + help="Directory to store report files when using config list", + ) @pytest.fixture(scope="session") @@ -48,26 +54,35 @@ def config(pytestconfig): return pytestconfig.getoption("--config") -@pytest.fixture(scope="session") -def report_output(pytestconfig): - return pytestconfig.getoption("--report_output") +@pytest.fixture(scope="function") +def report_output(pytestconfig, config_filename): + if pytestconfig.getoption("--config-list-file"): + report_dir = pytestconfig.getoption("--report-dir") + model_name = Path(config_filename).stem + report_path = Path(report_dir) / f"{model_name}.md" + report_path.parent.mkdir(parents=True, exist_ok=True) + return report_path + + output_path = pytestconfig.getoption("--report_output") + if output_path: + output_path = Path(output_path) + return output_path def pytest_generate_tests(metafunc): if "config_filename" in metafunc.fixturenames: - # If config specified, use the --config directly - single_config = metafunc.config.getoption("--config") - if single_config: - metafunc.parametrize("config_filename", - [Path(single_config).resolve()]) + if metafunc.config.getoption("--config-list-file"): + rel_path = metafunc.config.getoption("--config-list-file") + config_list_file = Path(rel_path).resolve() + config_dir = config_list_file.parent + with open(config_list_file, encoding="utf-8") as f: + configs = [ + config_dir / line.strip() for line in f + if line.strip() and not line.startswith("#") + ] + metafunc.parametrize("config_filename", configs) return - # Otherwise, check --config-list-file - rel_path = metafunc.config.getoption("--config-list-file") - config_list_file = Path(rel_path).resolve() - config_dir = config_list_file.parent - with open(config_list_file, encoding="utf-8") as f: - configs = [ - config_dir / line.strip() for line in f - if line.strip() and not line.startswith("#") - ] - metafunc.parametrize("config_filename", configs) + single_config = metafunc.config.getoption("--config") + config_path = Path(single_config).resolve() + + metafunc.parametrize("config_filename", [config_path])