Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added pytest.mark.xfail on test_distrib_gloo_cpu_or_gpu() #2454

Merged
merged 6 commits into from
Feb 8, 2022
Merged

Added pytest.mark.xfail on test_distrib_gloo_cpu_or_gpu() #2454

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4d0165e
pytest.approx corrected
nishantb06 Feb 7, 2022
72d623b
mark.xfail added on test_distrib_gloo_cpu_or_gpu
nishantb06 Feb 7, 2022
3a5baba
autopep8 fix
nishantb06 Feb 7, 2022
cd90982
added xfail to 3 tests
nishantb06 Feb 7, 2022
e76238a
autopep8 fix
nishantb06 Feb 7, 2022
ec3c921
Update test_classification_report.py
vfdev-5 Feb 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 18 additions & 12 deletions tests/ignite/metrics/test_classification_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ def update(engine, i):

for i in range(n_classes):
label_i = labels[i] if labels else str(i)
assert pytest.approx(res[label_i]["precision"] == sklearn_result[str(i)]["precision"])
assert pytest.approx(res[label_i]["f1-score"] == sklearn_result[str(i)]["f1-score"])
assert pytest.approx(res[label_i]["recall"] == sklearn_result[str(i)]["recall"])
assert pytest.approx(res["macro avg"]["precision"] == sklearn_result["macro avg"]["precision"])
assert pytest.approx(res["macro avg"]["recall"] == sklearn_result["macro avg"]["recall"])
assert pytest.approx(res["macro avg"]["f1-score"] == sklearn_result["macro avg"]["f1-score"])
assert sklearn_result[str(i)]["precision"] == pytest.approx(res[label_i]["precision"])
assert sklearn_result[str(i)]["f1-score"] == pytest.approx(res[label_i]["f1-score"])
assert sklearn_result[str(i)]["recall"] == pytest.approx(res[label_i]["recall"])
assert sklearn_result["macro avg"]["precision"] == pytest.approx(res["macro avg"]["precision"])
assert sklearn_result["macro avg"]["recall"] == pytest.approx(res["macro avg"]["recall"])
assert sklearn_result["macro avg"]["f1-score"] == pytest.approx(res["macro avg"]["f1-score"])

for _ in range(5):
# check multiple random inputs as random exact occurencies are rare
Expand Down Expand Up @@ -122,12 +122,12 @@ def update(engine, i):

for i in range(n_classes):
label_i = labels[i] if labels else str(i)
assert pytest.approx(res[label_i]["precision"] == sklearn_result[str(i)]["precision"])
assert pytest.approx(res[label_i]["f1-score"] == sklearn_result[str(i)]["f1-score"])
assert pytest.approx(res[label_i]["recall"] == sklearn_result[str(i)]["recall"])
assert pytest.approx(res["macro avg"]["precision"] == sklearn_result["macro avg"]["precision"])
assert pytest.approx(res["macro avg"]["recall"] == sklearn_result["macro avg"]["recall"])
assert pytest.approx(res["macro avg"]["f1-score"] == sklearn_result["macro avg"]["f1-score"])
assert sklearn_result[str(i)]["precision"] == pytest.approx(res[label_i]["precision"])
assert sklearn_result[str(i)]["f1-score"] == pytest.approx(res[label_i]["f1-score"])
assert sklearn_result[str(i)]["recall"] == pytest.approx(res[label_i]["recall"])
assert sklearn_result["macro avg"]["precision"] == pytest.approx(res["macro avg"]["precision"])
assert sklearn_result["macro avg"]["recall"] == pytest.approx(res["macro avg"]["recall"])
assert sklearn_result["macro avg"]["f1-score"] == pytest.approx(res["macro avg"]["f1-score"])

for _ in range(3):
# check multiple random inputs as random exact occurencies are rare
Expand All @@ -141,6 +141,7 @@ def update(engine, i):
_test(metric_device, 2, ["0", "1", "2", "3", "4", "5", "6"])


@pytest.mark.xfail
@pytest.mark.distributed
@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support")
@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU")
Expand All @@ -153,6 +154,7 @@ def test_distrib_nccl_gpu(distributed_context_single_node_nccl):
_test_integration_multilabel(device, False)


@pytest.mark.xfail
@pytest.mark.distributed
@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support")
def test_distrib_gloo_cpu_or_gpu(local_rank, distributed_context_single_node_gloo):
Expand All @@ -164,6 +166,7 @@ def test_distrib_gloo_cpu_or_gpu(local_rank, distributed_context_single_node_glo
_test_integration_multilabel(device, False)


@pytest.mark.xfail
@pytest.mark.distributed
@pytest.mark.skipif(not idist.has_hvd_support, reason="Skip if no Horovod dist support")
@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc")
Expand All @@ -187,6 +190,7 @@ def _test_distrib_xla_nprocs(index):
_test_integration_multilabel(device, False)


@pytest.mark.xfail
@pytest.mark.tpu
@pytest.mark.skipif("NUM_TPU_WORKERS" not in os.environ, reason="Skip if no NUM_TPU_WORKERS in env vars")
@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package")
Expand All @@ -203,6 +207,7 @@ def to_numpy_multilabel(y):
return y


@pytest.mark.xfail
@pytest.mark.multinode_distributed
@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support")
@pytest.mark.skipif("MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed")
Expand All @@ -215,6 +220,7 @@ def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo):
_test_integration_multilabel(device, False)


@pytest.mark.xfail
@pytest.mark.multinode_distributed
@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support")
@pytest.mark.skipif("GPU_MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed")
Expand Down