From 478b613ed8a95cb97541a3ac61c10f9ae9b4afae Mon Sep 17 00:00:00 2001 From: puhuk Date: Wed, 24 Aug 2022 17:50:31 +0900 Subject: [PATCH 1/6] Update rmse, ssim, top_k_categorical_accuracy in test for generating data with different rank Update `rmse`, `ssim`, `top_k_categorical_accuracy` --- .../metrics/test_root_mean_squared_error.py | 28 +++++++++---------- tests/ignite/metrics/test_ssim.py | 13 +++++---- .../test_top_k_categorical_accuracy.py | 15 ++++++---- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index de8fd558548..ed6cfa5bc5b 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -58,17 +58,19 @@ def _test_distrib_integration(device, tol=1e-6): from ignite.engine import Engine rank = idist.get_rank() - n_iters = 100 - s = 10 - offset = n_iters * s - y_true = torch.arange(0, offset * idist.get_world_size(), dtype=torch.float).to(device) - y_preds = (rank + 1) * torch.ones(offset, dtype=torch.float).to(device) + def _test(metric_device): + n_iters = 2 + batch_size = 3 - def update(engine, i): - return y_preds[i * s : (i + 1) * s], y_true[i * s + offset * rank : (i + 1) * s + offset * rank] + torch.manual_seed(12 + rank) + + y_true = torch.arange(0, n_iters * batch_size, dtype=torch.float).to(device) + y_preds = (rank + 1) * torch.ones(n_iters * batch_size, dtype=torch.float).to(device) + + def update(engine, i): + return y_preds[i * batch_size : (i + 1) * batch_size], y_true[i * batch_size : (i + 1) * batch_size] - def _test(metric_device): engine = Engine(update) m = RootMeanSquaredError(device=metric_device) @@ -77,15 +79,13 @@ def _test(metric_device): data = list(range(n_iters)) engine.run(data=data, max_epochs=1) + y_preds = idist.all_gather(y_preds) + y_true = idist.all_gather(y_true) + assert "rmse" in engine.state.metrics res = engine.state.metrics["rmse"] - y_preds_full = [] - for i in range(idist.get_world_size()): - y_preds_full.append((i + 1) * torch.ones(offset)) - y_preds_full = torch.stack(y_preds_full).to(device).flatten() - - true_res = np.sqrt(np.mean(np.square((y_true - y_preds_full).cpu().numpy()))) + true_res = np.sqrt(np.mean(np.square((y_true - y_preds).cpu().numpy()))) assert pytest.approx(res, rel=tol) == true_res diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index fe00d6c36e4..f49a86c44d6 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -131,17 +131,17 @@ def _test_distrib_integration(device, tol=1e-4): rank = idist.get_rank() n_iters = 100 - s = 10 - offset = n_iters * s + batch_size = 10 + offset = n_iters * batch_size def _test(metric_device): - y_pred = torch.rand(offset * idist.get_world_size(), 3, 28, 28, dtype=torch.float, device=device) + y_pred = torch.rand(n_iters * batch_size, 3, 28, 28, dtype=torch.float, device=device) y = y_pred * 0.65 def update(engine, i): return ( - y_pred[i * s + offset * rank : (i + 1) * s + offset * rank], - y[i * s + offset * rank : (i + 1) * s + offset * rank], + y_pred[i * batch_size : (i + 1) * batch_size], + y[i * batch_size : (i + 1) * batch_size], ) engine = Engine(update) @@ -150,6 +150,9 @@ def update(engine, i): data = list(range(n_iters)) engine.run(data=data, max_epochs=1) + y_pred = idist.all_gather(y_pred) + y = idist.all_gather(y) + assert "ssim" in engine.state.metrics res = engine.state.metrics["ssim"] diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index 2282248a96a..87233153cd3 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -63,17 +63,17 @@ def _test_distrib_integration(device): def _test(n_epochs, metric_device): n_iters = 100 - s = 16 + batch_size = 16 n_classes = 10 - offset = n_iters * s - y_true = torch.randint(0, n_classes, size=(offset * idist.get_world_size(),)).to(device) - y_preds = torch.rand(offset * idist.get_world_size(), n_classes).to(device) + offset = n_iters * batch_size + y_true = torch.randint(0, n_classes, size=(n_iters * batch_size,)).to(device) + y_preds = torch.rand(n_iters * batch_size, n_classes).to(device) def update(engine, i): return ( - y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, :], - y_true[i * s + rank * offset : (i + 1) * s + rank * offset], + y_preds[i * batch_size : (i + 1) * batch_size], + y_true[i * batch_size : (i + 1) * batch_size], ) engine = Engine(update) @@ -85,6 +85,9 @@ def update(engine, i): data = list(range(n_iters)) engine.run(data=data, max_epochs=n_epochs) + y_preds = idist.all_gather(y_preds) + y_true = idist.all_gather(y_true) + assert "acc" in engine.state.metrics res = engine.state.metrics["acc"] if isinstance(res, torch.Tensor): From 9713a544f835fdd49a1bd9c7270254c0074dd518 Mon Sep 17 00:00:00 2001 From: puhuk Date: Fri, 26 Aug 2022 17:39:40 +0900 Subject: [PATCH 2/6] Update with review --- tests/ignite/metrics/test_ssim.py | 2 +- tests/ignite/metrics/test_top_k_categorical_accuracy.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index f49a86c44d6..a24c73bcdff 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -130,9 +130,9 @@ def _test_distrib_integration(device, tol=1e-4): from ignite.engine import Engine rank = idist.get_rank() + torch.manual_seed(12 + rank) n_iters = 100 batch_size = 10 - offset = n_iters * batch_size def _test(metric_device): y_pred = torch.rand(n_iters * batch_size, 3, 28, 28, dtype=torch.float, device=device) diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index 87233153cd3..cda413a88fe 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -59,14 +59,12 @@ def _test_distrib_integration(device): from ignite.engine import Engine rank = idist.get_rank() - torch.manual_seed(12) def _test(n_epochs, metric_device): n_iters = 100 batch_size = 16 n_classes = 10 - offset = n_iters * batch_size y_true = torch.randint(0, n_classes, size=(n_iters * batch_size,)).to(device) y_preds = torch.rand(n_iters * batch_size, n_classes).to(device) @@ -100,7 +98,8 @@ def update(engine, i): metric_devices = ["cpu"] if device.type != "xla": metric_devices.append(idist.device()) - for _ in range(3): + for i in range(3): + torch.manual_seed(12 + rank + i) for metric_device in metric_devices: _test(n_epochs=1, metric_device=metric_device) _test(n_epochs=2, metric_device=metric_device) From 1a8e37406df522c1030a999c9dc08602b316d8eb Mon Sep 17 00:00:00 2001 From: puhuk Date: Sat, 27 Aug 2022 22:57:54 +0900 Subject: [PATCH 3/6] Update test_top_k_categorical_accuracy.py --- tests/ignite/metrics/test_top_k_categorical_accuracy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index cda413a88fe..35d146f30e3 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -58,8 +58,6 @@ def top_k_accuracy(y_true, y_pred, k=5, normalize=True): def _test_distrib_integration(device): from ignite.engine import Engine - rank = idist.get_rank() - def _test(n_epochs, metric_device): n_iters = 100 batch_size = 16 @@ -99,6 +97,7 @@ def update(engine, i): if device.type != "xla": metric_devices.append(idist.device()) for i in range(3): + rank = idist.get_rank() torch.manual_seed(12 + rank + i) for metric_device in metric_devices: _test(n_epochs=1, metric_device=metric_device) From 8486c0b12f7f9b8391bb594946d90415bd93f414 Mon Sep 17 00:00:00 2001 From: puhuk Date: Mon, 29 Aug 2022 12:01:13 +0900 Subject: [PATCH 4/6] Update test_top_k_categorical_accuracy.py --- tests/ignite/metrics/test_top_k_categorical_accuracy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index 35d146f30e3..71419b66751 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -96,8 +96,8 @@ def update(engine, i): metric_devices = ["cpu"] if device.type != "xla": metric_devices.append(idist.device()) + rank = idist.get_rank() for i in range(3): - rank = idist.get_rank() torch.manual_seed(12 + rank + i) for metric_device in metric_devices: _test(n_epochs=1, metric_device=metric_device) From 35ed03a783d2dd6862bea9a1b1adad29323dbea9 Mon Sep 17 00:00:00 2001 From: puhuk Date: Thu, 1 Sep 2022 23:33:22 +0900 Subject: [PATCH 5/6] Update test_top_k_categorical_accuracy.py --- tests/ignite/metrics/test_top_k_categorical_accuracy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index 71419b66751..17b35d9ef2f 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -69,7 +69,7 @@ def _test(n_epochs, metric_device): def update(engine, i): return ( y_preds[i * batch_size : (i + 1) * batch_size], - y_true[i * batch_size : (i + 1) * batch_size], + y_true[i * batch_size : (i + 1) * batch_size, :], ) engine = Engine(update) From 40318450a3027216fe836668f4bb7919b117283f Mon Sep 17 00:00:00 2001 From: vfdev Date: Thu, 1 Sep 2022 17:29:14 +0200 Subject: [PATCH 6/6] Apply suggestions from code review --- tests/ignite/metrics/test_ssim.py | 4 ++-- tests/ignite/metrics/test_top_k_categorical_accuracy.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/ignite/metrics/test_ssim.py b/tests/ignite/metrics/test_ssim.py index a24c73bcdff..f477892222c 100644 --- a/tests/ignite/metrics/test_ssim.py +++ b/tests/ignite/metrics/test_ssim.py @@ -140,8 +140,8 @@ def _test(metric_device): def update(engine, i): return ( - y_pred[i * batch_size : (i + 1) * batch_size], - y[i * batch_size : (i + 1) * batch_size], + y_pred[i * batch_size : (i + 1) * batch_size, ...], + y[i * batch_size : (i + 1) * batch_size, ...], ) engine = Engine(update) diff --git a/tests/ignite/metrics/test_top_k_categorical_accuracy.py b/tests/ignite/metrics/test_top_k_categorical_accuracy.py index 17b35d9ef2f..ed34994c6b7 100644 --- a/tests/ignite/metrics/test_top_k_categorical_accuracy.py +++ b/tests/ignite/metrics/test_top_k_categorical_accuracy.py @@ -68,8 +68,8 @@ def _test(n_epochs, metric_device): def update(engine, i): return ( - y_preds[i * batch_size : (i + 1) * batch_size], - y_true[i * batch_size : (i + 1) * batch_size, :], + y_preds[i * batch_size : (i + 1) * batch_size, :], + y_true[i * batch_size : (i + 1) * batch_size], ) engine = Engine(update)