Change data generating in classification, fbeta, mae (#2667)

* Change data generating in classification, fbeta, mae

Change data generating in `classification_report`, `fbeta`, `mae`

* Update with review

* Update test_classification_report.py

Co-authored-by: vfdev <vfdev.5@gmail.com>

Author: puhuk

tests/ignite/metrics/test_classification_report.py

@@ -12,21 +12,20 @@
 def _test_integration_multiclass(device, output_dict):
 
     rank = idist.get_rank()
-    torch.manual_seed(12)
 
     def _test(metric_device, n_classes, labels=None):
 
         classification_report = ClassificationReport(device=metric_device, output_dict=output_dict, labels=labels)
         n_iters = 80
-        s = 16
-        offset = n_iters * s
-        y_true = torch.randint(0, n_classes, size=(offset * idist.get_world_size(),)).to(device)
-        y_preds = torch.rand(offset * idist.get_world_size(), n_classes).to(device)
+        batch_size = 16
+
+        y_true = torch.randint(0, n_classes, size=(n_iters * batch_size,)).to(device)
+        y_preds = torch.rand(n_iters * batch_size, n_classes).to(device)
 
         def update(engine, i):
             return (
-                y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, :],
-                y_true[i * s + rank * offset : (i + 1) * s + rank * offset],
+                y_preds[i * batch_size : (i + 1) * batch_size, :],
+                y_true[i * batch_size : (i + 1) * batch_size],
             )
 
         engine = Engine(update)
@@ -36,6 +35,9 @@ def update(engine, i):
         data = list(range(n_iters))
         engine.run(data=data)
 
+        y_preds = idist.all_gather(y_preds)
+        y_true = idist.all_gather(y_true)
+
         assert "cr" in engine.state.metrics
         res = engine.state.metrics["cr"]
         res2 = classification_report.compute()
@@ -60,7 +62,8 @@ def update(engine, i):
         assert sklearn_result["macro avg"]["recall"] == pytest.approx(res["macro avg"]["recall"])
         assert sklearn_result["macro avg"]["f1-score"] == pytest.approx(res["macro avg"]["f1-score"])
 
-    for _ in range(5):
+    for i in range(5):
+        torch.manual_seed(12 + rank + i)
         # check multiple random inputs as random exact occurencies are rare
         metric_devices = ["cpu"]
         if device.type != "xla":
@@ -77,24 +80,22 @@ def update(engine, i):
 def _test_integration_multilabel(device, output_dict):
 
     rank = idist.get_rank()
-    torch.manual_seed(12)
 
     def _test(metric_device, n_epochs, labels=None):
 
         classification_report = ClassificationReport(device=metric_device, output_dict=output_dict, is_multilabel=True)
 
         n_iters = 10
-        s = 16
+        batch_size = 16
         n_classes = 7
 
-        offset = n_iters * s
-        y_true = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device)
-        y_preds = torch.randint(0, 2, size=(offset * idist.get_world_size(), n_classes, 6, 8)).to(device)
+        y_true = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 6, 8)).to(device)
+        y_preds = torch.randint(0, 2, size=(n_iters * batch_size, n_classes, 6, 8)).to(device)
 
         def update(engine, i):
             return (
-                y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, ...],
-                y_true[i * s + rank * offset : (i + 1) * s + rank * offset, ...],
+                y_preds[i * batch_size : (i + 1) * batch_size, ...],
+                y_true[i * batch_size : (i + 1) * batch_size, ...],
             )
 
         engine = Engine(update)
@@ -104,6 +105,9 @@ def update(engine, i):
         data = list(range(n_iters))
         engine.run(data=data, max_epochs=n_epochs)
 
+        y_preds = idist.all_gather(y_preds)
+        y_true = idist.all_gather(y_true)
+
         assert "cr" in engine.state.metrics
         res = engine.state.metrics["cr"]
         res2 = classification_report.compute()
@@ -121,6 +125,7 @@ def update(engine, i):
         sklearn_result = sklearn_classification_report(np_y_true, np_y_preds, output_dict=True, zero_division=1)
 
         for i in range(n_classes):
+            torch.manual_seed(12 + rank + i)
             label_i = labels[i] if labels else str(i)
             assert sklearn_result[str(i)]["precision"] == pytest.approx(res[label_i]["precision"])
             assert sklearn_result[str(i)]["f1-score"] == pytest.approx(res[label_i]["f1-score"])

tests/ignite/metrics/test_fbeta.py

@@ -91,21 +91,21 @@ def update_fn(engine, batch):
 def _test_distrib_integration(device):
 
     rank = idist.get_rank()
-    torch.manual_seed(12)
 
     def _test(p, r, average, n_epochs, metric_device):
         n_iters = 60
-        s = 16
+        batch_size = 16
         n_classes = 7
 
-        offset = n_iters * s
-        y_true = torch.randint(0, n_classes, size=(offset * idist.get_world_size(),)).to(device)
-        y_preds = torch.rand(offset * idist.get_world_size(), n_classes).to(device)
+        torch.manual_seed(12 + rank)
+
+        y_true = torch.randint(0, n_classes, size=(n_iters * batch_size,)).to(device)
+        y_preds = torch.rand(n_iters * batch_size, n_classes).to(device)
 
         def update(engine, i):
             return (
-                y_preds[i * s + rank * offset : (i + 1) * s + rank * offset, :],
-                y_true[i * s + rank * offset : (i + 1) * s + rank * offset],
+                y_preds[i * batch_size : (i + 1) * batch_size, :],
+                y_true[i * batch_size : (i + 1) * batch_size],
             )
 
         engine = Engine(update)
@@ -116,6 +116,9 @@ def update(engine, i):
         data = list(range(n_iters))
         engine.run(data=data, max_epochs=n_epochs)
 
+        y_preds = idist.all_gather(y_preds)
+        y_true = idist.all_gather(y_true)
+
         assert "f2.5" in engine.state.metrics
         res = engine.state.metrics["f2.5"]
         if isinstance(res, torch.Tensor):

tests/ignite/metrics/test_mean_absolute_error.py

@@ -59,20 +59,22 @@ def _test_distrib_integration(device):
     from ignite.engine import Engine
 
     rank = idist.get_rank()
-    n_iters = 80
-    s = 50
-    offset = n_iters * s
 
-    y_true = torch.arange(0, offset * idist.get_world_size(), dtype=torch.float).to(device)
-    y_preds = torch.ones(offset * idist.get_world_size(), dtype=torch.float).to(device)
+    def _test(metric_device):
 
-    def update(engine, i):
-        return (
-            y_preds[i * s + offset * rank : (i + 1) * s + offset * rank],
-            y_true[i * s + offset * rank : (i + 1) * s + offset * rank],
-        )
+        n_iters = 80
+        batch_size = 50
+        torch.manual_seed(12 + rank)
+
+        y_true = torch.arange(0, n_iters * batch_size, dtype=torch.float).to(device)
+        y_preds = torch.ones(n_iters * batch_size, dtype=torch.float).to(device)
+
+        def update(engine, i):
+            return (
+                y_preds[i * batch_size : (i + 1) * batch_size],
+                y_true[i * batch_size : (i + 1) * batch_size],
+            )
 
-    def _test(metric_device):
         engine = Engine(update)
 
         m = MeanAbsoluteError(device=metric_device)
@@ -81,6 +83,9 @@ def _test(metric_device):
         data = list(range(n_iters))
         engine.run(data=data, max_epochs=1)
 
+        y_preds = idist.all_gather(y_preds)
+        y_true = idist.all_gather(y_true)
+
         assert "mae" in engine.state.metrics
         res = engine.state.metrics["mae"]