[Data] Make test_dataset_throughput deterministic by increasing workload and applying tolerance

Author: dancingactor

python/ray/data/tests/test_stats.py

@@ -1752,18 +1752,20 @@ def test_dataset_throughput(shutdown_only):
     ray.shutdown()
     ray.init(num_cpus=2)
 
-    f = dummy_map_batches_sleep(0.01)
+    f = dummy_map_batches_sleep(0.03)
     ds = ray.data.range(100).map(f).materialize().map(f).materialize()
 
     operator_pattern = re.compile(
         r"Operator (\d+).*?\* Operator throughput:\s*.*?\* Ray Data throughput: (\d+\.\d+) rows/s.*?\* Estimated single node throughput: (\d+\.\d+) rows/s",
         re.DOTALL,
     )
 
+    Tolerance = 0.9
+
     # Ray data throughput should always be better than single node throughput for
     # multi-cpu case.
     for match in operator_pattern.findall(ds.stats()):
-        assert float(match[1]) >= float(match[2])
+        assert float(match[1]) >= float(match[2]) * Tolerance
 
     # Pattern to match dataset throughput
     dataset_pattern = re.compile(
@@ -1772,7 +1774,7 @@ def test_dataset_throughput(shutdown_only):
     )
 
     dataset_match = dataset_pattern.search(ds.stats())
-    assert float(dataset_match[1]) >= float(dataset_match[2])
+    assert float(dataset_match[1]) >= float(dataset_match[2]) * Tolerance
 
 
 def test_individual_operator_num_rows(shutdown_only):