[OBOD] Disable the ownership-based object directory for all tests tha…

…t use ray.objects(). (#14065)

python/ray/tests/test_advanced_3.py

@@ -21,9 +21,8 @@
 import setproctitle
 import subprocess
 
-from ray.test_utils import (check_call_ray, RayTestTimeoutException,
-                            wait_for_condition, wait_for_num_actors,
-                            new_scheduler_enabled)
+from ray.test_utils import (check_call_ray, wait_for_condition,
+                            wait_for_num_actors, new_scheduler_enabled)
 
 logger = logging.getLogger(__name__)
 
@@ -156,15 +155,6 @@ def f(x):
     assert ray.get(f.remote(non_local.remote())) == non_local_node.unique_id
 
 
-def wait_for_num_objects(num_objects, timeout=10):
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        if len(ray.objects()) >= num_objects:
-            return
-        time.sleep(0.1)
-    raise RayTestTimeoutException("Timed out while waiting for global state.")
-
-
 def test_global_state_api(shutdown_only):
 
     ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1})
@@ -624,7 +614,14 @@ def f(self):
 
 
 def test_lease_request_leak(shutdown_only):
-    ray.init(num_cpus=1, _system_config={"object_timeout_milliseconds": 200})
+    ray.init(
+        num_cpus=1,
+        _system_config={
+            # This test uses ray.objects(), which only works with the GCS-based
+            # object directory
+            "ownership_based_object_directory_enabled": False,
+            "object_timeout_milliseconds": 200
+        })
     assert len(ray.objects()) == 0
 
     @ray.remote

python/ray/tests/test_client_references.py

@@ -33,10 +33,17 @@ def test_cond():
 
 
 @pytest.mark.parametrize(
-    "ray_start_cluster", [{
+    "ray_start_cluster",
+    [{
         "num_nodes": 1,
-        "do_init": False
-    }], indirect=True)
+        "do_init": False,
+        # This test uses ray.objects(), which only works with the GCS-based
+        # object directory
+        "_system_config": {
+            "ownership_based_object_directory_enabled": False
+        },
+    }],
+    indirect=True)
 def test_delete_refs_on_disconnect(ray_start_cluster):
     cluster = ray_start_cluster
     with ray_start_cluster_client_server_pair(cluster.address) as pair:

python/ray/tests/test_multi_node.py

@@ -178,6 +178,16 @@ def f():
         assert "success" in out
 
 
+@pytest.mark.parametrize(
+    "call_ray_start",
+    [
+        "ray start --head --num-cpus=1 --min-worker-port=0 "
+        "--max-worker-port=0 --port 0 --system-config="
+        # This test uses ray.objects(), which only works with the GCS-based
+        # object directory
+        "{\"ownership_based_object_directory_enabled\":false}",
+    ],
+    indirect=True)
 def test_cleanup_on_driver_exit(call_ray_start):
     # This test will create a driver that creates a bunch of objects and then
     # exits. The entries in the object table should be cleaned up.

python/ray/tune/tests/test_trial_scheduler_pbt.py

@@ -29,7 +29,14 @@ def __call__(self, *args, **kwargs):
 
 class PopulationBasedTrainingMemoryTest(unittest.TestCase):
     def setUp(self):
-        ray.init(num_cpus=1, object_store_memory=100 * MB)
+        ray.init(
+            num_cpus=1,
+            object_store_memory=100 * MB,
+            _system_config={
+                # This test uses ray.objects(), which only works with the
+                # GCS-based object directory
+                "ownership_based_object_directory_enabled": False,
+            })
 
     def tearDown(self):
         ray.shutdown()
@@ -90,7 +97,13 @@ def save(self, *args, **kwargs):
 
 class PopulationBasedTrainingFileDescriptorTest(unittest.TestCase):
     def setUp(self):
-        ray.init(num_cpus=2)
+        ray.init(
+            num_cpus=2,
+            _system_config={
+                # This test uses ray.objects(), which only works with the
+                # GCS-based object directory
+                "ownership_based_object_directory_enabled": False,
+            })
         os.environ["TUNE_GLOBAL_CHECKPOINT_S"] = "0"
 
     def tearDown(self):