[core] Use psutil.process_iter to replace psutil.pids (ray-project#51123)

* Use `psutil.process_iter` to replace `psutil.pids`.
* Use `proc.info["name"]` instead of `proc.name()`.
* I'm not sure whether `proc.name()` uses the cache set by
`process_iter`, but I'm certain that using info is correct since the
official docs frequently use `proc.info[...]` with `process_iter`.
* I asked a question on giampaolo/psutil#2518,
but I'm not sure how long it will take to get an answer from the
community. For now, I think we can merge it, and I'll update the use of
psutil if the maintainers have any suggestions.

---------

Signed-off-by: kaihsun <[email protected]>

Author: kevin85421

python/ray/_private/memory_monitor.py

@@ -37,12 +37,12 @@ def get_top_n_memory_usage(n: int = 10):
     Returns:
         (str) The formatted string of top n process memory usage.
     """
-    pids = psutil.pids()
     proc_stats = []
-    for pid in pids:
+    for proc in psutil.process_iter(["memory_info", "cmdline"]):
         try:
-            proc = psutil.Process(pid)
-            proc_stats.append((get_rss(proc.memory_info()), pid, proc.cmdline()))
+            proc_stats.append(
+                (get_rss(proc.info["memory_info"]), proc.pid, proc.info["cmdline"])
+            )
         except psutil.NoSuchProcess:
             # We should skip the process that has exited. Refer this
             # issue for more detail:

python/ray/_private/services.py

@@ -340,23 +340,21 @@ def _find_address_from_flag(flag: str):
 
     # The --redis-address here is what is now called the --address, but it
     # appears in the default_worker.py and agent.py calls as --redis-address.
-    pids = psutil.pids()
     addresses = set()
-    for pid in pids:
+    for proc in psutil.process_iter(["cmdline"]):
         try:
-            proc = psutil.Process(pid)
             # HACK: Workaround for UNIX idiosyncrasy
             # Normally, cmdline() is supposed to return the argument list.
             # But it in some cases (such as when setproctitle is called),
             # an arbitrary string resembling a command-line is stored in
             # the first argument.
             # Explanation: https://unix.stackexchange.com/a/432681
             # More info: https://github.com/giampaolo/psutil/issues/1179
-            cmdline = proc.cmdline()
+            cmdline = proc.info["cmdline"]
             # NOTE(kfstorm): To support Windows, we can't use
             # `os.path.basename(cmdline[0]) == "raylet"` here.
 
-            if len(cmdline) > 0 and "raylet" in os.path.basename(cmdline[0]):
+            if _is_raylet_process(cmdline):
                 for arglist in cmdline:
                     # Given we're merely seeking --redis-address, we just split
                     # every argument on spaces for now.
@@ -2289,3 +2287,19 @@ def start_ray_client_server(
         fate_share=fate_share,
     )
     return process_info
+
+
+def _is_raylet_process(cmdline: Optional[List[str]]) -> bool:
+    """Check if the command line belongs to a raylet process.
+
+    Args:
+        cmdline: List of command line arguments or None
+
+    Returns:
+        bool: True if this is a raylet process, False otherwise
+    """
+    if cmdline is None or len(cmdline) == 0:
+        return False
+
+    executable = os.path.basename(cmdline[0])
+    return "raylet" in executable

python/ray/_private/test_utils.py

@@ -2130,11 +2130,11 @@ def get_gcs_memory_used():
     import psutil
 
     m = {
-        process.name(): process.memory_info().rss
-        for process in psutil.process_iter()
+        proc.info["name"]: proc.info["memory_info"].rss
+        for proc in psutil.process_iter(["status", "name", "memory_info"])
         if (
-            process.status() not in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD)
-            and process.name() in ("gcs_server", "redis-server")
+            proc.info["status"] not in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD)
+            and proc.info["name"] in ("gcs_server", "redis-server")
         )
     }
     assert "gcs_server" in m