diff --git a/ray.sub b/ray.sub index 32076d5fb3..442fe1968c 100644 --- a/ray.sub +++ b/ray.sub @@ -69,7 +69,12 @@ DASHBOARD_AGENT_LISTEN_PORT=${DASHBOARD_AGENT_LISTEN_PORT:-52365} # Setting ulimit is recommended by ray best practices page # @ https://docs.ray.io/en/latest/cluster/vms/user-guides/large-cluster-best-practices.html # It's session based and won't affect the system outside the script -ulimit -Sn 65535 +# Ensure that the soft limit isn't above the hard limit +if [[ $(ulimit -Hn) == "unlimited" ]] || [[ 65535 -lt $(ulimit -Hn) ]]; then + ulimit -Sn 65535 +elif [[ $(ulimit -Hn) != "unlimited" ]] && [[ $(ulimit -Hn) -lt 65535 ]]; then + echo "[WARNING]: Cannot increase ulimit on file descriptors to 65535 according ray recommendation: https://docs.ray.io/en/latest/cluster/vms/user-guides/large-cluster-best-practices.html. Speak to cluster admins to increase, otherwise ray may crash unexpectedly." +fi # On our clusters, the largest port range on an idle worker appeared between 52369-64607 # (not including the other ports set by this script). So this range is chosen to be