Skip to content

Commit

Permalink
Immediately raise exception when trying to connect to a closed cluster (
Browse files Browse the repository at this point in the history
  • Loading branch information
fjetter authored Feb 23, 2022
1 parent 8a99ac7 commit 985bff8
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 0 deletions.
6 changes: 6 additions & 0 deletions distributed/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
CommClosedError,
ConnectionPool,
PooledRPCCall,
Status,
clean_exception,
connect,
rpc,
Expand Down Expand Up @@ -825,6 +826,11 @@ def __init__(
elif isinstance(getattr(address, "scheduler_address", None), str):
# It's a LocalCluster or LocalCluster-compatible object
self.cluster = address
status = getattr(self.cluster, "status")
if status and status in [Status.closed, Status.closing]:
raise RuntimeError(
f"Trying to connect to an already closed or closing Cluster {self.cluster}."
)
with suppress(AttributeError):
loop = address.loop
if security is None:
Expand Down
15 changes: 15 additions & 0 deletions distributed/deploy/tests/test_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -1149,3 +1149,18 @@ async def test_cluster_host_used_throughout_cluster(host, use_nanny):
if use_nanny:
url = urlparse(worker.process.worker_address)
assert url.hostname == "127.0.0.1"


@gen_test()
async def test_connect_to_closed_cluster(cleanup):
async with LocalCluster(processes=False, asynchronous=True) as cluster:
async with Client(cluster, asynchronous=True) as c1:
assert await c1.submit(inc, 1) == 2

with pytest.raises(
RuntimeError,
match="Trying to connect to an already closed or closing Cluster",
):
# Raises during init without actually connecting since we're not
# awaiting anything
Client(cluster, asynchronous=True)

2 comments on commit 985bff8

@NHanser
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm using a YarnCLuster (from dask-yarn) and this type of Cluster is entering this part of the "if" (as it has a scheduler_address str starting by "tcp"), but YarnCluster has no "status" attribute
No sure if it is dask-yarn which has to be updated, or this commit to be reverted...

@RouvenG
Copy link

@RouvenG RouvenG commented on 985bff8 May 2, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 Please fix

Please sign in to comment.