-
Notifications
You must be signed in to change notification settings - Fork 52.9k
fix(tui_gateway): join _SlashWorker drain threads on close #53308
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
pasevin
wants to merge
3
commits into
NousResearch:main
Choose a base branch
from
pasevin:fix/slash-worker-drain-thread-leak
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
+164
−2
Open
Changes from all commits
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| """Tests for _SlashWorker drain thread cleanup (#53303).""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import threading | ||
| import time | ||
| from unittest.mock import MagicMock, patch | ||
|
|
||
| import pytest | ||
|
|
||
|
|
||
| class _FakeProc: | ||
| """Minimal subprocess.Popen stand-in for _SlashWorker tests.""" | ||
|
|
||
| def __init__(self): | ||
| self.stdin = MagicMock() | ||
| self.stdout = MagicMock() | ||
| self.stderr = MagicMock() | ||
| self._poll = None # None = still running | ||
|
|
||
| # Make stdout/stderr iteration return empty (drain threads exit) | ||
| self.stdout.__iter__ = lambda self: iter([]) | ||
| self.stderr.__iter__ = lambda self: iter([]) | ||
|
|
||
| def poll(self): | ||
| return self._poll | ||
|
|
||
| def terminate(self): | ||
| self._poll = 0 | ||
|
|
||
| def kill(self): | ||
| self._poll = -9 | ||
|
|
||
| def wait(self, timeout=None): | ||
| return self._poll or 0 | ||
|
|
||
|
|
||
| def test_slash_worker_close_joins_drain_threads(): | ||
| """_SlashWorker.close() must join its drain threads (#53303). | ||
|
|
||
| Prior to the fix, close() terminated the subprocess and closed | ||
| the pipes but never joined the _drain_stdout/_drain_stderr threads. | ||
| This left 2 leaked daemon threads per session on Linux, each holding | ||
| references to the _SlashWorker instance and its buffers. | ||
|
|
||
| The fix stores thread references and calls join(timeout=2) in close(). | ||
| In production, closing proc.stdout/proc.stderr causes the readline() | ||
| in the drain threads to hit EOF and exit, so join() returns quickly. | ||
| This test uses threads that exit promptly to verify the join path works. | ||
| """ | ||
| from tui_gateway.server import _SlashWorker | ||
|
|
||
| worker = _SlashWorker.__new__(_SlashWorker) | ||
| worker._lock = threading.Lock() | ||
| worker._seq = 0 | ||
| worker.stderr_tail = [] | ||
| worker.stdout_queue = __import__("queue").Queue() | ||
| worker._closed = False | ||
| worker.proc = _FakeProc() | ||
|
|
||
| # Use threads that exit quickly (simulating EOF on the pipe) | ||
| exit_event = threading.Event() | ||
| exit_event.set() # let them exit immediately | ||
|
|
||
| def quick_drain(): | ||
| exit_event.wait(timeout=5) | ||
|
|
||
| worker._drain_thread_stdout = threading.Thread( | ||
| target=quick_drain, daemon=True, name="test-drain-stdout" | ||
| ) | ||
| worker._drain_thread_stderr = threading.Thread( | ||
| target=quick_drain, daemon=True, name="test-drain-stderr" | ||
| ) | ||
| worker._drain_thread_stdout.start() | ||
| worker._drain_thread_stderr.start() | ||
|
|
||
| # Give threads time to exit | ||
| time.sleep(0.1) | ||
|
|
||
| # Call close() | ||
| worker.close() | ||
|
|
||
| # close() should have set _closed | ||
| assert worker._closed | ||
|
|
||
| # close() should have terminated the proc | ||
| assert worker.proc.poll() is not None | ||
|
|
||
| # close() should have closed stdin/stdout/stderr | ||
| worker.proc.stdin.close.assert_called() | ||
| worker.proc.stdout.close.assert_called() | ||
| worker.proc.stderr.close.assert_called() | ||
|
|
||
| # The drain threads should have exited (they exit on their own, and | ||
| # close() joins them — so they're definitely not alive after close()). | ||
| assert not worker._drain_thread_stdout.is_alive(), ( | ||
| "_drain_thread_stdout is still alive after close()" | ||
| ) | ||
| assert not worker._drain_thread_stderr.is_alive(), ( | ||
| "_drain_thread_stderr is still alive after close()" | ||
| ) | ||
|
|
||
|
|
||
| def test_slash_worker_close_is_idempotent(): | ||
| """close() can be called multiple times safely.""" | ||
| from tui_gateway.server import _SlashWorker | ||
|
|
||
| worker = _SlashWorker.__new__(_SlashWorker) | ||
| worker._closed = False | ||
| worker.proc = _FakeProc() | ||
|
|
||
| def noop(): | ||
| pass | ||
|
|
||
| worker._drain_thread_stdout = threading.Thread(target=noop, daemon=True) | ||
| worker._drain_thread_stderr = threading.Thread(target=noop, daemon=True) | ||
| worker._drain_thread_stdout.start() | ||
| worker._drain_thread_stderr.start() | ||
|
|
||
| worker.close() | ||
| assert worker._closed | ||
|
|
||
| # Second call should be a no-op (guard at top of close()) | ||
| worker.close() | ||
| assert worker._closed | ||
|
|
||
|
|
||
| def test_slash_worker_drain_threads_are_named(): | ||
| """Drain threads should have identifiable names for debugging.""" | ||
| # This is a regression guard: anonymous threads (no name) make it | ||
| # impossible to identify the source of leaked threads in py-spy dumps. | ||
| # The fix gives them explicit names: slash-drain-stdout, slash-drain-stderr. | ||
| import inspect | ||
|
|
||
| from tui_gateway import server | ||
|
|
||
| source = inspect.getsource(_SlashWorker := server._SlashWorker) | ||
| assert "slash-drain-stdout" in source, ( | ||
| "_SlashWorker should name its stdout drain thread 'slash-drain-stdout'" | ||
| ) | ||
| assert "slash-drain-stderr" in source, ( | ||
| "_SlashWorker should name its stderr drain thread 'slash-drain-stderr'" | ||
| ) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Because this event is set before either thread starts, both drain threads have already exited before
close()at line 81. The test therefore passes on current main even though currentclose()never callsjoin(). Please assertjoin(timeout=2)directly or keep a controlled thread alive until close().