Skip to content

[Data] Use tqdm_ray for progress reporting from worker #58277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 5, 2025
Merged

[Data] Use tqdm_ray for progress reporting from worker #58277

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e16d2bb
fix progress not showing for progress reporting on worker
kyuds Oct 29, 2025
17ad1b7
better logging
kyuds Oct 29, 2025
b7cb260
improve logging message
kyuds Oct 29, 2025
c911053
correct logging
kyuds Oct 29, 2025
64a80b7
Merge branch 'master' into progress-worker-case
kyuds Oct 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 13 additions & 8 deletions python/ray/data/_internal/execution/streaming_executor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,13 +83,6 @@ def __init__(
self._global_info: Optional[ProgressBar] = None
self._progress_manager: Optional[RichExecutionProgressManager] = None

if not self._use_rich_progress() and log_once("rich_progress_disabled"):
logger.info(
"A new progress UI is available. To enable, set "
"`ray.data.DataContext.get_current()."
"enable_rich_progress_bars = True`."
)

# The executor can be shutdown while still running.
self._shutdown_lock = threading.RLock()
self._execution_started = False
Expand Down Expand Up @@ -669,7 +662,19 @@ def _update_stats_metrics(self, state: str, force_update: bool = False):
)

def _use_rich_progress(self):
return self._data_context.enable_rich_progress_bars
rich_enabled = self._data_context.enable_rich_progress_bars
use_ray_tqdm = self._data_context.use_ray_tqdm

if not rich_enabled or use_ray_tqdm:
if log_once("ray_data_rich_progress_disabled"):
logger.info(
"[dataset]: A new progress UI is available. To enable, "
"set `ray.data.DataContext.get_current()."
"enable_rich_progress_bars = True` and `ray.data."
"DataContext.get_current().use_ray_tqdm = False`."
)
return False
return True


def _validate_dag(dag: PhysicalOperator, limits: ExecutionResources) -> None:
Expand Down