diff --git a/python/ray/tune/scripts.py b/python/ray/tune/scripts.py index 361dedfddffb..72db1453af19 100644 --- a/python/ray/tune/scripts.py +++ b/python/ray/tune/scripts.py @@ -3,13 +3,15 @@ from __future__ import print_function import click -import logging import glob import json import os +from datetime import datetime + + import pandas as pd -# from ray.tune.trial_runner import TrialRunner -import sys + +from ray.tune.trial import Trial def _flatten_dict(dt): @@ -31,60 +33,90 @@ def _flatten_dict(dt): def cli(): pass -@cli.command() -@click.argument("experiment_path", required=True, type=str) -def list_trials(experiment_path): - _list_trials(experiment_path) -def _list_trials(experiment_path): +DEFAULT_EXPERIMENT_INFO_KEYS = ( + "trial_name", + "trial_id", + "status", + "num_failures", + "logdir" +) + +DEFAULT_PROJECT_INFO_KEYS = ( + "name", + "timestamp", + "total_trials", + "running_trials", + "terminated_trials", + "error_trials", +) + + +def _list_trials(experiment_path, info_keys=DEFAULT_EXPERIMENT_INFO_KEYS): experiment_path = os.path.expanduser(experiment_path) - print("start glob") globs = glob.glob(os.path.join(experiment_path, "experiment_state*.json")) - print(globs) filename = max(list(globs)) - print("found") + with open(filename) as f: experiment_state = json.load(f) - for trial_state in experiment_state["checkpoints"]: - print("{trial_name}\t{trial_id}\t{status}\t{num_failures}\t{logdir}".format( - **trial_state)) + checkpoints = pd.DataFrame.from_records(experiment_state['checkpoints']) + # TODO(hartikainen): The logdir is often too verbose to be viewed in a + # table. + checkpoints['logdir'] = checkpoints['logdir'].str.replace( + experiment_path, '') + + print(checkpoints[list(info_keys)].to_string()) + @cli.command() -@click.argument("project_path", required=True, type=str) -def list_experiments(project_path): - _list_experiments(project_path) +@click.argument("experiment_path", required=True, type=str) +def list_trials(experiment_path): + _list_trials(experiment_path) + -def _list_experiments(project_path): +def _list_experiments(project_path, info_keys=DEFAULT_PROJECT_INFO_KEYS): base, experiment_paths, _ = list(os.walk(project_path))[0] # clean this - experiment_collection = {} + + experiment_data_collection = [] for experiment_path in experiment_paths: - experiment_state_path = glob.glob(os.path.join(base, experiment_path, "experiment_state*.json")) + experiment_state_path = glob.glob(os.path.join( + base, + experiment_path, + "experiment_state*.json")) + if not experiment_state_path: + # TODO(hartikainen): Print some warning? continue - else: - with open(experiment_state_path[0]) as f: - experiment_state = json.load(f) - # import ipdb; ipdb.set_trace() - experiment_collection[experiment_state_path[0]] = (pd.DataFrame(experiment_state["checkpoints"]), experiment_state["runner_data"], experiment_state["time_stamp"]) - - # total_ = pd.concat(experiment_collection.values()) - - from ray.tune.trial import Trial - all_values = [] - for experiment_path, (df, data, timestamp) in experiment_collection.items(): - status = {} - status["name"] = experiment_path - status["timestamp"] = timestamp - status["total_running"] = (df["status"] == Trial.RUNNING).sum() - status["total_terminated"] = (df["status"] == Trial.TERMINATED).sum() - status["total_errored"] = (df["status"] == Trial.ERROR).sum() - status["total_trials"] = df.shape[0] - all_values += [status] - - final_dataframe = pd.DataFrame(all_values) - print(final_dataframe.to_string()) + with open(experiment_state_path[0]) as f: + experiment_state = json.load(f) + + checkpoints = pd.DataFrame(experiment_state["checkpoints"]) + runner_data = experiment_state["runner_data"] + timestamp = experiment_state["timestamp"] + + experiment_data = { + "name": experiment_path, + "start_time": runner_data["_start_time"], + "timestamp": datetime.fromtimestamp(timestamp), + "total_trials": checkpoints.shape[0], + "running_trials": (checkpoints["status"] == Trial.RUNNING).sum(), + "terminated_trials": ( + checkpoints["status"] == Trial.TERMINATED).sum(), + "error_trials": (checkpoints["status"] == Trial.ERROR).sum(), + } + + experiment_data_collection.append(experiment_data) + + info_dataframe = pd.DataFrame(experiment_data_collection) + print(info_dataframe[list(info_keys)].to_string()) + + +@cli.command() +@click.argument("project_path", required=True, type=str) +def list_experiments(project_path): + _list_experiments(project_path) cli.add_command(list_trials, name="ls") diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 54d463f0b729..f97b4711eac5 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -112,7 +112,7 @@ def __init__(self, self._stop_queue = [] self._metadata_checkpoint_dir = metadata_checkpoint_dir - self._start_time = self.datetime.today() + self._start_time = datetime.today() self._session = self._start_time.strftime("%Y-%m-%d_%H-%M-%S") @classmethod