From 5f1d5a099e5623ef9ced3d00dc0b1433719388d0 Mon Sep 17 00:00:00 2001 From: Kristian Hartikainen Date: Thu, 7 Feb 2019 18:12:33 -0800 Subject: [PATCH 1/3] Fix TrialRunner._start_time --- python/ray/tune/trial_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/tune/trial_runner.py b/python/ray/tune/trial_runner.py index 54d463f0b729..f97b4711eac5 100644 --- a/python/ray/tune/trial_runner.py +++ b/python/ray/tune/trial_runner.py @@ -112,7 +112,7 @@ def __init__(self, self._stop_queue = [] self._metadata_checkpoint_dir = metadata_checkpoint_dir - self._start_time = self.datetime.today() + self._start_time = datetime.today() self._session = self._start_time.strftime("%Y-%m-%d_%H-%M-%S") @classmethod From d4a7b9afdbc56c28406cd6470514d77382fc8b48 Mon Sep 17 00:00:00 2001 From: Kristian Hartikainen Date: Thu, 7 Feb 2019 18:14:43 -0800 Subject: [PATCH 2/3] Clean tune cli --- python/ray/tune/scripts.py | 94 ++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 40 deletions(-) diff --git a/python/ray/tune/scripts.py b/python/ray/tune/scripts.py index 361dedfddffb..474fdd5f1554 100644 --- a/python/ray/tune/scripts.py +++ b/python/ray/tune/scripts.py @@ -3,13 +3,15 @@ from __future__ import print_function import click -import logging import glob import json import os +from datetime import datetime + + import pandas as pd -# from ray.tune.trial_runner import TrialRunner -import sys + +from ray.tune.trial import Trial def _flatten_dict(dt): @@ -31,60 +33,72 @@ def _flatten_dict(dt): def cli(): pass -@cli.command() -@click.argument("experiment_path", required=True, type=str) -def list_trials(experiment_path): - _list_trials(experiment_path) def _list_trials(experiment_path): experiment_path = os.path.expanduser(experiment_path) - print("start glob") globs = glob.glob(os.path.join(experiment_path, "experiment_state*.json")) - print(globs) filename = max(list(globs)) - print("found") + with open(filename) as f: experiment_state = json.load(f) - for trial_state in experiment_state["checkpoints"]: - print("{trial_name}\t{trial_id}\t{status}\t{num_failures}\t{logdir}".format( - **trial_state)) + checkpoints = pd.DataFrame.from_records(experiment_state['checkpoints']) + # TODO(hartikainen): The logdir is often too verbose to be viewed in a + # table. + checkpoints['logdir'] = checkpoints['logdir'].str.replace( + experiment_path, '') + + print(checkpoints.to_string()) + @cli.command() -@click.argument("project_path", required=True, type=str) -def list_experiments(project_path): - _list_experiments(project_path) +@click.argument("experiment_path", required=True, type=str) +def list_trials(experiment_path): + _list_trials(experiment_path) + def _list_experiments(project_path): base, experiment_paths, _ = list(os.walk(project_path))[0] # clean this - experiment_collection = {} + + experiment_data_collection = [] for experiment_path in experiment_paths: - experiment_state_path = glob.glob(os.path.join(base, experiment_path, "experiment_state*.json")) + experiment_state_path = glob.glob(os.path.join( + base, + experiment_path, + "experiment_state*.json")) + if not experiment_state_path: + # TODO(hartikainen): Print some warning? continue - else: - with open(experiment_state_path[0]) as f: - experiment_state = json.load(f) - # import ipdb; ipdb.set_trace() - experiment_collection[experiment_state_path[0]] = (pd.DataFrame(experiment_state["checkpoints"]), experiment_state["runner_data"], experiment_state["time_stamp"]) - - # total_ = pd.concat(experiment_collection.values()) - - from ray.tune.trial import Trial - all_values = [] - for experiment_path, (df, data, timestamp) in experiment_collection.items(): - status = {} - status["name"] = experiment_path - status["timestamp"] = timestamp - status["total_running"] = (df["status"] == Trial.RUNNING).sum() - status["total_terminated"] = (df["status"] == Trial.TERMINATED).sum() - status["total_errored"] = (df["status"] == Trial.ERROR).sum() - status["total_trials"] = df.shape[0] - all_values += [status] - - final_dataframe = pd.DataFrame(all_values) - print(final_dataframe.to_string()) + with open(experiment_state_path[0]) as f: + experiment_state = json.load(f) + + checkpoints = pd.DataFrame(experiment_state["checkpoints"]) + runner_data = experiment_state["runner_data"] + timestamp = experiment_state["timestamp"] + + experiment_data = { + "name": experiment_path, + "start_time": runner_data["_start_time"], + "timestamp": datetime.fromtimestamp(timestamp), + "total_trials": checkpoints.shape[0], + "running_trials": (checkpoints["status"] == Trial.RUNNING).sum(), + "terminated_trials": ( + checkpoints["status"] == Trial.TERMINATED).sum(), + "error_trials": (checkpoints["status"] == Trial.ERROR).sum(), + } + + experiment_data_collection.append(experiment_data) + + info_dataframe = pd.DataFrame(experiment_data_collection) + print(info_dataframe.to_string()) + + +@cli.command() +@click.argument("project_path", required=True, type=str) +def list_experiments(project_path): + _list_experiments(project_path) cli.add_command(list_trials, name="ls") From 400d382aa05f863ae1cc4d38d031c582becd9750 Mon Sep 17 00:00:00 2001 From: Kristian Hartikainen Date: Thu, 7 Feb 2019 18:22:51 -0800 Subject: [PATCH 3/3] Add explicit info keys for the experiment/project infos --- python/ray/tune/scripts.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/python/ray/tune/scripts.py b/python/ray/tune/scripts.py index 474fdd5f1554..72db1453af19 100644 --- a/python/ray/tune/scripts.py +++ b/python/ray/tune/scripts.py @@ -34,7 +34,25 @@ def cli(): pass -def _list_trials(experiment_path): +DEFAULT_EXPERIMENT_INFO_KEYS = ( + "trial_name", + "trial_id", + "status", + "num_failures", + "logdir" +) + +DEFAULT_PROJECT_INFO_KEYS = ( + "name", + "timestamp", + "total_trials", + "running_trials", + "terminated_trials", + "error_trials", +) + + +def _list_trials(experiment_path, info_keys=DEFAULT_EXPERIMENT_INFO_KEYS): experiment_path = os.path.expanduser(experiment_path) globs = glob.glob(os.path.join(experiment_path, "experiment_state*.json")) filename = max(list(globs)) @@ -48,7 +66,7 @@ def _list_trials(experiment_path): checkpoints['logdir'] = checkpoints['logdir'].str.replace( experiment_path, '') - print(checkpoints.to_string()) + print(checkpoints[list(info_keys)].to_string()) @cli.command() @@ -57,7 +75,7 @@ def list_trials(experiment_path): _list_trials(experiment_path) -def _list_experiments(project_path): +def _list_experiments(project_path, info_keys=DEFAULT_PROJECT_INFO_KEYS): base, experiment_paths, _ = list(os.walk(project_path))[0] # clean this experiment_data_collection = [] @@ -92,7 +110,7 @@ def _list_experiments(project_path): experiment_data_collection.append(experiment_data) info_dataframe = pd.DataFrame(experiment_data_collection) - print(info_dataframe.to_string()) + print(info_dataframe[list(info_keys)].to_string()) @cli.command()