From ccf6728cfc82178e782bb5066c68b2a5d7567507 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Thu, 18 Apr 2019 21:59:55 +0800 Subject: [PATCH 01/10] Add CommandLineArgs class. Refine Enable in worker runner. Fix Fix sh scripts/format.sh Add doc comment. Fix lint --- BUILD.bazel | 11 ++++ .../org/ray/runtime/runner/RunManager.java | 14 +++++ python/ray/services.py | 23 +++++++-- src/ray/raylet/main.cc | 51 ++++++++++--------- src/ray/util/command_line_args.cc | 48 +++++++++++++++++ src/ray/util/command_line_args.h | 47 +++++++++++++++++ src/ray/util/util_test.cc | 24 +++++++++ 7 files changed, 190 insertions(+), 28 deletions(-) create mode 100644 src/ray/util/command_line_args.cc create mode 100644 src/ray/util/command_line_args.h create mode 100644 src/ray/util/util_test.cc diff --git a/BUILD.bazel b/BUILD.bazel index 6dceb78a8626..23672f32a0c9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -149,6 +149,16 @@ cc_test( ], ) +cc_test( + name = "util_test", + srcs = ["src/ray/util/util_test.cc"], + copts = COPTS, + deps = [ + ":ray_util", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "task_dependency_manager_test", srcs = ["src/ray/raylet/task_dependency_manager_test.cc"], @@ -252,6 +262,7 @@ cc_library( exclude = [ "src/ray/util/logging_test.cc", "src/ray/util/signal_test.cc", + "src/ray/util/util_test.cc", ], ), hdrs = glob([ diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index 447da9d9e7f7..7128a6abdb29 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -253,19 +253,33 @@ private void startRaylet() { List command = ImmutableList.of( // The raylet executable file. getTempFile("/raylet").getAbsolutePath(), + "--raylet_socket_name", rayConfig.rayletSocketName, + "--store_socket_name", rayConfig.objectStoreSocketName, + "--object_manager_port", "0", // The object manager port. + "--node_manager_port", "0", // The node manager port. + "--node_ip_address", rayConfig.nodeIp, + "--redis_address", rayConfig.getRedisIp(), + "--redis_port", rayConfig.getRedisPort().toString(), + "--num_initial_workers", "0", // number of initial workers + "--maximum_startup_concurrency", String.valueOf(maximumStartupConcurrency), + "--static_resource_list", ResourceUtil.getResourcesStringFromMap(rayConfig.resources), + "--config_list", String.join(",", rayConfig.rayletConfigParameters), // The internal config list. + "--python_worker_command", buildPythonWorkerCommand(), // python worker command + "--java_worker_command", buildWorkerCommandRaylet(), // java worker command + "--redis_password", redisPasswordOption ); diff --git a/python/ray/services.py b/python/ray/services.py index b9d5867c4a96..e89386538a59 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -608,8 +608,8 @@ def start_redis(node_ip_address, # Put the redirect_worker_output bool in the Redis shard so that workers # can access it and know whether or not to redirect their output. - primary_redis_client.set("RedirectOutput", 1 - if redirect_worker_output else 0) + primary_redis_client.set("RedirectOutput", + 1 if redirect_worker_output else 0) # put the include_java bool to primary redis-server, so that other nodes # can access it and know whether or not to enable cross-languages. @@ -829,8 +829,8 @@ def _start_redis_instance(executable, # Increase the hard and soft limits for the redis client pubsub buffer to # 128MB. This is a hack to make it less likely for pubsub messages to be # dropped and for pubsub connections to therefore be killed. - cur_config = (redis_client.config_get("client-output-buffer-limit")[ - "client-output-buffer-limit"]) + cur_config = (redis_client.config_get("client-output-buffer-limit") + ["client-output-buffer-limit"]) cur_config_list = cur_config.split() assert len(cur_config_list) == 12 cur_config_list[8:] = ["pubsub", "134217728", "134217728", "60"] @@ -1177,20 +1177,35 @@ def start_raylet(redis_address, command = [ RAYLET_EXECUTABLE, + "--raylet_socket_name", raylet_name, + "--store_socket_name", plasma_store_name, + "--object_manager_port", str(object_manager_port), + "--node_manager_port", str(node_manager_port), + "--node_ip_address", node_ip_address, + "--redis_address", gcs_ip_address, + "--redis_port", gcs_port, + "--num_initial_workers", str(num_initial_workers), + "--maximum_startup_concurrency", str(maximum_startup_concurrency), + "--static_resource_list", resource_argument, + "--config_list", config_str, + "--python_worker_command", start_worker_command, + "--java_worker_command", java_worker_command, + "--redis_password", redis_password or "", + "--temp_dir", temp_dir, ] process_info = start_ray_process( diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 65923ae3d5e4..9c0aefa5222a 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -4,6 +4,7 @@ #include "ray/raylet/raylet.h" #include "ray/stats/stats.h" #include "ray/status.h" +#include "ray/util/command_line_args.h" #ifndef RAYLET_TEST @@ -21,30 +22,32 @@ int main(int argc, char *argv[]) { ray::RayLogLevel::INFO, /*log_dir=*/""); ray::RayLog::InstallFailureSignalHandler(); - RAY_CHECK(argc >= 14 && argc <= 19); - - const std::string raylet_socket_name = std::string(argv[1]); - const std::string store_socket_name = std::string(argv[2]); - int object_manager_port = std::stoi(argv[3]); - int node_manager_port = std::stoi(argv[4]); - const std::string node_ip_address = std::string(argv[5]); - const std::string redis_address = std::string(argv[6]); - int redis_port = std::stoi(argv[7]); - int num_initial_workers = std::stoi(argv[8]); - int maximum_startup_concurrency = std::stoi(argv[9]); - const std::string static_resource_list = std::string(argv[10]); - const std::string config_list = std::string(argv[11]); - const std::string python_worker_command = std::string(argv[12]); - const std::string java_worker_command = std::string(argv[13]); - const std::string redis_password = (argc >= 15 ? std::string(argv[14]) : ""); - const std::string temp_dir = (argc >= 16 ? std::string(argv[15]) : "/tmp/ray"); - const std::string disable_stats_str(argc >= 17 ? std::string(argv[16]) : "false"); - const bool disable_stats("true" == disable_stats_str); + const ray::CommandLineArgs command_line_args(argc, argv); + + const std::string raylet_socket_name = command_line_args.Get("raylet_socket_name"); + const std::string store_socket_name = command_line_args.Get("store_socket_name"); + int object_manager_port = std::stoi(command_line_args.Get("object_manager_port")); + int node_manager_port = std::stoi(command_line_args.Get("node_manager_port")); + const std::string node_ip_address = command_line_args.Get("node_ip_address"); + const std::string redis_address = command_line_args.Get("redis_address"); + int redis_port = std::stoi(command_line_args.Get("redis_port")); + int num_initial_workers = std::stoi(command_line_args.Get("num_initial_workers")); + int maximum_startup_concurrency = + std::stoi(command_line_args.Get("maximum_startup_concurrency")); + const std::string static_resource_list = command_line_args.Get("static_resource_list"); + const std::string config_list = command_line_args.Get("config_list"); + const std::string python_worker_command = + command_line_args.Get("python_worker_command"); + const std::string java_worker_command = command_line_args.Get("java_worker_command"); + const std::string redis_password = command_line_args.Get("redis_password", ""); + const std::string temp_dir = command_line_args.Get("temp_dir", "/tmp/ray"); + const std::string disable_stats_str = command_line_args.Get("disable_stats", "false"); + const bool disable_stats = ("true" == disable_stats_str); const std::string stat_address = - (argc >= 18 ? std::string(argv[17]) : "127.0.0.1:8888"); - const std::string disable_stdout_exporter_str(argc >= 19 ? std::string(argv[18]) - : "true"); - const bool disable_stdout_exporter("true" == disable_stdout_exporter_str); + command_line_args.Get("stat_address", "127.0.0.1:8888"); + const std::string disable_stdout_exporter_str = + command_line_args.Get("disable_stdout_exporter", "false"); + const bool disable_stdout_exporter = ("true" == disable_stdout_exporter_str); // Initialize stats. const ray::stats::TagsType global_tags = { @@ -151,7 +154,7 @@ int main(int argc, char *argv[]) { // instead of returning immediately. // We should stop the service and remove the local socket file. auto handler = [&main_service, &raylet_socket_name, &server, &gcs_client]( - const boost::system::error_code &error, int signal_number) { + const boost::system::error_code &error, int signal_number) { auto shutdown_callback = [&server, &main_service]() { server.reset(); main_service.stop(); diff --git a/src/ray/util/command_line_args.cc b/src/ray/util/command_line_args.cc new file mode 100644 index 000000000000..be60b06c4fa2 --- /dev/null +++ b/src/ray/util/command_line_args.cc @@ -0,0 +1,48 @@ +#include "ray/util/command_line_args.h" +#include "ray/util/logging.h" + +namespace ray { + +CommandLineArgs::CommandLineArgs(int argc, char **argv) { + RAY_CHECK(argc > 0) << "Number of command line arguments must be greater than 0."; + program_name_ = argv[0]; + for (size_t i = 1; i < argc; i += 2) { + RAY_CHECK(nullptr != argv[i]); + const std::string key(argv[i]); + if (key.size() <= 2 || "--" != key.substr(0, 2)) { + RAY_LOG(FATAL) << "The option must be started with `--`"; + } + + std::string value(argv[i + 1]); + RAY_CHECK(value.size() >= 0) << "The argument must be as pair like: " + << "--skip true"; + if (value.front() == '\"') { + RAY_CHECK(value.size() >= 2 && value.back() == '\"') + << "If the argument starts with \", it also must end with \"."; + value = value.substr(1, value.size() - 2); + } + arguments_.emplace(key.substr(2, key.size() - 2), value); + } +} + +std::string CommandLineArgs::GetProgramName() const { return program_name_; } + +bool CommandLineArgs::Contains(const std::string &key) const { + return arguments_.end() != arguments_.find(key); +} + +std::string CommandLineArgs::Get(const std::string &key) const { + RAY_CHECK(Contains(key)); + return arguments_.at(key); +} + +std::string CommandLineArgs::Get(const std::string &key, + const std::string &default_value) const { + if (Contains(key)) { + return arguments_.at(key); + } + + return default_value; +} + +} // namespace ray diff --git a/src/ray/util/command_line_args.h b/src/ray/util/command_line_args.h new file mode 100644 index 000000000000..c464d660cd6d --- /dev/null +++ b/src/ray/util/command_line_args.h @@ -0,0 +1,47 @@ +#ifndef RAY_UTIL_COMMAND_LINE_ARGS_H +#define RAY_UTIL_COMMAND_LINE_ARGS_H + +#include +#include + +namespace ray { + +/// A simple command line arguments util. +/// +/// It requires the options of command are as pair, like: +/// ./main --enable_x true --disable_y "false" +/// If the value of one option is empty, it should be specified a empty string, like: +/// ./main --enable_x "" +class CommandLineArgs final { + public: + /// Constructor. Note that the `argc` must equals the size of `argv` array. + CommandLineArgs(int args, char **argv); + + /// Get the program name of this command line statement. + std::string GetProgramName() const; + + /// Whether this command line statement contains the option `key`. + bool Contains(const std::string &key) const; + + /// Get the option value of the given `key`. + /// + /// Note that it will check fail if this command line statement doesn't contains the + /// `key`. + std::string Get(const std::string &key) const; + + /// Get the option value of the given `key`. + /// + /// Note that it will return the `default_value` if this command line statement doesn't + /// contains the `key`. + std::string Get(const std::string &key, const std::string &default_value) const; + + private: + /// The name of this program. + std::string program_name_; + /// The arguments of this command line. + std::unordered_map arguments_; +}; + +} // namespace ray + +#endif // RAY_UTIL_COMMAND_LINE_ARGS_H diff --git a/src/ray/util/util_test.cc b/src/ray/util/util_test.cc new file mode 100644 index 000000000000..60a0de744cb8 --- /dev/null +++ b/src/ray/util/util_test.cc @@ -0,0 +1,24 @@ +#include "gtest/gtest.h" +#include "ray/util/command_line_args.h" +#include "ray/util/logging.h" + +#include + +namespace ray { + +TEST(CommandLineArgsTest, TestParse) { + char *argv[] = {"test", "--k1", "v1", "--k2", "\"my_v2\"", "--k3", "\"\"", nullptr}; + CommandLineArgs command_line_args(7, argv); + ASSERT_EQ("test", command_line_args.GetProgramName()); + ASSERT_EQ("v1", command_line_args.Get("k1")); + ASSERT_EQ("my_v2", command_line_args.Get("k2")); + ASSERT_EQ("", command_line_args.Get("k3")); + ASSERT_EQ("default_v4", command_line_args.Get("k4", "default_v4")); +} + +} // namespace ray + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 8276ce1e2a0ce05b793e7387887ff9a356a36d29 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 14:49:49 +0800 Subject: [PATCH 02/10] Use gflag --- BUILD.bazel | 11 +---- src/ray/raylet/main.cc | 69 +++++++++++++++++++------------ src/ray/util/command_line_args.cc | 48 --------------------- src/ray/util/command_line_args.h | 47 --------------------- src/ray/util/util_test.cc | 24 ----------- 5 files changed, 43 insertions(+), 156 deletions(-) delete mode 100644 src/ray/util/command_line_args.cc delete mode 100644 src/ray/util/command_line_args.h delete mode 100644 src/ray/util/util_test.cc diff --git a/BUILD.bazel b/BUILD.bazel index 23672f32a0c9..bf1f1b0e1bd4 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -15,6 +15,7 @@ cc_binary( deps = [ ":ray_util", ":raylet_lib", + "@com_github_gflags_gflags//:gflags", ], ) @@ -149,16 +150,6 @@ cc_test( ], ) -cc_test( - name = "util_test", - srcs = ["src/ray/util/util_test.cc"], - copts = COPTS, - deps = [ - ":ray_util", - "@com_google_googletest//:gtest_main", - ], -) - cc_test( name = "task_dependency_manager_test", srcs = ["src/ray/raylet/task_dependency_manager_test.cc"], diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 9c0aefa5222a..15ab89bdeb54 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -4,7 +4,27 @@ #include "ray/raylet/raylet.h" #include "ray/stats/stats.h" #include "ray/status.h" -#include "ray/util/command_line_args.h" + +#include "gflags/gflags.h" + +DEFINE_string(raylet_socket_name, "", "The socket name of raylet."); +DEFINE_string(store_socket_name, "", "The socket name of object store."); +DEFINE_int32(object_manager_port, -1, "The port of object manager."); +DEFINE_int32(node_manager_port, -1, "The port of node manager."); +DEFINE_string(node_ip_address, "", "The ip address of this node."); +DEFINE_string(redis_address, "", "The ip address of redis server."); +DEFINE_int32(redis_port, -1, "The port of redis server."); +DEFINE_int32(num_initial_workers, 0, "Number of initial workers."); +DEFINE_int32(maximum_startup_concurrency, 1, "Maximum startup concurrency"); +DEFINE_string(static_resource_list, "", "The static resource list of this node."); +DEFINE_string(config_list, "", "The raylet config list of this node."); +DEFINE_string(python_worker_command, "", "Python worker command."); +DEFINE_string(java_worker_command, "", "Java worker command."); +DEFINE_string(redis_password, "", "The password of redis."); +DEFINE_string(temp_dir, "", "Temporary directory."); +DEFINE_bool(disable_stats, false, "Whether disable the stats."); +DEFINE_string(stat_address, "127.0.0.1:8888", "The address that we report metrics to."); +DEFINE_bool(disable_stdout_exporter, true, "Whether disable the stdout exporter for stats."); #ifndef RAYLET_TEST @@ -22,32 +42,27 @@ int main(int argc, char *argv[]) { ray::RayLogLevel::INFO, /*log_dir=*/""); ray::RayLog::InstallFailureSignalHandler(); - const ray::CommandLineArgs command_line_args(argc, argv); - - const std::string raylet_socket_name = command_line_args.Get("raylet_socket_name"); - const std::string store_socket_name = command_line_args.Get("store_socket_name"); - int object_manager_port = std::stoi(command_line_args.Get("object_manager_port")); - int node_manager_port = std::stoi(command_line_args.Get("node_manager_port")); - const std::string node_ip_address = command_line_args.Get("node_ip_address"); - const std::string redis_address = command_line_args.Get("redis_address"); - int redis_port = std::stoi(command_line_args.Get("redis_port")); - int num_initial_workers = std::stoi(command_line_args.Get("num_initial_workers")); - int maximum_startup_concurrency = - std::stoi(command_line_args.Get("maximum_startup_concurrency")); - const std::string static_resource_list = command_line_args.Get("static_resource_list"); - const std::string config_list = command_line_args.Get("config_list"); - const std::string python_worker_command = - command_line_args.Get("python_worker_command"); - const std::string java_worker_command = command_line_args.Get("java_worker_command"); - const std::string redis_password = command_line_args.Get("redis_password", ""); - const std::string temp_dir = command_line_args.Get("temp_dir", "/tmp/ray"); - const std::string disable_stats_str = command_line_args.Get("disable_stats", "false"); - const bool disable_stats = ("true" == disable_stats_str); - const std::string stat_address = - command_line_args.Get("stat_address", "127.0.0.1:8888"); - const std::string disable_stdout_exporter_str = - command_line_args.Get("disable_stdout_exporter", "false"); - const bool disable_stdout_exporter = ("true" == disable_stdout_exporter_str); + + gflags::ParseCommandLineFlags(&argc, &argv, true); + const std::string raylet_socket_name = FLAGS_raylet_socket_name; + const std::string store_socket_name = FLAGS_store_socket_name; + const int object_manager_port = static_cast(FLAGS_object_manager_port); + const int node_manager_port = static_cast(FLAGS_node_manager_port); + const std::string node_ip_address = FLAGS_node_ip_address; + const std::string redis_address = FLAGS_redis_address; + const int redis_port = static_cast(FLAGS_redis_port); + const int num_initial_workers = static_cast(FLAGS_num_initial_workers); + const int maximum_startup_concurrency = static_cast(FLAGS_maximum_startup_concurrency); + const std::string static_resource_list = FLAGS_static_resource_list; + const std::string config_list = FLAGS_config_list; + const std::string python_worker_command = FLAGS_python_worker_command; + const std::string java_worker_command = FLAGS_java_worker_command; + const std::string redis_password = FLAGS_redis_password; + const std::string temp_dir = FLAGS_temp_dir; + const bool disable_stats = FLAGS_disable_stats; + const std::string stat_address = FLAGS_stat_address; + const bool disable_stdout_exporter = FLAGS_disable_stdout_exporter; + gflags::ShutDownCommandLineFlags(); // Initialize stats. const ray::stats::TagsType global_tags = { diff --git a/src/ray/util/command_line_args.cc b/src/ray/util/command_line_args.cc deleted file mode 100644 index be60b06c4fa2..000000000000 --- a/src/ray/util/command_line_args.cc +++ /dev/null @@ -1,48 +0,0 @@ -#include "ray/util/command_line_args.h" -#include "ray/util/logging.h" - -namespace ray { - -CommandLineArgs::CommandLineArgs(int argc, char **argv) { - RAY_CHECK(argc > 0) << "Number of command line arguments must be greater than 0."; - program_name_ = argv[0]; - for (size_t i = 1; i < argc; i += 2) { - RAY_CHECK(nullptr != argv[i]); - const std::string key(argv[i]); - if (key.size() <= 2 || "--" != key.substr(0, 2)) { - RAY_LOG(FATAL) << "The option must be started with `--`"; - } - - std::string value(argv[i + 1]); - RAY_CHECK(value.size() >= 0) << "The argument must be as pair like: " - << "--skip true"; - if (value.front() == '\"') { - RAY_CHECK(value.size() >= 2 && value.back() == '\"') - << "If the argument starts with \", it also must end with \"."; - value = value.substr(1, value.size() - 2); - } - arguments_.emplace(key.substr(2, key.size() - 2), value); - } -} - -std::string CommandLineArgs::GetProgramName() const { return program_name_; } - -bool CommandLineArgs::Contains(const std::string &key) const { - return arguments_.end() != arguments_.find(key); -} - -std::string CommandLineArgs::Get(const std::string &key) const { - RAY_CHECK(Contains(key)); - return arguments_.at(key); -} - -std::string CommandLineArgs::Get(const std::string &key, - const std::string &default_value) const { - if (Contains(key)) { - return arguments_.at(key); - } - - return default_value; -} - -} // namespace ray diff --git a/src/ray/util/command_line_args.h b/src/ray/util/command_line_args.h deleted file mode 100644 index c464d660cd6d..000000000000 --- a/src/ray/util/command_line_args.h +++ /dev/null @@ -1,47 +0,0 @@ -#ifndef RAY_UTIL_COMMAND_LINE_ARGS_H -#define RAY_UTIL_COMMAND_LINE_ARGS_H - -#include -#include - -namespace ray { - -/// A simple command line arguments util. -/// -/// It requires the options of command are as pair, like: -/// ./main --enable_x true --disable_y "false" -/// If the value of one option is empty, it should be specified a empty string, like: -/// ./main --enable_x "" -class CommandLineArgs final { - public: - /// Constructor. Note that the `argc` must equals the size of `argv` array. - CommandLineArgs(int args, char **argv); - - /// Get the program name of this command line statement. - std::string GetProgramName() const; - - /// Whether this command line statement contains the option `key`. - bool Contains(const std::string &key) const; - - /// Get the option value of the given `key`. - /// - /// Note that it will check fail if this command line statement doesn't contains the - /// `key`. - std::string Get(const std::string &key) const; - - /// Get the option value of the given `key`. - /// - /// Note that it will return the `default_value` if this command line statement doesn't - /// contains the `key`. - std::string Get(const std::string &key, const std::string &default_value) const; - - private: - /// The name of this program. - std::string program_name_; - /// The arguments of this command line. - std::unordered_map arguments_; -}; - -} // namespace ray - -#endif // RAY_UTIL_COMMAND_LINE_ARGS_H diff --git a/src/ray/util/util_test.cc b/src/ray/util/util_test.cc deleted file mode 100644 index 60a0de744cb8..000000000000 --- a/src/ray/util/util_test.cc +++ /dev/null @@ -1,24 +0,0 @@ -#include "gtest/gtest.h" -#include "ray/util/command_line_args.h" -#include "ray/util/logging.h" - -#include - -namespace ray { - -TEST(CommandLineArgsTest, TestParse) { - char *argv[] = {"test", "--k1", "v1", "--k2", "\"my_v2\"", "--k3", "\"\"", nullptr}; - CommandLineArgs command_line_args(7, argv); - ASSERT_EQ("test", command_line_args.GetProgramName()); - ASSERT_EQ("v1", command_line_args.Get("k1")); - ASSERT_EQ("my_v2", command_line_args.Get("k2")); - ASSERT_EQ("", command_line_args.Get("k3")); - ASSERT_EQ("default_v4", command_line_args.Get("k4", "default_v4")); -} - -} // namespace ray - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} From a7e98a515badaef1be562f7f3789366a88b3bec4 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 15:33:39 +0800 Subject: [PATCH 03/10] Fix --- .../org/ray/runtime/runner/RunManager.java | 42 +++++++------------ src/ray/raylet/main.cc | 2 + 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index 7128a6abdb29..3115ff90a58e 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -253,34 +253,20 @@ private void startRaylet() { List command = ImmutableList.of( // The raylet executable file. getTempFile("/raylet").getAbsolutePath(), - "--raylet_socket_name", - rayConfig.rayletSocketName, - "--store_socket_name", - rayConfig.objectStoreSocketName, - "--object_manager_port", - "0", // The object manager port. - "--node_manager_port", - "0", // The node manager port. - "--node_ip_address", - rayConfig.nodeIp, - "--redis_address", - rayConfig.getRedisIp(), - "--redis_port", - rayConfig.getRedisPort().toString(), - "--num_initial_workers", - "0", // number of initial workers - "--maximum_startup_concurrency", - String.valueOf(maximumStartupConcurrency), - "--static_resource_list", - ResourceUtil.getResourcesStringFromMap(rayConfig.resources), - "--config_list", - String.join(",", rayConfig.rayletConfigParameters), // The internal config list. - "--python_worker_command", - buildPythonWorkerCommand(), // python worker command - "--java_worker_command", - buildWorkerCommandRaylet(), // java worker command - "--redis_password", - redisPasswordOption + String.format("-raylet_socket_name=\"%s\"", rayConfig.rayletSocketName), + String.format("-store_socket_name=\"%s\"", rayConfig.objectStoreSocketName), + String.format("-object_manager_port=%d", 0), // The object manager port. + String.format("-node_manager_port=%d", 0), // The node manager port. + String.format("-node_ip_address=\"%s\"",rayConfig.nodeIp), + String.format("-redis_address=\"%s\"", rayConfig.getRedisIp()), + String.format("-redis_port=%d", rayConfig.getRedisPort()), + String.format("-num_initial_workers=%d", 0), // number of initial workers + String.format("-maximum_startup_concurrency=%d", 0), + String.format("-static_resource_list=\"%s\"", ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), + String.format("-config_list=\"%s\"", String.join(",", rayConfig.rayletConfigParameters)), + String.format("-python_worker_command=\"%s\"", buildPythonWorkerCommand()), + String.format("-java_worker_command=\"%s\"", buildWorkerCommandRaylet()), + String.format("-redis_password=\"%s\"", redisPasswordOption) ); startProcess(command, null, "raylet"); diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 15ab89bdeb54..630070802e29 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -64,6 +64,8 @@ int main(int argc, char *argv[]) { const bool disable_stdout_exporter = FLAGS_disable_stdout_exporter; gflags::ShutDownCommandLineFlags(); + RAY_LOG(INFO) << ">>>> config list:" << config_list; + // Initialize stats. const ray::stats::TagsType global_tags = { {ray::stats::JobNameKey, "raylet"}, From d0edc948087e5750ab5a9637c081038a5860a07f Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 16:08:27 +0800 Subject: [PATCH 04/10] Update commands. --- .../org/ray/runtime/runner/RunManager.java | 21 ++++----- .../ray/api/benchmark/MicroBenchmarks.java | 4 +- python/ray/services.py | 45 +++++++------------ src/ray/raylet/main.cc | 8 ++-- 4 files changed, 32 insertions(+), 46 deletions(-) diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index 3115ff90a58e..e20baadbce67 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -253,20 +253,21 @@ private void startRaylet() { List command = ImmutableList.of( // The raylet executable file. getTempFile("/raylet").getAbsolutePath(), - String.format("-raylet_socket_name=\"%s\"", rayConfig.rayletSocketName), - String.format("-store_socket_name=\"%s\"", rayConfig.objectStoreSocketName), + String.format("-raylet_socket_name=%s", rayConfig.rayletSocketName), + String.format("-store_socket_name=%s", rayConfig.objectStoreSocketName), String.format("-object_manager_port=%d", 0), // The object manager port. String.format("-node_manager_port=%d", 0), // The node manager port. - String.format("-node_ip_address=\"%s\"",rayConfig.nodeIp), - String.format("-redis_address=\"%s\"", rayConfig.getRedisIp()), + String.format("-node_ip_address=%s",rayConfig.nodeIp), + String.format("-redis_address=%s", rayConfig.getRedisIp()), String.format("-redis_port=%d", rayConfig.getRedisPort()), String.format("-num_initial_workers=%d", 0), // number of initial workers - String.format("-maximum_startup_concurrency=%d", 0), - String.format("-static_resource_list=\"%s\"", ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), - String.format("-config_list=\"%s\"", String.join(",", rayConfig.rayletConfigParameters)), - String.format("-python_worker_command=\"%s\"", buildPythonWorkerCommand()), - String.format("-java_worker_command=\"%s\"", buildWorkerCommandRaylet()), - String.format("-redis_password=\"%s\"", redisPasswordOption) + String.format("-maximum_startup_concurrency=%d", maximumStartupConcurrency), + String.format("-static_resource_list=%s", + ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), + String.format("-config_list=%s", String.join(",", rayConfig.rayletConfigParameters)), + String.format("-python_worker_command=%s", buildPythonWorkerCommand()), + String.format("-java_worker_command=%s", buildWorkerCommandRaylet()), + String.format("-redis_password=%s", redisPasswordOption) ); startProcess(command, null, "raylet"); diff --git a/java/test/src/main/java/org/ray/api/benchmark/MicroBenchmarks.java b/java/test/src/main/java/org/ray/api/benchmark/MicroBenchmarks.java index 8a6d90562fbd..6ccec9a2e72b 100644 --- a/java/test/src/main/java/org/ray/api/benchmark/MicroBenchmarks.java +++ b/java/test/src/main/java/org/ray/api/benchmark/MicroBenchmarks.java @@ -19,8 +19,8 @@ private static void time(Runnable runnable, int numRepeats, String name) { runnable.run(); } final long duration = System.nanoTime() - start; - LOGGER.info( - "Benchmark \"{}\" finished, repeated {} times, total duration {} ms, average duration {} ns.", + LOGGER.info("Benchmark \"{}\" finished, repeated {} times, " + + "total duration {} ms, average duration {} ns.", name, numRepeats, duration / 1_000_000, duration / numRepeats); } diff --git a/python/ray/services.py b/python/ray/services.py index e89386538a59..1586c82b84c8 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -1177,36 +1177,21 @@ def start_raylet(redis_address, command = [ RAYLET_EXECUTABLE, - "--raylet_socket_name", - raylet_name, - "--store_socket_name", - plasma_store_name, - "--object_manager_port", - str(object_manager_port), - "--node_manager_port", - str(node_manager_port), - "--node_ip_address", - node_ip_address, - "--redis_address", - gcs_ip_address, - "--redis_port", - gcs_port, - "--num_initial_workers", - str(num_initial_workers), - "--maximum_startup_concurrency", - str(maximum_startup_concurrency), - "--static_resource_list", - resource_argument, - "--config_list", - config_str, - "--python_worker_command", - start_worker_command, - "--java_worker_command", - java_worker_command, - "--redis_password", - redis_password or "", - "--temp_dir", - temp_dir, + "-raylet_socket_name={}".format(raylet_name), + "-store_socket_name={}".format(plasma_store_name), + "-object_manager_port={}".format(object_manager_port), + "-node_manager_port={}".format(node_manager_port), + "-node_ip_address={}".format(node_ip_address), + "-redis_address={}".format(gcs_ip_address), + "-redis_port={}".format(gcs_port), + "-num_initial_workers={}".format(num_initial_workers), + "-maximum_startup_concurrency={}".format(maximum_startup_concurrency), + "-static_resource_list={}".format(resource_argument), + "-config_list={}".format(config_str), + "-python_worker_command={}".format(start_worker_command), + "-java_worker_command={}".format(java_worker_command), + "-redis_password={}".format(redis_password or ""), + "-temp_dir={}".format(temp_dir), ] process_info = start_ray_process( command, diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index 630070802e29..abfb6f6c22d0 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -24,7 +24,8 @@ DEFINE_string(redis_password, "", "The password of redis."); DEFINE_string(temp_dir, "", "Temporary directory."); DEFINE_bool(disable_stats, false, "Whether disable the stats."); DEFINE_string(stat_address, "127.0.0.1:8888", "The address that we report metrics to."); -DEFINE_bool(disable_stdout_exporter, true, "Whether disable the stdout exporter for stats."); +DEFINE_bool(disable_stdout_exporter, true, + "Whether disable the stdout exporter for stats."); #ifndef RAYLET_TEST @@ -52,7 +53,8 @@ int main(int argc, char *argv[]) { const std::string redis_address = FLAGS_redis_address; const int redis_port = static_cast(FLAGS_redis_port); const int num_initial_workers = static_cast(FLAGS_num_initial_workers); - const int maximum_startup_concurrency = static_cast(FLAGS_maximum_startup_concurrency); + const int maximum_startup_concurrency = + static_cast(FLAGS_maximum_startup_concurrency); const std::string static_resource_list = FLAGS_static_resource_list; const std::string config_list = FLAGS_config_list; const std::string python_worker_command = FLAGS_python_worker_command; @@ -64,8 +66,6 @@ int main(int argc, char *argv[]) { const bool disable_stdout_exporter = FLAGS_disable_stdout_exporter; gflags::ShutDownCommandLineFlags(); - RAY_LOG(INFO) << ">>>> config list:" << config_list; - // Initialize stats. const ray::stats::TagsType global_tags = { {ray::stats::JobNameKey, "raylet"}, From 069eabff282324059707ab067af69caee565952b Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 16:27:02 +0800 Subject: [PATCH 05/10] add gflags to raylet_monitor --- python/ray/services.py | 7 ++++++- src/ray/raylet/monitor_main.cc | 18 +++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python/ray/services.py b/python/ray/services.py index 1586c82b84c8..e6d9f56b3b59 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -1555,7 +1555,12 @@ def start_raylet_monitor(redis_address, redis_password = redis_password or "" config = config or {} config_str = ",".join(["{},{}".format(*kv) for kv in config.items()]) - command = [RAYLET_MONITOR_EXECUTABLE, gcs_ip_address, gcs_port, config_str] + command = [ + RAYLET_MONITOR_EXECUTABLE, + "-redis_address={}".format(gcs_ip_address), + "-redis_port={}".format(gcs_port), + "-config_list={}".format(config_str), + ] if redis_password: command += [redis_password] process_info = start_ray_process( diff --git a/src/ray/raylet/monitor_main.cc b/src/ray/raylet/monitor_main.cc index b352bca3828d..1539f01d1a25 100644 --- a/src/ray/raylet/monitor_main.cc +++ b/src/ray/raylet/monitor_main.cc @@ -4,17 +4,25 @@ #include "ray/raylet/monitor.h" #include "ray/util/util.h" +#include "gflags/gflags.h" + +DEFINE_string(redis_address, "", "The ip address of redis."); +DEFINE_int32(redis_port, -1, "The port of redis."); +DEFINE_string(config_list, "", "The config list of raylet."); +DEFINE_string(redis_password, "", "The password of redis."); + int main(int argc, char *argv[]) { InitShutdownRAII ray_log_shutdown_raii(ray::RayLog::StartRayLog, ray::RayLog::ShutDownRayLog, argv[0], ray::RayLogLevel::INFO, /*log_dir=*/""); ray::RayLog::InstallFailureSignalHandler(); - RAY_CHECK(argc == 4 || argc == 5); - const std::string redis_address = std::string(argv[1]); - int redis_port = std::stoi(argv[2]); - const std::string config_list = std::string(argv[3]); - const std::string redis_password = (argc == 5 ? std::string(argv[4]) : ""); + gflags::ParseCommandLineFlags(&argc, &argv, true); + const std::string redis_address = FLAGS_redis_address; + const int redis_port = static_cast(FLAGS_redis_port); + const std::string config_list = FLAGS_config_list; + const std::string redis_password = FLAGS_redis_password; + gflags::ShutDownCommandLineFlags(); std::unordered_map raylet_config; From f094cc294c977d35aad06716a9a6edc8b71a1488 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 17:45:34 +0800 Subject: [PATCH 06/10] Fix lint --- src/ray/raylet/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index abfb6f6c22d0..c78b69ad6e43 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -171,7 +171,7 @@ int main(int argc, char *argv[]) { // instead of returning immediately. // We should stop the service and remove the local socket file. auto handler = [&main_service, &raylet_socket_name, &server, &gcs_client]( - const boost::system::error_code &error, int signal_number) { + const boost::system::error_code &error, int signal_number) { auto shutdown_callback = [&server, &main_service]() { server.reset(); main_service.stop(); From 3b53d1d0c9a1f84e3b94c541b41597f4fe314761 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 21:41:37 +0800 Subject: [PATCH 07/10] Use double dashes. --- .../org/ray/runtime/runner/RunManager.java | 28 +++++++-------- python/ray/services.py | 36 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index e20baadbce67..15240e43e234 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -253,21 +253,21 @@ private void startRaylet() { List command = ImmutableList.of( // The raylet executable file. getTempFile("/raylet").getAbsolutePath(), - String.format("-raylet_socket_name=%s", rayConfig.rayletSocketName), - String.format("-store_socket_name=%s", rayConfig.objectStoreSocketName), - String.format("-object_manager_port=%d", 0), // The object manager port. - String.format("-node_manager_port=%d", 0), // The node manager port. - String.format("-node_ip_address=%s",rayConfig.nodeIp), - String.format("-redis_address=%s", rayConfig.getRedisIp()), - String.format("-redis_port=%d", rayConfig.getRedisPort()), - String.format("-num_initial_workers=%d", 0), // number of initial workers - String.format("-maximum_startup_concurrency=%d", maximumStartupConcurrency), - String.format("-static_resource_list=%s", + String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName), + String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName), + String.format("--object_manager_port=%d", 0), // The object manager port. + String.format("--node_manager_port=%d", 0), // The node manager port. + String.format("--node_ip_address=%s",rayConfig.nodeIp), + String.format("--redis_address=%s", rayConfig.getRedisIp()), + String.format("--redis_port=%d", rayConfig.getRedisPort()), + String.format("--num_initial_workers=%d", 0), // number of initial workers + String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency), + String.format("--static_resource_list=%s", ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), - String.format("-config_list=%s", String.join(",", rayConfig.rayletConfigParameters)), - String.format("-python_worker_command=%s", buildPythonWorkerCommand()), - String.format("-java_worker_command=%s", buildWorkerCommandRaylet()), - String.format("-redis_password=%s", redisPasswordOption) + String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)), + String.format("--python_worker_command=%s", buildPythonWorkerCommand()), + String.format("--java_worker_command=%s", buildWorkerCommandRaylet()), + String.format("--redis_password=%s", redisPasswordOption) ); startProcess(command, null, "raylet"); diff --git a/python/ray/services.py b/python/ray/services.py index e6d9f56b3b59..5727979c9e49 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -1177,21 +1177,21 @@ def start_raylet(redis_address, command = [ RAYLET_EXECUTABLE, - "-raylet_socket_name={}".format(raylet_name), - "-store_socket_name={}".format(plasma_store_name), - "-object_manager_port={}".format(object_manager_port), - "-node_manager_port={}".format(node_manager_port), - "-node_ip_address={}".format(node_ip_address), - "-redis_address={}".format(gcs_ip_address), - "-redis_port={}".format(gcs_port), - "-num_initial_workers={}".format(num_initial_workers), - "-maximum_startup_concurrency={}".format(maximum_startup_concurrency), - "-static_resource_list={}".format(resource_argument), - "-config_list={}".format(config_str), - "-python_worker_command={}".format(start_worker_command), - "-java_worker_command={}".format(java_worker_command), - "-redis_password={}".format(redis_password or ""), - "-temp_dir={}".format(temp_dir), + "--raylet_socket_name={}".format(raylet_name), + "--store_socket_name={}".format(plasma_store_name), + "--object_manager_port={}".format(object_manager_port), + "--node_manager_port={}".format(node_manager_port), + "--node_ip_address={}".format(node_ip_address), + "--redis_address={}".format(gcs_ip_address), + "--redis_port={}".format(gcs_port), + "--num_initial_workers={}".format(num_initial_workers), + "--maximum_startup_concurrency={}".format(maximum_startup_concurrency), + "--static_resource_list={}".format(resource_argument), + "--config_list={}".format(config_str), + "--python_worker_command={}".format(start_worker_command), + "--java_worker_command={}".format(java_worker_command), + "--redis_password={}".format(redis_password or ""), + "--temp_dir={}".format(temp_dir), ] process_info = start_ray_process( command, @@ -1557,9 +1557,9 @@ def start_raylet_monitor(redis_address, config_str = ",".join(["{},{}".format(*kv) for kv in config.items()]) command = [ RAYLET_MONITOR_EXECUTABLE, - "-redis_address={}".format(gcs_ip_address), - "-redis_port={}".format(gcs_port), - "-config_list={}".format(config_str), + "--redis_address={}".format(gcs_ip_address), + "--redis_port={}".format(gcs_port), + "--config_list={}".format(config_str), ] if redis_password: command += [redis_password] From bd0955a8ef56b9734af88671e0544a77d5de45e7 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Tue, 23 Apr 2019 21:47:09 +0800 Subject: [PATCH 08/10] address comments. --- src/ray/raylet/main.cc | 8 ++++---- src/ray/stats/stats.h | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ray/raylet/main.cc b/src/ray/raylet/main.cc index c78b69ad6e43..d5b498f23a0b 100644 --- a/src/ray/raylet/main.cc +++ b/src/ray/raylet/main.cc @@ -24,8 +24,8 @@ DEFINE_string(redis_password, "", "The password of redis."); DEFINE_string(temp_dir, "", "Temporary directory."); DEFINE_bool(disable_stats, false, "Whether disable the stats."); DEFINE_string(stat_address, "127.0.0.1:8888", "The address that we report metrics to."); -DEFINE_bool(disable_stdout_exporter, true, - "Whether disable the stdout exporter for stats."); +DEFINE_bool(enable_stdout_exporter, false, + "Whether enable the stdout exporter for stats."); #ifndef RAYLET_TEST @@ -63,7 +63,7 @@ int main(int argc, char *argv[]) { const std::string temp_dir = FLAGS_temp_dir; const bool disable_stats = FLAGS_disable_stats; const std::string stat_address = FLAGS_stat_address; - const bool disable_stdout_exporter = FLAGS_disable_stdout_exporter; + const bool enable_stdout_exporter = FLAGS_enable_stdout_exporter; gflags::ShutDownCommandLineFlags(); // Initialize stats. @@ -71,7 +71,7 @@ int main(int argc, char *argv[]) { {ray::stats::JobNameKey, "raylet"}, {ray::stats::VersionKey, "0.7.0"}, {ray::stats::NodeAddressKey, node_ip_address}}; - ray::stats::Init(stat_address, global_tags, disable_stats, disable_stdout_exporter); + ray::stats::Init(stat_address, global_tags, disable_stats, enable_stdout_exporter); // Configuration for the node manager. ray::raylet::NodeManagerConfig node_manager_config; diff --git a/src/ray/stats/stats.h b/src/ray/stats/stats.h index 08a8260a91c8..03b754980ed5 100644 --- a/src/ray/stats/stats.h +++ b/src/ray/stats/stats.h @@ -23,7 +23,7 @@ namespace stats { /// Initialize stats. static void Init(const std::string &address, const TagsType &global_tags, - bool disable_stats = false, bool disable_stdout_exporter = true) { + bool disable_stats = false, bool enable_stdout_exporter = false) { StatsConfig::instance().SetIsDisableStats(disable_stats); if (disable_stats) { RAY_LOG(INFO) << "Disabled stats."; @@ -36,7 +36,7 @@ static void Init(const std::string &address, const TagsType &global_tags, static auto exporter = std::make_shared(); - if (!disable_stdout_exporter) { + if (enable_stdout_exporter) { // Enable stdout exporter by default. opencensus::exporters::stats::StdoutExporter::Register(); } From 40c9b1d4d00295dcc11769959097ac67a86460c0 Mon Sep 17 00:00:00 2001 From: jovany-wang Date: Wed, 24 Apr 2019 10:58:57 +0800 Subject: [PATCH 09/10] Fix linting --- python/ray/services.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/ray/services.py b/python/ray/services.py index 5727979c9e49..3969019fcf6d 100644 --- a/python/ray/services.py +++ b/python/ray/services.py @@ -608,8 +608,8 @@ def start_redis(node_ip_address, # Put the redirect_worker_output bool in the Redis shard so that workers # can access it and know whether or not to redirect their output. - primary_redis_client.set("RedirectOutput", - 1 if redirect_worker_output else 0) + primary_redis_client.set("RedirectOutput", 1 + if redirect_worker_output else 0) # put the include_java bool to primary redis-server, so that other nodes # can access it and know whether or not to enable cross-languages. @@ -829,8 +829,8 @@ def _start_redis_instance(executable, # Increase the hard and soft limits for the redis client pubsub buffer to # 128MB. This is a hack to make it less likely for pubsub messages to be # dropped and for pubsub connections to therefore be killed. - cur_config = (redis_client.config_get("client-output-buffer-limit") - ["client-output-buffer-limit"]) + cur_config = (redis_client.config_get("client-output-buffer-limit")[ + "client-output-buffer-limit"]) cur_config_list = cur_config.split() assert len(cur_config_list) == 12 cur_config_list[8:] = ["pubsub", "134217728", "134217728", "60"] From e589a2e486d9473e82093e711e184033ae9bcd96 Mon Sep 17 00:00:00 2001 From: Qing Wang Date: Wed, 24 Apr 2019 12:24:26 +0800 Subject: [PATCH 10/10] Address comment. --- BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index bf1f1b0e1bd4..61484ba82f6b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -253,7 +253,6 @@ cc_library( exclude = [ "src/ray/util/logging_test.cc", "src/ray/util/signal_test.cc", - "src/ray/util/util_test.cc", ], ), hdrs = glob([