diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5b924789d872..69ef98aaef03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -73,7 +73,7 @@ repos: hooks: - id: cpplint args: ["--filter=-whitespace/braces,-whitespace/line_length,-build/c++11,-build/c++14,-build/c++17,-readability/braces,-whitespace/indent_namespace,-runtime/int,-runtime/references,-build/include_order"] - files: ^src/ray/(common/ray_syncer|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$ + files: ^src/ray/(common/cgroup2|common/ray_syncer|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$ exclude: | (?x)^( src/ray/raylet/scheduling/.*\.(h|cc)$ | diff --git a/src/ray/common/cgroup2/BUILD b/src/ray/common/cgroup2/BUILD new file mode 100644 index 000000000000..082f252f9e4c --- /dev/null +++ b/src/ray/common/cgroup2/BUILD @@ -0,0 +1,27 @@ +load("//bazel:ray.bzl", "ray_cc_library") + +ray_cc_library( + name = "cgroup_driver_interface", + hdrs = [ + "cgroup_driver_interface.h", + ], + deps = [ + "//src/ray/common:status", + "//src/ray/common:status_or", + ], +) + +ray_cc_library( + name = "sysfs_cgroup_driver", + srcs = ["sysfs_cgroup_driver.cc"], + hdrs = [ + "sysfs_cgroup_driver.h", + ], + deps = [ + ":cgroup_driver_interface", + "//src/ray/common:status", + "//src/ray/common:status_or", + "//src/ray/util:logging", + "@com_google_absl//absl/strings", + ], +) diff --git a/src/ray/common/cgroup2/cgroup_driver_interface.h b/src/ray/common/cgroup2/cgroup_driver_interface.h new file mode 100644 index 000000000000..132000c79ff5 --- /dev/null +++ b/src/ray/common/cgroup2/cgroup_driver_interface.h @@ -0,0 +1,206 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include + +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +namespace ray { + +/** + A utility that can be used to check if cgroupv2 is mounted correctly + and perform cgroup operations on the system. It supports the memory and cpu controllers + with the memory.min and cpu.weight constraints respectively. + + @see The cgroupv2 documentation for more details: + https://docs.kernel.org/admin-guide/cgroup-v2.html + */ +class CgroupDriverInterface { + public: + virtual ~CgroupDriverInterface() = default; + + /** + Checks to see if only cgroupv2 is enabled (known as unified mode) on the system. + If cgroupv2 is not enabled, or enabled along with cgroupv1, returns Invalid + with the appropriate error message. + + @see systemd's documentation for more information about unified mode: + https://github.com/systemd/systemd/blob/main/docs/CGROUP_DELEGATION.md#hierarchy-and-controller-support + + @see K8S documentation on how to enable cgroupv2 and check if it's enabled correctly: + https://kubernetes.io/docs/concepts/architecture/cgroups/#linux-distribution-cgroup-v2-support + + @return Status::OK if successful, + @return Status::Invalid if cgroupv2 is not enabled correctly. + */ + virtual Status CheckCgroupv2Enabled() = 0; + + /** + Checks that the cgroup is valid. See return values for details of which + invariants are checked. + + @param cgroup the absolute path of the cgroup. + + @return Status::OK if no errors are encounted. Otherwise, one of the following errors + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2. + */ + virtual Status CheckCgroup(const std::string &cgroup) = 0; + + /** + Creates a new cgroup at the specified path. + Expects all cgroups on the path from root -> the new cgroup to already exist. + Expects the user to have read, write, and execute privileges to parent cgroup. + + @param cgroup is an absolute path to the cgroup + + @return Status::OK if no errors are encounted. Otherwise, one of the following errors + @return Status::NotFound if an ancestor cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::AlreadyExists if the cgroup already exists. + */ + virtual Status CreateCgroup(const std::string &cgroup) = 0; + + /** + Move all processes from one cgroup to another. The process must have read, write, and + execute permissions for both cgroups and their lowest common ancestor. + + @see The relevant section of the cgroup documentation for more details: + https://docs.kernel.org/admin-guide/cgroup-v2.html#delegation-containment + + @param from the absolute path of the cgroup to migrate processes out of. + @param to the absolute path of the cgroup to migrate processes into. + + @return Status::OK if no errors are encounted. Otherwise, one of the following errors + @return Status::NotFound if to or from don't exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::Invalid if any errors occur while reading from and writing to + cgroups. + */ + virtual Status MoveAllProcesses(const std::string &from, const std::string &to) = 0; + + /** + Enables an available controller on a cgroup. A controller can be enabled if the + 1) controller is enabled in the parent of the cgroup. + 2) cgroup has no children i.e. it's a leaf node. + + @param cgroup is an absolute path to the cgroup. + @param controller is the name of the controller (e.g. "cpu" and not "+cpu") + + @see No Internal Process Constraint for more details: + https://docs.kernel.org/admin-guide/cgroup-v2.html#no-internal-process-constraint + + @return Status::OK if successful, otherwise one of the following + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions for the cgroup. + @return Status::InvalidArgument if the controller is not available or if cgroup is not + a cgroupv2. + @return Status::Invalid for all other failures. + */ + virtual Status EnableController(const std::string &cgroup, + const std::string &controller) = 0; + + /** + Disables an enabled controller in a cgroup. A controller can be disabled if the + controller is not enabled on a child cgroup. + + @param cgroup is an absolute path to the cgroup. + @param controller is the name of the controller (e.g. "cpu" and not "-cpu") + + @return Status::OK if successful, otherwise one of the following + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions for the cgroup. + @return Status::InvalidArgument if the controller is not enabled + or if cgroup is not a cgroupv2. Status::Invalid for all other failures. + */ + virtual Status DisableController(const std::string &cgroup, + const std::string &controller) = 0; + /** + Adds a resource constraint to the cgroup. To add a constraint + 1) the cgroup must have the relevant controller enabled e.g. memory.min cannot be + enabled if the memory controller is not enabled. + 2) the constraint must be supported in Ray (@see supported_constraints_). + 3) the constraint value must be in the correct range (@see supported_constraints_). + + @param cgroup is an absolute path to the cgroup. + @param constraint the name of the constraint. + @param value the value of the constraint. + + @return Status::OK if successful, otherwise one of the following + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions for the cgroup. + @return Status::InvalidArgument if the cgroup is not valid or constraint is not + supported or the value not correct. + */ + virtual Status AddConstraint(const std::string &cgroup, + const std::string &constraint, + const std::string &value) = 0; + /** + Returns a list of controllers that can be enabled on the given cgroup based on + what is enabled on the parent cgroup. + + @param cgroup absolute path of the cgroup. + + @return Status::OK with a set of controllers if successful, otherwise one of + following + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2 or malformed + controllers file. + */ + virtual StatusOr> GetAvailableControllers( + const std::string &cgroup) = 0; + + /** + Returns a list of controllers enabled on the cgroup. + + @param cgroup absolute path of the cgroup. + + @return Status::OK with a set of controllers if successful, otherwise one of following + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2 or malformed + controllers file. + */ + virtual StatusOr> GetEnabledControllers( + const std::string &cgroup) = 0; + + struct Constraint { + std::pair range; + std::string controller; + }; + + protected: + const std::unordered_map supported_constraints_ = { + {"cpu.weight", {{1, 10000}, "cpu"}}, + {"memory.min", {{0, std::numeric_limits::max()}, "memory"}}, + }; + const std::unordered_set supported_controllers_ = {"cpu", "memory"}; +}; +} // namespace ray diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.cc b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc new file mode 100644 index 000000000000..fc564dfb7fd9 --- /dev/null +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.cc @@ -0,0 +1,425 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/sysfs_cgroup_driver.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +namespace ray { +Status SysFsCgroupDriver::CheckCgroupv2Enabled() { + FILE *fp = setmntent(mount_file_path_.c_str(), "r"); + + if (!fp) { + return Status::Invalid( + absl::StrFormat("Failed to open mount file at %s. Could not verify that " + "cgroupv2 was mounted correctly. \n%s", + mount_file_path_, + strerror(errno))); + } + + bool found_cgroupv1 = false; + bool found_cgroupv2 = false; + + struct mntent *mnt; + while ((mnt = getmntent(fp)) != nullptr) { + found_cgroupv1 = found_cgroupv1 || strcmp(mnt->mnt_fsname, "cgroup") == 0; + found_cgroupv2 = found_cgroupv2 || strcmp(mnt->mnt_fsname, "cgroup2") == 0; + } + + // After parsing the mount file, the file should be at the EOF position. + // If it's not, getmntent encountered an error. + if (!feof(fp) || !endmntent(fp)) { + return Status::Invalid( + absl::StrFormat("Failed to parse mount file at %s. Could not verify that " + "cgroupv2 was mounted correctly.", + mount_file_path_)); + } + + if (found_cgroupv1 && found_cgroupv2) { + return Status::Invalid("Cgroupv1 and cgroupv2 are both mounted. Unmount cgroupv1."); + } else if (found_cgroupv1 && !found_cgroupv2) { + // TODO(#54703): provide a link to the ray documentation once it's been written + // for how to troubleshoot these. + return Status::Invalid( + "Cgroupv1 is mounted and cgroupv2 is not mounted. " + "Unmount cgroupv1 and mount cgroupv2."); + } else if (!found_cgroupv2) { + return Status::Invalid("Cgroupv2 is not mounted. Mount cgroupv2."); + } + return Status::OK(); +} + +Status SysFsCgroupDriver::CheckCgroup(const std::string &cgroup_path) { + struct statfs fs_stats {}; + if (statfs(cgroup_path.c_str(), &fs_stats) != 0) { + if (errno == ENOENT) { + return Status::NotFound( + absl::StrFormat("Cgroup at %s does not exist.", cgroup_path)); + } + if (errno == EACCES) { + return Status::PermissionDenied( + absl::StrFormat("The current user does not have read, write, and execute " + "permissions for the directory at path %s.\n%s", + cgroup_path, + strerror(errno))); + } + return Status::InvalidArgument( + absl::StrFormat("Failed to stat cgroup directory at path %s because of %s", + cgroup_path, + strerror(errno))); + } + if (fs_stats.f_type != CGROUP2_SUPER_MAGIC) { + return Status::InvalidArgument( + absl::StrFormat("Directory at path %s is not of type cgroupv2. " + "For instructions to mount cgroupv2 correctly, see:\n" + "https://kubernetes.io/docs/concepts/architecture/cgroups/" + "#linux-distribution-cgroup-v2-support.", + cgroup_path)); + } + + // NOTE: the process needs execute permissions for the cgroup directory + // to traverse the filesystem. + if (access(cgroup_path.c_str(), R_OK | W_OK | X_OK) == -1) { + return Status::PermissionDenied( + absl::StrFormat("The current user does not have read, write, and execute " + "permissions for the directory at path %s.\n%s", + cgroup_path, + strerror(errno))); + } + + return Status::OK(); +} + +Status SysFsCgroupDriver::CreateCgroup(const std::string &cgroup_path) { + if (mkdir(cgroup_path.c_str(), S_IRWXU) == -1) { + if (errno == ENOENT) { + return Status::NotFound( + absl::StrFormat("Failed to create cgroup at path %s with permissions %#o. " + "The parent cgroup does not exist.\n" + "Error: %s.", + cgroup_path, + S_IRWXU, + strerror(errno))); + } + if (errno == EACCES) { + return Status::PermissionDenied(absl::StrFormat( + "Failed to create cgroup at path %s with permissions %#o. " + "The current user does not have read, write, execute permissions " + "for the parent cgroup.\n" + "Error: %s.", + cgroup_path, + S_IRWXU, + strerror(errno))); + } + if (errno == EEXIST) { + return Status::AlreadyExists( + absl::StrFormat("Failed to create cgroup at path %s with permissions %#o. " + "The cgroup already exists.\n" + "Error: %s.", + cgroup_path, + S_IRWXU, + strerror(errno))); + } + return Status::InvalidArgument( + absl::StrFormat("Failed to create cgroup at path %s with permissions %#o.\n" + "Error: %s.", + cgroup_path, + S_IRWXU, + strerror(errno))); + } + return Status::OK(); +} + +StatusOr> SysFsCgroupDriver::GetAvailableControllers( + const std::string &cgroup_dir) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_dir)); + + std::string controller_file_path = cgroup_dir + + std::filesystem::path::preferred_separator + + std::string(kCgroupControllersFilename); + return ReadControllerFile(controller_file_path); +} + +StatusOr> SysFsCgroupDriver::GetEnabledControllers( + const std::string &cgroup_dir) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_dir)); + + std::string controller_file_path = cgroup_dir + + std::filesystem::path::preferred_separator + + std::string(kCgroupSubtreeControlFilename); + return ReadControllerFile(controller_file_path); +} + +Status SysFsCgroupDriver::MoveAllProcesses(const std::string &from, + const std::string &to) { + RAY_RETURN_NOT_OK(CheckCgroup(from)); + RAY_RETURN_NOT_OK(CheckCgroup(to)); + std::filesystem::path from_procs_file_path = + from / std::filesystem::path(kCgroupProcsFilename); + std::filesystem::path to_procs_file_path = + to / std::filesystem::path(kCgroupProcsFilename); + std::ifstream in_file(from_procs_file_path); + std::ofstream out_file(to_procs_file_path, std::ios::ate); + if (!in_file.is_open()) { + return Status::Invalid(absl::StrFormat("Could not open cgroup procs file at path %s.", + from_procs_file_path)); + } + if (!out_file.is_open()) { + return Status::Invalid( + absl::StrFormat("Could not open cgroup procs file %s", to_procs_file_path)); + } + pid_t pid = 0; + while (in_file >> pid) { + if (in_file.fail()) { + return Status::Invalid(absl::StrFormat( + "Could not read PID from cgroup procs file %s", from_procs_file_path)); + } + out_file << pid; + out_file.flush(); + if (out_file.fail()) { + return Status::Invalid(absl::StrFormat( + "Could not write pid to cgroup procs file %s", to_procs_file_path)); + } + } + return Status::OK(); +} + +Status SysFsCgroupDriver::EnableController(const std::string &cgroup_path, + const std::string &controller) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_path)); + + StatusOr> available_controllers_s = + GetAvailableControllers(cgroup_path); + + RAY_RETURN_NOT_OK(available_controllers_s.status()); + auto available_controllers = available_controllers_s.value(); + + if (available_controllers.find(controller) == available_controllers.end()) { + std::string enabled_controllers_str = + absl::StrCat("[", absl::StrJoin(available_controllers, ", "), "]"); + return Status::InvalidArgument(absl::StrFormat( + "Controller %s is not available for cgroup at path %s.\n" + "Current available controllers are %s. " + "To enable a controller in a cgroup X, all cgroups in the path from " + "the root cgroup to X must have the controller enabled.", + controller, + cgroup_path, + enabled_controllers_str)); + } + + std::filesystem::path enabled_ctrls_file = + std::filesystem::path(cgroup_path + std::filesystem::path::preferred_separator + + std::string(kCgroupSubtreeControlFilename)); + std::ofstream out_file(enabled_ctrls_file, std::ios::ate); + if (!out_file.is_open()) { + return Status::Invalid(absl::StrFormat("Could not open cgroup controllers file at %s", + enabled_ctrls_file)); + } + out_file << ("+" + controller); + out_file.flush(); + if (out_file.fail()) { + return Status::Invalid(absl::StrFormat( + "Could not open write to cgroup controllers file %s", enabled_ctrls_file)); + } + return Status::OK(); +} + +Status SysFsCgroupDriver::DisableController(const std::string &cgroup_path, + const std::string &controller) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup_path)); + std::string controller_file_path = cgroup_path + + std::filesystem::path::preferred_separator + + std::string(kCgroupSubtreeControlFilename); + + StatusOr> enabled_controllers_s = + ReadControllerFile(controller_file_path); + + RAY_RETURN_NOT_OK(enabled_controllers_s.status()); + + auto enabled_controllers = enabled_controllers_s.value(); + + if (enabled_controllers.find(controller) == enabled_controllers.end()) { + std::string enabled_controllers_str = + absl::StrCat("[", absl::StrJoin(enabled_controllers, ", "), "]"); + return Status::InvalidArgument( + absl::StrFormat("Controller %s is not enabled for cgroup at path %s.\n" + "Current enabled controllers are %s. ", + controller, + cgroup_path, + enabled_controllers_str)); + } + + std::ofstream out_file(controller_file_path, std::ios::ate); + if (!out_file.is_open()) { + return Status::Invalid(absl::StrFormat("Could not open cgroup controllers file at %s", + controller_file_path)); + } + out_file << ("-" + controller); + out_file.flush(); + if (!out_file.good()) { + return Status::Invalid(absl::StrFormat( + "Could not open write to cgroup controllers file %s", controller_file_path)); + } + return Status::OK(); +} + +Status SysFsCgroupDriver::AddConstraint(const std::string &cgroup, + const std::string &constraint, + const std::string &constraint_value) { + RAY_RETURN_NOT_OK(CheckCgroup(cgroup)); + auto constraint_it = supported_constraints_.find(constraint); + if (constraint_it == supported_constraints_.end()) { + std::string supported_constraint_names("["); + for (auto it = supported_constraints_.begin(); it != supported_constraints_.end(); + ++it) { + supported_constraint_names.append(it->first); + if (std::next(it) != supported_constraints_.end()) { + supported_constraint_names.append(", "); + } + } + supported_constraint_names.append("]"); + return Status::InvalidArgument(absl::StrFormat( + "Failed to apply constraint %s to cgroup %s. Ray only supports %s", + constraint, + cgroup, + supported_constraint_names)); + } + + // Check if the constraint value is out of range and therefore invalid. + auto [low, high] = constraint_it->second.range; + size_t value = static_cast(std::stoi(constraint_value)); + if (value < low || value > high) { + return Status::InvalidArgument(absl::StrFormat( + "Failed to apply constraint %s=%s to cgroup %s. %s can only have values " + "in the range[%i, %i].", + constraint, + constraint_value, + cgroup, + constraint, + low, + high)); + } + + // Check if the required controller for the constraint is enabled. + const std::string &controller = constraint_it->second.controller; + StatusOr> available_controllers_s = + GetEnabledControllers(cgroup); + RAY_RETURN_NOT_OK(available_controllers_s.status()); + const auto &controllers = available_controllers_s.value(); + if (controllers.find(controller) == controllers.end()) { + return Status::InvalidArgument(absl::StrFormat( + "Failed to apply %s to cgroup %s. To use %s, enable the %s controller.", + constraint, + cgroup, + constraint, + controller)); + } + + // Try to apply the constraint and propagate the appropriate failure error. + std::string file_path = + cgroup + std::filesystem::path::preferred_separator + constraint; + + int fd = open(file_path.c_str(), O_RDWR); + + if (fd == -1) { + return Status::InvalidArgument( + absl::StrFormat("Failed to apply %s=%s to cgroup %s.\n" + "Error: %s", + constraint, + constraint_value, + cgroup, + strerror(errno))); + } + + ssize_t bytes_written = write(fd, constraint_value.c_str(), constraint_value.size()); + + if (bytes_written != static_cast(constraint_value.size())) { + close(fd); + return Status::InvalidArgument( + absl::StrFormat("Failed to apply %s=%s to cgroup %s.\n" + "Error: %s", + constraint, + constraint_value, + cgroup, + strerror(errno))); + } + close(fd); + return Status::OK(); +} + +StatusOr> SysFsCgroupDriver::ReadControllerFile( + const std::string &controller_file_path) { + std::ifstream controllers_file(controller_file_path); + + if (!controllers_file.is_open()) { + return Status::InvalidArgument(absl::StrFormat( + "Failed to open controllers file at path %s.", controller_file_path)); + } + + std::unordered_set controllers; + + if (controllers_file.peek() == EOF) { + return StatusOr>(controllers); + } + + std::string line; + std::getline(controllers_file, line); + + if (!controllers_file.good()) { + return Status::InvalidArgument( + absl::StrFormat("Failed to parse controllers file %s.", controller_file_path)); + } + + std::istringstream input_ss(line); + std::string controller; + + while (input_ss >> controller) { + controllers.emplace(std::move(controller)); + } + + std::getline(controllers_file, line); + + // A well-formed controllers file should have just one line. + if (!controllers_file.eof()) { + return Status::InvalidArgument( + absl::StrFormat("Failed to parse controllers file %s.", controller_file_path)); + } + + return StatusOr>(controllers); +} + +} // namespace ray diff --git a/src/ray/common/cgroup2/sysfs_cgroup_driver.h b/src/ray/common/cgroup2/sysfs_cgroup_driver.h new file mode 100644 index 000000000000..fd56d129617b --- /dev/null +++ b/src/ray/common/cgroup2/sysfs_cgroup_driver.h @@ -0,0 +1,263 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include + +#include +#include +#include + +#include "ray/common/cgroup2/cgroup_driver_interface.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +// Used to identify if a filesystem is mounted using cgroupv2. +// See: https://docs.kernel.org/admin-guide/cgroup-v2.html#mounting +#ifndef CGROUP2_SUPER_MAGIC +#define CGROUP2_SUPER_MAGIC 0x63677270 +#endif + +namespace ray { + +/** + * Peforms cgroupv2 operations using the pseudo filesystem documented + * here https://docs.kernel.org/admin-guide/cgroup-v2.html#interface-files. + * + * Usage: + * std::unique_ptr driver = + * std::make_unique(); + * if (driver->CheckCgroupv2Enabled.ok()) { + * // perform operations + * } + */ +class SysFsCgroupDriver : public CgroupDriverInterface { + public: + /** + * MOUNTED is defined in mntent.h (and typically refers to /etc/mtab) + * @see https://www.gnu.org/software/libc/manual/2.24/html_node/Mount-Information.html + * + * @param mount_file_path only used for testing. + */ + explicit SysFsCgroupDriver(std::string mount_file_path = MOUNTED) + : mount_file_path_(std::move(mount_file_path)) {} + + ~SysFsCgroupDriver() override = default; + SysFsCgroupDriver(const SysFsCgroupDriver &other) = delete; + SysFsCgroupDriver(const SysFsCgroupDriver &&other) = delete; + SysFsCgroupDriver &operator=(const SysFsCgroupDriver &other) = delete; + SysFsCgroupDriver &operator=(const SysFsCgroupDriver &&other) = delete; + + /** + The recommended way to mount cgroupv2 is with cgroupv1 disabled. This prevents + cgroup controllers from being migrated between the two modes. This follows + the recommendation from systemd and K8S. + + Parses the mount file at /etc/mstab and returns Ok if only cgroupv2 is + mounted. + + Example Mountfile that is correct: + /dev/root / ext4 rw,relatime,discard + /dev/nvme2n1 /home/ubuntu ext4 rw,noatime,discard + cgroup2 /sys/fs/cgroup cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate + + Example Mountfile that is incorrect (both v2 and v1 are mounted): + /dev/root / ext4 rw,relatime,discard + /dev/nvme2n1 /home/ubuntu ext4 rw,noatime,discard + cgroup /sys/fs/cgroup cgroup rw,nosuid,nodev,noexec,relatime,nsdelegate + cgroup2 /sys/fs/cgroup/unified/ cgroup2 rw,nosuid,nodev,noexec,relatime,nsdelegate + + @return OK if no errors + @return Status::Invalid if cgroupv2 is not enabled correctly. + */ + Status CheckCgroupv2Enabled() override; + + /** + Checks to see if the cgroup_path is mounted in the cgroupv2 filesystem + and that the current process has read, write, and execute permissions for + the directory. Uses the CGROUP_SUPER_MAGIC to detect that the filesystem + is mounted as cgroupv2. + + @param cgroup_path the path of a cgroup directory. + + @see The kernel documentation for CGROUP2_SUPER_MAGIC + https://www.kernel.org/doc/html/v5.4/admin-guide/cgroup-v2.html#mounting + + @return Status::OK if no errors are encounted. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2. + */ + Status CheckCgroup(const std::string &cgroup_path) override; + + /** + To create a cgroup using the cgroupv2 vfs, the current user needs to read, write, and + execute permissions for the parent cgroup. This can be achieved through cgroup + delegation. + + @see The relevant manpage section on delegation for more details + https://docs.kernel.org/admin-guide/cgroup-v2.html#delegation + + @param cgroup_path the absolute path of the cgroup directory to create. + + @return Status::OK if no errors are encounted. + @return Status::NotFound if an ancestor cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::AlreadyExists if the cgroup already exists. + */ + Status CreateCgroup(const std::string &cgroup_path) override; + + /** + Parses the cgroup.controllers file which has a space separated list of all controllers + available to the cgroup. + + @see For details of the cgroup.controllers file + https://docs.kernel.org/admin-guide/cgroup-v2.html#enabling-and-disabling. + + @param cgroup_path absolute path of the cgroup. + @return Status::OK with a set of controllers if successful. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2 or malformed + controllers file. + */ + StatusOr> GetAvailableControllers( + const std::string &cgroup_dir) override; + + /** + Parses the cgroup.subtree_control file which has a space separated list of all + controllers enabled in the cgroup. + + @see For details of the cgroup.subtree_control file + https://docs.kernel.org/admin-guide/cgroup-v2.html#enabling-and-disabling. + + @param cgroup_path absolute path of the cgroup. + @return Status::OK with a set of controllers if successful. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2 or if the + cgroup.subtree_control is malformed. + */ + StatusOr> GetEnabledControllers( + const std::string &cgroup_dir) override; + + /** + Reads the cgroup.procs of "from" and writes them out to the given file. + The cgroup.procs file is newline seperated. The current user must have + read-write permissions to both cgroup.procs file as well as the common ancestor + of the source and destination cgroups. + + @see The cgroup.procs section for more information + https://docs.kernel.org/admin-guide/cgroup-v2.html#core-interface-files + + @return Status::OK with if successful. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2. + @return Status::Invalid if files could not be opened, read from, or written to + correctly. + */ + Status MoveAllProcesses(const std::string &from, const std::string &to) override; + + /** + Enables a controller by writing to the cgroup.subtree_control file. This can + only happen if + + 1. The controller is not enabled in the parent see cgroup. + 2. The cgroup is not a leaf node i.e. it has children. This is called the no internal + process constraint + + @see the cgroup documentation for the cgroup.subtree_control file + https://docs.kernel.org/admin-guide/cgroup-v2.html#controlling-controllers + + @param cgroup_path absolute path of the cgroup. + @param controller name of the controller i.e. "cpu" or "memory" from + @ref CgroupDriverInterface::supported_controllers_ "supported controllers". + + @return Status::OK if successful + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2, if the controller + is not available i.e not enabled on the parent. + @return Status::Invalid if cannot open or write to cgroup.subtree_control. + */ + Status EnableController(const std::string &cgroup_path, + const std::string &controller) override; + + /** + Disables a controller by writing to the cgroup.subtree_control file. This can + only happen if the controller is not enabled in child cgroups. + + @see the cgroup documentation for the cgroup.subtree_control file + https://docs.kernel.org/admin-guide/cgroup-v2.html#controlling-controllers + + @param cgroup_path absolute path of the cgroup. + @param controller name of the controller i.e. "cpu" or "memory" from + @ref CgroupDriverInterface::supported_controllers_ "supported controllers". + + @return Status::OK if successful. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2, if the controller + is not available i.e not enabled on the parent. + @return Status::Invalid if cannot open or write to cgroup.subtree_control. + */ + Status DisableController(const std::string &cgroup_path, + const std::string &controller) override; + + /** + Adds a constraint to the respective cgroup file. See + @ref CgroupDriverInterface::supported_constraints_ "supported constraints" and valid + values. + + @return Status::OK if no errors are encounted. + @return Status::NotFound if the cgroup does not exist. + @return Status::PermissionDenied if current user doesn't have read, write, and execute + permissions. + @return Status::InvalidArgument if the cgroup is not using cgroupv2, the constraint + is not supported in ray, the constraint value is out of range, or if cannot write + to the relevant constraint file. + */ + Status AddConstraint(const std::string &cgroup, + const std::string &constraint, + const std::string &constraint_value) override; + + private: + /** + @param controller_file_path the absolute path of the controller file to read which is + one of cgroup.subtree_control or cgroup.controllers. + + @return Status::OK with a list of controllers in the file. + @return Status::InvalidArgument if failed to read file or file was malformed. + */ + StatusOr> ReadControllerFile( + const std::string &controller_file_path); + + // Used for unit testing through the constructor. + std::string mount_file_path_; + + static constexpr std::string_view kCgroupProcsFilename = "cgroup.procs"; + static constexpr std::string_view kCgroupSubtreeControlFilename = + "cgroup.subtree_control"; + static constexpr std::string_view kCgroupControllersFilename = "cgroup.controllers"; +}; +} // namespace ray diff --git a/src/ray/common/cgroup2/test/BUILD b/src/ray/common/cgroup2/test/BUILD new file mode 100644 index 000000000000..7157feb4a7dd --- /dev/null +++ b/src/ray/common/cgroup2/test/BUILD @@ -0,0 +1,29 @@ +load("//bazel:ray.bzl", "ray_cc_library", "ray_cc_test") + +ray_cc_library( + name = "cgroup_test_utils", + srcs = ["cgroup_test_utils.cc"], + hdrs = ["cgroup_test_utils.h"], + deps = [ + "//src/ray/common:status", + "//src/ray/common:status_or", + "@com_google_absl//absl/strings:str_format", + ], +) + +ray_cc_test( + name = "sysfs_cgroup_driver_test", + srcs = ["sysfs_cgroup_driver_test.cc"], + tags = [ + "team:core", + ], + deps = [ + ":cgroup_test_utils", + "//src/ray/common:status", + "//src/ray/common:status_or", + "//src/ray/common/cgroup2:sysfs_cgroup_driver", + "//src/ray/common/test:testing", + "@com_google_absl//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/src/ray/common/cgroup2/test/cgroup_test_utils.cc b/src/ray/common/cgroup2/test/cgroup_test_utils.cc new file mode 100644 index 000000000000..e61ad82e633c --- /dev/null +++ b/src/ray/common/cgroup2/test/cgroup_test_utils.cc @@ -0,0 +1,107 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/test/cgroup_test_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" +#include "ray/util/logging.h" + +ray::StatusOr> TempDirectory::Create() { + std::string path = "/tmp/XXXXXX"; + char *ret = mkdtemp(path.data()); + if (ret == nullptr) { + return ray::Status::UnknownError( + absl::StrFormat("Failed to create a temp directory. " + "Cgroup tests expect tmpfs to be mounted and only run on Linux.\n" + "Error: %s", + strerror(errno))); + } + std::unique_ptr temp_dir = + std::make_unique(std::move(path)); + return ray::StatusOr>(std::move(temp_dir)); +} + +TempDirectory::~TempDirectory() { std::filesystem::remove_all(path_); } + +TempFile::TempFile(std::string path) { + path_ = path; + fd_ = open(path_.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); // NOLINT + if (fd_ == -1) { + throw std::runtime_error( + absl::StrFormat("Failed to create a temp file. Cgroup tests expect " + "tmpfs to be mounted " + "and only run on Linux. Error: %s", + strerror(errno))); + } + file_output_stream_ = std::ofstream(path_, std::ios::trunc); + if (!file_output_stream_.is_open()) { + throw std::runtime_error("Could not open file on tmpfs."); + } +} + +TempFile::TempFile() { + fd_ = mkstemp(path_.data()); // NOLINT + if (fd_ == -1) { + throw std::runtime_error( + "Failed to create a temp file. Cgroup tests expect tmpfs to be " + "mounted " + "and only run on Linux"); + } + if (unlink(path_.c_str()) == -1) { + close(fd_); + throw std::runtime_error("Failed to unlink temporary file."); + } + file_output_stream_ = std::ofstream(path_, std::ios::trunc); + if (!file_output_stream_.is_open()) { + throw std::runtime_error("Could not open mount file on tmpfs."); + } +} + +TempFile::~TempFile() { + close(fd_); + file_output_stream_.close(); +} + +void TempFile::AppendLine(const std::string &line) { + file_output_stream_ << line; + file_output_stream_.flush(); + if (file_output_stream_.fail()) { + throw std::runtime_error("Could not write to mount file on tmpfs"); + } +} diff --git a/src/ray/common/cgroup2/test/cgroup_test_utils.h b/src/ray/common/cgroup2/test/cgroup_test_utils.h new file mode 100644 index 000000000000..f1622d413573 --- /dev/null +++ b/src/ray/common/cgroup2/test/cgroup_test_utils.h @@ -0,0 +1,71 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include +#include +#include +#include + +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +/** + RAII style class for creating and destroying temporary directory for testing. + TODO(irabbani): add full documentation once complete. + */ +class TempDirectory { + public: + static ray::StatusOr> Create(); + explicit TempDirectory(std::string &&path) : path_(path) {} + + TempDirectory(const TempDirectory &) = delete; + TempDirectory(TempDirectory &&) = delete; + TempDirectory &operator=(const TempDirectory &) = delete; + TempDirectory &operator=(TempDirectory &&) = delete; + + const std::string &GetPath() const { return path_; } + + ~TempDirectory(); + + private: + const std::string path_; +}; + +/** + RAII wrapper that creates a file that can be written to. + TODO(irabbani): Add full documentation once the API is complete. +*/ +class TempFile { + public: + explicit TempFile(std::string path); + TempFile(); + + TempFile(TempFile &other) = delete; + TempFile(TempFile &&other) = delete; + TempFile operator=(TempFile &other) = delete; + TempFile &operator=(TempFile &&other) = delete; + + ~TempFile(); + void AppendLine(const std::string &line); + + const std::string &GetPath() const { return path_; } + + private: + std::string path_ = "/tmp/XXXXXX"; + std::ofstream file_output_stream_; + int fd_; +}; diff --git a/src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc b/src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc new file mode 100644 index 000000000000..275a122e808f --- /dev/null +++ b/src/ray/common/cgroup2/test/sysfs_cgroup_driver_test.cc @@ -0,0 +1,132 @@ +// Copyright 2025 The Ray Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ray/common/cgroup2/sysfs_cgroup_driver.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "ray/common/cgroup2/test/cgroup_test_utils.h" +#include "ray/common/status.h" +#include "ray/common/status_or.h" + +namespace ray { + +TEST(SysFsCgroupDriverTest, CheckCgroupv2EnabledFailsIfEmptyMountFile) { + TempFile temp_mount_file; + SysFsCgroupDriver driver(temp_mount_file.GetPath()); + Status s = driver.CheckCgroupv2Enabled(); + EXPECT_TRUE(s.IsInvalid()) << s.ToString(); +} + +TEST(SysFsCgroupDriverTest, CheckCgroupv2EnabledFailsIfMalformedMountFile) { + TempFile temp_mount_file; + temp_mount_file.AppendLine("cgroup /sys/fs/cgroup rw 0 0\n"); + temp_mount_file.AppendLine("cgroup2 /sys/fs/cgroup/unified/ rw 0 0\n"); + temp_mount_file.AppendLine("oopsie"); + SysFsCgroupDriver driver(temp_mount_file.GetPath()); + Status s = driver.CheckCgroupv2Enabled(); + EXPECT_TRUE(s.IsInvalid()) << s.ToString(); +} + +TEST(SysFsCgroupDriverTest, + CheckCgroupv2EnabledFailsIfCgroupv1MountedAndCgroupv2NotMounted) { + TempFile temp_mount_file; + temp_mount_file.AppendLine("cgroup /sys/fs/cgroup rw 0 0\n"); + SysFsCgroupDriver driver(temp_mount_file.GetPath()); + Status s = driver.CheckCgroupv2Enabled(); + ASSERT_TRUE(s.IsInvalid()) << s.ToString(); +} + +TEST(SysFsCgroupDriverTest, + CheckCgroupv2EnabledFailsIfCgroupv1MountedAndCgroupv2Mounted) { + TempFile temp_mount_file; + temp_mount_file.AppendLine("cgroup /sys/fs/cgroup rw 0 0\n"); + temp_mount_file.AppendLine("cgroup2 /sys/fs/cgroup/unified/ rw 0 0\n"); + SysFsCgroupDriver driver(temp_mount_file.GetPath()); + Status s = driver.CheckCgroupv2Enabled(); + ASSERT_TRUE(s.IsInvalid()) << s.ToString(); +} + +TEST(SysFsCgroupDriverTest, CheckCgroupv2EnabledSucceedsIfOnlyCgroupv2Mounted) { + TempFile temp_mount_file; + temp_mount_file.AppendLine("cgroup2 /sys/fs/cgroup rw 0 0\n"); + SysFsCgroupDriver driver(temp_mount_file.GetPath()); + Status s = driver.CheckCgroupv2Enabled(); + EXPECT_TRUE(s.ok()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, CheckCgroupFailsIfNotCgroupv2Path) { + // This is not a directory on the cgroupv2 vfs. + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup(temp_dir->GetPath()); + EXPECT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, CheckCgroupFailsIfCgroupDoesNotExist) { + // This is not a directory on the cgroupv2 vfs. + SysFsCgroupDriver driver; + Status s = driver.CheckCgroup("/some/path/that/doesnt/exist"); + EXPECT_TRUE(s.IsNotFound()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, GetAvailableControllersFailsIfNotCgroup2Path) { + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + std::filesystem::path controller_file_path = + std::filesystem::path(temp_dir->GetPath()) / + std::filesystem::path("cgroup.controllers"); + TempFile controller_file(controller_file_path); + controller_file.AppendLine("cpuset cpu io memory hugetlb pids rdma misc"); + SysFsCgroupDriver driver; + StatusOr> s = + driver.GetAvailableControllers(temp_dir->GetPath()); + EXPECT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, EnableControllerFailsIfNotCgroupv2Path) { + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.EnableController(temp_dir->GetPath(), "cpu"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, DisableControllerFailsIfNotCgroupv2Path) { + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.DisableController(temp_dir->GetPath(), "cpu"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +TEST(SysFsCgroupDriver, AddConstraintFailsIfNotCgroupv2Path) { + auto temp_dir_or_status = TempDirectory::Create(); + ASSERT_TRUE(temp_dir_or_status.ok()) << temp_dir_or_status.ToString(); + std::unique_ptr temp_dir = std::move(temp_dir_or_status.value()); + SysFsCgroupDriver driver; + Status s = driver.AddConstraint(temp_dir->GetPath(), "memory.min", "1"); + ASSERT_TRUE(s.IsInvalidArgument()) << s.ToString(); +} + +}; // namespace ray diff --git a/src/ray/common/status.cc b/src/ray/common/status.cc index 3e8f51261585..3500ddaf3b80 100644 --- a/src/ray/common/status.cc +++ b/src/ray/common/status.cc @@ -78,6 +78,7 @@ const absl::flat_hash_map kCodeToStr = { {StatusCode::InvalidArgument, "InvalidArgument"}, {StatusCode::ChannelError, "ChannelError"}, {StatusCode::ChannelTimeoutError, "ChannelTimeoutError"}, + {StatusCode::PermissionDenied, "PermissionDenied"}, }; const absl::flat_hash_map kStrToCode = []() { diff --git a/src/ray/common/status.h b/src/ray/common/status.h index 2c5c82a856cb..0ba9a37478ae 100644 --- a/src/ray/common/status.h +++ b/src/ray/common/status.h @@ -76,6 +76,8 @@ enum class StatusCode : char { IntentionalSystemExit = 14, UnexpectedSystemExit = 15, CreationTaskError = 16, + // Indicates that the caller request a resource that could not be found. A common + // example is that a request file does not exist. NotFound = 17, Disconnected = 18, SchedulingCancelled = 19, @@ -101,6 +103,9 @@ enum class StatusCode : char { ChannelError = 35, // Indicates that a read or write on a channel (a mutable plasma object) timed out. ChannelTimeoutError = 36, + // Indicates that the executing user does not have permissions to perform the + // requested operation. A common example is filesystem permissions. + PermissionDenied = 37, // If you add to this list, please also update kCodeToStr in status.cc. }; @@ -254,6 +259,10 @@ class RAY_EXPORT Status { return Status(StatusCode::ChannelTimeoutError, msg); } + static Status PermissionDenied(const std::string &msg) { + return Status(StatusCode::PermissionDenied, msg); + } + static StatusCode StringToCode(const std::string &str); // Returns true iff the status indicates success. @@ -303,6 +312,7 @@ class RAY_EXPORT Status { bool IsChannelError() const { return code() == StatusCode::ChannelError; } bool IsChannelTimeoutError() const { return code() == StatusCode::ChannelTimeoutError; } + bool IsPermissionDenied() const { return code() == StatusCode::PermissionDenied; } // Return a string representation of this status suitable for printing. // Returns the string "OK" for success. diff --git a/src/ray/common/status_or.h b/src/ray/common/status_or.h index 88eb99a7a386..991b03961df0 100644 --- a/src/ray/common/status_or.h +++ b/src/ray/common/status_or.h @@ -153,6 +153,12 @@ class StatusOr { std::string StatusString() const { return status_.StatusString(); } + std::string ToString() const { return status_.ToString(); } + + bool IsNotFound() const { return code() == StatusCode::NotFound; } + bool IsInvalidArgument() const { return code() == StatusCode::InvalidArgument; } + bool IsPermissionDenied() const { return code() == StatusCode::PermissionDenied; } + // Returns a reference to the current `ray::Status` contained within the // `ray::StatusOr`. If `ray::StatusOr` contains a `T`, then this // function returns `ray::Ok()`.