Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions mooncake-common/include/environment_variables.h
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,11 @@ struct TransferSubmitterEnvironmentVariables {
MC_DEFINE_ENV_VAR(std::string, MC_STORE_MEMCPY);
};

struct NoFRegisterEnvironmentVariables {
// Keep the raw string to preserve case normalization and warning behavior.
MC_DEFINE_ENV_VAR(std::string, MC_NOF_TRTYPE);
};

#undef MC_DEFINE_ENV_VAR

} // namespace mooncake
1 change: 1 addition & 0 deletions mooncake-store/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ set(MOONCAKE_STORE_CLIENT_SOURCES
config/file_per_key_config.cpp
config/client_auto_port_config.cpp
config/local_hot_cache_config.cpp
config/nof_register_config.cpp
config/offset_allocator_backend_config.cpp
config/replica_selection_config.cpp
config/rpc_timeout_config.cpp
Expand Down
30 changes: 30 additions & 0 deletions mooncake-store/src/config/nof_register_config.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#include "nof_register_config.h"

#include <glog/logging.h>

#include <algorithm>
#include <cctype>

#include "environ.h"
#include "environment_variables.h"

namespace mooncake {

NoFRegisterConfig NoFRegisterConfig::FromEnvironment() {
NoFRegisterConfig config;
config.transport_type =
Environ::Read(NoFRegisterEnvironmentVariables::MC_NOF_TRTYPE)
.value_or("RDMA");
std::transform(config.transport_type.begin(), config.transport_type.end(),
config.transport_type.begin(), [](unsigned char c) {
return static_cast<char>(std::toupper(c));
});
if (config.transport_type != "RDMA" && config.transport_type != "TCP") {
LOG(WARNING) << "Invalid MC_NOF_TRTYPE=" << config.transport_type
<< ", fallback to RDMA";
config.transport_type = "RDMA";
}
return config;
}

} // namespace mooncake
13 changes: 13 additions & 0 deletions mooncake-store/src/config/nof_register_config.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#pragma once

#include <string>

namespace mooncake {

struct NoFRegisterConfig {
std::string transport_type = "RDMA";

static NoFRegisterConfig FromEnvironment();
};

} // namespace mooncake
43 changes: 12 additions & 31 deletions mooncake-store/src/ssd_register_client.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#include "ssd_register_client.h"
#include <algorithm>
#include <cctype>
#include <cstdlib>

#include "config/nof_register_config.h"

namespace mooncake {

Expand All @@ -25,21 +24,12 @@ int NoFRegisterClient::set_register(const std::string &nqn, size_t nsid,
return OPERATION_FAILED;
}

const char *trtype_env = std::getenv("MC_NOF_TRTYPE");
std::string trtype = trtype_env ? trtype_env : "RDMA";
std::transform(
trtype.begin(), trtype.end(), trtype.begin(),
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
if (trtype != "RDMA" && trtype != "TCP") {
LOG(WARNING) << "Invalid MC_NOF_TRTYPE=" << trtype
<< ", fallback to RDMA";
trtype = "RDMA";
}
const auto config = NoFRegisterConfig::FromEnvironment();

std::string te_endpoint = "traddr:" + traddr +
" trsvcid:" + std::to_string(trsvcid) +
" subnqn:" + nqn + " trtype:" + trtype +
" adrfam:IPv4 ns:" + std::to_string(nsid);
std::string te_endpoint =
"traddr:" + traddr + " trsvcid:" + std::to_string(trsvcid) +
" subnqn:" + nqn + " trtype:" + config.transport_type +
" adrfam:IPv4 ns:" + std::to_string(nsid);

NoFSegment segment;
segment.base = base;
Expand Down Expand Up @@ -70,22 +60,13 @@ int NoFRegisterClient::set_unregister_by_endpoint(
return OPERATION_FAILED;
}

const char *trtype_env = std::getenv("MC_NOF_TRTYPE");
std::string trtype = trtype_env ? trtype_env : "RDMA";
std::transform(
trtype.begin(), trtype.end(), trtype.begin(),
[](unsigned char c) { return static_cast<char>(std::toupper(c)); });
if (trtype != "RDMA" && trtype != "TCP") {
LOG(WARNING) << "Invalid MC_NOF_TRTYPE=" << trtype
<< ", fallback to RDMA";
trtype = "RDMA";
}
const auto config = NoFRegisterConfig::FromEnvironment();

// Build the te_endpoint string to match registered segments
std::string te_endpoint = "traddr:" + traddr +
" trsvcid:" + std::to_string(trsvcid) +
" subnqn:" + nqn + " trtype:" + trtype +
" adrfam:IPv4 ns:" + std::to_string(nsid);
std::string te_endpoint =
"traddr:" + traddr + " trsvcid:" + std::to_string(trsvcid) +
" subnqn:" + nqn + " trtype:" + config.transport_type +
" adrfam:IPv4 ns:" + std::to_string(nsid);

LOG(INFO) << "Built te_endpoint: " << te_endpoint;

Expand Down
1 change: 1 addition & 0 deletions mooncake-store/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ add_store_test(master_service_promotion_test_for_snapshot
if(USE_NOF)
add_store_test(nof_heartbeat_test nof_heartbeat_test.cpp)
endif()
add_store_test(nof_register_config_test nof_register_config_test.cpp)
add_store_test(client_integration_test client_integration_test.cpp)
add_test(
NAME object_checksum_client_test
Expand Down
112 changes: 112 additions & 0 deletions mooncake-store/tests/nof_register_config_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
#include "../src/config/nof_register_config.h"

#include <glog/logging.h>
#include <gtest/gtest.h>

#include <cstdlib>
#include <optional>
#include <string>

namespace mooncake::test {
namespace {

class NoFRegisterConfigTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {
google::InitGoogleLogging("NoFRegisterConfigTest");
}

static void TearDownTestSuite() { google::ShutdownGoogleLogging(); }

void SetUp() override {
original_logtostderr_ = FLAGS_logtostderr;
FLAGS_logtostderr = true;
if (const char* value = std::getenv("MC_NOF_TRTYPE")) {
original_ = value;
}
ASSERT_EQ(unsetenv("MC_NOF_TRTYPE"), 0);
}

void TearDown() override {
if (original_.has_value()) {
EXPECT_EQ(setenv("MC_NOF_TRTYPE", original_->c_str(), 1), 0);
} else {
EXPECT_EQ(unsetenv("MC_NOF_TRTYPE"), 0);
}
FLAGS_logtostderr = original_logtostderr_;
}

private:
std::optional<std::string> original_;
bool original_logtostderr_ = false;
};

TEST_F(NoFRegisterConfigTest, UnsetTransportDefaultsToRdmaSilently) {
testing::internal::CaptureStderr();
const auto config = NoFRegisterConfig::FromEnvironment();
const auto diagnostics = testing::internal::GetCapturedStderr();

EXPECT_EQ(config.transport_type, "RDMA");
EXPECT_TRUE(diagnostics.empty()) << diagnostics;
}

TEST_F(NoFRegisterConfigTest, ValidTransportIsCaseInsensitive) {
struct Case {
const char* value;
const char* expected;
};
const Case cases[] = {{"RDMA", "RDMA"}, {"rdma", "RDMA"}, {"RdMa", "RDMA"},
{"TCP", "TCP"}, {"tcp", "TCP"}, {"TcP", "TCP"}};
for (const auto& entry : cases) {
SCOPED_TRACE(entry.value);
ASSERT_EQ(setenv("MC_NOF_TRTYPE", entry.value, 1), 0);

testing::internal::CaptureStderr();
const auto config = NoFRegisterConfig::FromEnvironment();
const auto diagnostics = testing::internal::GetCapturedStderr();

EXPECT_EQ(config.transport_type, entry.expected);
EXPECT_TRUE(diagnostics.empty()) << diagnostics;
}
}

TEST_F(NoFRegisterConfigTest, EmptyAndInvalidValuesWarnAndFallbackToRdma) {
struct Case {
const char* value;
const char* normalized;
};
const Case cases[] = {{"", ""},
{"udp", "UDP"},
{" tcp", " TCP"},
{"tcp ", "TCP "},
{"1", "1"}};
for (const auto& entry : cases) {
SCOPED_TRACE(entry.value);
ASSERT_EQ(setenv("MC_NOF_TRTYPE", entry.value, 1), 0);

testing::internal::CaptureStderr();
const auto config = NoFRegisterConfig::FromEnvironment();
const auto diagnostics = testing::internal::GetCapturedStderr();

EXPECT_EQ(config.transport_type, "RDMA");
EXPECT_NE(diagnostics.find(std::string("Invalid MC_NOF_TRTYPE=") +
entry.normalized + ", fallback to RDMA"),
std::string::npos);
}
}

TEST_F(NoFRegisterConfigTest, NewConfigsReadCurrentEnvironment) {
ASSERT_EQ(setenv("MC_NOF_TRTYPE", "tcp", 1), 0);
const auto first = NoFRegisterConfig::FromEnvironment();
ASSERT_EQ(setenv("MC_NOF_TRTYPE", "rdma", 1), 0);
const auto second = NoFRegisterConfig::FromEnvironment();
ASSERT_EQ(unsetenv("MC_NOF_TRTYPE"), 0);
const auto third = NoFRegisterConfig::FromEnvironment();

EXPECT_EQ(first.transport_type, "TCP");
EXPECT_EQ(second.transport_type, "RDMA");
EXPECT_EQ(third.transport_type, "RDMA");
}

} // namespace
} // namespace mooncake::test
Loading