From fb8ffd66f9e7ce36a297eb919d35566a940e0f3d Mon Sep 17 00:00:00 2001 From: Schatten Date: Sat, 29 Aug 2026 01:17:15 +0800 Subject: [PATCH 1/2] [Store] Structure DistributedStorageConfig environment settings Signed-off-by: Schatten --- .../include/environment_variables.h | 17 + mooncake-store/src/CMakeLists.txt | 1 + .../src/config/distributed_storage_config.cpp | 143 +++++++++ .../distributed_storage_backend.cpp | 122 ------- mooncake-store/tests/CMakeLists.txt | 2 + mooncake-store/tests/dfs_posix_test.cpp | 41 --- .../tests/distributed_storage_config_test.cpp | 297 ++++++++++++++++++ 7 files changed, 460 insertions(+), 163 deletions(-) create mode 100644 mooncake-store/src/config/distributed_storage_config.cpp create mode 100644 mooncake-store/tests/distributed_storage_config_test.cpp diff --git a/mooncake-common/include/environment_variables.h b/mooncake-common/include/environment_variables.h index 90739e9ec6..23d3efc3f4 100644 --- a/mooncake-common/include/environment_variables.h +++ b/mooncake-common/include/environment_variables.h @@ -44,6 +44,23 @@ struct ClientAutoPortEnvironmentVariables { MC_DEFINE_ENV_VAR(int, MC_STORE_CLIENT_MAX_PORT); }; +struct DistributedStorageEnvironmentVariables { + MC_DEFINE_ENV_VAR(std::string, MOONCAKE_DFS_ROOT_DIR); + MC_DEFINE_ENV_VAR(std::string, MOONCAKE_DISTRIBUTED_ROOT_DIR); + MC_DEFINE_ENV_VAR(std::string, MOONCAKE_DFS_FS_ADAPTER); + MC_DEFINE_ENV_VAR(std::string, MOONCAKE_DISTRIBUTED_FS_TYPE); + MC_DEFINE_ENV_VAR(bool, MOONCAKE_DISTRIBUTED_HEALTH_CHECK); + MC_DEFINE_ENV_VAR(int, MOONCAKE_DFS_SHARD_COUNT); + MC_DEFINE_ENV_VAR(uint64_t, MOONCAKE_DFS_SHARD_CAPACITY); + MC_DEFINE_ENV_VAR(uint64_t, MOONCAKE_DFS_ALIGNMENT); + MC_DEFINE_ENV_VAR(bool, MOONCAKE_DFS_SINGLE_TENANT); + MC_DEFINE_ENV_VAR(bool, MOONCAKE_DFS_EVICTION_ENABLED); + MC_DEFINE_ENV_VAR(double, MOONCAKE_DFS_EVICTION_HIGH_WATERMARK); + MC_DEFINE_ENV_VAR(double, MOONCAKE_DFS_EVICTION_LOW_WATERMARK); + MC_DEFINE_ENV_VAR(int, MOONCAKE_DFS_DEFERRED_FREE_SECONDS); + MC_DEFINE_ENV_VAR(int, MOONCAKE_DFS_EVICTION_CHECK_INTERVAL); +}; + #undef MC_DEFINE_ENV_VAR } // namespace mooncake diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 4eb689df05..8aad513353 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -8,6 +8,7 @@ set(MOONCAKE_STORE_SHARED_SOURCES thread_pool.cpp etcd_helper.cpp uds_transport.cpp + config/distributed_storage_config.cpp storage/distributed/distributed_storage_backend.cpp storage/distributed/posix_fs_adapter.cpp utils/file_util.cpp) diff --git a/mooncake-store/src/config/distributed_storage_config.cpp b/mooncake-store/src/config/distributed_storage_config.cpp new file mode 100644 index 0000000000..f0fbe5be6e --- /dev/null +++ b/mooncake-store/src/config/distributed_storage_config.cpp @@ -0,0 +1,143 @@ +#include "storage/distributed/distributed_storage_backend.h" + +#include +#include +#include + +#include "environ.h" +#include "environment_variables.h" + +namespace mooncake { + +bool DistributedStorageConfig::Validate() const { + if (fsdir.empty()) { + LOG(ERROR) << "DistributedStorageConfig: fsdir is empty"; + return false; + } + if (!std::filesystem::path(fsdir).is_absolute()) { + LOG(ERROR) + << "DistributedStorageConfig: fsdir must be an absolute path: " + << fsdir; + return false; + } + if (fs_adapter_type != "hf3fs" && fs_adapter_type != "posix") { + LOG(ERROR) << "DistributedStorageConfig: unsupported fs_adapter_type: " + << fs_adapter_type; + return false; + } + if (shard_count <= 0) { + LOG(ERROR) << "DistributedStorageConfig: shard_count must > 0"; + return false; + } + if (shard_capacity == 0) { + LOG(ERROR) << "DistributedStorageConfig: shard_capacity must > 0"; + return false; + } + if (alignment == 0 || (alignment & (alignment - 1)) != 0) { + LOG(ERROR) << "DistributedStorageConfig: alignment must be power of 2"; + return false; + } + if (shard_capacity % alignment != 0) { + LOG(ERROR) << "DistributedStorageConfig: shard_capacity must align"; + return false; + } + if (!single_tenant) { + LOG(ERROR) << "DistributedStorageConfig: Currently, DFS requires " + "single_tenant=true"; + return false; + } + return true; +} + +bool DistributedStorageConfig::ValidateForAllocator() const { + if (!Validate()) return false; + + if (eviction_low_watermark < 0.0 || eviction_low_watermark > 1.0 || + eviction_high_watermark < 0.0 || eviction_high_watermark > 1.0 || + eviction_low_watermark >= eviction_high_watermark) { + LOG(ERROR) << "DistributedStorageConfig: eviction watermarks must " + "satisfy 0 <= low < high <= 1, low=" + << eviction_low_watermark + << ", high=" << eviction_high_watermark; + return false; + } + if (deferred_free_duration.count() < 0) { + LOG(ERROR) << "DistributedStorageConfig: deferred_free_duration must " + "be non-negative, seconds=" + << deferred_free_duration.count(); + return false; + } + if (eviction_enabled && eviction_check_interval.count() <= 0) { + LOG(ERROR) << "DistributedStorageConfig: eviction_check_interval must " + "be positive when eviction is enabled, seconds=" + << eviction_check_interval.count(); + return false; + } + return true; +} + +DistributedStorageConfig DistributedStorageConfig::FromEnvironment() { + DistributedStorageConfig config; + using Variables = DistributedStorageEnvironmentVariables; + + const auto legacy_root_dir = + Environ::ReadOr(Variables::MOONCAKE_DISTRIBUTED_ROOT_DIR, config.fsdir); + config.fsdir = + Environ::ReadOr(Variables::MOONCAKE_DFS_ROOT_DIR, legacy_root_dir); + if (!std::filesystem::path(config.fsdir).is_absolute()) { + config.fsdir = std::filesystem::absolute(config.fsdir).string(); + } + + const auto legacy_fs_adapter = Environ::ReadOr( + Variables::MOONCAKE_DISTRIBUTED_FS_TYPE, config.fs_adapter_type); + config.fs_adapter_type = + Environ::ReadOr(Variables::MOONCAKE_DFS_FS_ADAPTER, legacy_fs_adapter); + config.enable_health_check = + Environ::ReadOr(Variables::MOONCAKE_DISTRIBUTED_HEALTH_CHECK, + config.enable_health_check); + config.shard_count = Environ::ReadOr(Variables::MOONCAKE_DFS_SHARD_COUNT, + config.shard_count); + config.shard_capacity = Environ::ReadOr( + Variables::MOONCAKE_DFS_SHARD_CAPACITY, config.shard_capacity); + config.alignment = + Environ::ReadOr(Variables::MOONCAKE_DFS_ALIGNMENT, config.alignment); + config.single_tenant = Environ::ReadOr( + Variables::MOONCAKE_DFS_SINGLE_TENANT, config.single_tenant); + config.eviction_enabled = Environ::ReadOr( + Variables::MOONCAKE_DFS_EVICTION_ENABLED, config.eviction_enabled); + + // GetDouble silently falls back for an empty value; ReadOr emits a + // warning. Keep the existing diagnostics while this refactor is + // behavior-preserving. + config.eviction_high_watermark = + Environ::GetDouble(Variables::MOONCAKE_DFS_EVICTION_HIGH_WATERMARK.name, + config.eviction_high_watermark); + config.eviction_low_watermark = + Environ::GetDouble(Variables::MOONCAKE_DFS_EVICTION_LOW_WATERMARK.name, + config.eviction_low_watermark); + config.deferred_free_duration = std::chrono::seconds(Environ::ReadOr( + Variables::MOONCAKE_DFS_DEFERRED_FREE_SECONDS, + static_cast(config.deferred_free_duration.count()))); + config.eviction_check_interval = std::chrono::seconds(Environ::ReadOr( + Variables::MOONCAKE_DFS_EVICTION_CHECK_INTERVAL, + static_cast(config.eviction_check_interval.count()))); + return config; +} + +std::string DistributedStorageConfig::FormatStr() const { + std::ostringstream oss; + oss << "fsdir=" << fsdir << ", fs_adapter_type=" << fs_adapter_type + << ", enable_health_check=" << enable_health_check + << ", shard_count=" << shard_count + << ", shard_capacity=" << shard_capacity << ", alignment=" << alignment + << ", single_tenant=" << single_tenant + << ", eviction_enabled=" << eviction_enabled + << ", eviction_high_watermark=" << eviction_high_watermark + << ", eviction_low_watermark=" << eviction_low_watermark + << ", deferred_free_seconds=" << deferred_free_duration.count() + << ", eviction_check_interval_seconds=" + << eviction_check_interval.count(); + return oss.str(); +} + +} // namespace mooncake diff --git a/mooncake-store/src/storage/distributed/distributed_storage_backend.cpp b/mooncake-store/src/storage/distributed/distributed_storage_backend.cpp index d49f3adcd2..46b93d55ed 100644 --- a/mooncake-store/src/storage/distributed/distributed_storage_backend.cpp +++ b/mooncake-store/src/storage/distributed/distributed_storage_backend.cpp @@ -3,9 +3,7 @@ #include #include #include -#include -#include "environ.h" #include "storage/distributed/dfs_global_allocator.h" #include "types.h" #include "utils.h" @@ -35,126 +33,6 @@ bool IsDfsDescriptorRangeValid(const DistributedFSDescriptor& desc, } // namespace -bool DistributedStorageConfig::Validate() const { - if (fsdir.empty()) { - LOG(ERROR) << "DistributedStorageConfig: fsdir is empty"; - return false; - } - if (!std::filesystem::path(fsdir).is_absolute()) { - LOG(ERROR) - << "DistributedStorageConfig: fsdir must be an absolute path: " - << fsdir; - return false; - } - if (fs_adapter_type != "hf3fs" && fs_adapter_type != "posix") { - LOG(ERROR) << "DistributedStorageConfig: unsupported fs_adapter_type: " - << fs_adapter_type; - return false; - } - if (shard_count <= 0) { - LOG(ERROR) << "DistributedStorageConfig: shard_count must > 0"; - return false; - } - if (shard_capacity == 0) { - LOG(ERROR) << "DistributedStorageConfig: shard_capacity must > 0"; - return false; - } - if (alignment == 0 || (alignment & (alignment - 1)) != 0) { - LOG(ERROR) << "DistributedStorageConfig: alignment must be power of 2"; - return false; - } - if (shard_capacity % alignment != 0) { - LOG(ERROR) << "DistributedStorageConfig: shard_capacity must align"; - return false; - } - if (!single_tenant) { - LOG(ERROR) << "DistributedStorageConfig: Currently, DFS requires " - "single_tenant=true"; - return false; - } - return true; -} - -bool DistributedStorageConfig::ValidateForAllocator() const { - if (!Validate()) return false; - - if (eviction_low_watermark < 0.0 || eviction_low_watermark > 1.0 || - eviction_high_watermark < 0.0 || eviction_high_watermark > 1.0 || - eviction_low_watermark >= eviction_high_watermark) { - LOG(ERROR) << "DistributedStorageConfig: eviction watermarks must " - "satisfy 0 <= low < high <= 1, low=" - << eviction_low_watermark - << ", high=" << eviction_high_watermark; - return false; - } - if (deferred_free_duration.count() < 0) { - LOG(ERROR) << "DistributedStorageConfig: deferred_free_duration must " - "be non-negative, seconds=" - << deferred_free_duration.count(); - return false; - } - if (eviction_enabled && eviction_check_interval.count() <= 0) { - LOG(ERROR) << "DistributedStorageConfig: eviction_check_interval must " - "be positive when eviction is enabled, seconds=" - << eviction_check_interval.count(); - return false; - } - return true; -} - -DistributedStorageConfig DistributedStorageConfig::FromEnvironment() { - DistributedStorageConfig config; - config.fsdir = Environ::GetString( - "MOONCAKE_DFS_ROOT_DIR", - Environ::GetString("MOONCAKE_DISTRIBUTED_ROOT_DIR", config.fsdir)); - if (!std::filesystem::path(config.fsdir).is_absolute()) { - config.fsdir = std::filesystem::absolute(config.fsdir).string(); - } - config.fs_adapter_type = - Environ::GetString("MOONCAKE_DFS_FS_ADAPTER", - Environ::GetString("MOONCAKE_DISTRIBUTED_FS_TYPE", - config.fs_adapter_type)); - config.enable_health_check = - Environ::GetBool("MOONCAKE_DISTRIBUTED_HEALTH_CHECK", false); - config.shard_count = - Environ::GetInt("MOONCAKE_DFS_SHARD_COUNT", config.shard_count); - config.shard_capacity = Environ::GetUInt64("MOONCAKE_DFS_SHARD_CAPACITY", - config.shard_capacity); - config.alignment = - Environ::GetUInt64("MOONCAKE_DFS_ALIGNMENT", config.alignment); - config.single_tenant = - Environ::GetBool("MOONCAKE_DFS_SINGLE_TENANT", config.single_tenant); - config.eviction_enabled = Environ::GetBool("MOONCAKE_DFS_EVICTION_ENABLED", - config.eviction_enabled); - config.eviction_high_watermark = Environ::GetDouble( - "MOONCAKE_DFS_EVICTION_HIGH_WATERMARK", config.eviction_high_watermark); - config.eviction_low_watermark = Environ::GetDouble( - "MOONCAKE_DFS_EVICTION_LOW_WATERMARK", config.eviction_low_watermark); - config.deferred_free_duration = std::chrono::seconds(Environ::GetInt( - "MOONCAKE_DFS_DEFERRED_FREE_SECONDS", - static_cast(config.deferred_free_duration.count()))); - config.eviction_check_interval = std::chrono::seconds(Environ::GetInt( - "MOONCAKE_DFS_EVICTION_CHECK_INTERVAL", - static_cast(config.eviction_check_interval.count()))); - return config; -} - -std::string DistributedStorageConfig::FormatStr() const { - std::ostringstream oss; - oss << "fsdir=" << fsdir << ", fs_adapter_type=" << fs_adapter_type - << ", enable_health_check=" << enable_health_check - << ", shard_count=" << shard_count - << ", shard_capacity=" << shard_capacity << ", alignment=" << alignment - << ", single_tenant=" << single_tenant - << ", eviction_enabled=" << eviction_enabled - << ", eviction_high_watermark=" << eviction_high_watermark - << ", eviction_low_watermark=" << eviction_low_watermark - << ", deferred_free_seconds=" << deferred_free_duration.count() - << ", eviction_check_interval_seconds=" - << eviction_check_interval.count(); - return oss.str(); -} - DistributedStorageBackend::DistributedStorageBackend( const FileStorageConfig& file_storage_config, const DistributedStorageConfig& distributed_config, diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index ed3c035f97..9a194e681a 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -158,6 +158,8 @@ add_store_test(client_metrics_test client_metrics_test.cpp) add_store_test(ssd_metrics_test ssd_metrics_test.cpp) add_store_test(serializer_test serializer_test.cpp) add_store_test(dfs_posix_test dfs_posix_test.cpp) +add_store_test(distributed_storage_config_test + distributed_storage_config_test.cpp) add_store_test(dfs_sync_client_test dfs_sync_client_test.cpp) add_test(NAME dfs_batch_read_checksum_test COMMAND dfs_sync_client_test diff --git a/mooncake-store/tests/dfs_posix_test.cpp b/mooncake-store/tests/dfs_posix_test.cpp index 1e5ec533d6..e792b81a08 100644 --- a/mooncake-store/tests/dfs_posix_test.cpp +++ b/mooncake-store/tests/dfs_posix_test.cpp @@ -331,47 +331,6 @@ TEST(DfsGlobalAllocatorTest, AllocateFreeAndFormatShardIdx) { EXPECT_EQ(DfsGlobalAllocator::FormatShardIdx(100, 1000), "100"); } -TEST(DistributedStorageConfigTest, ReadsValidatesAndFormatsEnvironment) { - EnvGuard env; - TempDir tmp("dfs_config"); - env.Set("MOONCAKE_DFS_ROOT_DIR", tmp.path().c_str()); - env.Set("MOONCAKE_DFS_FS_ADAPTER", "posix"); - env.Set("MOONCAKE_DFS_SHARD_COUNT", "8"); - env.Set("MOONCAKE_DFS_SHARD_CAPACITY", "1048576"); - env.Set("MOONCAKE_DFS_ALIGNMENT", "4096"); - env.Set("MOONCAKE_DFS_SINGLE_TENANT", "1"); - env.Set("MOONCAKE_DFS_EVICTION_ENABLED", "1"); - env.Set("MOONCAKE_DFS_EVICTION_HIGH_WATERMARK", "0.85"); - env.Set("MOONCAKE_DFS_EVICTION_LOW_WATERMARK", "0.65"); - env.Set("MOONCAKE_DFS_DEFERRED_FREE_SECONDS", "12"); - env.Set("MOONCAKE_DFS_EVICTION_CHECK_INTERVAL", "3"); - - const auto config = DistributedStorageConfig::FromEnvironment(); - EXPECT_EQ(config.fsdir, tmp.path()); - EXPECT_EQ(config.fs_adapter_type, "posix"); - EXPECT_EQ(config.shard_count, 8); - EXPECT_EQ(config.shard_capacity, 1048576); - EXPECT_EQ(config.alignment, 4096); - EXPECT_TRUE(config.eviction_enabled); - EXPECT_DOUBLE_EQ(config.eviction_high_watermark, 0.85); - EXPECT_DOUBLE_EQ(config.eviction_low_watermark, 0.65); - EXPECT_EQ(config.deferred_free_duration, std::chrono::seconds(12)); - EXPECT_EQ(config.eviction_check_interval, std::chrono::seconds(3)); - EXPECT_TRUE(config.Validate()); - EXPECT_TRUE(config.ValidateForAllocator()); - - const std::string formatted = config.FormatStr(); - EXPECT_NE(formatted.find("fs_adapter_type=posix"), std::string::npos); - EXPECT_NE(formatted.find("shard_count=8"), std::string::npos); - EXPECT_NE(formatted.find("eviction_high_watermark=0.85"), - std::string::npos); - - auto invalid_eviction = config; - invalid_eviction.eviction_low_watermark = 0.9; - EXPECT_TRUE(invalid_eviction.Validate()); - EXPECT_FALSE(invalid_eviction.ValidateForAllocator()); -} - TEST(DfsGlobalAllocatorTest, InitReturnsSpecificErrors) { EnvGuard env; ConfigurePosixDfs(env); diff --git a/mooncake-store/tests/distributed_storage_config_test.cpp b/mooncake-store/tests/distributed_storage_config_test.cpp new file mode 100644 index 0000000000..d12c5e9d4b --- /dev/null +++ b/mooncake-store/tests/distributed_storage_config_test.cpp @@ -0,0 +1,297 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "storage/distributed/distributed_storage_backend.h" + +namespace mooncake { + +namespace { + +class ScopedEnvVar { + public: + explicit ScopedEnvVar(const char* name) : name_(name) { + const char* value = std::getenv(name); + if (value != nullptr) { + original_ = value; + } + unsetenv(name); + } + + ~ScopedEnvVar() { + if (original_.has_value()) { + setenv(name_.c_str(), original_->c_str(), 1); + } else { + unsetenv(name_.c_str()); + } + } + + ScopedEnvVar(const ScopedEnvVar&) = delete; + ScopedEnvVar& operator=(const ScopedEnvVar&) = delete; + + void Set(const char* value) { setenv(name_.c_str(), value, 1); } + + private: + std::string name_; + std::optional original_; +}; + +struct DistributedStorageEnvironment { + ScopedEnvVar root_dir{"MOONCAKE_DFS_ROOT_DIR"}; + ScopedEnvVar legacy_root_dir{"MOONCAKE_DISTRIBUTED_ROOT_DIR"}; + ScopedEnvVar fs_adapter{"MOONCAKE_DFS_FS_ADAPTER"}; + ScopedEnvVar legacy_fs_adapter{"MOONCAKE_DISTRIBUTED_FS_TYPE"}; + ScopedEnvVar health_check{"MOONCAKE_DISTRIBUTED_HEALTH_CHECK"}; + ScopedEnvVar shard_count{"MOONCAKE_DFS_SHARD_COUNT"}; + ScopedEnvVar shard_capacity{"MOONCAKE_DFS_SHARD_CAPACITY"}; + ScopedEnvVar alignment{"MOONCAKE_DFS_ALIGNMENT"}; + ScopedEnvVar single_tenant{"MOONCAKE_DFS_SINGLE_TENANT"}; + ScopedEnvVar eviction_enabled{"MOONCAKE_DFS_EVICTION_ENABLED"}; + ScopedEnvVar eviction_high_watermark{ + "MOONCAKE_DFS_EVICTION_HIGH_WATERMARK"}; + ScopedEnvVar eviction_low_watermark{"MOONCAKE_DFS_EVICTION_LOW_WATERMARK"}; + ScopedEnvVar deferred_free_seconds{"MOONCAKE_DFS_DEFERRED_FREE_SECONDS"}; + ScopedEnvVar eviction_check_interval{ + "MOONCAKE_DFS_EVICTION_CHECK_INTERVAL"}; +}; + +void ExpectDefaultConfig(const DistributedStorageConfig& config) { + EXPECT_EQ(config.fsdir, "/mnt/3fs/mooncake"); + EXPECT_EQ(config.fs_adapter_type, "hf3fs"); + EXPECT_FALSE(config.enable_health_check); + EXPECT_EQ(config.shard_count, 64); + EXPECT_EQ(config.shard_capacity, 4ULL * 1024 * 1024 * 1024); + EXPECT_EQ(config.alignment, 4096); + EXPECT_TRUE(config.single_tenant); + EXPECT_TRUE(config.eviction_enabled); + EXPECT_DOUBLE_EQ(config.eviction_high_watermark, 0.9); + EXPECT_DOUBLE_EQ(config.eviction_low_watermark, 0.7); + EXPECT_EQ(config.deferred_free_duration, std::chrono::seconds(30)); + EXPECT_EQ(config.eviction_check_interval, std::chrono::seconds(5)); +} + +DistributedStorageConfig ValidConfig() { + DistributedStorageConfig config; + config.fsdir = "/tmp/mooncake-distributed-storage"; + config.fs_adapter_type = "posix"; + config.shard_count = 8; + config.shard_capacity = 1024 * 1024; + config.alignment = 4096; + config.single_tenant = true; + config.eviction_enabled = true; + config.eviction_high_watermark = 0.85; + config.eviction_low_watermark = 0.65; + config.deferred_free_duration = std::chrono::seconds(12); + config.eviction_check_interval = std::chrono::seconds(3); + return config; +} + +class DistributedStorageConfigTest : public ::testing::Test { + protected: + DistributedStorageEnvironment env; +}; + +TEST_F(DistributedStorageConfigTest, UsesDefaultsWhenEnvironmentIsUnset) { + const auto config = DistributedStorageConfig::FromEnvironment(); + + ExpectDefaultConfig(config); + EXPECT_TRUE(config.Validate()); + EXPECT_TRUE(config.ValidateForAllocator()); +} + +TEST_F(DistributedStorageConfigTest, ReadsValidEnvironmentValues) { + env.root_dir.Set("/tmp/mooncake-dfs"); + env.fs_adapter.Set("posix"); + env.health_check.Set("true"); + env.shard_count.Set("8"); + env.shard_capacity.Set("1048576"); + env.alignment.Set("4096"); + env.single_tenant.Set("1"); + env.eviction_enabled.Set("1"); + env.eviction_high_watermark.Set("0.85"); + env.eviction_low_watermark.Set("0.65"); + env.deferred_free_seconds.Set("12"); + env.eviction_check_interval.Set("3"); + + const auto config = DistributedStorageConfig::FromEnvironment(); + + EXPECT_EQ(config.fsdir, "/tmp/mooncake-dfs"); + EXPECT_EQ(config.fs_adapter_type, "posix"); + EXPECT_TRUE(config.enable_health_check); + EXPECT_EQ(config.shard_count, 8); + EXPECT_EQ(config.shard_capacity, 1048576); + EXPECT_EQ(config.alignment, 4096); + EXPECT_TRUE(config.single_tenant); + EXPECT_TRUE(config.eviction_enabled); + EXPECT_DOUBLE_EQ(config.eviction_high_watermark, 0.85); + EXPECT_DOUBLE_EQ(config.eviction_low_watermark, 0.65); + EXPECT_EQ(config.deferred_free_duration, std::chrono::seconds(12)); + EXPECT_EQ(config.eviction_check_interval, std::chrono::seconds(3)); + EXPECT_TRUE(config.Validate()); + EXPECT_TRUE(config.ValidateForAllocator()); + + const std::string formatted = config.FormatStr(); + EXPECT_NE(formatted.find("fs_adapter_type=posix"), std::string::npos); + EXPECT_NE(formatted.find("shard_count=8"), std::string::npos); + EXPECT_NE(formatted.find("eviction_high_watermark=0.85"), + std::string::npos); +} + +TEST_F(DistributedStorageConfigTest, PreservesAliasPrecedence) { + env.legacy_root_dir.Set("/tmp/legacy-dfs"); + env.legacy_fs_adapter.Set("posix"); + + const auto legacy = DistributedStorageConfig::FromEnvironment(); + EXPECT_EQ(legacy.fsdir, "/tmp/legacy-dfs"); + EXPECT_EQ(legacy.fs_adapter_type, "posix"); + + env.root_dir.Set("/tmp/preferred-dfs"); + env.fs_adapter.Set("hf3fs"); + + const auto preferred = DistributedStorageConfig::FromEnvironment(); + EXPECT_EQ(preferred.fsdir, "/tmp/preferred-dfs"); + EXPECT_EQ(preferred.fs_adapter_type, "hf3fs"); +} + +TEST_F(DistributedStorageConfigTest, EmptyPreferredRootOverridesAlias) { + env.legacy_root_dir.Set("/tmp/legacy-dfs"); + env.root_dir.Set(""); + + EXPECT_THROW(DistributedStorageConfig::FromEnvironment(), + std::filesystem::filesystem_error); +} + +TEST_F(DistributedStorageConfigTest, EmptyPreferredAdapterOverridesAlias) { + env.legacy_fs_adapter.Set("posix"); + env.fs_adapter.Set(""); + + const auto config = DistributedStorageConfig::FromEnvironment(); + + EXPECT_TRUE(config.fs_adapter_type.empty()); + EXPECT_FALSE(config.Validate()); +} + +TEST_F(DistributedStorageConfigTest, ConvertsRelativeRootToAbsolutePath) { + env.root_dir.Set("relative-dfs-root"); + + const auto config = DistributedStorageConfig::FromEnvironment(); + + EXPECT_EQ(config.fsdir, + std::filesystem::absolute("relative-dfs-root").string()); +} + +TEST_F(DistributedStorageConfigTest, + InvalidValuesUseDefaultsAndPreserveDiagnostics) { + env.health_check.Set("invalid"); + env.shard_count.Set("invalid"); + env.shard_capacity.Set("-1"); + env.alignment.Set("18446744073709551616"); + env.single_tenant.Set("invalid"); + env.eviction_enabled.Set("invalid"); + env.eviction_high_watermark.Set("invalid"); + env.deferred_free_seconds.Set("invalid"); + env.eviction_check_interval.Set("invalid"); + + ::testing::internal::CaptureStderr(); + const auto config = DistributedStorageConfig::FromEnvironment(); + const std::string logs = ::testing::internal::GetCapturedStderr(); + + ExpectDefaultConfig(config); + for (const char* name : { + "MOONCAKE_DISTRIBUTED_HEALTH_CHECK", + "MOONCAKE_DFS_SHARD_COUNT", + "MOONCAKE_DFS_SHARD_CAPACITY", + "MOONCAKE_DFS_ALIGNMENT", + "MOONCAKE_DFS_SINGLE_TENANT", + "MOONCAKE_DFS_EVICTION_ENABLED", + "MOONCAKE_DFS_EVICTION_HIGH_WATERMARK", + "MOONCAKE_DFS_DEFERRED_FREE_SECONDS", + "MOONCAKE_DFS_EVICTION_CHECK_INTERVAL", + }) { + EXPECT_NE(logs.find(name), std::string::npos) << name; + } +} + +TEST_F(DistributedStorageConfigTest, + EmptyWatermarksUseDefaultsWithoutDiagnostics) { + env.eviction_high_watermark.Set(""); + env.eviction_low_watermark.Set(""); + + ::testing::internal::CaptureStderr(); + const auto config = DistributedStorageConfig::FromEnvironment(); + const std::string logs = ::testing::internal::GetCapturedStderr(); + + EXPECT_DOUBLE_EQ(config.eviction_high_watermark, 0.9); + EXPECT_DOUBLE_EQ(config.eviction_low_watermark, 0.7); + EXPECT_TRUE(logs.empty()); +} + +TEST(DistributedStorageConfigValidationTest, RejectsInvalidBaseSettings) { + using Mutation = + std::pair>; + const std::vector mutations{ + {"empty root", [](auto& config) { config.fsdir.clear(); }}, + {"relative root", [](auto& config) { config.fsdir = "relative/path"; }}, + {"unsupported adapter", + [](auto& config) { config.fs_adapter_type = "unsupported"; }}, + {"zero shards", [](auto& config) { config.shard_count = 0; }}, + {"zero capacity", [](auto& config) { config.shard_capacity = 0; }}, + {"zero alignment", [](auto& config) { config.alignment = 0; }}, + {"non-power-of-two alignment", + [](auto& config) { config.alignment = 3; }}, + {"unaligned capacity", + [](auto& config) { config.shard_capacity += 1; }}, + {"multi tenant", [](auto& config) { config.single_tenant = false; }}, + }; + + for (const auto& [name, mutate] : mutations) { + SCOPED_TRACE(name); + auto config = ValidConfig(); + mutate(config); + EXPECT_FALSE(config.Validate()); + } +} + +TEST(DistributedStorageConfigValidationTest, RejectsInvalidAllocatorSettings) { + using Mutation = + std::pair>; + const std::vector mutations{ + {"negative low watermark", + [](auto& config) { config.eviction_low_watermark = -0.1; }}, + {"high watermark above one", + [](auto& config) { config.eviction_high_watermark = 1.1; }}, + {"unordered watermarks", + [](auto& config) { config.eviction_low_watermark = 0.85; }}, + {"negative deferred free", + [](auto& config) { + config.deferred_free_duration = std::chrono::seconds(-1); + }}, + {"zero eviction interval", + [](auto& config) { + config.eviction_check_interval = std::chrono::seconds(0); + }}, + }; + + for (const auto& [name, mutate] : mutations) { + SCOPED_TRACE(name); + auto config = ValidConfig(); + mutate(config); + EXPECT_FALSE(config.ValidateForAllocator()); + } + + auto eviction_disabled = ValidConfig(); + eviction_disabled.eviction_enabled = false; + eviction_disabled.eviction_check_interval = std::chrono::seconds(0); + EXPECT_TRUE(eviction_disabled.ValidateForAllocator()); +} + +} // namespace + +} // namespace mooncake From 8a50872a5583cb1bbc5cf755cf6ef7731d36615d Mon Sep 17 00:00:00 2001 From: Schatten Date: Mon, 31 Aug 2026 17:45:50 +0800 Subject: [PATCH 2/2] [Store] Separate DistributedStorageConfig from backend Signed-off-by: Schatten --- .../config/distributed_storage_config.h | 29 +++++++++++++++++++ .../distributed/distributed_storage_backend.h | 22 +------------- .../src/config/distributed_storage_config.cpp | 3 +- .../distributed/dfs_global_allocator.cpp | 2 +- .../tests/distributed_storage_config_test.cpp | 2 +- 5 files changed, 34 insertions(+), 24 deletions(-) create mode 100644 mooncake-store/include/config/distributed_storage_config.h diff --git a/mooncake-store/include/config/distributed_storage_config.h b/mooncake-store/include/config/distributed_storage_config.h new file mode 100644 index 0000000000..bce5facebc --- /dev/null +++ b/mooncake-store/include/config/distributed_storage_config.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include +#include + +namespace mooncake { + +struct DistributedStorageConfig { + std::string fsdir = "/mnt/3fs/mooncake"; + std::string fs_adapter_type = "hf3fs"; + bool enable_health_check = false; + int shard_count = 64; + uint64_t shard_capacity = 4ULL * 1024 * 1024 * 1024; + uint64_t alignment = 4096; + bool single_tenant = true; + bool eviction_enabled = true; + double eviction_high_watermark = 0.9; + double eviction_low_watermark = 0.7; + std::chrono::seconds deferred_free_duration{30}; + std::chrono::seconds eviction_check_interval{5}; + + bool Validate() const; + bool ValidateForAllocator() const; + static DistributedStorageConfig FromEnvironment(); + std::string FormatStr() const; +}; + +} // namespace mooncake diff --git a/mooncake-store/include/storage/distributed/distributed_storage_backend.h b/mooncake-store/include/storage/distributed/distributed_storage_backend.h index 7bc512625e..252cde5a2b 100644 --- a/mooncake-store/include/storage/distributed/distributed_storage_backend.h +++ b/mooncake-store/include/storage/distributed/distributed_storage_backend.h @@ -1,12 +1,12 @@ #pragma once -#include #include #include #include #include #include +#include "config/distributed_storage_config.h" #include "fs_adapter.h" #include "replica.h" #include "storage/distributed/object_storage_adapter.h" @@ -21,26 +21,6 @@ enum class DistributedStorageMode { kObjectStorage, }; -struct DistributedStorageConfig { - std::string fsdir = "/mnt/3fs/mooncake"; - std::string fs_adapter_type = "hf3fs"; - bool enable_health_check = false; - int shard_count = 64; - uint64_t shard_capacity = 4ULL * 1024 * 1024 * 1024; - uint64_t alignment = 4096; - bool single_tenant = true; - bool eviction_enabled = true; - double eviction_high_watermark = 0.9; - double eviction_low_watermark = 0.7; - std::chrono::seconds deferred_free_duration{30}; - std::chrono::seconds eviction_check_interval{5}; - - bool Validate() const; - bool ValidateForAllocator() const; - static DistributedStorageConfig FromEnvironment(); - std::string FormatStr() const; -}; - struct DfsWriteRequest { std::string key; DistributedFSDescriptor descriptor; diff --git a/mooncake-store/src/config/distributed_storage_config.cpp b/mooncake-store/src/config/distributed_storage_config.cpp index f0fbe5be6e..e2e8ae14de 100644 --- a/mooncake-store/src/config/distributed_storage_config.cpp +++ b/mooncake-store/src/config/distributed_storage_config.cpp @@ -1,5 +1,6 @@ -#include "storage/distributed/distributed_storage_backend.h" +#include "config/distributed_storage_config.h" +#include #include #include #include diff --git a/mooncake-store/src/storage/distributed/dfs_global_allocator.cpp b/mooncake-store/src/storage/distributed/dfs_global_allocator.cpp index 3554d704b3..99ae56427f 100644 --- a/mooncake-store/src/storage/distributed/dfs_global_allocator.cpp +++ b/mooncake-store/src/storage/distributed/dfs_global_allocator.cpp @@ -7,7 +7,7 @@ #include #include -#include "storage/distributed/distributed_storage_backend.h" +#include "config/distributed_storage_config.h" #include "storage/distributed/fs_adapter.h" #include "storage/distributed/posix_fs_adapter.h" #include "utils.h" diff --git a/mooncake-store/tests/distributed_storage_config_test.cpp b/mooncake-store/tests/distributed_storage_config_test.cpp index d12c5e9d4b..bdd1c53320 100644 --- a/mooncake-store/tests/distributed_storage_config_test.cpp +++ b/mooncake-store/tests/distributed_storage_config_test.cpp @@ -9,7 +9,7 @@ #include #include -#include "storage/distributed/distributed_storage_backend.h" +#include "config/distributed_storage_config.h" namespace mooncake {