From 621318831ed1040f1729d554aeacec93558429a9 Mon Sep 17 00:00:00 2001 From: CAICAIIs <3360776475@qq.com> Date: Thu, 28 May 2026 10:36:52 +0800 Subject: [PATCH] [Store] Clarify cache stats semantics Signed-off-by: CAICAIIs <3360776475@qq.com> --- .../mooncake-store-deployment-guide.md | 15 +++ docs/source/design/tent/overview.md | 8 ++ mooncake-store/include/master_client.h | 6 +- .../include/master_metric_manager.h | 14 ++- mooncake-store/src/master_metric_manager.cpp | 61 ++++++---- mooncake-store/tests/master_metrics_test.cpp | 113 ++++++++++++++---- 6 files changed, 167 insertions(+), 50 deletions(-) diff --git a/docs/source/deployment/mooncake-store-deployment-guide.md b/docs/source/deployment/mooncake-store-deployment-guide.md index 58d56a4db5..641695f63e 100644 --- a/docs/source/deployment/mooncake-store-deployment-guide.md +++ b/docs/source/deployment/mooncake-store-deployment-guide.md @@ -126,6 +126,21 @@ curl -s http://:9003/metrics curl -s http://:9003/metrics/summary ``` +Mooncake Store can report Store-observed cache reuse signals, such as +completed `GetReplicaList` results served from memory/SSD and current cached +object counts. These signals help operators understand reuse inside the Store, +but they are not the final request-level or token-level cache hit ratio for an +inference system. That end-to-end hit ratio should be calculated by Conductor or +the inference engine, which can observe the full request path across GPU, CPU, +and Mooncake tiers. + +For `CalcCacheStats()`, prefer the `MEMORY_CURRENT_CACHED_OBJECTS`, +`SSD_CURRENT_CACHED_OBJECTS`, and `*_HITS_PER_CURRENT_CACHED_OBJECT` enum +aliases when consuming Store-side values. The older `*_TOTAL` and `*_HIT_RATE` +names are retained for compatibility. The `*_HIT_RATE` values divide cumulative +Store-observed hits by current cached object counts, so they are not bounded +request-level hit ratios and may exceed `1.0`. + ## Client/Engine Tuning (Env Vars, with defaults) - Topology discovery (Store Client → Transfer Engine) diff --git a/docs/source/design/tent/overview.md b/docs/source/design/tent/overview.md index 0c19cdc774..13dc295eb4 100644 --- a/docs/source/design/tent/overview.md +++ b/docs/source/design/tent/overview.md @@ -89,6 +89,14 @@ cpp-api metrics ::: +## TENT Transport Selection + +:::{toctree} +:maxdepth: 1 + +transport-selector +::: + ## TENT Quality of Service :::{toctree} diff --git a/mooncake-store/include/master_client.h b/mooncake-store/include/master_client.h index 038b16180f..5da96fbac5 100644 --- a/mooncake-store/include/master_client.h +++ b/mooncake-store/include/master_client.h @@ -101,9 +101,11 @@ class MasterClient { const std::vector& object_keys); /** - * @brief Calculate cache hit rate metrics + * @brief Calculate Store-observed cache reuse metrics * @param object_keys None - * @return Map containing metrics + * @return Map containing metrics. Legacy hit-rate keys describe cumulative + * Store-side hits normalized by current cached object counts, not + * end-to-end request/token hit ratios. */ [[nodiscard]] tl::expected CalcCacheStats(); diff --git a/mooncake-store/include/master_metric_manager.h b/mooncake-store/include/master_metric_manager.h index 34ad14e84b..d50c61e61b 100644 --- a/mooncake-store/include/master_metric_manager.h +++ b/mooncake-store/include/master_metric_manager.h @@ -58,7 +58,16 @@ class MasterMetricManager { MEMORY_HIT_RATE, SSD_HIT_RATE, OVERALL_HIT_RATE, - VALID_GET_RATE + VALID_GET_RATE, + // Clearer aliases for Store-observed counters. The legacy names above + // are kept to preserve RPC/API enum values. + MEMORY_CURRENT_CACHED_OBJECTS = MEMORY_TOTAL, + SSD_CURRENT_CACHED_OBJECTS = SSD_TOTAL, + // These values may exceed 1.0 because the numerator is cumulative + // while the denominator is the current cached object count. + MEMORY_HITS_PER_CURRENT_CACHED_OBJECT = MEMORY_HIT_RATE, + SSD_HITS_PER_CURRENT_CACHED_OBJECT = SSD_HIT_RATE, + OVERALL_HITS_PER_CURRENT_CACHED_OBJECT = OVERALL_HIT_RATE }; using CacheHitStatDict = std::unordered_map; void add_stat_to_dict(CacheHitStatDict&, CacheHitStat, double); @@ -581,7 +590,8 @@ class MasterMetricManager { ylt::metric::counter_t batch_put_revoke_items_; ylt::metric::counter_t batch_put_revoke_failed_items_; - // cache hit Statistics + // Store-observed cache reuse statistics. These counters do not represent + // end-to-end request/token-level cache hit ratio. ylt::metric::counter_t mem_cache_hit_nums_; ylt::metric::counter_t file_cache_hit_nums_; ylt::metric::gauge_t mem_cache_nums_; diff --git a/mooncake-store/src/master_metric_manager.cpp b/mooncake-store/src/master_metric_manager.cpp index da1ef239fc..d82d6be13d 100644 --- a/mooncake-store/src/master_metric_manager.cpp +++ b/mooncake-store/src/master_metric_manager.cpp @@ -269,18 +269,23 @@ MasterMetricManager::MasterMetricManager() "master_batch_put_revoke_failed_items_total", "Total number of failed items in BatchPutRevoke requests"), - // Initialize cache hit rate metrics + // Initialize Store-observed cache reuse metrics. These are not + // end-to-end request/token-level cache hit ratio metrics. mem_cache_hit_nums_("mem_cache_hit_nums_", - "Total number of cache hits in the memory pool"), + "Total number of GetReplicaList results served from " + "the memory pool"), file_cache_hit_nums_("file_cache_hit_nums_", - "Total number of cache hits in the ssd"), + "Total number of GetReplicaList results served from " + "the SSD cache"), mem_cache_nums_("mem_cache_nums_", - "Total number of cached values in the memory pool"), + "Current number of cached values in the memory pool"), file_cache_nums_("file_cache_nums_", - "Total number of cached values in the ssd"), + "Current number of cached values in the SSD cache"), valid_get_nums_("valid_get_nums_", - "Total number of valid get operations"), - total_get_nums_("total_get_nums_", "Total number of get operations"), + "Total number of GetReplicaList operations that returned " + "at least one completed replica"), + total_get_nums_("total_get_nums_", + "Total number of GetReplicaList operations"), // Initialize Eviction Counters // total eviction @@ -506,7 +511,7 @@ void MasterMetricManager::update_metrics_for_zero_output() { batch_put_revoke_items_.inc(0); batch_put_revoke_failed_items_.inc(0); - // Update cache hit rate metrics + // Update Store-observed cache reuse metrics mem_cache_hit_nums_.inc(0); file_cache_hit_nums_.inc(0); valid_get_nums_.inc(0); @@ -762,7 +767,7 @@ int64_t MasterMetricManager::get_active_clients() { return active_clients_.value(); } -// cache hit rate metrics +// Store-observed cache reuse metrics void MasterMetricManager::inc_mem_cache_hit_nums(int64_t val) { mem_cache_hit_nums_.inc(val); } @@ -1665,25 +1670,33 @@ MasterMetricManager::calculate_cache_stats() { int64_t valid_get_nums = valid_get_nums_.value(); int64_t total_get_nums = total_get_nums_.value(); - double mem_hit_rate = 0.0; + // These values divide cumulative Store-observed hits by the current cached + // object count. They are not bounded request/token-level hit ratios; that + // end-to-end metric belongs to Conductor or the inference engine. + double mem_hits_per_current_cached_object = 0.0; if (mem_total_cache > 0) { - mem_hit_rate = static_cast(mem_cache_hits) / - static_cast(mem_total_cache); - mem_hit_rate = std::round(mem_hit_rate * 100.0) / 100.0; + mem_hits_per_current_cached_object = + static_cast(mem_cache_hits) / + static_cast(mem_total_cache); + mem_hits_per_current_cached_object = + std::round(mem_hits_per_current_cached_object * 100.0) / 100.0; } - double ssd_hit_rate = 0.0; + double ssd_hits_per_current_cached_object = 0.0; if (ssd_total_cache > 0) { - ssd_hit_rate = static_cast(ssd_cache_hits) / - static_cast(ssd_total_cache); - ssd_hit_rate = std::round(ssd_hit_rate * 100.0) / 100.0; + ssd_hits_per_current_cached_object = + static_cast(ssd_cache_hits) / + static_cast(ssd_total_cache); + ssd_hits_per_current_cached_object = + std::round(ssd_hits_per_current_cached_object * 100.0) / 100.0; } - double total_hit_rate = 0.0; + double overall_hits_per_current_cached_object = 0.0; if (total_cache > 0) { - total_hit_rate = + overall_hits_per_current_cached_object = static_cast(total_hits) / static_cast(total_cache); - total_hit_rate = std::round(total_hit_rate * 100.0) / 100.0; + overall_hits_per_current_cached_object = + std::round(overall_hits_per_current_cached_object * 100.0) / 100.0; } double valid_get_rate = 0.0; @@ -1697,10 +1710,12 @@ MasterMetricManager::calculate_cache_stats() { add_stat_to_dict(stats_dict, CacheHitStat::SSD_HITS, ssd_cache_hits); add_stat_to_dict(stats_dict, CacheHitStat::MEMORY_TOTAL, mem_total_cache); add_stat_to_dict(stats_dict, CacheHitStat::SSD_TOTAL, ssd_total_cache); - add_stat_to_dict(stats_dict, CacheHitStat::MEMORY_HIT_RATE, mem_hit_rate); - add_stat_to_dict(stats_dict, CacheHitStat::SSD_HIT_RATE, ssd_hit_rate); + add_stat_to_dict(stats_dict, CacheHitStat::MEMORY_HIT_RATE, + mem_hits_per_current_cached_object); + add_stat_to_dict(stats_dict, CacheHitStat::SSD_HIT_RATE, + ssd_hits_per_current_cached_object); add_stat_to_dict(stats_dict, CacheHitStat::OVERALL_HIT_RATE, - total_hit_rate); + overall_hits_per_current_cached_object); add_stat_to_dict(stats_dict, CacheHitStat::VALID_GET_RATE, valid_get_rate); return stats_dict; } diff --git a/mooncake-store/tests/master_metrics_test.cpp b/mooncake-store/tests/master_metrics_test.cpp index b57dcc2db4..467fca69f9 100644 --- a/mooncake-store/tests/master_metrics_test.cpp +++ b/mooncake-store/tests/master_metrics_test.cpp @@ -1,7 +1,9 @@ #include #include +#include #include +#include #include #include #include @@ -270,6 +272,55 @@ TEST_F(MasterMetricsTest, BasicRequestTest) { TEST_F(MasterMetricsTest, CalcCacheStatsTest) { const uint64_t default_kv_lease_ttl = 100; auto& metrics = MasterMetricManager::instance(); + using CacheHitStat = MasterMetricManager::CacheHitStat; + + // These values are part of the RPC/API contract. New enum entries should + // be appended instead of renumbering existing values. + ASSERT_EQ(static_cast(CacheHitStat::MEMORY_HITS), 0); + ASSERT_EQ(static_cast(CacheHitStat::SSD_HITS), 1); + ASSERT_EQ(static_cast(CacheHitStat::MEMORY_TOTAL), 2); + ASSERT_EQ(static_cast(CacheHitStat::SSD_TOTAL), 3); + ASSERT_EQ(static_cast(CacheHitStat::MEMORY_HIT_RATE), 4); + ASSERT_EQ(static_cast(CacheHitStat::SSD_HIT_RATE), 5); + ASSERT_EQ(static_cast(CacheHitStat::OVERALL_HIT_RATE), 6); + ASSERT_EQ(static_cast(CacheHitStat::VALID_GET_RATE), 7); + + auto round_to_2 = [](double value) { + return std::round(value * 100.0) / 100.0; + }; + auto expected_ratio = [&round_to_2](double hits, double total) { + return total > 0.0 ? round_to_2(hits / total) : 0.0; + }; + auto expect_aliases = + [](const MasterMetricManager::CacheHitStatDict& stats) { + ASSERT_EQ(stats.at(CacheHitStat::MEMORY_CURRENT_CACHED_OBJECTS), + stats.at(CacheHitStat::MEMORY_TOTAL)); + ASSERT_EQ(stats.at(CacheHitStat::SSD_CURRENT_CACHED_OBJECTS), + stats.at(CacheHitStat::SSD_TOTAL)); + ASSERT_EQ( + stats.at(CacheHitStat::MEMORY_HITS_PER_CURRENT_CACHED_OBJECT), + stats.at(CacheHitStat::MEMORY_HIT_RATE)); + ASSERT_EQ( + stats.at(CacheHitStat::SSD_HITS_PER_CURRENT_CACHED_OBJECT), + stats.at(CacheHitStat::SSD_HIT_RATE)); + ASSERT_EQ( + stats.at(CacheHitStat::OVERALL_HITS_PER_CURRENT_CACHED_OBJECT), + stats.at(CacheHitStat::OVERALL_HIT_RATE)); + }; + auto expect_reuse_ratios = + [&expected_ratio](const MasterMetricManager::CacheHitStatDict& stats) { + const double memory_hits = stats.at(CacheHitStat::MEMORY_HITS); + const double ssd_hits = stats.at(CacheHitStat::SSD_HITS); + const double memory_total = stats.at(CacheHitStat::MEMORY_TOTAL); + const double ssd_total = stats.at(CacheHitStat::SSD_TOTAL); + ASSERT_EQ(stats.at(CacheHitStat::MEMORY_HIT_RATE), + expected_ratio(memory_hits, memory_total)); + ASSERT_EQ(stats.at(CacheHitStat::SSD_HIT_RATE), + expected_ratio(ssd_hits, ssd_total)); + ASSERT_EQ(stats.at(CacheHitStat::OVERALL_HIT_RATE), + expected_ratio(memory_hits + ssd_hits, + memory_total + ssd_total)); + }; // Use a wrapped master service to test the metrics manager WrappedMasterServiceConfig service_config; service_config.default_kv_lease_ttl = default_kv_lease_ttl; @@ -292,17 +343,16 @@ TEST_F(MasterMetricsTest, CalcCacheStatsTest) { ReplicateConfig config; config.replica_num = 1; - auto stats_dict = metrics.calculate_cache_stats(); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_HITS], 1); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_HITS], 0); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_TOTAL], 2); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_TOTAL], 0); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_HIT_RATE], - 0.5); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_HIT_RATE], 0); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::OVERALL_HIT_RATE], - 0.5); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::VALID_GET_RATE], 1); + // MasterMetricManager is a process-wide singleton and these counters do + // not have reset APIs, so assert deltas from the current baseline. + const auto base_stats = metrics.calculate_cache_stats(); + expect_aliases(base_stats); + expect_reuse_ratios(base_stats); + + const double base_memory_hits = base_stats.at(CacheHitStat::MEMORY_HITS); + const double base_memory_total = base_stats.at(CacheHitStat::MEMORY_TOTAL); + const double base_valid_get_rate = + base_stats.at(CacheHitStat::VALID_GET_RATE); auto mount_result = service_.MountSegment(segment, client_id); ASSERT_TRUE(mount_result.has_value()); @@ -311,22 +361,39 @@ TEST_F(MasterMetricsTest, CalcCacheStatsTest) { ASSERT_TRUE(put_start_result1.has_value()); auto put_end_result1 = service_.PutEnd(client_id, key, ReplicaType::MEMORY); ASSERT_TRUE(put_end_result1.has_value()); - stats_dict = metrics.calculate_cache_stats(); + auto stats_dict = metrics.calculate_cache_stats(); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_TOTAL], 3); + expect_aliases(stats_dict); + expect_reuse_ratios(stats_dict); + ASSERT_EQ(stats_dict[CacheHitStat::MEMORY_HITS], base_memory_hits); + ASSERT_EQ(stats_dict[CacheHitStat::MEMORY_TOTAL], base_memory_total + 1); auto get_replica_result = service_.GetReplicaList(key); + ASSERT_TRUE(get_replica_result.has_value()); + stats_dict = metrics.calculate_cache_stats(); + expect_aliases(stats_dict); + expect_reuse_ratios(stats_dict); + ASSERT_EQ(stats_dict[CacheHitStat::MEMORY_HITS], base_memory_hits + 1); + ASSERT_EQ(stats_dict[CacheHitStat::MEMORY_TOTAL], base_memory_total + 1); + ASSERT_GE(stats_dict[CacheHitStat::VALID_GET_RATE], base_valid_get_rate); + ASSERT_LE(stats_dict[CacheHitStat::VALID_GET_RATE], 1.0); + + // This value is not a bounded hit ratio: hits are cumulative while cached + // objects are a current gauge. + // Keep fetching until cumulative hits exceed current cached objects. + const auto extra_gets = std::max( + 1, static_cast(stats_dict[CacheHitStat::MEMORY_TOTAL] - + stats_dict[CacheHitStat::MEMORY_HITS]) + + 1); + for (int64_t i = 0; i < extra_gets; ++i) { + get_replica_result = service_.GetReplicaList(key); + ASSERT_TRUE(get_replica_result.has_value()); + } stats_dict = metrics.calculate_cache_stats(); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_HITS], 2); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_HITS], 0); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_TOTAL], 3); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_TOTAL], 0); - ASSERT_NEAR(stats_dict[MasterMetricManager::CacheHitStat::MEMORY_HIT_RATE], - 0.67, 0.01); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::SSD_HIT_RATE], 0); - ASSERT_NEAR(stats_dict[MasterMetricManager::CacheHitStat::OVERALL_HIT_RATE], - 0.67, 0.01); - ASSERT_EQ(stats_dict[MasterMetricManager::CacheHitStat::VALID_GET_RATE], 1); + expect_aliases(stats_dict); + expect_reuse_ratios(stats_dict); + ASSERT_GT(stats_dict[CacheHitStat::MEMORY_HITS_PER_CURRENT_CACHED_OBJECT], + 1.0); std::this_thread::sleep_for( std::chrono::milliseconds(default_kv_lease_ttl));