diff --git a/src/umbp/distributed/master/evict_strategy.cpp b/src/umbp/distributed/master/evict_strategy.cpp index bd62987f5..424ce0bbb 100644 --- a/src/umbp/distributed/master/evict_strategy.cpp +++ b/src/umbp/distributed/master/evict_strategy.cpp @@ -26,7 +26,7 @@ namespace mori::umbp { -std::unordered_map> LruMasterEvictStrategy::SelectVictims( +std::unordered_map> LruMasterEvictStrategy::SelectVictims( std::vector candidates, std::unordered_map> bytes_to_free) { // Oldest-access first (LRU); no tiebreak (peers don't ship depth in KvEvent). @@ -36,16 +36,17 @@ std::unordered_map> LruMasterEvictStrategy }); // Walk oldest-first, taking a candidate while its (node, tier) has budget - // left. Group by node so each peer gets a single EvictKey keys[]. - std::unordered_map> per_node_keys; + // left. Group by node so each peer gets a single EvictKey. The victim keeps + // the tier it was charged to: that is the medium the peer must free. + std::unordered_map> per_node_victims; for (const auto& c : candidates) { auto& tier_budget = bytes_to_free[c.location.node_id]; auto it = tier_budget.find(c.location.tier); if (it == tier_budget.end() || it->second <= 0) continue; - per_node_keys[c.location.node_id].push_back(c.key); + per_node_victims[c.location.node_id].push_back(EvictionVictim{c.key, c.location.tier}); it->second -= static_cast(c.size); } - return per_node_keys; + return per_node_victims; } } // namespace mori::umbp diff --git a/src/umbp/distributed/master/eviction_manager.cpp b/src/umbp/distributed/master/eviction_manager.cpp index 7e732528d..c96311eb0 100644 --- a/src/umbp/distributed/master/eviction_manager.cpp +++ b/src/umbp/distributed/master/eviction_manager.cpp @@ -21,6 +21,7 @@ // SOFTWARE. #include "umbp/distributed/master/eviction_manager.h" +#include #include #include #include @@ -49,12 +50,59 @@ EvictionManager::EvictionManager(IMasterMetadataStore& store, const EvictionConf EvictionManager::~EvictionManager() { Stop(); } +void EvictionManager::SetPeerDriven(const std::string& node_id, bool peer_driven) { + std::lock_guard lock(peer_driven_mutex_); + if (peer_driven) { + if (peer_driven_.insert(node_id).second) { + MORI_UMBP_INFO("[EvictionManager] node {} evicts for itself; no EvictKey rounds", node_id); + } + } else { + peer_driven_.erase(node_id); + } +} + +bool EvictionManager::PeerDriven(const std::string& node_id) const { + std::lock_guard lock(peer_driven_mutex_); + return peer_driven_.count(node_id) != 0; +} + +std::vector SelectPeerEvictionCandidates( + const IMasterMetadataStore& store, const std::string& node_id, TierType tier, + const std::string& logical_tier, uint64_t bytes, size_t max_keys, + std::chrono::system_clock::time_point now) { + if (max_keys == 0 || max_keys > kMaxPeerEvictionCandidates) { + max_keys = kMaxPeerEvictionCandidates; + } + const NodeTierKey bucket{node_id, tier}; + // A logical-tier filter drops rows after the store returns them, so it + // cannot cap. + auto by_bucket = store.EnumerateEvictionCandidates( + {bucket}, EvictionOrder::kLeastRecentlyAccessed, logical_tier.empty() ? max_keys : 0, now); + auto it = by_bucket.find(bucket); + if (it == by_bucket.end()) return {}; + auto& rows = it->second; + // The order is only a hint to the store. + std::sort(rows.begin(), rows.end(), + [](const auto& a, const auto& b) { return a.last_accessed_at < b.last_accessed_at; }); + std::vector out; + uint64_t covered = 0; + for (auto& row : rows) { + if (out.size() >= max_keys || covered >= bytes) break; + if (!logical_tier.empty() && row.location.logical_tier != logical_tier) continue; + covered += row.size; + out.push_back(std::move(row)); + } + return out; +} + void EvictionManager::Start() { if (running_.load(std::memory_order_relaxed)) return; running_.store(true, std::memory_order_relaxed); thread_ = std::thread(&EvictionManager::EvictionLoop, this); - MORI_UMBP_INFO("[EvictionManager] Started (interval={}s, high={}, low={})", - config_.check_interval.count(), config_.high_watermark, config_.low_watermark); + MORI_UMBP_INFO( + "[EvictionManager] Started (interval={}s, high={}, low={}, max_bytes_per_round={})", + config_.check_interval.count(), config_.high_watermark, config_.low_watermark, + config_.max_evict_bytes_per_round); } void EvictionManager::Stop() { @@ -97,6 +145,7 @@ void EvictionManager::RunOnce() { std::map bytes_to_free; for (const auto& client : clients) { + if (PeerDriven(client.node_id)) continue; for (const auto& [tier, cap] : client.tier_capacities) { // No tier is special-cased here any more (backend-agnostic refactor // Phase 4). The old `if (tier == SSD) continue` existed because EvictKey @@ -118,20 +167,30 @@ void EvictionManager::RunOnce() { } if (bytes_to_free.empty()) return; - MORI_UMBP_INFO("[EvictionManager] {} overloaded node-tiers detected", bytes_to_free.size()); + + // A round frees each bucket's overage in bytes, optionally capped per round. + if (config_.max_evict_bytes_per_round > 0) { + for (auto& [ntk, bytes] : bytes_to_free) { + bytes = std::min(bytes, config_.max_evict_bytes_per_round); + } + } + uint64_t budget_total = 0; + for (const auto& [ntk, bytes] : bytes_to_free) budget_total += bytes; + MORI_UMBP_INFO("[EvictionManager] {} overloaded node-tiers detected, {} bytes to free", + bytes_to_free.size(), budget_total); // Ask the store for eviction-eligible candidates in the overloaded buckets. // The store is policy-neutral: it returns rows ordered by the hint (LRU here, // so an indexed backend can push the ordering down) but makes no eviction - // decision and never sees the byte budget. Limit each bucket to the configured - // round size; a still-overloaded bucket is revisited on the next interval. + // decision and never sees the byte budget; the strategy trims to it, so + // there is no row cap. std::vector buckets; buckets.reserve(bytes_to_free.size()); for (const auto& [ntk, _bytes] : bytes_to_free) buckets.push_back(ntk); - auto candidates_by_bucket = store_.EnumerateEvictionCandidates( - buckets, EvictionOrder::kLeastRecentlyAccessed, config_.evict_batch_size, - std::chrono::system_clock::now()); + auto candidates_by_bucket = + store_.EnumerateEvictionCandidates(buckets, EvictionOrder::kLeastRecentlyAccessed, + /*max_per_bucket=*/0, std::chrono::system_clock::now()); if (candidates_by_bucket.empty()) { MORI_UMBP_DEBUG("[EvictionManager] No eviction candidates found"); return; @@ -139,7 +198,8 @@ void EvictionManager::RunOnce() { // Flatten the candidates and translate the per-(node, tier) byte budget into // the strategy's shape, then let the strategy pick victims (default LRU), - // grouped by node so each peer gets a single EvictKey keys[]. + // grouped by node so each peer gets a single EvictKey. Victims keep their + // tier, so the peer frees the copy that is over budget. std::vector candidates; for (auto& [bucket, rows] : candidates_by_bucket) { for (auto& c : rows) candidates.push_back(std::move(c)); @@ -149,15 +209,16 @@ void EvictionManager::RunOnce() { strategy_budget[ntk.node_id][ntk.tier] = static_cast(bytes); } - auto per_node_keys = strategy_->SelectVictims(std::move(candidates), std::move(strategy_budget)); + auto per_node_victims = + strategy_->SelectVictims(std::move(candidates), std::move(strategy_budget)); size_t selected = 0; - for (const auto& [node_id, keys] : per_node_keys) selected += keys.size(); + for (const auto& [node_id, victims] : per_node_victims) selected += victims.size(); if (selected == 0) return; MORI_UMBP_INFO("[EvictionManager] Selected {} victims across {} nodes", selected, - per_node_keys.size()); + per_node_victims.size()); // Look up peer addresses once per dispatch round. ClientRegistry // owns the (node_id -> peer_address) mapping; we can't ship an @@ -165,16 +226,16 @@ void EvictionManager::RunOnce() { std::unordered_map node_to_peer; for (const auto& client : clients) node_to_peer[client.node_id] = client.peer_address; - for (auto& [node_id, keys] : per_node_keys) { + for (auto& [node_id, victims] : per_node_victims) { auto it = node_to_peer.find(node_id); if (it == node_to_peer.end() || it->second.empty()) { MORI_UMBP_WARN("[EvictionManager] no peer_address for node={} — skipping {} keys", node_id, - keys.size()); + victims.size()); continue; } if (dispatcher_ == nullptr) { MORI_UMBP_DEBUG("[EvictionManager] dispatcher unset; would EvictKey on node={} ({} keys)", - node_id, keys.size()); + node_id, victims.size()); continue; } // Chunked to the shared gRPC message limit. An eviction round frees a @@ -189,19 +250,23 @@ void EvictionManager::RunOnce() { // next heartbeat are what shrink the index), and a chunk that fails is // retried next round. So N chunks reach the same state as one message // would have, and a partial round still makes progress. + // The one-byte tier per key fits in GrpcMaxItemsPerBatch's item overhead. + std::vector keys; + keys.reserve(victims.size()); + for (const auto& victim : victims) keys.push_back(victim.key); size_t sent = 0; size_t chunks = 0; - while (sent < keys.size()) { + while (sent < victims.size()) { const size_t take = GrpcMaxItemsPerBatch(keys, sent); - std::vector chunk(std::make_move_iterator(keys.begin() + sent), - std::make_move_iterator(keys.begin() + sent + take)); + std::vector chunk(std::make_move_iterator(victims.begin() + sent), + std::make_move_iterator(victims.begin() + sent + take)); sent += take; ++chunks; dispatcher_->DispatchEvictKey(node_id, it->second, std::move(chunk)); } if (chunks > 1) { MORI_UMBP_DEBUG("[EvictionManager] node={} {} keys dispatched in {} chunks", node_id, - keys.size(), chunks); + victims.size(), chunks); } } } diff --git a/src/umbp/distributed/master/in_memory_master_metadata_store.cpp b/src/umbp/distributed/master/in_memory_master_metadata_store.cpp index d51bd85a2..e439b9cd1 100644 --- a/src/umbp/distributed/master/in_memory_master_metadata_store.cpp +++ b/src/umbp/distributed/master/in_memory_master_metadata_store.cpp @@ -546,6 +546,23 @@ std::vector> InMemoryMasterMetadataStore::BatchLookupBlock return out; } +void InMemoryMasterMetadataStore::RecordLocalReads(const std::vector& keys, + std::chrono::system_clock::time_point now) { + if (keys.empty()) return; + // An access bump is atomic, so the shard's shared lock is enough. + std::vector> idx_by_shard(num_shards_); + for (size_t i = 0; i < keys.size(); ++i) idx_by_shard[shard_index(keys[i])].push_back(i); + for (size_t si = 0; si < num_shards_; ++si) { + if (idx_by_shard[si].empty()) continue; + auto& sh = shard_at(si); + std::shared_lock lock(sh.mutex); + for (size_t i : idx_by_shard[si]) { + auto it = sh.entries.find(keys[i]); + if (it != sh.entries.end()) it->second.RecordAccessAtomic(now); + } + } +} + std::vector InMemoryMasterMetadataStore::BatchExistsBlock( const std::vector& keys) const { std::vector results(keys.size(), false); diff --git a/src/umbp/distributed/master/master_client.cpp b/src/umbp/distributed/master/master_client.cpp index f15b5112d..aa83899d1 100644 --- a/src/umbp/distributed/master/master_client.cpp +++ b/src/umbp/distributed/master/master_client.cpp @@ -204,6 +204,7 @@ grpc::Status MasterClient::RegisterSelf( FillTierCapacities(req.mutable_tier_capacities(), tier_capacities); FillLogicalTierCapacities(req.mutable_logical_tier_capacities(), logical_tier_capacities); for (const auto& tag : config_.tags) req.add_tags(tag); + req.set_peer_driven_eviction(peer_driven_requested_); ::umbp::RegisterClientResponse resp; grpc::ClientContext ctx; @@ -217,6 +218,7 @@ grpc::Status MasterClient::RegisterSelf( if (resp.heartbeat_interval_ms() > 0) heartbeat_interval_ms_ = resp.heartbeat_interval_ms(); supports_max_allocatable_capacity_ = resp.supports_max_allocatable_bytes(); supports_logical_tiers_ = resp.supports_logical_tiers(); + supports_peer_driven_eviction_ = resp.supports_peer_driven_eviction(); aggregate_backend_capacities_ = aggregate_backend_capacities_requested_ && supports_max_allocatable_capacity_; { @@ -445,6 +447,57 @@ grpc::Status MasterClient::BatchLookup(const std::vector& keys, return grpc::Status::OK; } +std::optional> MasterClient::GetEvictionCandidates( + TierType tier, const std::string& logical_tier, uint64_t bytes, uint32_t max_keys) { + ScopedRpcTimer _rpc_timer(this, "GetEvictionCandidates"); + ::umbp::GetEvictionCandidatesRequest req; + req.set_node_id(config_.node_id); + req.set_tier(ToProtoTier(tier)); + req.set_logical_tier(logical_tier); + req.set_bytes(bytes); + req.set_max_keys(max_keys); + + ::umbp::GetEvictionCandidatesResponse resp; + grpc::ClientContext ctx; + // Eviction does not wait on this (PeerPool::RankedCandidates); past the + // age a peer still uses, an answer is wasted work. + ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(3)); + auto status = GetStub(stub_.get())->GetEvictionCandidates(&ctx, req, &resp); + _rpc_timer.SetStatus(status); + if (!status.ok()) { + static std::atomic failures{0}; + const uint64_t n = failures.fetch_add(1, std::memory_order_relaxed) + 1; + if (n <= 8 || n % 1000 == 0) { + MORI_UMBP_WARN("[Client] GetEvictionCandidates failed ({} so far): {}", n, + status.error_message()); + } + return std::nullopt; + } + const int n = std::min(resp.keys_size(), resp.sizes_size()); + std::vector out; + out.reserve(static_cast(n)); + for (int i = 0; i < n; ++i) out.push_back(EvictionOffer{resp.keys(i), resp.sizes(i)}); + return out; +} + +void MasterClient::NoteLocalReads(const std::vector& keys) { + // Past kFlushAt distinct keys, ship them now rather than wait out the + // interval; past kMax (master unreachable) the rest are dropped. + constexpr size_t kFlushAt = 4096; + constexpr size_t kMax = 65536; + if (keys.empty() || !ReportsLocalReads()) return; + bool flush = false; + { + std::lock_guard lock(local_reads_mutex_); + for (const auto& key : keys) { + if (local_reads_.size() >= kMax) break; + local_reads_.insert(key); + } + flush = local_reads_.size() >= kFlushAt; + } + if (flush) FlushHeartbeat(); +} + namespace { // Auto-flush the heartbeat once this many KvEvents accumulate at the peer, so a // completed batch of puts becomes visible at the master without the application @@ -742,6 +795,7 @@ bool MasterClient::SendDeltaHeartbeatLocked(const std::mapLogicalTierCapacities()); } for (const auto& tag : config_.tags) re_req.add_tags(tag); + re_req.set_peer_driven_eviction(peer_driven_requested_); ::umbp::RegisterClientResponse re_resp; grpc::ClientContext re_ctx; grpc::Status re_status; @@ -754,6 +808,7 @@ bool MasterClient::SendDeltaHeartbeatLocked(const std::map reads; + { + std::lock_guard lock(local_reads_mutex_); + reads.swap(local_reads_); + } + req.mutable_local_reads()->Reserve(static_cast(reads.size())); + while (!reads.empty()) req.add_local_reads(std::move(reads.extract(reads.begin()).value())); grpc::ClientContext ctx; ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::milliseconds(HeartbeatRpcTimeoutMs())); diff --git a/src/umbp/distributed/master/master_server.cpp b/src/umbp/distributed/master/master_server.cpp index 89b53e9a9..3da880c16 100644 --- a/src/umbp/distributed/master/master_server.cpp +++ b/src/umbp/distributed/master/master_server.cpp @@ -128,14 +128,21 @@ int EvictKeyDeadlineMs() { class MasterPeerStubPool : public EvictKeyDispatcher { public: void DispatchEvictKey(const std::string& node_id, const std::string& peer_address, - std::vector keys) override { - if (keys.empty() || peer_address.empty()) return; + std::vector victims) override { + if (victims.empty() || peer_address.empty()) return; auto stub = GetOrCreateStub(node_id, peer_address); if (stub == nullptr) return; + // keys and tiers are parallel, so a peer that predates `tiers` still reads + // a complete key list. ::umbp::EvictKeyRequest req; - for (auto& k : keys) req.add_keys(std::move(k)); + req.mutable_keys()->Reserve(static_cast(victims.size())); + req.mutable_tiers()->Reserve(static_cast(victims.size())); + for (auto& victim : victims) { + req.add_keys(std::move(victim.key)); + req.add_tiers(static_cast<::umbp::TierType>(victim.tier)); + } ::umbp::EvictKeyResponse resp; grpc::ClientContext ctx; ctx.set_deadline(std::chrono::system_clock::now() + @@ -284,6 +291,7 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser "node is already alive and cannot be re-registered"); } + NotePeerDriven(request->node_id(), request->peer_driven_eviction()); UpdateClientCountMetric(); UpdateClientCapacityMetrics(request->node_id(), caps); UpdateClientLogicalTierMetrics(request->node_id(), registration.logical_tier_capacities); @@ -294,6 +302,7 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser response->set_ack_seq(0); response->set_supports_max_allocatable_bytes(config_.advertise_max_allocatable_bytes); response->set_supports_logical_tiers(config_.advertise_logical_tiers); + response->set_supports_peer_driven_eviction(config_.advertise_peer_driven_eviction); return grpc::Status::OK; }); } @@ -303,6 +312,7 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser ::umbp::UnregisterClientResponse* /*response*/) override { return GuardStore("UnregisterClient", [&]() -> grpc::Status { store_.UnregisterClient(request->node_id()); + NotePeerDriven(request->node_id(), false); UpdateClientCountMetric(); return grpc::Status::OK; }); @@ -313,6 +323,7 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser grpc::Status Heartbeat(grpc::ServerContext* /*ctx*/, const ::umbp::HeartbeatRequest* request, ::umbp::HeartbeatResponse* response) override { return GuardStore("Heartbeat", [&]() -> grpc::Status { + NotePeerDriven(request->node_id(), request->peer_driven_eviction()); std::map caps; for (const auto& tc : request->tier_capacities()) { TierCapacity c; @@ -397,6 +408,12 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser } } + // After the bundles, so a key added by this heartbeat is already there. + if (client_status != ClientStatus::UNKNOWN && request->local_reads_size() > 0) { + store_.RecordLocalReads({request->local_reads().begin(), request->local_reads().end()}, + now); + } + response->set_status(static_cast<::umbp::ClientStatus>(client_status)); response->set_acked_seq(acked_seq); response->set_request_full_sync(request_full_sync); @@ -602,6 +619,30 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser }); } + grpc::Status GetEvictionCandidates(grpc::ServerContext* /*ctx*/, + const ::umbp::GetEvictionCandidatesRequest* request, + ::umbp::GetEvictionCandidatesResponse* response) override { + if (!config_.advertise_peer_driven_eviction) { + return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "peer-driven eviction is disabled"); + } + return GuardStore("GetEvictionCandidates", [&]() -> grpc::Status { + if (request->node_id().empty()) { + return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "node_id must not be empty"); + } + const auto rows = SelectPeerEvictionCandidates( + store_, request->node_id(), static_cast(request->tier()), + request->logical_tier(), request->bytes(), request->max_keys(), + std::chrono::system_clock::now()); + response->mutable_keys()->Reserve(static_cast(rows.size())); + response->mutable_sizes()->Reserve(static_cast(rows.size())); + for (const auto& row : rows) { + response->add_keys(row.key); + response->add_sizes(row.size); + } + return grpc::Status::OK; + }); + } + // -------- External KV mutation/query -------- grpc::Status ReportExternalKvBlocks( @@ -824,8 +865,17 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser } void SetMetrics(mori::metrics::MetricsServer* metrics) { metrics_ = metrics; } + void SetEvictionManager(EvictionManager* eviction_manager) { + eviction_manager_ = eviction_manager; + } private: + void NotePeerDriven(const std::string& node_id, bool peer_driven) { + if (eviction_manager_ == nullptr || node_id.empty()) return; + eviction_manager_->SetPeerDriven(node_id, + peer_driven && config_.advertise_peer_driven_eviction); + } + void UpdateClientCountMetric() { if (!metrics_) return; metrics_->setGauge(MORI_UMBP_METRIC_CLIENT_COUNT, MORI_UMBP_METRIC_CLIENT_COUNT_HELP, @@ -896,6 +946,7 @@ class MasterServer::UMBPMasterServiceImpl final : public ::umbp::UMBPMaster::Ser Router& router_; ClientRegistryConfig config_; mori::metrics::MetricsServer* metrics_ = nullptr; + EvictionManager* eviction_manager_ = nullptr; // non-owning; set before Run() }; // --------------------------------------------------------------------------- @@ -912,6 +963,7 @@ MasterServer::MasterServer(MasterServerConfig config) peer_stub_pool_.get(), std::move(config_.evict_strategy))) { router_.SetLeaseDuration(config_.eviction_config.lease_duration); + service_->SetEvictionManager(eviction_manager_.get()); } MasterServer::~MasterServer() { diff --git a/src/umbp/distributed/master/redis_master_metadata_store.cpp b/src/umbp/distributed/master/redis_master_metadata_store.cpp index 6ed7ebd7b..67e2a76ec 100644 --- a/src/umbp/distributed/master/redis_master_metadata_store.cpp +++ b/src/umbp/distributed/master/redis_master_metadata_store.cpp @@ -799,6 +799,23 @@ std::vector> RedisMasterMetadataStore::BatchLookupBlockFor return out; } +void RedisMasterMetadataStore::RecordLocalReads(const std::vector& keys, + std::chrono::system_clock::time_point now) { + if (keys.empty()) return; + ScopedStoreOp _op(metrics_, "RecordLocalReads"); + const redis::ShardedBatch batch = redis::GroupKeysByShard(keys_, keys); + try { + redis::RunShardedRead( + batch, redis::kRecordLocalReadsLua, {std::to_string(ToEpochMs(now))}, "RecordLocalReads", + fanout_pool_.get(), /*tolerate_shard_failures=*/mode_ != Mode::kSingle, metrics_, + [&](size_t group) { return &client_for_shard(batch.shard_of_group[group]); }, + [](size_t, const RespValue&) {}); + } catch (const std::exception& e) { + // Recency is a hint; losing it must not fail the heartbeat carrying it. + MORI_UMBP_WARN("[RedisStore] RecordLocalReads failed: {}", e.what()); + } +} + std::vector RedisMasterMetadataStore::BatchExistsBlock( const std::vector& keys) const { std::vector results(keys.size(), false); diff --git a/src/umbp/distributed/peer/backend/page_backend.cpp b/src/umbp/distributed/peer/backend/page_backend.cpp index 479866e53..8b858a54b 100644 --- a/src/umbp/distributed/peer/backend/page_backend.cpp +++ b/src/umbp/distributed/peer/backend/page_backend.cpp @@ -359,17 +359,8 @@ void HostPageMemorySource::Release() { // --------------------------------------------------------------------------- AllocateResult PageBackend::Allocate(const std::string& key, uint64_t size) { - AllocateResult out; - { - std::unique_lock lock(mutex_); - out = AllocateLocked(key, size); - } - // A full pool is the one state the commit-time round cannot have prevented - // (one object larger than the headroom, or a burst that never reached - // commit). Evicting here does not rescue THIS allocation -- it is not - // retried -- but it stops the pool staying wedged for every put after it. - if (out.outcome == AllocateOutcome::kFailedNoSpace) MaybeEvictToLowWatermark(); - return out; + std::unique_lock lock(mutex_); + return AllocateLocked(key, size); } AllocateResult PageBackend::AllocateLocked(const std::string& key, uint64_t size, @@ -433,18 +424,13 @@ std::vector PageBackend::BatchAllocate( const std::vector& entries) { std::vector out(entries.size()); if (entries.empty()) return out; - bool no_space = false; - { - std::unique_lock lock(mutex_); - for (size_t i = 0; i < entries.size(); ++i) { - out[i] = AllocateLocked(entries[i].key, entries[i].size, entries[i].preferred_numa_node); - if (out[i].outcome == AllocateOutcome::kSuccessAllocated) { - out[i].descs = BuildBufferDescsLocked(out[i].pages); - } - no_space = no_space || out[i].outcome == AllocateOutcome::kFailedNoSpace; + std::unique_lock lock(mutex_); + for (size_t i = 0; i < entries.size(); ++i) { + out[i] = AllocateLocked(entries[i].key, entries[i].size, entries[i].preferred_numa_node); + if (out[i].outcome == AllocateOutcome::kSuccessAllocated) { + out[i].descs = BuildBufferDescsLocked(out[i].pages); } } - if (no_space) MaybeEvictToLowWatermark(); return out; } @@ -453,16 +439,8 @@ std::vector PageBackend::BatchAllocate( // --------------------------------------------------------------------------- bool PageBackend::Commit(uint64_t slot_id, const std::string& key, uint64_t& bytes_committed) { - bool ok = false; - { - std::unique_lock lock(mutex_); - ok = CommitLocked(slot_id, key, bytes_committed); - } - // Check-after-write, on the committing thread: there is no eviction thread, - // and a commit is the only event that grows usage. Outside the lock because - // a round takes mutex_ itself. No-op unless local eviction is enabled. - MaybeEvictToLowWatermark(); - return ok; + std::unique_lock lock(mutex_); + return CommitLocked(slot_id, key, bytes_committed); } bool PageBackend::CommitLocked(uint64_t slot_id, const std::string& key, @@ -510,15 +488,10 @@ bool PageBackend::CommitLocked(uint64_t slot_id, const std::string& key, std::vector PageBackend::BatchCommit(const std::vector& entries) { std::vector out(entries.size()); if (entries.empty()) return out; - { - std::unique_lock lock(mutex_); - for (size_t i = 0; i < entries.size(); ++i) { - out[i].success = CommitLocked(entries[i].slot_id, entries[i].key, out[i].bytes_committed); - } + std::unique_lock lock(mutex_); + for (size_t i = 0; i < entries.size(); ++i) { + out[i].success = CommitLocked(entries[i].slot_id, entries[i].key, out[i].bytes_committed); } - // One round for the whole batch, not one per key: a batch is a single burst - // of growth and a round already frees down to the low watermark. - MaybeEvictToLowWatermark(); return out; } @@ -583,7 +556,7 @@ std::vector PageBackend::BatchResolve(const std::vector> touched; - if (local_evict_enabled_) touched.reserve(keys.size()); + if (track_eviction_order_) touched.reserve(keys.size()); for (size_t i = 0; i < keys.size(); ++i) { auto it = owned_.find(keys[i]); if (it == owned_.end()) continue; @@ -595,7 +568,7 @@ std::vector PageBackend::BatchResolve(const std::vectorsecond.pages); RenewLeaseAtomic(it->second, lease_deadline); - if (local_evict_enabled_) touched.emplace_back(&keys[i], &it->second); + if (track_eviction_order_) touched.emplace_back(&keys[i], &it->second); } if (!touched.empty()) { std::lock_guard lru_lock(lru_mutex_); @@ -614,7 +587,7 @@ bool PageBackend::AcquireMigrationRead(const std::string& key, ResolvedEntry* re resolved->pages = owned->second.pages; resolved->size = owned->second.size; resolved->page_size = page_size_; - pins_[key] = PinState{next_pin_token_++, std::chrono::steady_clock::now()}; + pins_[key] = PinState{next_pin_token_++, std::chrono::steady_clock::now(), /*migration=*/true}; return true; } @@ -831,35 +804,41 @@ void PageBackend::SetEventPublishing(bool enabled) { } // --------------------------------------------------------------------------- -// Local eviction (no-master deployments; see EnableLocalEviction) +// Eviction order (the policy that consumes it lives in PeerPool) // --------------------------------------------------------------------------- -void PageBackend::EnableLocalEviction(double high_watermark, double low_watermark) { - if (!(high_watermark > 0.0 && high_watermark <= 1.0 && low_watermark > 0.0 && - low_watermark < high_watermark)) { - MORI_UMBP_WARN( - "[PageBackend] ignoring local eviction watermarks high={} low={} " - "(require 0 < low < high <= 1) — local eviction stays OFF", - high_watermark, low_watermark); - return; - } +void PageBackend::TrackEvictionOrder() { std::unique_lock lock(mutex_); - local_evict_enabled_ = true; - local_evict_high_wm_ = high_watermark; - local_evict_low_wm_ = low_watermark; - // Keys committed before this call are not in lru_ and would therefore be - // invisible to a round. Enabling before Init is the contract, but seed - // anyway so a late enable degrades to "wrong order", not "never evicted". - { - std::lock_guard lru_lock(lru_mutex_); - for (auto& [key, slot] : owned_) TouchLruLocked(key, slot); + if (track_eviction_order_) return; + track_eviction_order_ = true; + // Seed keys committed before this call, as coldest. + std::lock_guard lru_lock(lru_mutex_); + for (auto& [key, slot] : owned_) TouchLruLocked(key, slot); +} + +std::vector PageBackend::EvictionCandidates(size_t max_count) { + std::vector out; + if (max_count == 0) return out; + std::shared_lock lock(mutex_); + if (!track_eviction_order_) return out; + const auto now = std::chrono::steady_clock::now(); + std::lock_guard lru_lock(lru_mutex_); + for (auto it = lru_.rbegin(); it != lru_.rend() && out.size() < max_count; ++it) { + auto owned_it = owned_.find(*it); + if (owned_it == owned_.end()) continue; // defensive; lru_/owned_ stay in sync + // Skip what Evict() refuses (read lease, copy pin); a migration pin is + // offered so the caller can count it as in flight. + if (HasActiveReadLeaseLocked(owned_it->second, now)) continue; + if (auto pin = pins_.find(*it); pin != pins_.end() && !pin->second.migration) continue; + // Whole pages, as Capacity() counts them. + out.push_back( + EvictionOffer{*it, static_cast(owned_it->second.pages.size()) * page_size_}); } - MORI_UMBP_INFO("[PageBackend] local eviction ON tier={} high={} low={}", static_cast(tier_), - high_watermark, low_watermark); + return out; } void PageBackend::TouchLruLocked(const std::string& key, OwnedSlot& slot) { - if (!local_evict_enabled_) return; + if (!track_eviction_order_) return; if (slot.in_lru) { lru_.splice(lru_.begin(), lru_, slot.lru_it); return; @@ -876,7 +855,7 @@ void PageBackend::RemoveFromLruLocked(OwnedSlot& slot) { } void PageBackend::TouchLru(const std::string& key, OwnedSlot& slot) { - if (!local_evict_enabled_) return; + if (!track_eviction_order_) return; std::lock_guard lru_lock(lru_mutex_); TouchLruLocked(key, slot); } @@ -886,72 +865,6 @@ void PageBackend::RemoveFromLru(OwnedSlot& slot) { RemoveFromLruLocked(slot); } -size_t PageBackend::MaybeEvictToLowWatermark() { - // Only one round at a time. A second committing thread backs off rather - // than over-evicting, and try_lock keeps the put path non-blocking. - std::unique_lock round(local_evict_round_mutex_, std::try_to_lock); - if (!round.owns_lock()) return 0; - - std::unique_lock lock(mutex_); - if (!local_evict_enabled_ || !allocator_) return 0; - - const uint64_t total = allocator_->TotalBytes(); - if (total == 0) return 0; - const uint64_t used = allocator_->UsedBytes(); - if (static_cast(used) < local_evict_high_wm_ * static_cast(total)) return 0; - - const auto low_bytes = static_cast(local_evict_low_wm_ * static_cast(total)); - if (used <= low_bytes) return 0; - const uint64_t to_free = used - low_bytes; - - // Oldest first. Two passes so the list is not mutated under its own - // reverse iterator; the selection pass is where the skips happen. - const auto now = std::chrono::steady_clock::now(); - uint64_t selected_bytes = 0; - std::vector victims; - { - std::lock_guard lru_lock(lru_mutex_); - for (auto it = lru_.rbegin(); it != lru_.rend() && selected_bytes < to_free; ++it) { - auto owned_it = owned_.find(*it); - if (owned_it == owned_.end()) continue; // defensive; lru_/owned_ stay in sync - // Same two protections master-driven Evict honours: an in-flight read - // owns the pages until its lease expires, and a copy pin until the - // worker is done. A round that finds only protected keys frees what it - // can and stops — it must not spin waiting for a reader. - if (HasActiveReadLeaseLocked(owned_it->second, now)) continue; - if (HasActivePinLocked(*it)) continue; - victims.push_back(*it); - selected_bytes += static_cast(owned_it->second.pages.size()) * page_size_; - } - } - - size_t freed = 0; - for (const auto& key : victims) { - auto owned_it = owned_.find(key); - if (owned_it == owned_.end()) continue; - if (allocator_) allocator_->Deallocate(owned_it->second.pages); - // A REMOVE is queued for symmetry with master-driven Evict; with no master - // event publishing is off and this is a no-op, so nothing accumulates. - QueueEventLocked(KvEvent{KvEvent::Kind::REMOVE, key, tier_, 0}); - RemoveFromLru(owned_it->second); - owned_.erase(owned_it); - ++freed; - } - if (freed > 0) { - MORI_UMBP_DEBUG("[PageBackend] local eviction freed {} key(s) ({} bytes) tier={}", freed, - selected_bytes, static_cast(tier_)); - local_evict_warned_ = false; - } else if (!local_evict_warned_) { - // Latched: a wedged pool would otherwise log this on every commit. - local_evict_warned_ = true; - MORI_UMBP_WARN( - "[PageBackend] local eviction found no reclaimable key at {}/{} bytes used " - "(all leased or pinned) tier={}", - used, total, static_cast(tier_)); - } - return freed; -} - void PageBackend::SetAutoFlushHook(size_t threshold, std::function cb) { std::unique_lock lock(mutex_); auto_flush_threshold_ = threshold; diff --git a/src/umbp/distributed/peer/backend/ssd_backend.cpp b/src/umbp/distributed/peer/backend/ssd_backend.cpp index 6e14857e8..07b3058bf 100644 --- a/src/umbp/distributed/peer/backend/ssd_backend.cpp +++ b/src/umbp/distributed/peer/backend/ssd_backend.cpp @@ -718,6 +718,20 @@ std::vector SsdBackend::Evict(const std::vector& keys) return results; } +std::vector SsdBackend::EvictionCandidates(size_t max_count) { + if (ssd_ == nullptr || max_count == 0) return {}; + auto candidates = ssd_->EvictionCandidates(max_count); + std::lock_guard lock(mutex_); + if (read_leases_.empty() && migration_reads_.empty()) return candidates; + candidates.erase(std::remove_if(candidates.begin(), candidates.end(), + [&](const EvictionOffer& offer) { + return read_leases_.count(offer.key) != 0 || + migration_reads_.count(offer.key) != 0; + }), + candidates.end()); + return candidates; +} + // --------------------------------------------------------------------------- // Bootstrap // --------------------------------------------------------------------------- @@ -862,10 +876,8 @@ std::vector SsdBackend::SampleMetrics() const { event("single_flight_dup", ssd_->ReadDup()); event("single_flight_merged", ssd_->ReadMerged()); - // Local high-watermark eviction. The victims and the freed bytes are the - // decorator's (Evict is an interface call); the ROUNDS are not — master never - // asked for them — and neither is a backend delete that refused. - event("eviction_round", ssd_->EvictionRounds()); + // Evictions are counted by the decorator and the pool; only a delete the + // drive refused is counted here. event("eviction_backend_failed", ssd_->EvictionBackendFailures()); // Staging pressure. slot_full_reject is the one to watch: it counts resolves diff --git a/src/umbp/distributed/peer/peer_service.cpp b/src/umbp/distributed/peer/peer_service.cpp index 7e68b9813..c2188c04d 100644 --- a/src/umbp/distributed/peer/peer_service.cpp +++ b/src/umbp/distributed/peer/peer_service.cpp @@ -265,18 +265,22 @@ class PeerServiceServer::UMBPPeerServiceImpl final : public ::umbp::UMBPPeer::Se grpc::Status EvictKey(grpc::ServerContext* /*ctx*/, const ::umbp::EvictKeyRequest* request, ::umbp::EvictKeyResponse* response) override { - // Eviction carries no tier either. Master drives this to reclaim capacity - // it measured per (node, tier), so the pool is free to demote where a tier - // configures on_evict offload; otherwise a key mirrored across media is - // dropped from ALL of them. Either way the freed bytes are summed per key - // — master sizes its next eviction round off this total. - std::vector keys(request->keys().begin(), request->keys().end()); - if (keys.empty()) return grpc::Status::OK; - auto evicted = - pool_ == nullptr ? std::vector{} : pool_->Evict(keys, PoolEvictMode::kReclaim); - for (size_t i = 0; i < keys.size(); ++i) { + // Free each key's copy on the tier the master named (an older master names + // none: the medium holding it). bytes_freed is informational; a queued + // demotion reports 0. + const int n = request->keys_size(); + if (n == 0) return grpc::Status::OK; + const bool scoped = request->tiers_size() == n; + std::vector requests(static_cast(n)); + for (int i = 0; i < n; ++i) { + requests[i].key = request->keys(i); + if (scoped) requests[i].tier = static_cast(request->tiers(i)); + } + auto evicted = pool_ == nullptr ? std::vector{} + : pool_->Evict(requests, PoolEvictMode::kReclaim); + for (size_t i = 0; i < requests.size(); ++i) { auto* entry = response->add_evicted(); - entry->set_key(keys[i]); + entry->set_key(requests[i].key); entry->set_bytes_freed(i < evicted.size() ? evicted[i].bytes_freed : 0); } return grpc::Status::OK; diff --git a/src/umbp/distributed/peer/ssd/peer_ssd_manager.cpp b/src/umbp/distributed/peer/ssd/peer_ssd_manager.cpp index b6e3f823a..57a27b50f 100644 --- a/src/umbp/distributed/peer/ssd/peer_ssd_manager.cpp +++ b/src/umbp/distributed/peer/ssd/peer_ssd_manager.cpp @@ -38,14 +38,6 @@ namespace mori::umbp { -namespace { -// Watermarks must satisfy 0 < low < high <= 1. We fail fast on a bad value -// rather than silently clamping, so a misconfigured env surfaces immediately. -bool WatermarksValid(double high, double low) { - return high > 0.0 && high <= 1.0 && low > 0.0 && low < high; -} -} // namespace - // --------------------------------------------------------------------------- // Construction // --------------------------------------------------------------------------- @@ -56,14 +48,7 @@ PeerSsdManager::PeerSsdManager(const PeerSsdConfig& cfg) { return; } const auto& ssd = cfg.ssd; - - if (!WatermarksValid(ssd.high_watermark, ssd.low_watermark)) { - throw std::runtime_error( - "[PeerSsdManager] invalid SSD watermarks (require 0 < low_watermark < " - "high_watermark <= 1)"); - } - high_watermark_ = ssd.high_watermark; - low_watermark_ = ssd.low_watermark; + // `ssd`'s watermarks drive PeerPool's local eviction, not this class. single_flight_ = ssd.single_flight_reads; // Explicit backend selection — an unknown value is a configuration error, not @@ -100,18 +85,8 @@ PeerSsdManager::PeerSsdManager(const PeerSsdConfig& cfg) { "' (expected one of: file, spdk, spdk_proxy)"); } -PeerSsdManager::PeerSsdManager(std::unique_ptr backend, double high_watermark, - double low_watermark, bool single_flight) - : backend_(std::move(backend)), - high_watermark_(high_watermark), - low_watermark_(low_watermark), - single_flight_(single_flight) { - if (!WatermarksValid(high_watermark_, low_watermark_)) { - throw std::runtime_error( - "[PeerSsdManager] invalid SSD watermarks (require 0 < low_watermark < " - "high_watermark <= 1)"); - } -} +PeerSsdManager::PeerSsdManager(std::unique_ptr backend, bool single_flight) + : backend_(std::move(backend)), single_flight_(single_flight) {} PeerSsdManager::~PeerSsdManager() = default; @@ -215,14 +190,10 @@ bool PeerSsdManager::Write(const std::string& key, } // Backend IO outside our mutex_ — SSDTier is internally synchronized. A - // failure may be ENOSPC: run one eviction round to reclaim space and retry - // once before giving up (best-effort, no event/record on final failure). + // failure (ENOSPC included) records nothing; this never evicts. if (!backend_->Write(key, data, total_size)) { - EvictToLowWatermark(); - if (!backend_->Write(key, data, total_size)) { - MORI_UMBP_WARN("[PeerSsdManager] backend Write failed key={} size={}", key, total_size); - return false; - } + MORI_UMBP_WARN("[PeerSsdManager] backend Write failed key={} size={}", key, total_size); + return false; } // Bytes physically written to the SSD device (write IO bandwidth source). @@ -244,19 +215,7 @@ bool PeerSsdManager::Write(const std::string& key, pending_events_.push_back(KvEvent{KvEvent::Kind::ADD, key, TierType::SSD, total_size}); } } - // Keep the just-written key out of this write's own watermark victim set. - // A successful Write must imply the key still exists when it returns. - auto& slot = inflight_reads_[key]; - if (!slot) slot = std::make_unique(); - ++slot->refs; - } - - // Check-after-write trigger, on this copy worker (no dedicated thread). - auto [used, total] = Capacity(); - if (total > 0 && static_cast(used) >= high_watermark_ * static_cast(total)) { - EvictToLowWatermark(); } - UnpinForMigration(key); return true; } @@ -274,7 +233,7 @@ std::vector PeerSsdManager::WriteBatch(const std::vector& key // Stage timers for the [SsdPerf/peer] PUT breakdown (no-ops unless // UMBP_SSD_TIMING is set). const auto t_begin = ssdperf::Now(); - double dedup_ms = 0.0, backend_ms = 0.0, retry_ms = 0.0, record_ms = 0.0, evict_ms = 0.0; + double dedup_ms = 0.0, backend_ms = 0.0, record_ms = 0.0; // Pass 1 — dedup under one lock. Already-owned keys succeed with no IO (same // rationale as Write: a sequential re-put must not repeat the device write). @@ -296,46 +255,24 @@ std::vector PeerSsdManager::WriteBatch(const std::vector& key const size_t dedup_hits = keys.size() - todo.size(); if (todo.empty()) return results; - auto gather = [&](const std::vector& idxs) { - std::vector k; - std::vector d; - std::vector z; - k.reserve(idxs.size()); - d.reserve(idxs.size()); - z.reserve(idxs.size()); - for (size_t i : idxs) { - k.push_back(keys[i]); - d.push_back(srcs[i]); - z.push_back(sizes[i]); - } - return std::make_tuple(std::move(k), std::move(d), std::move(z)); - }; - - // Pass 2 — one batched backend write outside the lock. Failures are most - // likely ENOSPC, so retry exactly those once after a single eviction round - // (mirrors Write's evict-and-retry, amortized over the whole batch). + // Pass 2 — one batched backend write outside the lock. A failed key + // (ENOSPC included) is recorded nowhere; this never evicts. const auto t_backend = ssdperf::Now(); - auto [bk, bd, bz] = gather(todo); - auto ok = backend_->BatchWrite(bk, bd, bz); + std::vector batch_keys; + std::vector batch_srcs; + std::vector batch_sizes; + batch_keys.reserve(todo.size()); + batch_srcs.reserve(todo.size()); + batch_sizes.reserve(todo.size()); + for (size_t i : todo) { + batch_keys.push_back(keys[i]); + batch_srcs.push_back(srcs[i]); + batch_sizes.push_back(sizes[i]); + } + auto ok = backend_->BatchWrite(batch_keys, batch_srcs, batch_sizes); ok.resize(todo.size(), false); backend_ms = ssdperf::MsSince(t_backend); - std::vector retry; - for (size_t j = 0; j < todo.size(); ++j) { - if (!ok[j]) retry.push_back(todo[j]); - } - if (!retry.empty()) { - const auto t_retry = ssdperf::Now(); - EvictToLowWatermark(); - auto [rk, rd, rz] = gather(retry); - auto retry_ok = backend_->BatchWrite(rk, rd, rz); - retry_ok.resize(retry.size(), false); - for (size_t j = 0, r = 0; j < todo.size(); ++j) { - if (!ok[j]) ok[j] = retry_ok[r++]; - } - retry_ms = ssdperf::MsSince(t_retry); - } - // Pass 3 — record locations + ADD events under one lock. The owned_ re-check // matches Write: a concurrent writer may have landed the same content-addressed // bytes, and only the first recorder emits ADD SSD. @@ -369,29 +306,21 @@ std::vector PeerSsdManager::WriteBatch(const std::vector& key } } - // One check-after-write for the whole batch instead of one per key. - const auto t_evict = ssdperf::Now(); - auto [used, total] = Capacity(); - if (total > 0 && static_cast(used) >= high_watermark_ * static_cast(total)) { - EvictToLowWatermark(); - } - evict_ms = ssdperf::MsSince(t_evict); - if (ssdperf::Enabled()) { const double total_ms = ssdperf::MsSince(t_begin); + const auto [used, total] = Capacity(); MORI_UMBP_INFO( "[SsdPerf/peer] PUT keys={} dedup_hits={} written={} bytes={} total_ms={:.3f} GB_s={:.2f} " - "| dedup={:.3f}ms backend={:.3f}ms ({:.0f}%) evict_retry={:.3f}ms record={:.3f}ms " - "evict_check={:.3f}ms | used={}/{}B", + "| dedup={:.3f}ms backend={:.3f}ms ({:.0f}%) record={:.3f}ms | used={}/{}B", keys.size(), dedup_hits, todo.size(), written_bytes, total_ms, ssdperf::GbPerSec(written_bytes, total_ms), dedup_ms, backend_ms, - ssdperf::Pct(backend_ms, total_ms), retry_ms, record_ms, evict_ms, used, total); + ssdperf::Pct(backend_ms, total_ms), record_ms, used, total); } return results; } // --------------------------------------------------------------------------- -// Eviction (local, read-priority) +// Eviction (the medium's one implementation; read-priority) // --------------------------------------------------------------------------- bool PeerSsdManager::Evict(const std::string& key) { @@ -431,50 +360,21 @@ bool PeerSsdManager::Evict(const std::string& key) { return true; } -std::vector PeerSsdManager::SelectVictims(size_t bytes_to_free) { +std::vector PeerSsdManager::EvictionCandidates(size_t max_count) { std::lock_guard lock(mutex_); - std::vector victims; - if (bytes_to_free == 0) return victims; + std::vector candidates; + if (max_count == 0) return candidates; // Oldest first (LRU tail -> MRU front), skipping keys being read or evicted. - size_t freed = 0; - for (auto it = lru_.rbegin(); it != lru_.rend(); ++it) { + for (auto it = lru_.rbegin(); it != lru_.rend() && candidates.size() < max_count; ++it) { const std::string& key = *it; if (inflight_reads_.count(key) != 0) continue; if (evicting_.count(key) != 0) continue; auto owned_it = owned_.find(key); if (owned_it == owned_.end()) continue; // defensive; lru_/owned_ stay in sync - victims.push_back(key); - freed += owned_it->second.size; - if (freed >= bytes_to_free) break; - } - return victims; -} - -void PeerSsdManager::EvictToLowWatermark() { - if (!backend_) return; - - // Only one eviction round at a time: a second concurrent worker (or a worker - // racing ClearLocal) backs off instead of over-evicting. try_lock keeps the - // copy path non-blocking. - std::unique_lock round(eviction_mu_, std::try_to_lock); - if (!round.owns_lock()) return; - - auto [used, total] = Capacity(); - if (total == 0) return; - double low_bytes = low_watermark_ * static_cast(total); - if (static_cast(used) <= low_bytes) return; - size_t bytes_to_free = used - static_cast(low_bytes); - - // A real round is about to run (we own the round lock and are over the low - // watermark). Count it before selecting victims. - metrics_.evict_rounds.fetch_add(1, std::memory_order_relaxed); - - // Single pass, no retry loop: if everything reclaimable is in use we free - // what we can and stop, so a fully-pinned tier cannot starve the worker. - for (const auto& key : SelectVictims(bytes_to_free)) { - Evict(key); + candidates.push_back(EvictionOffer{key, owned_it->second.size}); } + return candidates; } // --------------------------------------------------------------------------- @@ -486,10 +386,8 @@ void PeerSsdManager::ClearLocal() { // in-flight reads (below) but not in-flight Writes, so a racing Write could // re-add owned_/ADD SSD after backend->Clear() wipes its bytes. // PoolClient::Clear() Quiesce()s first; new callers must too. - // - // Exclude eviction rounds for the whole operation (lock order: eviction_mu_ - // before mutex_, matching EvictToLowWatermark). - std::lock_guard round(eviction_mu_); + // No Evict overlaps this: PeerPool::Evict holds the lifecycle lock that + // PeerPool::ClearLocal takes exclusively. { std::unique_lock lock(mutex_); owned_.clear(); diff --git a/src/umbp/distributed/pool/peer_pool.cpp b/src/umbp/distributed/pool/peer_pool.cpp index 50d6a5048..9127f5ef6 100644 --- a/src/umbp/distributed/pool/peer_pool.cpp +++ b/src/umbp/distributed/pool/peer_pool.cpp @@ -25,6 +25,7 @@ #include "umbp/distributed/pool/peer_pool.h" #include +#include #include #include #include @@ -44,6 +45,24 @@ constexpr size_t kWatermarkCandidateBatch = 64; // watermark would re-walk the placements. constexpr auto kWatermarkScanInterval = std::chrono::milliseconds(5); +// Keys per Evict() call, so one call never holds the pool lock for a whole drain. +constexpr size_t kLocalEvictBatch = 64; +// Wakes are the main signal; the poll catches growth no wake reported. +constexpr auto kLocalEvictPollInterval = std::chrono::milliseconds(100); +constexpr auto kLocalEvictMinGap = std::chrono::milliseconds(1); // between passes +// Per backend, between candidate-source queries. +constexpr auto kRankedCandidatesMinInterval = std::chrono::milliseconds(200); +// How long a pass waits on its first query (a small index answers in time); a +// slower answer is taken when ready, unless older than the max age. +constexpr auto kRankedCandidatesWait = std::chrono::milliseconds(20); +constexpr auto kRankedCandidatesMaxAge = std::chrono::seconds(3); + +uint64_t UsedBytes(const TierCapacity& capacity) { + return capacity.total_bytes > capacity.available_bytes + ? capacity.total_bytes - capacity.available_bytes + : 0; +} + } // namespace PeerPool::PeerPool(BackendRegistry* backends, std::unique_ptr policy, @@ -75,14 +94,31 @@ PeerPool::PeerPool(BackendRegistry* backends, std::unique_ptr policy } } -PeerPool::~PeerPool() { StopTransitionWorker(); } +PeerPool::~PeerPool() { + StopLocalEviction(); // first: a pass may still be using the transition worker + StopTransitionWorker(); +} -void PeerPool::EnqueueTransition(TransitionJob job) { +bool PeerPool::EnqueueTransition(TransitionJob job) { std::lock_guard lock(transition_mutex_); - if (stop_transition_worker_ || !queued_transition_keys_.insert(job.key).second) return; + if (stop_transition_worker_ || !queued_transition_keys_.insert(job.key).second) return false; if (job.source_tier < queued_by_tier_.size()) ++queued_by_tier_[job.source_tier]; transition_queue_.push_back(std::move(job)); transition_cv_.notify_one(); + return true; +} + +bool PeerPool::TransitionQueued(const std::string& key) { + std::lock_guard lock(transition_mutex_); + return queued_transition_keys_.count(key) != 0; +} + +void PeerPool::DropEvictedCopyLocked(const TransitionJob& job) { + if (migrating_keys_.count(job.key) != 0) return; + auto* source = backends_ == nullptr ? nullptr : backends_->Get(job.source_backend_id); + if (source == nullptr) return; + source->Evict({job.key}); + RepairAfterDropLocked(job.key); } void PeerPool::StopTransitionWorker() { @@ -198,7 +234,8 @@ void PeerPool::MaybeEnqueueWatermarkOffloadLocked() { } bool PeerPool::WatermarkDrivenLocked(const TransitionJob& job) const { - return job.kind == TierTransitionKind::kOffload && tier_graph_ != nullptr && + // An eviction's demotion runs whatever the tier's own watermark says. + return !job.evict && job.kind == TierTransitionKind::kOffload && tier_graph_ != nullptr && tier_graph_->NodeAt(job.source_tier).trigger == PoolOffloadTrigger::kWatermark; } @@ -230,6 +267,8 @@ void PeerPool::FinishTransitionLocked(const TransitionJob& job, return; } ++transition_metrics_.succeeded; + // The target medium just grew without passing through BatchCommit. + WakeLocalEviction(); if (job.kind == TierTransitionKind::kPromotion) { transition_metrics_.promoted_bytes += result.bytes_moved; } else { @@ -251,6 +290,12 @@ void PeerPool::FinishTransitionLocked(const TransitionJob& job, if (draining->second.empty()) draining_sources_.erase(draining); } } + // A parked copy reused as the target (ALREADY_EXISTS) is durable again. + auto draining = draining_sources_.find(job.key); + if (draining != draining_sources_.end()) { + draining->second.erase(result.target_backend_id); + if (draining->second.empty()) draining_sources_.erase(draining); + } } TierTransitionResult PeerPool::RunTransition(std::unique_lock& lock, @@ -286,6 +331,7 @@ void PeerPool::TransitionWorkerLoop() { const bool watermark_driven = WatermarkDrivenLocked(job); if (!watermark_driven || tier_graph_->AtOrAbove(job.source_tier, tier.low_watermark)) { const auto result = RunTransition(lock, job); + if (job.evict && !result.success) DropEvictedCopyLocked(job); const bool still_pressured = watermark_driven && tier_graph_->AtOrAbove(job.source_tier, tier.low_watermark); retry = still_pressured && !result.success && job.attempts < 3; @@ -442,6 +488,7 @@ std::vector PeerPool::BatchAllocate( pending_keys_.erase(requests[index].key); break; case AllocateOutcome::kFailedNoSpace: + WakeLocalEviction(); // don't wait for the poll; the put never evicts advance(index); break; } @@ -523,6 +570,7 @@ std::vector PeerPool::BatchCommit( } } MaybeEnqueueWatermarkOffloadLocked(); + WakeLocalEviction(); return out; } @@ -835,88 +883,414 @@ std::vector PeerPool::BatchContains(const std::vector& keys) return out; } -std::vector PeerPool::Evict(const std::vector& keys, PoolEvictMode mode) { - std::unique_lock operation_lock(operation_mutex_); - std::vector out; - out.reserve(keys.size()); - for (const auto& key : keys) out.push_back(EvictResult{key, 0}); - if (backends_ == nullptr || keys.empty()) return out; +uint32_t PeerPool::EvictSourceLocked(const PoolEvictRequest& request) const { + constexpr uint32_t kNone = BackendRegistry::kMaxBackends; + const auto holds = [&](uint32_t backend_id) { + if (backend_id >= kNone) return false; + auto* backend = backends_->Get(backend_id); + return backend != nullptr && + (request.tier == TierType::UNKNOWN || backend->Tier() == request.tier) && + backend->Contains(request.key); + }; + if (request.backend_id < kNone) return holds(request.backend_id) ? request.backend_id : kNone; + // Prefer the placement: the only copy a demotion can start from. + auto placement = placements_.find(request.key); + if (placement != placements_.end() && holds(placement->second)) return placement->second; + if (request.tier == TierType::UNKNOWN) return FindOwnerLocked(request.key); + for (auto* backend : backends_->All()) { + const uint32_t backend_id = backends_->BackendId(backend); + if (holds(backend_id)) return backend_id; + } + return kNone; +} - std::vector handled(keys.size(), false); - for (size_t i = 0; i < keys.size(); ++i) { +void PeerPool::RepairAfterDropLocked(const std::string& key) { + auto draining = draining_sources_.find(key); + if (draining != draining_sources_.end()) { + for (auto it = draining->second.begin(); it != draining->second.end();) { + auto* source = backends_->Get(*it); + it = source == nullptr || !source->Contains(key) ? draining->second.erase(it) : std::next(it); + } + if (draining->second.empty()) draining_sources_.erase(draining); + } + auto placement = placements_.find(key); + if (placement != placements_.end()) { + auto* backend = backends_->Get(placement->second); + if (backend != nullptr && backend->Contains(key)) return; + } + const uint32_t owner = FindOwnerLocked(key); + if (owner != BackendRegistry::kMaxBackends) { + placements_[key] = owner; + return; + } + placements_.erase(key); + ForgetAccessLocked(key); + draining_sources_.erase(key); +} + +std::vector PeerPool::Evict(const std::vector& keys, + PoolEvictMode mode) { + std::vector requests(keys.size()); + for (size_t i = 0; i < keys.size(); ++i) requests[i].key = keys[i]; + return Evict(requests, mode); +} + +std::vector PeerPool::Evict(const std::vector& requests, + PoolEvictMode mode) { + // Drops run without operation_mutex_; this keeps ClearLocal out meanwhile. + std::shared_lock lifecycle_lock(lifecycle_mutex_); + std::vector out; + out.reserve(requests.size()); + for (const auto& request : requests) out.push_back(PoolEvictResult{request.key}); + if (backends_ == nullptr || requests.empty()) return out; + + // (request index, backend), decided under the pool lock. + std::vector> drops; + std::vector> demotions; + + std::unique_lock operation_lock(operation_mutex_); + for (size_t i = 0; i < requests.size(); ++i) { + const auto& request = requests[i]; // A copy already owns this key. Freeing nothing makes the caller retry, // which is the same contract as a read lease delaying a source delete, and // it keeps a discard from racing a target commit that would resurrect it. - if (migrating_keys_.count(keys[i]) != 0) { - handled[i] = true; + if (migrating_keys_.count(request.key) != 0) { + if (mode == PoolEvictMode::kReclaim) out[i].outcome = PoolEvictOutcome::kInFlight; + continue; + } + // A discard drops every copy, ignoring scope. + if (mode == PoolEvictMode::kDiscard) { + for (auto* backend : backends_->All()) drops.emplace_back(i, backends_->BackendId(backend)); continue; } - // A discard has to leave no copy behind, so it skips demotion and the - // drain shortcut and goes straight to the delete pass below. - if (mode == PoolEvictMode::kDiscard) continue; - auto draining = draining_sources_.find(keys[i]); - if (draining != draining_sources_.end()) { - std::vector drained; - for (uint32_t source_id : draining->second) { - auto* source = backends_->Get(source_id); - if (source != nullptr) { - auto retried = source->Evict({keys[i]}); - if (!retried.empty()) out[i].bytes_freed += retried.front().bytes_freed; - } - if (source == nullptr || !source->Contains(keys[i])) { - drained.push_back(source_id); - } - } - for (uint32_t source_id : drained) draining->second.erase(source_id); - if (draining->second.empty()) draining_sources_.erase(draining); - handled[i] = true; + // A source parked by a lease-blocked offload is only ever deleted; an + // unscoped request means "finish the drain". + auto draining = draining_sources_.find(request.key); + const bool unscoped = + request.tier == TierType::UNKNOWN && request.backend_id >= BackendRegistry::kMaxBackends; + if (draining != draining_sources_.end() && unscoped) { + for (uint32_t source_id : draining->second) drops.emplace_back(i, source_id); + continue; + } + const uint32_t source_id = EvictSourceLocked(request); + if (source_id == BackendRegistry::kMaxBackends) continue; + if (draining != draining_sources_.end() && draining->second.count(source_id) != 0) { + drops.emplace_back(i, source_id); continue; } - auto placement = placements_.find(keys[i]); - const uint32_t source_id = - placement != placements_.end() ? placement->second : FindOwnerLocked(keys[i]); - if (tier_graph_ == nullptr) continue; + // Demote the placement copy if its tier has an offload target, whatever + // the trigger. + bool demote = false; + if (tier_graph_ != nullptr) { + auto placement = placements_.find(request.key); + const auto tier_index = tier_graph_->TierIndexForBackendId(source_id); + demote = placement != placements_.end() && placement->second == source_id && + tier_index.has_value() && !tier_graph_->NodeAt(*tier_index).offload_to.empty(); + } + if (demote) { + demotions.emplace_back(i, source_id); + } else { + drops.emplace_back(i, source_id); + } + } + + // Queued, so EvictKey never waits on a copy. + for (const auto& [index, source_id] : demotions) { const auto tier_index = tier_graph_->TierIndexForBackendId(source_id); - if (!tier_index.has_value()) continue; - const auto& tier = tier_graph_->NodeAt(*tier_index); - if (tier.trigger != PoolOffloadTrigger::kOnEvict || tier.offload_to.empty()) { + if (!tier_index.has_value()) { + drops.emplace_back(index, source_id); continue; } - auto migrated = RunTransition(operation_lock, - {TierTransitionKind::kOffload, keys[i], source_id, *tier_index}); - handled[i] = migrated.success; - out[i].bytes_freed = migrated.bytes_freed; + TransitionJob job{TierTransitionKind::kOffload, requests[index].key, source_id, *tier_index}; + job.evict = true; + if (EnqueueTransition(std::move(job))) { + out[index].outcome = PoolEvictOutcome::kDemotionQueued; + } else if (TransitionQueued(requests[index].key)) { + // Another job will move it; deleting the source could lose the key. + out[index].outcome = PoolEvictOutcome::kInFlight; + } } + if (drops.empty()) return out; - std::vector delete_keys; - std::vector delete_indices; - for (size_t i = 0; i < keys.size(); ++i) { - if (!handled[i]) { - delete_keys.push_back(keys[i]); - delete_indices.push_back(i); + // Unlocked: an SSD delete is device IO. One Evict call per backend. + operation_lock.unlock(); + std::map> by_backend; + for (const auto& [index, backend_id] : drops) by_backend[backend_id].push_back(index); + for (const auto& [backend_id, indices] : by_backend) { + auto* backend = backends_->Get(backend_id); + if (backend == nullptr) continue; + std::vector keys; + keys.reserve(indices.size()); + for (size_t index : indices) keys.push_back(requests[index].key); + auto results = backend->Evict(keys); + for (size_t j = 0; j < indices.size() && j < results.size(); ++j) { + out[indices[j]].bytes_freed += results[j].bytes_freed; + if (results[j].bytes_freed > 0) out[indices[j]].outcome = PoolEvictOutcome::kFreed; } } - for (auto* backend : backends_->All()) { - auto results = backend->Evict(delete_keys); - for (size_t i = 0; i < results.size() && i < delete_indices.size(); ++i) { - out[delete_indices[i]].bytes_freed += results[i].bytes_freed; + operation_lock.lock(); + + std::unordered_set repaired; + for (const auto& [index, backend_id] : drops) { + if (repaired.insert(requests[index].key).second) RepairAfterDropLocked(requests[index].key); + } + return out; +} + +// ---- Local watermark eviction: picks when and what, then calls Evict(). ---- + +bool PeerPool::EnableLocalEviction(uint32_t backend_id, double high_watermark, + double low_watermark) { + if (!(high_watermark > 0.0 && high_watermark <= 1.0 && low_watermark > 0.0 && + low_watermark < high_watermark)) { + MORI_UMBP_WARN( + "[PeerPool] ignoring local eviction watermarks high={} low={} for backend {} " + "(require 0 < low < high <= 1)", + high_watermark, low_watermark, backend_id); + return false; + } + auto* backend = backends_ == nullptr ? nullptr : backends_->Get(backend_id); + if (backend == nullptr) return false; + // Before the worker can ask for candidates. + backend->TrackEvictionOrder(); + + std::lock_guard lock(local_evict_mutex_); + if (stop_local_evict_) return false; + local_evict_[backend_id] = LocalEvictionWatermarks{high_watermark, low_watermark}; + if (!local_evict_worker_.joinable()) { + local_evict_worker_ = std::thread(&PeerPool::LocalEvictionLoop, this); + } + const auto* entry = backends_->GetEntry(backend_id); + MORI_UMBP_INFO("[PeerPool] local eviction ON backend={} tier={} high={} low={}", + entry == nullptr ? std::string(backend->Name()) : entry->name, + TierTypeName(backend->Tier()), high_watermark, low_watermark); + return true; +} + +LocalEvictionMetrics PeerPool::LocalEvictionStats() const { + LocalEvictionMetrics metrics; + metrics.rounds = local_evict_rounds_.load(std::memory_order_relaxed); + metrics.keys = local_evict_keys_.load(std::memory_order_relaxed); + metrics.no_candidate = local_evict_no_candidate_.load(std::memory_order_relaxed); + metrics.stalled = local_evict_stalled_.load(std::memory_order_relaxed); + metrics.master_keys = local_evict_master_keys_.load(std::memory_order_relaxed); + return metrics; +} + +void PeerPool::WakeLocalEviction() { + // A pending wake covers this one (with eviction off, nothing clears it). + if (local_evict_wake_.exchange(true, std::memory_order_acq_rel)) return; + // Under the worker's lock, so the notify can't slip between check and wait. + std::lock_guard lock(local_evict_mutex_); + local_evict_cv_.notify_one(); +} + +void PeerPool::StopLocalEviction() { + { + std::lock_guard lock(local_evict_mutex_); + stop_local_evict_ = true; + } + local_evict_cv_.notify_all(); + if (local_evict_worker_.joinable()) local_evict_worker_.join(); +} + +void PeerPool::LocalEvictionLoop() { + for (;;) { + { + std::unique_lock lock(local_evict_mutex_); + local_evict_cv_.wait_for(lock, kLocalEvictPollInterval, [&] { + return stop_local_evict_ || local_evict_wake_.load(std::memory_order_acquire); + }); + if (stop_local_evict_) return; } + // Cleared first, so a commit during the pass schedules another. + local_evict_wake_.store(false, std::memory_order_release); + LocalEvictionPass(); + std::this_thread::sleep_for(kLocalEvictMinGap); } +} - for (size_t i = 0; i < keys.size(); ++i) { - if (handled[i]) continue; - const uint32_t owner = FindOwnerLocked(keys[i]); - if (owner != BackendRegistry::kMaxBackends) { - placements_[keys[i]] = owner; - } else { - placements_.erase(keys[i]); - ForgetAccessLocked(keys[i]); - draining_sources_.erase(keys[i]); +size_t PeerPool::LocalEvictionPass() { + std::lock_guard pass(local_evict_pass_mutex_); + std::vector> targets; + { + std::lock_guard lock(local_evict_mutex_); + targets.assign(local_evict_.begin(), local_evict_.end()); + } + if (targets.empty() || backends_ == nullptr) return 0; + + // Lowest tier first, so upper tiers' demotions find room below. + const auto depth = [this](uint32_t backend_id) { + auto* backend = backends_->Get(backend_id); + return backend == nullptr ? 0 : static_cast(backend->Tier()); + }; + std::stable_sort(targets.begin(), targets.end(), + [&](const auto& a, const auto& b) { return depth(a.first) > depth(b.first); }); + + size_t freed = 0; + for (const auto& [backend_id, watermarks] : targets) { + freed += ReclaimBackend(backend_id, watermarks); + } + return freed; +} + +size_t PeerPool::ReclaimBackend(uint32_t backend_id, const LocalEvictionWatermarks& watermarks) { + auto* backend = backends_->Get(backend_id); + if (backend == nullptr) return 0; + const TierCapacity capacity = backend->Capacity(); + if (capacity.total_bytes == 0) return 0; + const double total = static_cast(capacity.total_bytes); + const uint64_t used = UsedBytes(capacity); + if (static_cast(used) < watermarks.high * total) return 0; + const auto low_bytes = static_cast(watermarks.low * total); + if (used <= low_bytes) return 0; + local_evict_rounds_.fetch_add(1, std::memory_order_relaxed); + + // Stop once offers cover the starting overage (usage can lag deletes) or + // usage reaches the low watermark. Queued demotions count as covered. + const uint64_t budget = used - low_bytes; + const TierType tier = backend->Tier(); + uint64_t covered = 0; + size_t evicted_keys = 0; + // The medium keeps offering settled keys until they leave; look past them. + std::unordered_set settled; + const auto stopping = [this] { + std::lock_guard lock(local_evict_mutex_); + return stop_local_evict_; + }; + const auto drained = [&] { return UsedBytes(backend->Capacity()) <= low_bytes; }; + + // Evicts the coldest of offers[*next...] the budget still needs, at most one + // batch. False if none of them freed or moved anything. + const auto evict_some = [&](const std::vector& offers, size_t* next) { + std::vector requests; + std::vector offered_bytes; + uint64_t taken = 0; + for (; *next < offers.size(); ++*next) { + const auto& offer = offers[*next]; + if (requests.size() >= kLocalEvictBatch) break; + if (!requests.empty() && covered + taken >= budget) break; + if (settled.count(offer.key) != 0) continue; + taken += offer.bytes; + offered_bytes.push_back(offer.bytes); + requests.push_back(PoolEvictRequest{offer.key, tier, backend_id}); + } + const auto results = Evict(requests, PoolEvictMode::kReclaim); + bool progressed = false; + for (size_t i = 0; i < results.size() && i < offered_bytes.size(); ++i) { + settled.insert(results[i].key); + switch (results[i].outcome) { + case PoolEvictOutcome::kFreed: + case PoolEvictOutcome::kDemotionQueued: + ++evicted_keys; + [[fallthrough]]; + case PoolEvictOutcome::kInFlight: + covered += offered_bytes[i]; // capacity units, as offered + progressed = true; + break; + case PoolEvictOutcome::kNone: + break; + } + } + return progressed; + }; + + // The master's ranking whenever an answer is at hand, asked again for what + // the budget still needs as each one runs out; the medium's own order + // covers the rest meanwhile. A pass can span a whole band, far more keys + // than one answer carries. + auto ranked = RankedCandidates(backend_id, tier, budget, kRankedCandidatesWait); + size_t ranked_next = 0; + size_t master_keys = 0; + size_t window = kLocalEvictBatch; + while (covered < budget && !drained()) { + if (stopping()) break; + if (!ranked.has_value() || ranked_next >= ranked->size()) { + ranked = RankedCandidates(backend_id, tier, budget - covered, std::chrono::milliseconds(0)); + ranked_next = 0; + } + if (ranked.has_value() && ranked_next < ranked->size()) { + const size_t before = evicted_keys; + evict_some(*ranked, &ranked_next); // stale entries just yield kNone + master_keys += evicted_keys - before; + continue; + } + auto offers = backend->EvictionCandidates(window); + const bool exhausted = offers.size() < window; + offers.erase(std::remove_if(offers.begin(), offers.end(), + [&](const EvictionOffer& o) { return settled.count(o.key) != 0; }), + offers.end()); + if (offers.empty()) { + if (!exhausted) { + window *= 2; + continue; + } + if (settled.empty()) { + local_evict_no_candidate_.fetch_add(1, std::memory_order_relaxed); + MORI_UMBP_DEBUG( + "[PeerPool] local eviction: backend {} over its watermark but offers no " + "candidate (all leased or pinned)", + backend_id); + } + break; + } + size_t next = 0; + if (!evict_some(offers, &next)) { + // All got leased or pinned since offered; retry on the next wake. + local_evict_stalled_.fetch_add(1, std::memory_order_relaxed); + break; } } - return out; + local_evict_master_keys_.fetch_add(master_keys, std::memory_order_relaxed); + local_evict_keys_.fetch_add(evicted_keys, std::memory_order_relaxed); + return evicted_keys; +} + +std::optional> PeerPool::RankedCandidates( + uint32_t backend_id, TierType tier, uint64_t bytes, std::chrono::milliseconds wait) { + EvictionCandidateSource source; + { + std::lock_guard lock(local_evict_mutex_); + source = candidate_source_; + } + if (!source) return std::nullopt; + // Only the pass thread touches this (local_evict_pass_mutex_). The master + // answers in time proportional to its index, so the pass never waits long + // (`wait` only on a query it just sent): it evicts by the medium's order + // meanwhile. The interval keeps the master from being asked continuously. + auto& query = ranked_queries_[backend_id]; + if (!query.answer.valid()) { + if (std::chrono::steady_clock::now() - query.started < kRankedCandidatesMinInterval) { + return std::nullopt; + } + query.started = std::chrono::steady_clock::now(); + const std::string logical = + tier_graph_ == nullptr ? "" : tier_graph_->NameForBackend(backend_id); + query.answer = std::async(std::launch::async, [source = std::move(source), tier, logical, + bytes] { return source(tier, logical, bytes); }); + } else { + wait = std::chrono::milliseconds(0); + } + if (query.answer.wait_for(wait) != std::future_status::ready) return std::nullopt; + auto offers = query.answer.get(); + if (std::chrono::steady_clock::now() - query.started > kRankedCandidatesMaxAge) { + return std::nullopt; // the index has moved on since + } + // The source reports value sizes; the budget counts whole pages. + auto* backend = backends_->Get(backend_id); + const uint64_t page = backend == nullptr ? 0 : backend->PageSize(); + if (offers.has_value() && page > 0) { + for (auto& offer : *offers) + offer.bytes = std::max(1, (offer.bytes + page - 1) / page) * page; + } + return offers; +} + +void PeerPool::SetEvictionCandidateSource(EvictionCandidateSource source) { + std::lock_guard lock(local_evict_mutex_); + candidate_source_ = std::move(source); } void PeerPool::ClearLocal() { diff --git a/src/umbp/distributed/pool_client.cpp b/src/umbp/distributed/pool_client.cpp index 266f8d84b..6b72c52d4 100644 --- a/src/umbp/distributed/pool_client.cpp +++ b/src/umbp/distributed/pool_client.cpp @@ -1063,19 +1063,8 @@ bool PoolClient::Init() { // every named backend, including multiple instances of the same medium. backend = MakeInstrumentedBackend(std::move(backend)); - // A node with no master owns its whole storage policy, and both halves of - // that follow from the same fact: nobody will ever call Evict() on this - // backend, and nobody will ever drain its heartbeat outbox. So the backend - // evicts for itself, and stops recording events addressed to a master that - // does not exist. Both are no-ops in a cluster -- master-driven eviction - // stays the only policy there, and the outbox keeps feeding the heartbeat. - // - // Before Init(), which is the contract EnableLocalEviction states. - if (!master_client_) { - backend->SetEventPublishing(false); - backend->EnableLocalEviction(config_.local_evict_high_watermark, - config_.local_evict_low_watermark); - } + // With no master, nothing drains the heartbeat outbox. + if (!master_client_) backend->SetEventPublishing(false); // Narrowed to MemoryRegistrar: a backend publishes endpoints, it does not // move bytes, and that is a compile-time fact (design doc §5 Rule C). @@ -1102,6 +1091,43 @@ bool PoolClient::Init() { } default_pool_ = std::make_unique(®istry_, std::move(placement_policy), transfer_engine_.get()); + + // Local watermark eviction, for every medium unless a master's rounds own + // DRAM/HBM (eviction_mode kMaster; SSD always evicts locally). A node that + // evicts for itself uses a tiered policy's per-tier watermarks when there + // is one; otherwise SSD's own pair, or the node pair for paged media. + { + static constexpr const char* kModes[] = {"local", "master_ranked", "master"}; + const std::string mode = + GetEnvEnum("UMBP_EVICTION_MODE", kModes[static_cast(config_.eviction_mode)], + {kModes[0], kModes[1], kModes[2]}); + config_.eviction_mode = mode == "master" ? PeerEvictionMode::kMaster + : mode == "master_ranked" ? PeerEvictionMode::kMasterRanked + : PeerEvictionMode::kLocal; + } + const bool self_evicting = !master_client_ || config_.eviction_mode != PeerEvictionMode::kMaster; + for (const auto& backend_config : backend_configs) { + auto* backend = registry_.Get(backend_config.name); + if (backend == nullptr) continue; + const bool ssd = backend_config.tier == TierType::SSD; + if (!ssd && !self_evicting) continue; + const uint32_t backend_id = registry_.BackendId(backend); + double high = ssd ? backend_config.ssd.ssd.high_watermark : config_.local_evict_high_watermark; + double low = ssd ? backend_config.ssd.ssd.low_watermark : config_.local_evict_low_watermark; + if (self_evicting && tier_graph != nullptr) { + if (const auto index = tier_graph->TierIndexForBackendId(backend_id)) { + high = tier_graph->NodeAt(*index).high_watermark; + low = tier_graph->NodeAt(*index).low_watermark; + } + } + if (!default_pool_->EnableLocalEviction(backend_id, high, low) && ssd) { + MORI_UMBP_ERROR("[PoolClient] backend '{}': invalid SSD watermarks high={} low={}", + backend_config.name, high, low); + Shutdown(); + return false; + } + } + const bool weighted_placement = config_.placement_policy == PoolPlacementPolicy::WEIGHTED || config_.placement_policy == PoolPlacementPolicy::TIERED; if (master_client_) { @@ -1206,6 +1232,11 @@ bool PoolClient::Init() { const auto logical_caps = default_pool_ == nullptr ? std::map{} : default_pool_->LogicalTierCapacities(); + // A self-evicting node asks the master to leave it out of its rounds. + master_client_->RequestPeerDrivenEviction(self_evicting); + // Unless it evicts by its own LRU, its victims follow the master's + // recency, which must then hear about the reads served here. + master_client_->EnableLocalReadReports(config_.eviction_mode != PeerEvictionMode::kLocal); auto status = master_client_->RegisterSelf(tier_caps, peer_address, engine_desc_bytes, logical_caps); if (!status.ok()) { @@ -1213,6 +1244,21 @@ bool PoolClient::Init() { Shutdown(); return false; } + if (self_evicting && !master_client_->PeerDrivenEviction()) { + MORI_UMBP_WARN( + "[PoolClient] master predates peer-driven eviction: it may also evict this node in its " + "rounds, and master_ranked falls back to local LRU"); + } else if (config_.eviction_mode == PeerEvictionMode::kMasterRanked && + default_pool_ != nullptr) { + default_pool_->SetEvictionCandidateSource( + [client = master_client_.get()](TierType tier, const std::string& logical_tier, + uint64_t bytes) { + return client->GetEvictionCandidates(tier, logical_tier, bytes, /*max_keys=*/0); + }); + } + static constexpr const char* kModeNames[] = {"local", "master_ranked", "master"}; + MORI_UMBP_INFO("[PoolClient] eviction mode: {}", + kModeNames[static_cast(config_.eviction_mode)]); if (config_.master_config.auto_heartbeat) master_client_->StartHeartbeat(); if (weighted_placement && !master_client_->SupportsMaxAllocatableCapacity()) { MORI_UMBP_WARN( @@ -1287,6 +1333,7 @@ void PoolClient::Shutdown() { if (!status.ok()) { MORI_UMBP_WARN("[PoolClient] UnregisterSelf failed: {}", status.error_message()); } + if (default_pool_) default_pool_->SetEvictionCandidateSource(nullptr); } { @@ -1371,6 +1418,10 @@ TierTransitionMetrics PoolClient::TransitionMetrics() const { return default_pool_ == nullptr ? TierTransitionMetrics{} : default_pool_->TransitionMetrics(); } +LocalEvictionMetrics PoolClient::LocalEvictionStats() const { + return default_pool_ == nullptr ? LocalEvictionMetrics{} : default_pool_->LocalEvictionStats(); +} + // --------------------------------------------------------------------------- // Memory registration // --------------------------------------------------------------------------- @@ -1712,15 +1763,19 @@ void PoolClient::ResolveLocalBatch(const std::vector& keys, } } } + const bool report = master_client_ != nullptr && master_client_->ReportsLocalReads(); + std::vector read_keys; for (size_t j = 0; j < candidates.size() && j < found.size(); ++j) { if (!found[j].resolved.found) continue; auto* backend = registry_.Get(found[j].backend_id); if (backend == nullptr) continue; (*holders)[candidates[j]] = backend; + if (report) read_keys.push_back((*batch)[j]); if (resolutions != nullptr) { (*resolutions)[candidates[j]] = std::move(found[j].resolved); } } + if (report) master_client_->NoteLocalReads(read_keys); } // --------------------------------------------------------------------------- @@ -5079,6 +5134,18 @@ void PoolClient::PublishComponentMetrics() { sample(MORI_UMBP_METRIC_CLIENT_TIER_READ_HITS, MORI_UMBP_METRIC_CLIENT_TIER_READ_HITS_HELP, {{"tier", tier}}, hits); } + + const LocalEvictionMetrics local = default_pool_->LocalEvictionStats(); + sample(MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT, MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP, + {{"event", "round"}}, local.rounds); + sample(MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT, MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP, + {{"event", "key"}}, local.keys); + sample(MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT, MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP, + {{"event", "no_candidate"}}, local.no_candidate); + sample(MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT, MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP, + {{"event", "stalled"}}, local.stalled); + sample(MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT, MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP, + {{"event", "master_key"}}, local.master_keys); metric_publisher_.Publish("pool", {}, samples, sink); } } diff --git a/src/umbp/distributed/proto/umbp.proto b/src/umbp/distributed/proto/umbp.proto index aceb5d5ff..115c5bd11 100644 --- a/src/umbp/distributed/proto/umbp.proto +++ b/src/umbp/distributed/proto/umbp.proto @@ -89,12 +89,16 @@ message RegisterClientRequest { bytes engine_desc = 5; // packed EngineDesc repeated string tags = 13; // opaque key=value labels (e.g. "sgl_role=prefill") repeated LogicalTierCapacity logical_tier_capacities = 14; + // The peer evicts at its own watermarks (GetEvictionCandidates); the master + // stops pushing EvictKey to it. Re-sent on every heartbeat. + bool peer_driven_eviction = 15; } message RegisterClientResponse { uint64 heartbeat_interval_ms = 1; // Recommended heartbeat interval uint64 ack_seq = 2; // Initial heartbeat seq peer should send (= 0) bool supports_max_allocatable_bytes = 3; bool supports_logical_tiers = 4; + bool supports_peer_driven_eviction = 5; } message UnregisterClientRequest { @@ -142,6 +146,26 @@ message HeartbeatRequest { uint64 delta_seq_baseline = 6; // valid for full-sync; master adopts this cursor repeated TierKvCount tier_kv_counts = 7; // live owned-key count per tier (client-reported) repeated LogicalTierCapacity logical_tier_capacities = 8; + bool peer_driven_eviction = 9; // as in RegisterClientRequest + // Keys this node read locally since its last heartbeat (best effort, deduped). + // Sent only by a node whose eviction follows the master's recency, which + // local reads would otherwise never reach. + repeated string local_reads = 10; +} + +// A peer over its watermark asks for its own keys on one medium, coldest first +// by the master's cluster-wide recency, until their sizes cover `bytes`. +// Master-leased keys are skipped. +message GetEvictionCandidatesRequest { + string node_id = 1; + TierType tier = 2; + string logical_tier = 3; // empty: any logical tier on that medium + uint64 bytes = 4; + uint32 max_keys = 5; // 0: server cap +} +message GetEvictionCandidatesResponse { + repeated string keys = 1; + repeated uint64 sizes = 2; // parallel to keys } message HeartbeatResponse { ClientStatus status = 1; @@ -371,6 +395,10 @@ service UMBPMaster { rpc UnregisterClient(UnregisterClientRequest) returns (UnregisterClientResponse); rpc Heartbeat (HeartbeatRequest) returns (HeartbeatResponse); + // Peer-driven eviction: ranked victims for a peer over its watermark + rpc GetEvictionCandidates(GetEvictionCandidatesRequest) + returns (GetEvictionCandidatesResponse); + // Router (read-only) rpc RouteGet (RouteGetRequest) returns (RouteGetResponse); rpc RoutePut (RoutePutRequest) returns (RoutePutResponse); diff --git a/src/umbp/distributed/proto/umbp_peer.proto b/src/umbp/distributed/proto/umbp_peer.proto index 54680d9b3..a4d3cdaf9 100644 --- a/src/umbp/distributed/proto/umbp_peer.proto +++ b/src/umbp/distributed/proto/umbp_peer.proto @@ -142,6 +142,9 @@ message EvictKeyEntry { } message EvictKeyRequest { repeated string keys = 1; + // The medium to free keys[i] from. Used only if sized like `keys`; older + // masters send none. + repeated TierType tiers = 2; } message EvictKeyResponse { repeated EvictKeyEntry evicted = 1; diff --git a/src/umbp/doc/runtime-env-vars.md b/src/umbp/doc/runtime-env-vars.md index 384a58eb6..ca10751e7 100644 --- a/src/umbp/doc/runtime-env-vars.md +++ b/src/umbp/doc/runtime-env-vars.md @@ -60,6 +60,7 @@ Read by the **master process** (`bin/master_main.cpp` via | `UMBP_LEASE_DURATION_SEC` | `2` | sec | Master-side read-lease length granted by `Router::RouteGet`: `IsLeased()` keys are skipped by the eviction scan, keeping a key alive from the moment the master returns its location until the reader connects to the owning peer. Only needs to cover the master→reader gRPC round trip + reach the peer (the actual RDMA transfer is covered peer-side by `UMBP_DRAM_READ_LEASE_MS`), so seconds is already generous; larger values pin actively-read (hot) keys against eviction. | | `UMBP_HEARTBEAT_INTERVAL_DIVISOR` | `2` | count | Recommended client heartbeat interval = `heartbeat_ttl / divisor`. `min_allowed=1` guards against div-by-zero. Read by the master and echoed in `RegisterClientResponse.heartbeat_interval_ms`. | | `UMBP_EVICTKEY_DEADLINE_MS` | `1000` | ms | Per-call gRPC deadline applied to outbound `EvictKey` RPCs from `MasterPeerStubPool`. | +| `UMBP_EVICT_MAX_BYTES_PER_ROUND` | `0` | bytes | Most bytes one eviction round frees from one `(node, tier)`. `0` = its whole overage above the low watermark. Set it to throttle eviction, e.g. how much a demoting tier copies downstream per round. | | `UMBP_HIT_INDEX_TTL_SEC` | `7200` | sec | External KV hit-count entry TTL. A hash with no counted match for longer than this is removed from the hit index. | | `UMBP_HIT_INDEX_GC_INTERVAL_SEC` | `60` | sec | External KV hit-count GC sweep interval. | | `UMBP_HIT_QUERY_MAX_BATCH` | `4096` | count | Maximum hashes accepted by one `GetExternalKvHitCounts` request. Oversized requests return gRPC `INVALID_ARGUMENT`; the server does not truncate. | @@ -74,6 +75,7 @@ that has loaded `libmori_pybinds.so`). | Env var | Default | Unit | Description | |---|---|---|---| +| `UMBP_EVICTION_MODE` | `local` | enum | How a node with a master evicts. `local`: every medium at this node's own watermarks (a tiered policy's per-tier pair, else the SSD / DRAM config pair), by its own LRU, without asking the master; the master leaves it out of its `EvictKey` rounds. `master_ranked`: the same, with victims ranked by the master (`GetEvictionCandidates`), falling back to local LRU. `master`: DRAM/HBM wait for the master's rounds (SSD still evicts locally). In both master modes the node reports the keys it reads locally on its heartbeats, since the master's recency otherwise sees only `RouteGet`. A node without a master always evicts locally, and every mode still serves `EvictKey`. | | `UMBP_DRAM_READ_LEASE_MS` | `500` | ms | Peer-side DRAM/HBM read lease: how long a single `PeerDramAllocator::Resolve` protects its key's pages from concurrent local `Evict`, covering one RDMA read of those pages. Only needs to exceed one DRAM RDMA round trip (sub-ms), so 500 ms is ~100x margin. Read once at `PoolClient::Init`; `min_allowed=1`. | | `UMBP_SSD_READ_LEASE_MS` | `3000` | ms | Peer-side SSD read-staging slot lease: how long a claimed staging slot is reserved before the peer reclaims it by TTL (the fallback when the reader's best-effort `ReleaseSsdLease` is lost), and, echoed back in `PrepareSsdReadResponse.lease_ttl_ms`, the reader's validity window anchored at `t_send`. Must exceed one SSD read + RDMA (slower than DRAM), but too long pins a configured staging slot on a lost release. Also the fallback for the `PrepareSsdRead` RPC deadline when `UMBP_SSD_PREPARE_TIMEOUT_MS` is unset. Read once at `PoolClient::Init`; `min_allowed=1`. Also the read lease of a policy-declared `SsdBackend`, where it governs how long a contiguous staging span stays claimed and so sets that backend's read concurrency against a fixed arena — see `staging_slots`. | | `UMBP_RESOLVE_BUSY_TIMEOUT_MS` | `30000` | ms | Overall deadline for retrying a peer-local or remote batch resolve that returned `BUSY` because SSD staging was temporarily full. Retries discard the whole response and use exponential backoff capped at 50 ms. A batch whose own working set exceeds the arena returns a permanent failure immediately and is not retried. Values above 300000 ms are clamped. | diff --git a/src/umbp/include/umbp/common/env_time.h b/src/umbp/include/umbp/common/env_time.h index 7d17a1b81..87b5abd0c 100644 --- a/src/umbp/include/umbp/common/env_time.h +++ b/src/umbp/include/umbp/common/env_time.h @@ -146,6 +146,18 @@ inline uint32_t GetEnvUint32(const char* name, uint32_t def, uint32_t min_allowe return static_cast(v); } +// Byte counts and other values past uint32. The parser is signed, so the +// ceiling is INT64_MAX -- far beyond any byte count this reads. +inline uint64_t GetEnvUint64(const char* name, uint64_t def, uint64_t min_allowed = 0) { + const int64_t v = env_time_detail::ResolveEnvInt(name, static_cast(def), + static_cast(min_allowed)); + if (v < 0) { + env_time_detail::WarnOnce(name, "negative", std::getenv(name)); + return def; + } + return static_cast(v); +} + // Resolve a string-enum env var against a fixed set of allowed values. // - Unset / empty / whitespace-only -> return `def` silently. // - After trimming leading/trailing whitespace, exact (case-sensitive, diff --git a/src/umbp/include/umbp/distributed/config.h b/src/umbp/include/umbp/distributed/config.h index 351d18b24..da394954c 100644 --- a/src/umbp/include/umbp/distributed/config.h +++ b/src/umbp/include/umbp/distributed/config.h @@ -92,6 +92,9 @@ struct ClientRegistryConfig { // single backend can hold. Tests set this false to emulate that Master. bool advertise_max_allocatable_bytes = true; bool advertise_logical_tiers = true; + // Serve GetEvictionCandidates and stop EvictKey rounds for peers that evict + // for themselves. False emulates an older Master. + bool advertise_peer_driven_eviction = true; }; struct EvictionConfig { @@ -99,16 +102,20 @@ struct EvictionConfig { double low_watermark = 0.7; std::chrono::seconds check_interval{5}; std::chrono::seconds lease_duration{2}; - size_t evict_batch_size = 32; - // Only timing fields are env-overridable here; watermarks and batch size - // have dedicated tuning paths and are intentionally excluded. + // Most bytes one round evicts from one (node, tier). 0 = its whole overage + // above the low watermark. + uint64_t max_evict_bytes_per_round = 0; + + // Watermarks have dedicated tuning paths and are not env-overridable. static EvictionConfig FromEnvironment() { EvictionConfig cfg; cfg.check_interval = GetEnvSeconds("UMBP_EVICTION_CHECK_INTERVAL_SEC", cfg.check_interval, /*min_allowed=*/1); cfg.lease_duration = GetEnvSeconds("UMBP_LEASE_DURATION_SEC", cfg.lease_duration, /*min_allowed=*/1); + cfg.max_evict_bytes_per_round = + GetEnvUint64("UMBP_EVICT_MAX_BYTES_PER_ROUND", cfg.max_evict_bytes_per_round); return cfg; } }; @@ -226,6 +233,12 @@ struct BackendInstanceConfig { uint32_t placement_weight = 1; }; +enum class PeerEvictionMode { + kLocal, // at its own watermarks, by its own LRU; never asks the master + kMasterRanked, // at its own watermarks, victims ranked by the master + kMaster, // the master's EvictKey rounds (SSD still evicts locally) +}; + struct PoolClientConfig { UMBPMasterClientConfig master_config; UMBPIoEngineConfig io_engine; @@ -303,16 +316,17 @@ struct PoolClientConfig { // (2 MiB by default). Set to an explicit byte count to override. uint64_t dram_page_size = 0; - // Watermarks for the paged medium's own eviction, used ONLY when this node - // has no master (PoolClient::Init enables it exactly then). With a master, - // eviction is a cluster-wide decision shipped as Evict RPCs and these are - // ignored; without one, nothing else would ever free a page. Lowered from - // UMBPDramConfig::high_watermark / low_watermark, which is where the - // deleted local backend read the same policy from. The SSD medium has its - // own pair inside UMBPSsdConfig and does not consult these. + // Local eviction watermarks for DRAM/HBM (SSD has its own pair). Unless + // eviction_mode is kMaster, a tiered policy's per-tier pair takes + // precedence. From UMBPDramConfig. double local_evict_high_watermark = 0.9; double local_evict_low_watermark = 0.7; + // How this node evicts when it has a master (env UMBP_EVICTION_MODE = + // local | master_ranked | master). Without a master it always evicts + // locally. In every mode it still serves the master's EvictKey. + PeerEvictionMode eviction_mode = PeerEvictionMode::kLocal; + UMBPCopyPipelineConfig copy_pipeline = [] { UMBPCopyPipelineConfig c; c.worker_threads = 1; diff --git a/src/umbp/include/umbp/distributed/master/evict_strategy.h b/src/umbp/include/umbp/distributed/master/evict_strategy.h index 4da9aaec9..c0aacf5eb 100644 --- a/src/umbp/include/umbp/distributed/master/evict_strategy.h +++ b/src/umbp/include/umbp/distributed/master/evict_strategy.h @@ -27,23 +27,20 @@ #include #include -#include "umbp/distributed/types.h" // EvictionCandidate, TierType +#include "umbp/distributed/types.h" // EvictionCandidate, EvictionVictim, TierType namespace mori::umbp { -// Master-side victim selection for DRAM/HBM eviction. EvictionManager handles -// watermark detection, candidate gathering, and EvictKey dispatch; the strategy -// only ranks candidates and picks victims within the per-(node,tier) budget. -// (SSD eviction is peer-local; master never evicts SSD.) +// Master-side victim selection within each (node, tier) budget. class MasterEvictStrategy { public: virtual ~MasterEvictStrategy() = default; // Pick victims from @p candidates within the per-(node,tier) byte budget // @p bytes_to_free. Both are by value so the impl may sort/decrement freely. - // Returns victims grouped by node_id (one keys[] per peer); empty groups - // omitted. - virtual std::unordered_map> SelectVictims( + // Victims grouped by node_id (empty groups omitted), each with the tier it + // was charged to. + virtual std::unordered_map> SelectVictims( std::vector candidates, std::unordered_map> bytes_to_free) = 0; }; @@ -51,7 +48,7 @@ class MasterEvictStrategy { // Default policy: pure LRU (evict oldest-access-first until each budget is met). class LruMasterEvictStrategy : public MasterEvictStrategy { public: - std::unordered_map> SelectVictims( + std::unordered_map> SelectVictims( std::vector candidates, std::unordered_map> bytes_to_free) override; }; diff --git a/src/umbp/include/umbp/distributed/master/eviction_manager.h b/src/umbp/include/umbp/distributed/master/eviction_manager.h index caa2737af..a135d1b4f 100644 --- a/src/umbp/include/umbp/distributed/master/eviction_manager.h +++ b/src/umbp/include/umbp/distributed/master/eviction_manager.h @@ -22,14 +22,18 @@ #pragma once #include +#include #include +#include #include #include #include #include +#include #include #include "umbp/distributed/config.h" +#include "umbp/distributed/types.h" namespace mori::umbp { @@ -42,6 +46,8 @@ class MasterEvictStrategy; // REMOVE events arrive on the peer's next heartbeat and that's where // the index actually shrinks. // +// Each victim names the medium to free the key from. +// // The implementation lives in master_server.cpp (MasterPeerStubPool) // where the gRPC stub plumbing belongs. Tests can swap in a fake. class EvictKeyDispatcher { @@ -49,7 +55,7 @@ class EvictKeyDispatcher { virtual ~EvictKeyDispatcher() = default; virtual void DispatchEvictKey(const std::string& node_id, const std::string& peer_address, - std::vector keys) = 0; + std::vector victims) = 0; }; class EvictionManager { @@ -72,6 +78,15 @@ class EvictionManager { void Start(); void Stop(); + // A peer-driven node evicts at its own watermarks and asks for candidates + // (SelectPeerEvictionCandidates); rounds skip it. In memory only: peers + // re-assert it on every heartbeat. + void SetPeerDriven(const std::string& node_id, bool peer_driven); + bool PeerDriven(const std::string& node_id) const; + + // Test seam: one synchronous round. + void RunOnceForTest() { RunOnce(); } + private: void EvictionLoop(); @@ -88,6 +103,19 @@ class EvictionManager { std::atomic running_{false}; std::mutex cv_mutex_; std::condition_variable cv_; + mutable std::mutex peer_driven_mutex_; + std::unordered_set peer_driven_; }; +// Server cap on one GetEvictionCandidates reply. +constexpr size_t kMaxPeerEvictionCandidates = 4096; + +// `node_id`'s keys on `tier` (and `logical_tier`, if non-empty), least +// recently accessed first, until their sizes cover `bytes` or `max_keys` is +// reached (0: kMaxPeerEvictionCandidates). Master-leased keys are skipped. +std::vector SelectPeerEvictionCandidates( + const IMasterMetadataStore& store, const std::string& node_id, TierType tier, + const std::string& logical_tier, uint64_t bytes, size_t max_keys, + std::chrono::system_clock::time_point now); + } // namespace mori::umbp diff --git a/src/umbp/include/umbp/distributed/master/in_memory_master_metadata_store.h b/src/umbp/include/umbp/distributed/master/in_memory_master_metadata_store.h index 95b7f2199..8298daf28 100644 --- a/src/umbp/include/umbp/distributed/master/in_memory_master_metadata_store.h +++ b/src/umbp/include/umbp/distributed/master/in_memory_master_metadata_store.h @@ -112,6 +112,8 @@ class InMemoryMasterMetadataStore : public IMasterMetadataStore { const std::vector& keys, const std::unordered_set& exclude_nodes, std::chrono::system_clock::time_point now, std::chrono::system_clock::duration lease_duration) override; + void RecordLocalReads(const std::vector& keys, + std::chrono::system_clock::time_point now) override; std::vector BatchExistsBlock(const std::vector& keys) const override; std::map> EnumerateEvictionCandidates( const std::vector& buckets, EvictionOrder order, size_t max_per_bucket, diff --git a/src/umbp/include/umbp/distributed/master/master_client.h b/src/umbp/include/umbp/distributed/master/master_client.h index 0b5eb3d7d..f3c669c31 100644 --- a/src/umbp/include/umbp/distributed/master/master_client.h +++ b/src/umbp/include/umbp/distributed/master/master_client.h @@ -122,6 +122,28 @@ class MasterClient { // wants to know "is the key resident?" and is not about to RDMA-read. grpc::Status BatchLookup(const std::vector& keys, std::vector* out); + // --- Peer-driven eviction --- + // Ask the master to stop EvictKey rounds for this node; set before + // RegisterSelf. In effect only once the master confirms support. + void RequestPeerDrivenEviction(bool enabled) { peer_driven_requested_ = enabled; } + bool PeerDrivenEviction() const { + return peer_driven_requested_ && supports_peer_driven_eviction_.load(); + } + // This node's keys on `tier`, coldest first by the master's recency, until + // their sizes cover `bytes`. nullopt when the RPC fails. + std::optional> GetEvictionCandidates(TierType tier, + const std::string& logical_tier, + uint64_t bytes, + uint32_t max_keys); + // Keys read locally, shipped (deduped, best effort) on the next heartbeat: + // the master's recency otherwise sees only RouteGet. For a node whose + // eviction follows that recency; a no-op until enabled and supported. + void EnableLocalReadReports(bool enabled) { local_read_reports_requested_ = enabled; } + bool ReportsLocalReads() const { + return local_read_reports_requested_ && supports_peer_driven_eviction_.load(); + } + void NoteLocalReads(const std::vector& keys); + // --- Heartbeat (event-driven) --- // Bind every storage medium live on this peer. The heartbeat drains each // backend's outbox and concatenates them into ONE bundle under a single @@ -243,6 +265,12 @@ class MasterClient { bool aggregate_backend_capacities_ = false; bool supports_max_allocatable_capacity_ = false; bool supports_logical_tiers_ = false; + bool peer_driven_requested_ = false; + // Rewritten on re-registration (heartbeat thread), read by eviction. + std::atomic supports_peer_driven_eviction_{false}; + std::atomic local_read_reports_requested_{false}; + std::mutex local_reads_mutex_; + std::unordered_set local_reads_; // Initial conservative registration payload, retained so an UNKNOWN response // can rebuild the full peer record after Master state loss. std::map registration_capacities_; diff --git a/src/umbp/include/umbp/distributed/master/master_metadata_store.h b/src/umbp/include/umbp/distributed/master/master_metadata_store.h index 8ea1ff32d..ec80be17f 100644 --- a/src/umbp/include/umbp/distributed/master/master_metadata_store.h +++ b/src/umbp/include/umbp/distributed/master/master_metadata_store.h @@ -415,6 +415,12 @@ class IMasterMetadataStore { std::chrono::system_clock::time_point now, std::chrono::system_clock::duration lease_duration) = 0; + // Records an access (last_accessed_at = now, access_count + 1) for every key + // that still exists: reads a peer served locally (heartbeat local_reads), + // which RouteGet never sees. No lease. One round trip. + virtual void RecordLocalReads(const std::vector& keys, + std::chrono::system_clock::time_point now) = 0; + // Batched existence — pure read, no lease grant, no access record. // Used by BatchRoutePut for dedup (router.cpp:112): a writer // landing on an existing key isn't a "read" and must not extend diff --git a/src/umbp/include/umbp/distributed/master/master_metrics.h b/src/umbp/include/umbp/distributed/master/master_metrics.h index cd4f7b2dc..1660e6272 100644 --- a/src/umbp/include/umbp/distributed/master/master_metrics.h +++ b/src/umbp/include/umbp/distributed/master/master_metrics.h @@ -181,6 +181,14 @@ #define MORI_UMBP_METRIC_CLIENT_TIER_READ_HITS_HELP \ "Reads served by each logical tier of this client's pool" +// --- Local watermark eviction ---------------------------------------------- +// event=round|key|no_candidate|stalled|master_key (LocalEvictionMetrics); bytes +// freed are in the backend evict series. + +#define MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT "mori_umbp_client_local_evict_total" +#define MORI_UMBP_METRIC_CLIENT_LOCAL_EVICT_HELP \ + "Local watermark eviction on this client's pool, by event" + // --- Per-client RPC call counters ------------------------------------------ // Full name: prefix + sanitized_node_id diff --git a/src/umbp/include/umbp/distributed/master/redis/lua_scripts.h b/src/umbp/include/umbp/distributed/master/redis/lua_scripts.h index 07cd9d653..65139580f 100644 --- a/src/umbp/include/umbp/distributed/master/redis/lua_scripts.h +++ b/src/umbp/include/umbp/distributed/master/redis/lua_scripts.h @@ -266,6 +266,21 @@ end return out )LUA"; +// record_local_reads: +// KEYS[1..n] = block keys (single-slot per invocation, grouped by shard, as +// route_get_batch). ARGV = [now_ms]. +// An access bump without a lease for every key that still exists. +inline const std::string kRecordLocalReadsLua = R"LUA( +local now = ARGV[1] +for i = 1, #KEYS do + if redis.call('EXISTS', KEYS[i]) == 1 then + redis.call('HSET', KEYS[i], '_lacc', now) + redis.call('HINCRBY', KEYS[i], '_acnt', 1) + end +end +return 0 +)LUA"; + // exists_batch: // KEYS[1..n] = block keys (single-slot per invocation, grouped by shard by // the caller, same as route_get_batch). diff --git a/src/umbp/include/umbp/distributed/master/redis_master_metadata_store.h b/src/umbp/include/umbp/distributed/master/redis_master_metadata_store.h index c93a0922e..b651f563b 100644 --- a/src/umbp/include/umbp/distributed/master/redis_master_metadata_store.h +++ b/src/umbp/include/umbp/distributed/master/redis_master_metadata_store.h @@ -125,6 +125,8 @@ class RedisMasterMetadataStore : public IMasterMetadataStore { const std::vector& keys, const std::unordered_set& exclude_nodes, std::chrono::system_clock::time_point now, std::chrono::system_clock::duration lease_duration) override; + void RecordLocalReads(const std::vector& keys, + std::chrono::system_clock::time_point now) override; std::vector BatchExistsBlock(const std::vector& keys) const override; std::map> EnumerateEvictionCandidates( const std::vector& buckets, EvictionOrder order, size_t max_per_bucket, diff --git a/src/umbp/include/umbp/distributed/peer/backend/instrumented_backend.h b/src/umbp/include/umbp/distributed/peer/backend/instrumented_backend.h index 05eafbd77..1e1e36ef1 100644 --- a/src/umbp/include/umbp/distributed/peer/backend/instrumented_backend.h +++ b/src/umbp/include/umbp/distributed/peer/backend/instrumented_backend.h @@ -91,8 +91,11 @@ class InstrumentedBackend final : public MediumBackend { void ClearFullSyncAcked() override { inner_->ClearFullSyncAcked(); } bool IsClearFullSyncPending() const override { return inner_->IsClearFullSyncPending(); } void SetEventPublishing(bool enabled) override { inner_->SetEventPublishing(enabled); } - void EnableLocalEviction(double high_watermark, double low_watermark) override { - inner_->EnableLocalEviction(high_watermark, low_watermark); + // Ordering only, not an operation: nothing is freed, so nothing is counted. + // The eviction that follows is an Evict() call and is measured there. + void TrackEvictionOrder() override { inner_->TrackEvictionOrder(); } + std::vector EvictionCandidates(size_t max_count) override { + return inner_->EvictionCandidates(max_count); } void SetAutoFlushHook(size_t threshold, std::function cb) override { diff --git a/src/umbp/include/umbp/distributed/peer/backend/medium_backend.h b/src/umbp/include/umbp/distributed/peer/backend/medium_backend.h index 401d64ab1..3a9bdd7e4 100644 --- a/src/umbp/include/umbp/distributed/peer/backend/medium_backend.h +++ b/src/umbp/include/umbp/distributed/peer/backend/medium_backend.h @@ -276,15 +276,15 @@ class MediumBackend : public MetricSource { // call (the default) simply has no outbox worth gating. virtual void SetEventPublishing(bool /*enabled*/) {} - // Turn on the backend's OWN eviction, for a node with no master. - // - // Evict(keys) above is a master decision: it picks victims with a - // cluster-wide view of who else holds the key. Nothing in the system calls - // it otherwise, so a masterless node needs some medium-local policy or it - // simply fills up. A backend that already self-evicts (SsdBackend, whose - // PeerSsdManager runs its own watermark loop from UMBPSsdConfig) ignores - // this; the default is to ignore it. - virtual void EnableLocalEviction(double /*high_watermark*/, double /*low_watermark*/) {} + // ---- eviction ordering: the pool decides; a medium only supplies order ---- + + // Start the recency order EvictionCandidates() reads. Off by default (it + // costs every read a touch). Idempotent; seeds keys already owned. + virtual void TrackEvictionOrder() {} + + // Up to `max_count` unprotected keys, coldest first, with the bytes each + // frees. Frees nothing; Evict() re-checks. Empty if no order is kept. + virtual std::vector EvictionCandidates(size_t /*max_count*/) { return {}; } // ---- observability ---- // @@ -363,9 +363,8 @@ class MediumBackend : public MetricSource { return out; } - // Master-driven eviction. Idempotent; see EvictResult::bytes_freed. One - // result per key, in request order — the peer service relies on that to sum - // freed bytes for a key mirrored across media. + // The medium's only eviction path, reached via PeerPool::Evict: frees keys a + // lease or pin doesn't protect and queues a REMOVE for each. Idempotent. virtual std::vector Evict(const std::vector& keys) = 0; // ---- bootstrap (GetPeerInfo) ---- diff --git a/src/umbp/include/umbp/distributed/peer/backend/page_backend.h b/src/umbp/include/umbp/distributed/peer/backend/page_backend.h index f337f66fb..190763a57 100644 --- a/src/umbp/include/umbp/distributed/peer/backend/page_backend.h +++ b/src/umbp/include/umbp/distributed/peer/backend/page_backend.h @@ -331,38 +331,11 @@ class PageBackend : public MediumBackend { // keeping one is a leak of one event per put. void SetEventPublishing(bool enabled) override; - // ==================== Local (masterless) eviction ==================== - // - // Eviction here is normally the MASTER's policy: it chooses victims with a - // cluster-wide view and calls Evict(keys) through the peer service. Nothing - // else ever calls Evict, so a node with no master fills its pool and - // Allocate starts answering NO_SPACE forever — where the deleted local - // backend's LocalStorageManager would have evicted on a watermark. This is - // that watermark loop, restored on the backend that now serves embedded - // deployments. - // - // Opt-in so the cluster path is bit-for-bit unchanged: with a master, a - // second local policy would free keys the master still indexes and hands out - // routes for. PoolClient::Init enables it exactly when it builds no - // MasterClient. - // - // Enabling it turns on an LRU ordering (inserted by Commit, renewed by - // Resolve) that is not maintained otherwise, so the read path pays nothing - // in a cluster. A round runs on the committing thread — there is no - // eviction thread — and frees oldest-first down to `low` once usage reaches - // `high`. Leased and copy-pinned keys are skipped, exactly as master-driven - // Evict skips them; a round that can only find protected keys frees what it - // can and stops rather than spinning. - // - // Must be called before Init(). Watermarks must satisfy - // 0 < low < high <= 1; anything else is rejected with a warning and leaves - // local eviction off. - void EnableLocalEviction(double high_watermark, double low_watermark) override; - - // Test seam: run one local-eviction round synchronously. Returns the number - // of keys freed (0 when local eviction is off or usage is below the high - // watermark). - size_t RunLocalEvictionOnceForTest() { return MaybeEvictToLowWatermark(); } + // ==================== Eviction ordering ==================== + // An LRU of owned keys (Commit inserts, Resolve renews), kept only after + // TrackEvictionOrder(). Candidates skip what Evict() would refuse. + void TrackEvictionOrder() override; + std::vector EvictionCandidates(size_t max_count) override; // ==================== Reaper ==================== @@ -398,7 +371,7 @@ class PageBackend : public MediumBackend { // (RenewLeaseAtomic) so a slower thread can never shorten a lease another // thread already extended. std::atomic read_lease_ns{0}; - // Position in lru_, valid only while in_lru is true (local eviction on). + // Position in lru_, valid only while in_lru is true (order tracked). // Held here for the same reason as the lease: the alternative is a second // map keyed by the same string. // @@ -487,10 +460,10 @@ class PageBackend : public MediumBackend { void ReaperLoop(); void ReaperSweep(); - // ---- local eviction (no-master deployments; see EnableLocalEviction) ---- + // ---- eviction order (see TrackEvictionOrder) ---- // Caller MUST hold `lru_mutex_`. Move `key` to the MRU end, inserting it if - // it is not yet tracked. No-op when local eviction is off. + // it is not yet tracked. No-op until the order is tracked. void TouchLruLocked(const std::string& key, OwnedSlot& slot); // Caller MUST hold `lru_mutex_`. Drop `key` from the LRU if tracked. @@ -501,11 +474,6 @@ class PageBackend : public MediumBackend { void TouchLru(const std::string& key, OwnedSlot& slot); void RemoveFromLru(OwnedSlot& slot); - // Run one round if local eviction is on and usage has reached the high - // watermark. Takes `mutex_` itself, so callers must NOT hold it. Returns - // the number of keys freed. - size_t MaybeEvictToLowWatermark(); - // Owned pages held back at ClearLocal() because of an active read lease. // Released by ReaperSweep() when release_at <= now. struct DeferredFree { @@ -558,31 +526,26 @@ class PageBackend : public MediumBackend { struct PinState { uint64_t token = 0; std::chrono::steady_clock::time_point acquired_at; // for long-running warning only + // Set by AcquireMigrationRead; the key stays an eviction candidate. + bool migration = false; }; std::unordered_map pins_; uint64_t next_pin_token_ = 1; std::vector deferred_frees_; - // ---- local eviction state (see EnableLocalEviction) ---- + // ---- eviction order (see TrackEvictionOrder) ---- // Front = most recently committed/resolved, back = eviction candidate. - // Empty and unmaintained unless local_evict_enabled_. + // Empty and unmaintained until track_eviction_order_. std::list lru_; // Guards lru_ and the per-slot lru_it/in_lru fields. Separate from mutex_ // so a resolve can record an access while holding only a shared lock; the // critical section is a list splice, not a hash lookup plus a page-vector // copy. Always taken after mutex_ when both are held. mutable std::mutex lru_mutex_; - bool local_evict_enabled_ = false; - double local_evict_high_wm_ = 0.9; - double local_evict_low_wm_ = 0.7; - // Serializes rounds without holding mutex_ across one: a second committing - // thread backs off instead of over-evicting (same shape as PeerSsdManager). - std::mutex local_evict_round_mutex_; - // Latches the "nothing reclaimable" warning so a wedged pool reports once - // per episode rather than once per commit. Cleared by the next round that - // frees something. - bool local_evict_warned_ = false; + // Written under mutex_ held exclusively, read under it held shared or + // exclusively -- every LRU touch already holds one or the other. + bool track_eviction_order_ = false; // False stops QueueEventLocked from recording anything (no master to ship // to). True is the cluster default. diff --git a/src/umbp/include/umbp/distributed/peer/backend/ssd_backend.h b/src/umbp/include/umbp/distributed/peer/backend/ssd_backend.h index 75c3e7fd1..ebc1fc16a 100644 --- a/src/umbp/include/umbp/distributed/peer/backend/ssd_backend.h +++ b/src/umbp/include/umbp/distributed/peer/backend/ssd_backend.h @@ -167,6 +167,8 @@ class SsdBackend : public MediumBackend { void ReleaseMigrationRead(const std::string& key) override; bool Contains(const std::string& key) const override; std::vector Evict(const std::vector& keys) override; + // The manager's LRU (always kept) minus keys a staged or migration read holds. + std::vector EvictionCandidates(size_t max_count) override; void SetAutoFlushHook(size_t threshold, std::function cb) override; diff --git a/src/umbp/include/umbp/distributed/peer/ssd/peer_ssd_manager.h b/src/umbp/include/umbp/distributed/peer/ssd/peer_ssd_manager.h index 3810c4d25..f2d1da996 100644 --- a/src/umbp/include/umbp/distributed/peer/ssd/peer_ssd_manager.h +++ b/src/umbp/include/umbp/distributed/peer/ssd/peer_ssd_manager.h @@ -52,9 +52,9 @@ struct SsdReadOutcome { }; // Peer-side owner of the local SSD tier: one SSD TierBackend + the -// key->SSD-location map, capacity, the owned-location event outbox, and the -// read-prepare and local-eviction paths. Reached from the distributed data -// plane through SsdBackend : MediumBackend, which forwards to it. +// key->SSD-location map, capacity, the owned-location event outbox, the +// read-prepare path, and Evict(). Reached through SsdBackend. It never +// decides to evict; it only offers its LRU as candidates. // // It reuses ONLY the low-level TierBackend (SSDTier). Peer DRAM is owned by // PageBackend, and a second thing here that also held DRAM and demoted between @@ -64,11 +64,8 @@ class PeerSsdManager { public: explicit PeerSsdManager(const PeerSsdConfig& cfg); - // Test-only: inject a ready-made backend and explicit watermarks so unit - // tests can drive eviction with a controllable (e.g. blocking) fake backend. - // Production code must use the config constructor. - PeerSsdManager(std::unique_ptr backend, double high_watermark, double low_watermark, - bool single_flight = true); + // Test-only: inject a fake backend. + explicit PeerSsdManager(std::unique_ptr backend, bool single_flight = true); ~PeerSsdManager(); @@ -86,8 +83,8 @@ class PeerSsdManager { // `owned_` has carried this all along; only the accessor was missing. uint64_t SizeOf(const std::string& key) const; - // Hold a key against local watermark eviction across a peer-local migration. - // Uses the same refcount as active reads so eviction has one protection rule. + // Hold a key against eviction across a peer-local migration. Uses the same + // refcount as active reads so eviction has one protection rule. bool PinForMigration(const std::string& key); void UnpinForMigration(const std::string& key); @@ -111,44 +108,28 @@ class PeerSsdManager { // drive but one. // // Result length and order match @p keys. An already-owned key reports true - // without any device IO (LRU refreshed). On a partial failure the failed keys - // are retried once after a single eviction round, so a batch that trips the - // high watermark mid-way still lands. + // without any device IO (LRU refreshed). A key with no room fails; this + // never evicts to make room. std::vector WriteBatch(const std::vector& keys, const std::vector& srcs, const std::vector& sizes); - // Local eviction of a single key. Read priority: a key with an - // in-flight PrepareRead (inflight_reads_ > 0) is NOT evicted (returns false). - // Concurrency: marks the key in evicting_ under the lock, runs the backend - // evict outside the lock, and only on backend success removes owned_/lru_ and - // queues a REMOVE SSD event (so REMOVE is never emitted while the bytes still - // exist, and two workers cannot double-evict the same victim). - // - // Does NOT itself hold eviction_mu_ (EvictToLowWatermark already holds it - // while looping over victims, so taking it here would deadlock). The normal - // caller is EvictToLowWatermark; a direct caller must not run concurrently - // with ClearLocal (production relies on PoolClient::Clear quiescing the copy - // pipeline first, so no eviction is in flight during a Clear). + // Frees one key; false if it is being read or pinned (inflight_reads_ > 0). + // Marks the key in evicting_ under the lock, evicts outside it, and only on + // success removes owned_/lru_ and queues a REMOVE (never while the bytes + // still exist; no double evict). PeerPool::Evict holds the lifecycle lock, + // so this never overlaps a Clear. bool Evict(const std::string& key); - // Local LRU victim selection (oldest first), skipping keys that are - // being read (inflight_reads_ > 0) or already being evicted (evicting_). - // Accumulates sizes until >= bytes_to_free; returns fewer if not enough - // free-able keys exist (never blocks). - // - // Not pluggable yet (only master-side eviction is). A real SSD plugin must - // own the per-algorithm state that lives here in lru_/owned_, not just the - // selection step; target design is a stateful policy with - // OnAdd/OnTouch/OnRemove/Clear hooks plus a SelectVictims called under mutex_. - std::vector SelectVictims(size_t bytes_to_free); + // Coldest first, skipping keys being read, pinned or evicted. Frees nothing. + std::vector EvictionCandidates(size_t max_count); // Distributed Clear: drop the logical owned-location map + undrained events, // then delete the physical SSD bytes (a user Clear means the cache is no // longer wanted). Read priority: clears the logical map first (so new reads // immediately miss with kNotFound), waits for any in-flight PrepareRead to // finish (SSD reads cannot be safely aborted), and only then wipes the - // backend. Serializes against eviction rounds via eviction_mu_. + // backend. No eviction runs concurrently: see Evict. // Precondition: callers (PoolClient::Clear) MUST quiesce the SSD copy // pipeline first so no in-flight copy re-populates owned_ right after this // returns. Crash-restart leftover (metadata gone, files remain) is a @@ -217,7 +198,6 @@ class PeerSsdManager { // Byte counters for SSD IO bandwidth (rate() in Grafana = bytes/s). uint64_t CopyBytes() const { return metrics_.copy_bytes.load(std::memory_order_relaxed); } uint64_t ReadBytes() const { return metrics_.read_bytes.load(std::memory_order_relaxed); } - uint64_t EvictionRounds() const { return metrics_.evict_rounds.load(std::memory_order_relaxed); } uint64_t EvictionVictims() const { return metrics_.evict_victims.load(std::memory_order_relaxed); } @@ -243,19 +223,8 @@ class PeerSsdManager { // Build the full ADD list for every owned SSD key. Caller MUST hold mutex_. std::vector SnapshotOwnedKeysLocked() const; - // Evict oldest keys until used <= low_watermark * total. Runs the backend - // IO outside mutex_ (via Evict); serialized by eviction_mu_ so concurrent - // copy workers do not run overlapping eviction rounds (and never over-evict). - void EvictToLowWatermark(); - - // Serializes eviction rounds (EvictToLowWatermark) and excludes ClearLocal. - // Always acquired BEFORE mutex_ to keep a single lock order. - std::mutex eviction_mu_; - mutable std::mutex mutex_; std::unique_ptr backend_; // null when cfg.enabled == false - double high_watermark_ = 0.9; - double low_watermark_ = 0.7; // key -> {size, lru position}. The authoritative owned-location map. std::unordered_map owned_; @@ -311,7 +280,7 @@ class PeerSsdManager { // inflight_reads_: key -> InflightRead (entry exists only while refs > 0). // Eviction skips keys with a live read. // evicting_: keys currently inside Evict's backend-evict window; new reads - // of these miss (kNotFound) and SelectVictims skips them. + // of these miss (kNotFound) and EvictionCandidates skips them. std::unordered_map> inflight_reads_; std::unordered_set evicting_; std::condition_variable reads_drained_cv_; // notified when inflight_reads_ empties @@ -348,7 +317,6 @@ class PeerSsdManager { std::atomic read_merged{0}; std::atomic copy_bytes{0}; // bytes written to SSD (write IO) std::atomic read_bytes{0}; // bytes read from SSD (read IO) - std::atomic evict_rounds{0}; std::atomic evict_victims{0}; std::atomic evict_bytes_freed{0}; std::atomic evict_backend_failures{0}; diff --git a/src/umbp/include/umbp/distributed/pool/peer_pool.h b/src/umbp/include/umbp/distributed/pool/peer_pool.h index c8e18ff74..689082484 100644 --- a/src/umbp/include/umbp/distributed/pool/peer_pool.h +++ b/src/umbp/include/umbp/distributed/pool/peer_pool.h @@ -24,10 +24,13 @@ // MIT License #pragma once +#include #include #include #include #include +#include +#include #include #include #include @@ -71,20 +74,39 @@ struct PoolResolvedEntry { ResolvedEntry resolved; }; +// What PeerPool::Evict did with one request. +enum class PoolEvictOutcome { + kNone, // nothing freed: no such copy, or a read lease / pin holds it + kFreed, // the copy was dropped + kDemotionQueued, // queued for the transition worker, which frees the source + kInFlight, // another transition owns the key and will move or free it +}; + +struct PoolEvictResult { + std::string key; + uint64_t bytes_freed = 0; // by the time Evict returned; 0 for a queued demotion + PoolEvictOutcome outcome = PoolEvictOutcome::kNone; +}; + // Why a key is being removed. A tiered pool answers the two intents // differently, so the caller has to say which one it means instead of leaving // it to the tier configuration. enum class PoolEvictMode { - // Free bytes on the tier that holds the key. Where a tier configures - // on_evict offload the key is demoted, not dropped: the master budgets per - // (node, tier), so moving the bytes downstream does relieve the pressure it - // measured, and the data stays reachable. + // Free the one copy the request names: demoted if its tier has an offload + // target, else dropped. Other copies are left alone. kReclaim, // Drop the key from every backend on the peer regardless of tier - // configuration, for when the value itself must stop existing. + // configuration or scope, for when the value itself must stop existing. kDiscard, }; +// One key to evict, and which of its copies (a key can live on several media). +struct PoolEvictRequest { + std::string key; + TierType tier = TierType::UNKNOWN; // UNKNOWN: the medium holding the placement + uint32_t backend_id = BackendRegistry::kMaxBackends; // overrides tier when set +}; + // The implicit peer-local default Pool. It owns logical key placement and // delegates backend choice to PoolPolicy; BackendRegistry continues to own the // physical storage instances. This keeps the first Pool layer behaviorally @@ -107,7 +129,13 @@ class PeerPool { // check discards. Takes no read lease and does not touch recency: asking // whether a key is here is not a read of it. std::vector BatchContains(const std::vector& keys); - std::vector Evict(const std::vector& keys, PoolEvictMode mode); + // The single eviction entry point (master EvictKey and local eviction). + // Drops happen before it returns; demotions are only queued. One result per + // request, in order. + std::vector Evict(const std::vector& requests, + PoolEvictMode mode); + // Every key unscoped: freed from the medium that currently holds it. + std::vector Evict(const std::vector& keys, PoolEvictMode mode); void ClearLocal(); std::vector DrainPendingEvents(); std::vector SnapshotOwnedKeysForFullSync(); @@ -121,6 +149,21 @@ class PeerPool { // a tier policy is actually keeping hot data where it was supposed to. std::map TierReadHits() const; + // Background watermark eviction for one backend: at `high` usage, Evict() + // its coldest keys down to `low`. Puts only wake the worker. False for an + // unknown backend or watermarks outside 0 < low < high <= 1. + bool EnableLocalEviction(uint32_t backend_id, double high_watermark, double low_watermark); + LocalEvictionMetrics LocalEvictionStats() const; + // Test seam: one synchronous pass. Returns keys dropped or queued. + size_t RunLocalEvictionOnceForTest() { return LocalEvictionPass(); } + + // Victims for a backend over its watermark, coldest first, before the + // medium's own order: (tier, logical tier, bytes) -> offers, or nullopt when + // unavailable. The master's GetEvictionCandidates in peer-driven mode. + using EvictionCandidateSource = std::function>( + TierType tier, const std::string& logical_tier, uint64_t bytes)>; + void SetEvictionCandidateSource(EvictionCandidateSource source); + private: struct PendingPlacement { uint32_t backend_id = BackendRegistry::kMaxBackends; @@ -134,6 +177,7 @@ class PeerPool { uint32_t source_backend_id = BackendRegistry::kMaxBackends; LogicalTierGraph::TierIndex source_tier = 0; uint8_t attempts = 0; + bool evict = false; // from Evict(): ignores watermarks; drops on failure }; // What the copy phase needs, resolved while the pool lock is still held. @@ -143,9 +187,26 @@ class PeerPool { std::vector targets; }; - void EnqueueTransition(TransitionJob job); + // False if the key already has a job or the worker is stopping. + bool EnqueueTransition(TransitionJob job); + bool TransitionQueued(const std::string& key); // queued or running void TransitionWorkerLoop(); void StopTransitionWorker(); + void DropEvictedCopyLocked(const TransitionJob& job); + + struct LocalEvictionWatermarks { + double high = 0.0; + double low = 0.0; + }; + void WakeLocalEviction(); // put path: one atomic exchange, no capacity read + void LocalEvictionLoop(); + void StopLocalEviction(); + size_t LocalEvictionPass(); // lowest tier first; returns keys evicted + size_t ReclaimBackend(uint32_t backend_id, const LocalEvictionWatermarks& watermarks); + std::optional> RankedCandidates(uint32_t backend_id, TierType tier, + uint64_t bytes, + std::chrono::milliseconds wait); + void TouchLocked(const std::string& key); void ForgetAccessLocked(const std::string& key); // Whether this read should promote the key out of `source_tier`, per that @@ -156,6 +217,11 @@ class PeerPool { // is process-local and can be stale or empty (after a restart, or once // another path evicted the key), so the backends are the authority. uint32_t FindOwnerLocked(const std::string& key) const; + // The backend holding the copy a request names, or kMaxBackends. + uint32_t EvictSourceLocked(const PoolEvictRequest& request) const; + // Re-points the placement at a surviving copy (or forgets the key); drops + // run unlocked, so it re-checks the backends. + void RepairAfterDropLocked(const std::string& key); // Queues at most `max_count` offload candidates for one tier, oldest first, // and reports how many it queued. Bounded on purpose: a peer can hold // millions of placements, so neither the scan nor the queue may be @@ -230,6 +296,29 @@ class PeerPool { std::vector queued_by_tier_; bool stop_transition_worker_ = false; std::thread transition_worker_; + + // Local watermark eviction. local_evict_mutex_ guards local_evict_, + // stop_local_evict_ and the worker's start. + mutable std::mutex local_evict_mutex_; + std::condition_variable local_evict_cv_; + std::map local_evict_; + bool stop_local_evict_ = false; + std::atomic local_evict_wake_{false}; // lets a put skip the lock + std::thread local_evict_worker_; + std::mutex local_evict_pass_mutex_; // one pass at a time (worker vs test seam) + std::atomic local_evict_rounds_{0}; + std::atomic local_evict_keys_{0}; + std::atomic local_evict_no_candidate_{0}; + std::atomic local_evict_stalled_{0}; + std::atomic local_evict_master_keys_{0}; + EvictionCandidateSource candidate_source_; // guarded by local_evict_mutex_ + // At most one source query in flight per backend; pass thread only. Last + // member, so a pending query is joined before anything else goes away. + struct RankedQuery { + std::chrono::steady_clock::time_point started{}; + std::future>> answer; + }; + std::map ranked_queries_; }; } // namespace mori::umbp diff --git a/src/umbp/include/umbp/distributed/pool_client.h b/src/umbp/include/umbp/distributed/pool_client.h index 12abc2037..14880566b 100644 --- a/src/umbp/include/umbp/distributed/pool_client.h +++ b/src/umbp/include/umbp/distributed/pool_client.h @@ -203,6 +203,7 @@ class PoolClient { // MediumBackend — no concrete backend type is named outside PoolClient::Init. BackendRegistry& Backends(); TierTransitionMetrics TransitionMetrics() const; + LocalEvictionMetrics LocalEvictionStats() const; std::map TierReadHits() const; std::string LogicalTierForBackend(uint32_t backend_id) const; diff --git a/src/umbp/include/umbp/distributed/types.h b/src/umbp/include/umbp/distributed/types.h index 97ddf7227..bc3e95b44 100644 --- a/src/umbp/include/umbp/distributed/types.h +++ b/src/umbp/include/umbp/distributed/types.h @@ -84,6 +84,16 @@ struct TierTransitionMetrics { uint64_t promoted_bytes = 0; }; +// Local watermark eviction counters; bytes freed are counted by +// InstrumentedBackend. +struct LocalEvictionMetrics { + uint64_t rounds = 0; // passes that found a backend over its high watermark + uint64_t keys = 0; // keys dropped or queued for demotion + uint64_t no_candidate = 0; // passes where every key was leased or pinned + uint64_t stalled = 0; // batches that freed nothing; the next wake retries + uint64_t master_keys = 0; // of `keys`, those the candidate source ranked +}; + struct ExternalKvHitCountEntry { std::string hash; uint64_t hit_count_total = 0; @@ -160,6 +170,26 @@ struct EvictionCandidate { uint64_t size; }; +// One master eviction decision: free `key`'s copy on the `tier` medium. +struct EvictionVictim { + std::string key; + TierType tier = TierType::UNKNOWN; + + bool operator==(const EvictionVictim& other) const { + return key == other.key && tier == other.tier; + } +}; + +// A key offered for local eviction, with the (page-rounded) bytes it frees. +struct EvictionOffer { + std::string key; + uint64_t bytes = 0; + + bool operator==(const EvictionOffer& other) const { + return key == other.key && bytes == other.bytes; + } +}; + // Ordering hint for IMasterMetadataStore::EnumerateEvictionCandidates. This is // a performance affordance, NOT eviction policy: it only tells the store what // order to return rows in so a backend with an index (e.g. a Redis ZSET keyed diff --git a/src/umbp/include/umbp/local/tiers/spdk_ssd_tier.h b/src/umbp/include/umbp/local/tiers/spdk_ssd_tier.h index 6b1725ce0..27a1cf9e8 100644 --- a/src/umbp/include/umbp/local/tiers/spdk_ssd_tier.h +++ b/src/umbp/include/umbp/local/tiers/spdk_ssd_tier.h @@ -25,7 +25,8 @@ // Metadata management follows mooncake-store's OffsetAllocatorStorageBackend: // - Sharded map (kNumShards) with std::shared_mutex for concurrent reads // - RefCounted allocation handles (AllocationPtr) for safe concurrent access -// - Auto LRU eviction on allocation failure +// - NO eviction on allocation failure: a write that does not fit fails, and +// freeing space is the owner's decision (PeerSsdManager calls Evict()). #pragma once #include @@ -207,9 +208,6 @@ class SpdkSsdTier : public TierBackend { const std::vector& sizes, std::vector& results); - // -- LRU eviction -------------------------------------------------------- - size_t EvictLRU(size_t needed); - // -- Member data --------------------------------------------------------- bool initialized_ = false; std::shared_ptr<::umbp::offset_allocator::OffsetAllocator> allocator_; diff --git a/src/umbp/local/tiers/spdk_ssd_tier.cpp b/src/umbp/local/tiers/spdk_ssd_tier.cpp index 558e6734e..f717ae295 100644 --- a/src/umbp/local/tiers/spdk_ssd_tier.cpp +++ b/src/umbp/local/tiers/spdk_ssd_tier.cpp @@ -242,51 +242,14 @@ std::vector SpdkSsdTier::GetLRUCandidates(size_t max_candidates) co return result; } -// =========================================================================== -// LRU eviction — evicts entries until `needed` bytes are freed. -// Lock ordering: lru_mu_ → shard.mutex (consistent with all other methods). -// Returns actual bytes freed (may be 0 if LRU is empty). -// =========================================================================== -size_t SpdkSsdTier::EvictLRU(size_t needed) { - size_t freed = 0; - int evicted = 0; - while (freed < needed) { - std::lock_guard lru_lk(lru_mu_); - if (lru_list_.empty()) break; - - std::string key = lru_list_.back(); - auto& shard = shards_[ShardForKey(key)]; - std::unique_lock slk(shard.mutex); - auto it = shard.map.find(key); - if (it != shard.map.end()) { - size_t entry_size = AlignUp(it->second.data_size); - bool immediate = (it->second.allocation.use_count() == 1); - freed += entry_size; - evicted_bytes_.fetch_add(entry_size, std::memory_order_relaxed); - shard.map.erase(it); - if (!immediate) { - MORI_UMBP_WARN( - "EvictLRU: key '{}' ({}KB) has in-flight readers, " - "space reclaim deferred", - key, entry_size / 1024); - } - } - lru_list_.pop_back(); - ++evicted; - } - if (evicted > 0) { - MORI_UMBP_INFO("EvictLRU: evicted {} entries, freed {}MB (requested {}MB)", evicted, - freed / (1024 * 1024), needed / (1024 * 1024)); - } - return freed; -} - // =========================================================================== // PrepareWriteAlloc — common Phase 1 for all BatchWrite* variants. // // 1) Checks existing keys (shard shared lock + LRU update) // 2) Batch allocates space for new keys -// 3) On allocation failure, evicts LRU entries and retries +// +// A key that does not fit fails. The tier never evicts on its own: its owner +// (PeerSsdManager) tracks the keys and would keep listing ones it dropped. // =========================================================================== std::vector SpdkSsdTier::PrepareWriteAlloc( const std::vector& keys, const std::vector& sizes, @@ -322,34 +285,18 @@ std::vector SpdkSsdTier::PrepareWriteAlloc( if (alloc_sizes.empty()) return pending; - // Allocate with auto-eviction retry - size_t success_count = 0; - constexpr int kMaxEvictRetries = 3; - - for (int retry = 0; retry <= kMaxEvictRetries; ++retry) { - std::vector remaining(alloc_sizes.begin() + static_cast(success_count), - alloc_sizes.end()); - - auto handles = allocator_->batch_allocate(remaining); - - for (size_t j = 0; j < handles.size(); ++j) { - if (!handles[j].has_value()) break; - PendingWrite pw; - pw.idx = new_indices[success_count + j]; - pw.aligned_size = alloc_sizes[success_count + j]; - pw.allocation = std::make_shared(std::move(handles[j].value())); - pending.push_back(std::move(pw)); - } - - success_count = pending.size(); - if (success_count >= new_indices.size()) break; - - size_t needed = 0; - for (size_t k = success_count; k < alloc_sizes.size(); ++k) needed += alloc_sizes[k]; - - size_t freed = EvictLRU(needed); - if (freed == 0) break; + // One attempt. The allocator stops at the first request it cannot place, + // so everything from there on fails with it. + auto handles = allocator_->batch_allocate(alloc_sizes); + for (size_t j = 0; j < handles.size(); ++j) { + if (!handles[j].has_value()) break; + PendingWrite pw; + pw.idx = new_indices[j]; + pw.aligned_size = alloc_sizes[j]; + pw.allocation = std::make_shared(std::move(handles[j].value())); + pending.push_back(std::move(pw)); } + const size_t success_count = pending.size(); if (success_count < new_indices.size()) { size_t failed = new_indices.size() - success_count; @@ -439,7 +386,7 @@ std::vector SpdkSsdTier::PrepareReadLookup( // =========================================================================== // BatchWrite — deep-queue NVMe write pipeline // -// Phase 1: PrepareWriteAlloc (check existing, allocate with auto-eviction) +// Phase 1: PrepareWriteAlloc (check existing, allocate; no eviction) // Phase 2: memcpy + submit + drain pipeline on calling thread // Phase 3: CommitWriteEntries (update sharded map + LRU) // =========================================================================== diff --git a/tests/cpp/umbp/distributed/CMakeLists.txt b/tests/cpp/umbp/distributed/CMakeLists.txt index 452a2466d..9f5e35c3f 100644 --- a/tests/cpp/umbp/distributed/CMakeLists.txt +++ b/tests/cpp/umbp/distributed/CMakeLists.txt @@ -277,6 +277,18 @@ target_link_libraries( gtest_main) add_test(NAME umbp_local_first COMMAND test_umbp_local_first) +# test_umbp_peer_driven_eviction — a real master and DRAM node in each eviction +# mode: local (default), master_ranked, and master (the master's rounds). +add_executable(test_umbp_peer_driven_eviction test_peer_driven_eviction.cpp) +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_link_options(test_umbp_peer_driven_eviction PRIVATE -Wl,--no-as-needed) +endif() +target_link_libraries( + test_umbp_peer_driven_eviction + PRIVATE umbp_core umbp_common ${_TEST_PROTOBUF_LIB} ${_TEST_GRPCPP_LIB} + gtest_main) +add_test(NAME umbp_peer_driven_eviction COMMAND test_umbp_peer_driven_eviction) + # test_umbp_no_master — the distributed client with master_address empty: same # data plane, no routing/registration/heartbeat, a local miss is final. No # master is started in that file, so reaching for one hangs or crashes rather diff --git a/tests/cpp/umbp/distributed/test_cross_node_smoke.cpp b/tests/cpp/umbp/distributed/test_cross_node_smoke.cpp index 253ce9364..645cf732f 100644 --- a/tests/cpp/umbp/distributed/test_cross_node_smoke.cpp +++ b/tests/cpp/umbp/distributed/test_cross_node_smoke.cpp @@ -98,6 +98,8 @@ class CrossNodeSmoke : public ::testing::Test { cfg_a.auto_peer_service_port = true; cfg_a.dram.buffer_sizes = {kBlockSize}; cfg_a.dram_page_size = kBlockSize; + // These tiers are filled on purpose; leave eviction to the master's rounds. + cfg_a.eviction_mode = PeerEvictionMode::kMaster; client_a_ = std::make_unique(std::move(cfg_a)); ASSERT_TRUE(client_a_->Init()); @@ -110,6 +112,7 @@ class CrossNodeSmoke : public ::testing::Test { cfg_b.auto_peer_service_port = true; cfg_b.dram.buffer_sizes = {kBufSize}; cfg_b.dram_page_size = kBlockSize; + cfg_b.eviction_mode = PeerEvictionMode::kMaster; client_b_ = std::make_unique(std::move(cfg_b)); ASSERT_TRUE(client_b_->Init()); @@ -242,6 +245,8 @@ class CrossNodeMultiPage : public ::testing::Test { cfg.auto_peer_service_port = true; cfg.dram_page_size = kPageSize; cfg.dram.buffer_sizes = setup.buffer_sizes; + // These tiers are filled on purpose; leave eviction to the master's rounds. + cfg.eviction_mode = PeerEvictionMode::kMaster; auto cli = std::make_unique(std::move(cfg)); EXPECT_TRUE(cli->Init()); return cli; @@ -542,6 +547,8 @@ class CrossNodeOverlap : public ::testing::Test { cfg.dram_page_size = kPageSize; if (staging_buffer_size > 0) cfg.staging_buffer_size = staging_buffer_size; cfg.dram.buffer_sizes = buffer_sizes; + // These tiers are filled on purpose; leave eviction to the master's rounds. + cfg.eviction_mode = PeerEvictionMode::kMaster; auto cli = std::make_unique(std::move(cfg)); EXPECT_TRUE(cli->Init()); clients_.push_back(std::move(cli)); diff --git a/tests/cpp/umbp/distributed/test_env_time.cpp b/tests/cpp/umbp/distributed/test_env_time.cpp index 622a74ab4..9e5b6f8d4 100644 --- a/tests/cpp/umbp/distributed/test_env_time.cpp +++ b/tests/cpp/umbp/distributed/test_env_time.cpp @@ -40,6 +40,7 @@ using mori::umbp::GetEnvMicroseconds; using mori::umbp::GetEnvMilliseconds; using mori::umbp::GetEnvSeconds; using mori::umbp::GetEnvUint32; +using mori::umbp::GetEnvUint64; using mori::umbp::ResetEnvWarnStateForTesting; constexpr const char* kName = "UMBP_TEST_ENV_TIME_XYZ"; @@ -107,6 +108,15 @@ TEST_F(EnvTimeTest, ZeroIsAllowedWhenMinIsZero) { EXPECT_EQ(GetEnvUint32(kName, 7), 0u); } +TEST_F(EnvTimeTest, Uint64ParsesPastUint32AndRejectsGarbage) { + ::setenv(kName, "8589934592", 1); // 8 GiB + EXPECT_EQ(GetEnvUint64(kName, 1), 8589934592ULL); + ::setenv(kName, "-1", 1); + EXPECT_EQ(GetEnvUint64(kName, 5), 5u); + ::setenv(kName, "1G", 1); + EXPECT_EQ(GetEnvUint64(kName, 6), 6u); +} + TEST_F(EnvTimeTest, FallsBackWhenAboveUint32Max) { // A value that exceeds uint32 range must fall back to default. ::setenv(kName, "99999999999", 1); diff --git a/tests/cpp/umbp/distributed/test_master_evict_strategy.cpp b/tests/cpp/umbp/distributed/test_master_evict_strategy.cpp index a5bc7ac45..af8107523 100644 --- a/tests/cpp/umbp/distributed/test_master_evict_strategy.cpp +++ b/tests/cpp/umbp/distributed/test_master_evict_strategy.cpp @@ -25,6 +25,7 @@ // byte budget. #include +#include #include #include #include @@ -67,8 +68,31 @@ TEST(LruMasterEvictStrategy, PicksOldestFirstUntilBudgetMet) { ASSERT_EQ(victims.count("n1"), 1u); // Oldest-first: a, then b; c (newest) is spared once the 150-byte budget met. ASSERT_EQ(victims["n1"].size(), 2u); - EXPECT_EQ(victims["n1"][0], "a"); - EXPECT_EQ(victims["n1"][1], "b"); + EXPECT_EQ(victims["n1"][0].key, "a"); + EXPECT_EQ(victims["n1"][1].key, "b"); +} + +TEST(LruMasterEvictStrategy, KeyOverBudgetInTwoTiersIsOneVictimPerTier) { + // A key in DRAM and SSD, both over budget: one victim per (key, tier), each + // carrying the tier its copy was charged to. + LruMasterEvictStrategy strategy; + auto now = Clock::now(); + std::vector candidates = { + MakeCandidate("k", "n1", TierType::DRAM, 100, now - std::chrono::seconds(1)), + MakeCandidate("k", "n1", TierType::SSD, 100, now - std::chrono::seconds(1)), + }; + std::unordered_map> budget; + budget["n1"][TierType::DRAM] = 100; + budget["n1"][TierType::SSD] = 100; + + auto victims = strategy.SelectVictims(candidates, budget); + ASSERT_EQ(victims["n1"].size(), 2u); + EXPECT_NE( + std::find(victims["n1"].begin(), victims["n1"].end(), EvictionVictim{"k", TierType::DRAM}), + victims["n1"].end()); + EXPECT_NE( + std::find(victims["n1"].begin(), victims["n1"].end(), EvictionVictim{"k", TierType::SSD}), + victims["n1"].end()); } TEST(LruMasterEvictStrategy, HonoursPerNodeTierBudgetIndependently) { @@ -85,9 +109,9 @@ TEST(LruMasterEvictStrategy, HonoursPerNodeTierBudgetIndependently) { auto victims = strategy.SelectVictims(candidates, budget); ASSERT_EQ(victims["n1"].size(), 1u); - EXPECT_EQ(victims["n1"][0], "n1-old"); + EXPECT_EQ(victims["n1"][0].key, "n1-old"); ASSERT_EQ(victims["n2"].size(), 1u); - EXPECT_EQ(victims["n2"][0], "n2-old"); + EXPECT_EQ(victims["n2"][0].key, "n2-old"); } TEST(LruMasterEvictStrategy, SkipsTiersWithNoBudget) { diff --git a/tests/cpp/umbp/distributed/test_master_eviction_chain.cpp b/tests/cpp/umbp/distributed/test_master_eviction_chain.cpp index ca5e0c70d..1f6f976d5 100644 --- a/tests/cpp/umbp/distributed/test_master_eviction_chain.cpp +++ b/tests/cpp/umbp/distributed/test_master_eviction_chain.cpp @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -114,12 +115,18 @@ class PoolDispatcher final : public EvictKeyDispatcher { explicit PoolDispatcher(PeerPool* pool) : pool_(pool) {} void DispatchEvictKey(const std::string& node_id, const std::string& /*peer_address*/, - std::vector keys) override { - auto results = pool_->Evict(keys, PoolEvictMode::kReclaim); + std::vector victims) override { + std::vector requests(victims.size()); + for (size_t i = 0; i < victims.size(); ++i) { + requests[i].key = victims[i].key; + requests[i].tier = victims[i].tier; + } + auto results = pool_->Evict(requests, PoolEvictMode::kReclaim); std::lock_guard lock(mutex_); ++rounds_; - for (size_t i = 0; i < keys.size(); ++i) { - dispatched_.push_back(keys[i]); + for (size_t i = 0; i < victims.size(); ++i) { + dispatched_.push_back(victims[i].key); + victims_.push_back(victims[i]); if (i < results.size()) freed_bytes_ += results[i].bytes_freed; } node_ids_.push_back(node_id); @@ -129,6 +136,10 @@ class PoolDispatcher final : public EvictKeyDispatcher { std::lock_guard lock(mutex_); return dispatched_; } + std::vector Victims() const { + std::lock_guard lock(mutex_); + return victims_; + } size_t Rounds() const { std::lock_guard lock(mutex_); return rounds_; @@ -146,6 +157,7 @@ class PoolDispatcher final : public EvictKeyDispatcher { PeerPool* pool_; mutable std::mutex mutex_; std::vector dispatched_; + std::vector victims_; std::vector node_ids_; size_t rounds_ = 0; uint64_t freed_bytes_ = 0; @@ -247,13 +259,21 @@ TEST(MasterEvictionChain, OverWatermarkTierDemotesIntoTheColdTier) { EXPECT_EQ(dispatcher.NodeIds().front(), kNodeId); // Every key the master named must have moved down a tier rather than vanished. + // The peer only queues the demotion, so wait for the move. const auto evicted = dispatcher.Dispatched(); ASSERT_FALSE(evicted.empty()); + const bool moved = WaitFor( + [&] { + for (const auto& key : evicted) { + if (registry.Get("hot")->Contains(key)) return false; + } + return true; + }, + std::chrono::seconds(5)); + ASSERT_TRUE(moved) << "a queued demotion never completed"; for (const auto& key : evicted) { - EXPECT_FALSE(registry.Get("hot")->Contains(key)) << key << " still in hot"; EXPECT_TRUE(registry.Get("cold")->Contains(key)) << key << " was dropped, not demoted"; } - EXPECT_GT(dispatcher.FreedBytes(), 0u); // Keys the master did not name are untouched: eviction is bounded by the // budget, so a round must not drain the whole tier. @@ -351,8 +371,16 @@ TEST(MasterEvictionChain, MasterRoundReclaimsTheParkedSourceOfAMovePromotion) { EXPECT_TRUE(registry.Get("cold")->Contains("moved")) << "source was deleted during the promotion, so there is nothing to reclaim"; + // Only SSD (the parked source) is over its watermark, so the victim must name + // SSD, not the promoted DRAM copy the placement points at. InMemoryMasterMetadataStore store; - PublishNode(&store, {"moved"}, /*total_bytes=*/4 * kPageSize, /*used_bytes=*/4 * kPageSize); + const auto now = Clock::now(); + RegisterNode(&store, now); + std::map caps; + caps[TierType::SSD] = TierCapacity{4 * kPageSize, 0, kPageSize}; + caps[TierType::DRAM] = TierCapacity{4 * kPageSize, 3 * kPageSize, kPageSize}; + ReportHeartbeat(&store, /*seq=*/1, now, caps, TierType::SSD, "cold", {"moved"}, + /*full_sync=*/true); PoolDispatcher dispatcher(&pool); EvictionManager manager(store, OneSecondRounds(), &dispatcher); @@ -362,6 +390,8 @@ TEST(MasterEvictionChain, MasterRoundReclaimsTheParkedSourceOfAMovePromotion) { manager.Stop(); EXPECT_TRUE(reclaimed) << "the parked source outlived a master eviction round that named it"; + ASSERT_FALSE(dispatcher.Victims().empty()); + EXPECT_EQ(dispatcher.Victims().front(), (EvictionVictim{"moved", TierType::SSD})); // The promoted copy is the only one left, so the key is still servable. EXPECT_TRUE(registry.Get("hot")->Contains("moved")) << "the evict drained the promoted target instead of the parked source"; @@ -511,5 +541,84 @@ TEST(MasterEvictionChain, ParkedMoveSourceIsNamedLastAmongColderKeys) { EXPECT_TRUE(registry.Get("hot")->Contains("moved")); } +// Records what the manager sends, one entry per EvictKey; executes nothing. +class RecordingDispatcher final : public EvictKeyDispatcher { + public: + void DispatchEvictKey(const std::string& /*node_id*/, const std::string& /*peer_address*/, + std::vector victims) override { + std::lock_guard lock(mutex_); + calls_.push_back(std::move(victims)); + } + std::vector> Calls() const { + std::lock_guard lock(mutex_); + return calls_; + } + + private: + mutable std::mutex mutex_; + std::vector> calls_; +}; + +// Keys in the first EvictKey for a full 256-page DRAM bucket of one-page keys. +size_t FirstRoundVictims(const EvictionConfig& config) { + std::vector keys; + for (int i = 0; i < 256; ++i) keys.push_back("k" + std::to_string(i)); + InMemoryMasterMetadataStore store; + PublishNode(&store, keys, 256 * kPageSize, 256 * kPageSize); + RecordingDispatcher dispatcher; + EvictionManager manager(store, config, &dispatcher); + manager.Start(); + const bool sent = WaitFor([&] { return !dispatcher.Calls().empty(); }, std::chrono::seconds(8)); + manager.Stop(); + return sent ? dispatcher.Calls().front().size() : 0; +} + +TEST(MasterEvictionBytes, ARoundFreesTheOverageUpToTheByteLimit) { + // 77 keys over the 0.7 low watermark; rounds used to stop at 32. + EXPECT_EQ(FirstRoundVictims(OneSecondRounds()), 77u); + + ::setenv("UMBP_EVICT_MAX_BYTES_PER_ROUND", std::to_string(10 * kPageSize).c_str(), 1); + auto limited = OneSecondRounds(); + limited.max_evict_bytes_per_round = EvictionConfig::FromEnvironment().max_evict_bytes_per_round; + ::unsetenv("UMBP_EVICT_MAX_BYTES_PER_ROUND"); + EXPECT_EQ(FirstRoundVictims(limited), 10u); +} + +// Peer-driven eviction: the master ranks a node's keys on request and leaves +// the node out of its own rounds. +TEST(MasterPeerDrivenEviction, RanksANodesKeysAndSkipsItInRounds) { + InMemoryMasterMetadataStore store; + const auto t0 = Clock::now(); + RegisterNode(&store, t0); + std::map full; + full[TierType::DRAM] = TierCapacity{6 * kPageSize, 0, kPageSize}; + // k5 is the oldest; k0..k3 are "hot", k4/k5 "warm". + for (int i = 0; i < 6; ++i) { + ReportHeartbeat(&store, /*seq=*/i + 1, t0 + std::chrono::seconds(6 - i), full, TierType::DRAM, + i < 4 ? "hot" : "warm", {"k" + std::to_string(i)}, /*full_sync=*/i == 0); + } + const auto now = t0 + std::chrono::seconds(10); + const auto keys = [&](const std::string& logical, uint64_t pages, size_t max_keys) { + std::vector out; + for (const auto& row : SelectPeerEvictionCandidates(store, kNodeId, TierType::DRAM, logical, + pages * kPageSize, max_keys, now)) { + out.push_back(row.key); + } + return out; + }; + EXPECT_EQ(keys("", 3, 0), (std::vector{"k5", "k4", "k3"})); + EXPECT_EQ(keys("hot", 2, 0), (std::vector{"k3", "k2"})); + EXPECT_EQ(keys("", 100, 2), (std::vector{"k5", "k4"})); + + RecordingDispatcher dispatcher; + EvictionManager manager(store, OneSecondRounds(), &dispatcher); + manager.SetPeerDriven(kNodeId, true); + manager.RunOnceForTest(); + EXPECT_TRUE(dispatcher.Calls().empty()) << "a peer-driven node still got EvictKey"; + manager.SetPeerDriven(kNodeId, false); + manager.RunOnceForTest(); + EXPECT_FALSE(dispatcher.Calls().empty()); +} + } // namespace } // namespace mori::umbp diff --git a/tests/cpp/umbp/distributed/test_peer_driven_eviction.cpp b/tests/cpp/umbp/distributed/test_peer_driven_eviction.cpp new file mode 100644 index 000000000..a80da154f --- /dev/null +++ b/tests/cpp/umbp/distributed/test_peer_driven_eviction.cpp @@ -0,0 +1,191 @@ +// Copyright © Advanced Micro Devices, Inc. All rights reserved. +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// Eviction modes end to end: a real master and a DRAM PoolClient over gRPC. +// The master's own rounds run every 5 s, so anything evicted well inside that +// window was evicted by the node itself. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "umbp/distributed/config.h" +#include "umbp/distributed/master/master_server.h" +#include "umbp/distributed/pool_client.h" + +namespace mori::umbp { +namespace { + +constexpr size_t kPage = 4096; +constexpr size_t kPages = 16; + +std::string Key(int i) { return "pde-" + std::to_string(i); } + +bool WaitFor(const std::function& done, std::chrono::milliseconds budget) { + const auto deadline = std::chrono::steady_clock::now() + budget; + while (std::chrono::steady_clock::now() < deadline) { + if (done()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + return done(); +} + +class PeerDrivenEvictionTest : public ::testing::Test { + protected: + void TearDown() override { Stop(); } + + void Stop() { + if (client_) client_->Shutdown(); + client_.reset(); + if (master_) master_->Shutdown(); + if (server_thread_.joinable()) server_thread_.join(); + master_.reset(); + } + + // A master (advertising peer-driven eviction or not) and one 16-page DRAM + // node with 0.9 / 0.5 watermarks. + PoolClient* Start(bool master_supports, PeerEvictionMode mode) { + MasterServerConfig master_cfg; + master_cfg.listen_address = "0.0.0.0:0"; + master_cfg.registry_config.advertise_peer_driven_eviction = master_supports; + master_ = std::make_unique(std::move(master_cfg)); + server_thread_ = std::thread([this] { master_->Run(); }); + if (!WaitFor([&] { return master_->GetBoundPort() != 0; }, std::chrono::seconds(5))) { + return nullptr; + } + + PoolClientConfig cfg; + cfg.master_config.node_id = "peer-driven"; + cfg.master_config.node_address = "127.0.0.1"; + cfg.master_config.master_address = "localhost:" + std::to_string(master_->GetBoundPort()); + cfg.io_engine.host = "0.0.0.0"; + cfg.io_engine.port = 0; + cfg.auto_peer_service_port = true; + cfg.dram_page_size = kPage; + cfg.medium = TierType::DRAM; + cfg.dram.buffer_sizes = {kPages * kPage}; + cfg.cache_remote_fetches = false; + cfg.local_evict_high_watermark = 0.9; + cfg.local_evict_low_watermark = 0.5; + cfg.eviction_mode = mode; + client_ = std::make_unique(std::move(cfg)); + if (!client_->Init()) client_.reset(); + return client_.get(); + } + + void PutKeys(int first, int count) { + const std::vector value(kPage, 'x'); + for (int i = first; i < first + count; ++i) { + ASSERT_TRUE(client_->Put(Key(i), value.data(), value.size())) << i; + } + } + + MediumBackend* Dram() { return client_->Backends().Get(TierType::DRAM); } + + std::unique_ptr master_; + std::thread server_thread_; + std::unique_ptr client_; +}; + +TEST_F(PeerDrivenEvictionTest, MasterRankedNodeEvictsTheMastersVictims) { + auto* node = Start(/*master_supports=*/true, PeerEvictionMode::kMasterRanked); + ASSERT_NE(node, nullptr); + ASSERT_TRUE(node->Master().PeerDrivenEviction()); + + // Just under the watermark, and known to the master. + PutKeys(0, 14); + std::vector known; + for (int i = 0; i < 14; ++i) known.push_back(Key(i)); + node->Master().FlushHeartbeat(); + ASSERT_TRUE(WaitFor( + [&] { + std::vector found; + return node->Master().BatchLookup(known, &found).ok() && + std::all_of(found.begin(), found.end(), [](bool f) { return f; }); + }, + std::chrono::seconds(5))); + + // Local reads, which the master sees only through the heartbeat, must move + // these keys behind every other in its ranking. + const std::vector read = {Key(0), Key(1), Key(2)}; + std::vector buf(kPage); + for (const auto& key : read) ASSERT_TRUE(node->Get(key, buf.data(), buf.size())) << key; + ASSERT_TRUE(WaitFor( + [&] { + node->Master().FlushHeartbeat(); + auto ranked = node->Master().GetEvictionCandidates(TierType::DRAM, "", 14 * kPage, 0); + if (!ranked.has_value() || ranked->size() != 14) return false; + std::vector last; + for (size_t i = 11; i < 14; ++i) last.push_back((*ranked)[i].key); + std::sort(last.begin(), last.end()); + return last == read; + }, + std::chrono::seconds(5))) + << "local reads did not reach the master's ranking"; + + PutKeys(14, 1); // 15 of 16 pages + ASSERT_TRUE( + WaitFor([&] { return Dram()->OwnedKeyCount() <= kPages / 2; }, std::chrono::seconds(3))) + << "the node did not evict for itself"; + EXPECT_EQ(Dram()->OwnedKeyCount(), kPages / 2); + EXPECT_EQ(node->LocalEvictionStats().master_keys, 7u) << "victims did not come from the master"; + for (const auto& key : read) EXPECT_TRUE(Dram()->Contains(key)) << key; +} + +TEST_F(PeerDrivenEvictionTest, EvictsLocallyUnlessTheModeIsMaster) { + struct Case { + bool master_supports; + PeerEvictionMode mode; + bool self_evicts; + }; + // The default; master_ranked against an older master (falls back to local + // LRU); and master mode, which leaves DRAM to the master's rounds. + const std::vector cases = {{true, PeerEvictionMode::kLocal, true}, + {false, PeerEvictionMode::kMasterRanked, true}, + {true, PeerEvictionMode::kMaster, false}}; + for (const auto& c : cases) { + SCOPED_TRACE(::testing::Message() << "master_supports=" << c.master_supports + << " mode=" << static_cast(c.mode)); + auto* node = Start(c.master_supports, c.mode); + ASSERT_NE(node, nullptr); + PutKeys(0, 15); // 15 of 16 pages + if (c.self_evicts) { + EXPECT_TRUE( + WaitFor([&] { return Dram()->OwnedKeyCount() <= kPages / 2; }, std::chrono::seconds(3))); + } else { + std::this_thread::sleep_for(std::chrono::milliseconds(1500)); + EXPECT_EQ(Dram()->OwnedKeyCount(), 15u); + } + EXPECT_EQ(node->LocalEvictionStats().master_keys, 0u) << "asked the master"; + Stop(); + } +} + +} // namespace +} // namespace mori::umbp diff --git a/tests/cpp/umbp/distributed/test_peer_pool.cpp b/tests/cpp/umbp/distributed/test_peer_pool.cpp index b181e6ec6..8882d0fce 100644 --- a/tests/cpp/umbp/distributed/test_peer_pool.cpp +++ b/tests/cpp/umbp/distributed/test_peer_pool.cpp @@ -24,9 +24,11 @@ // MIT License #include +#include #include #include #include +#include #include #include #include @@ -111,6 +113,58 @@ bool WaitForKey(BackendRegistry* registry, const std::string& backend, const std return registry->Get(backend)->Contains(key); } +// Eviction demotions run on the transition worker, so wait for the source. +bool WaitForKeyGone(BackendRegistry* registry, const std::string& backend, const std::string& key) { + for (int attempt = 0; attempt < 500 && registry->Get(backend)->Contains(key); ++attempt) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + return !registry->Get(backend)->Contains(key); +} + +// Holds every copy until Release(), or 10 s so a failed test can't hang. +class GatedEngine final : public TransferEngine { + public: + const char* Name() const override { return "gated"; } + TransferRef RegisterMemory(void* base, size_t size, mori::io::MemoryLocationType loc, + int device) override { + return inner_.RegisterMemory(base, size, loc, device); + } + void Deregister(const TransferRef& ref) override { inner_.Deregister(ref); } + bool CanHandle(const TransferRef& src, const TransferRef& dst) const override { + return inner_.CanHandle(src, dst); + } + TransferPlanSet Plan(const std::vector& items) const override { + return inner_.Plan(items); + } + std::unique_ptr Submit(std::vector plans) override { + { + std::unique_lock lock(mutex_); + ++held_; + cv_.notify_all(); + cv_.wait_for(lock, std::chrono::seconds(10), [&] { return open_; }); + } + return inner_.Submit(std::move(plans)); + } + + void Release() { + std::lock_guard lock(mutex_); + open_ = true; + cv_.notify_all(); + } + // True once `n` copies are being held. + bool WaitForHeld(int n) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, std::chrono::seconds(5), [&] { return held_ >= n; }); + } + + private: + LocalCopyEngine inner_; + std::mutex mutex_; + std::condition_variable cv_; + bool open_ = false; + int held_ = 0; +}; + class NoSpaceBackend final : public MockBackend { public: explicit NoSpaceBackend(TierType tier) : MockBackend(tier) {} @@ -550,7 +604,7 @@ TEST(TieredPlacementPolicy, HonorsExplicitLogicalTierAndTagsEvents) { TEST(PeerPool, OnEvictMigratesBytesToConfiguredBackend) { constexpr uint64_t kPageSize = 64; - LocalCopyEngine engine; + GatedEngine engine; BackendRegistry registry; RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 2); RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 2); @@ -571,10 +625,18 @@ TEST(PeerPool, OnEvictMigratesBytesToConfiguredBackend) { .front() .commit.success); + // Evict returns while the copy is held, and does not queue it twice. auto evicted = pool.Evict({"move"}, PoolEvictMode::kReclaim); ASSERT_EQ(evicted.size(), 1u); - EXPECT_EQ(evicted.front().bytes_freed, payload.size()); - EXPECT_FALSE(registry.Get("hot")->Contains("move")); + EXPECT_EQ(evicted.front().outcome, PoolEvictOutcome::kDemotionQueued); + EXPECT_EQ(evicted.front().bytes_freed, 0u); + ASSERT_TRUE(engine.WaitForHeld(1)) << "the demotion never reached the copy"; + EXPECT_TRUE(registry.Get("hot")->Contains("move")); + EXPECT_EQ(pool.Evict({"move"}, PoolEvictMode::kReclaim).front().outcome, + PoolEvictOutcome::kInFlight); + + engine.Release(); + ASSERT_TRUE(WaitForKeyGone(®istry, "hot", "move")); EXPECT_TRUE(registry.Get("cold")->Contains("move")); EXPECT_EQ(pool.PlacementBackend("move"), std::optional{registry.BackendId(registry.Get("cold"))}); @@ -591,11 +653,12 @@ TEST(PeerPool, OnEvictMigratesBytesToConfiguredBackend) { TEST(PeerPool, EvictionRetryDrainsLeasedSourceWithoutDeletingTarget) { constexpr uint64_t kPageSize = 64; - constexpr auto kShortLease = std::chrono::milliseconds(1); + // Outlives the queued demotion, so its source delete is refused. + constexpr auto kLease = std::chrono::milliseconds(300); LocalCopyEngine engine; BackendRegistry registry; - RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 2, kShortLease); - RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 2, kShortLease); + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 2, kLease); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 2, kLease); auto compiled = LogicalTierGraph::Compile(HotColdTiers(PoolOffloadTrigger::kOnEvict), registry); ASSERT_TRUE(compiled.ok()) << compiled.error; PeerPool pool(®istry, MakeTieredPlacementPolicy(compiled.graph), &engine); @@ -605,10 +668,10 @@ TEST(PeerPool, EvictionRetryDrainsLeasedSourceWithoutDeletingTarget) { auto first = pool.Evict({"leased"}, PoolEvictMode::kReclaim); ASSERT_EQ(first.front().bytes_freed, 0u); - EXPECT_TRUE(registry.Get("hot")->Contains("leased")); - EXPECT_TRUE(registry.Get("cold")->Contains("leased")); + ASSERT_TRUE(WaitForKey(®istry, "cold", "leased")); + EXPECT_TRUE(registry.Get("hot")->Contains("leased")) << "the lease did not hold the source"; - std::this_thread::sleep_for(std::chrono::milliseconds(5)); + std::this_thread::sleep_for(kLease + std::chrono::milliseconds(50)); auto retry = pool.Evict({"leased"}, PoolEvictMode::kReclaim); EXPECT_EQ(retry.front().bytes_freed, 64u); EXPECT_FALSE(registry.Get("hot")->Contains("leased")); @@ -636,6 +699,133 @@ TEST(PeerPool, DiscardDeletesInsteadOfDemoting) { EXPECT_EQ(pool.PlacementCount(), 0u); } +// Copy-promotes `key`: both media hold it, the placement is the hot copy, and +// the promotion's read lease on the cold copy has lapsed. +void PromoteByCopy(PeerPool* pool, BackendRegistry* registry, const std::string& key) { + auto request = PutRequest(key); + request.logical_tier = "cold"; + auto allocation = pool->BatchAllocate({request}).front(); + ASSERT_EQ(allocation.allocation.outcome, AllocateOutcome::kSuccessAllocated); + ASSERT_TRUE(pool->BatchCommit({PoolCommitRequest{ + {allocation.backend_id, allocation.allocation.slot_id}, key}}) + .front() + .commit.success); + ASSERT_TRUE(pool->BatchResolve({key}, false).front().resolved.found); + ASSERT_TRUE(WaitForKey(registry, "hot", key)); + ASSERT_TRUE(registry->Get("cold")->Contains(key)); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); +} + +std::shared_ptr CopyPromotingHotCold(BackendRegistry* registry) { + auto tiers = HotColdTiers(PoolOffloadTrigger::kOnEvict); + tiers.back().promote_trigger = PoolPromoteTrigger::kOnRead; + tiers.back().promote_mode = PoolTransitionMode::kCopy; + auto compiled = LogicalTierGraph::Compile(tiers, *registry); + EXPECT_TRUE(compiled.ok()) << compiled.error; + return compiled.graph; +} + +TEST(PeerPool, TierScopedEvictFreesOnlyThatMediumsCopy) { + constexpr uint64_t kPageSize = 64; + constexpr auto kShortLease = std::chrono::milliseconds(1); + LocalCopyEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 4, kShortLease); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 4, kShortLease); + PeerPool pool(®istry, MakeTieredPlacementPolicy(CopyPromotingHotCold(®istry)), &engine); + const uint32_t hot = registry.BackendId(registry.Get("hot")); + const uint32_t cold = registry.BackendId(registry.Get("cold")); + PromoteByCopy(&pool, ®istry, "a"); + PromoteByCopy(&pool, ®istry, "b"); + + // SSD scope: only the SSD copy goes; the promoted DRAM copy keeps serving. + auto evicted = pool.Evict({PoolEvictRequest{"a", TierType::SSD}}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().bytes_freed, 64u); + EXPECT_FALSE(registry.Get("cold")->Contains("a")); + EXPECT_TRUE(registry.Get("hot")->Contains("a")); + EXPECT_EQ(pool.PlacementBackend("a"), std::optional{hot}); + + // DRAM scope: cold already holds the key, so DRAM frees without a copy. + evicted = pool.Evict({PoolEvictRequest{"b", TierType::DRAM}}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().outcome, PoolEvictOutcome::kDemotionQueued); + ASSERT_TRUE(WaitForKeyGone(®istry, "hot", "b")); + EXPECT_EQ(pool.PlacementBackend("b"), std::optional{cold}); + EXPECT_TRUE(pool.BatchResolve({"b"}, false).front().resolved.found); +} + +TEST(PeerPool, ScopedEvictFreesOnlyTheNamedCopy) { + // The same key on two DRAM instances. + BackendRegistry registry; + ASSERT_TRUE(registry.Register("dram_a", std::make_unique(TierType::DRAM))); + ASSERT_TRUE(registry.Register("dram_b", std::make_unique(TierType::DRAM))); + PeerPool pool(®istry, MakeSingleBackendPolicy()); + for (const char* name : {"dram_a", "dram_b"}) { + auto* backend = registry.Get(name); + auto allocated = backend->BatchAllocate({AllocateRequest{"k", 32}}).front(); + ASSERT_TRUE(backend->BatchCommit({CommitRequest{allocated.slot_id, "k"}}).front().success); + } + + // A tier with no copy (stale master metadata) frees nothing. + auto evicted = pool.Evict({PoolEvictRequest{"k", TierType::SSD}}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().bytes_freed, 0u); + + // A backend scope frees only that instance. + PoolEvictRequest request; + request.key = "k"; + request.backend_id = registry.BackendId(registry.Get("dram_b")); + evicted = pool.Evict({request}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().bytes_freed, 32u); + EXPECT_TRUE(registry.Get("dram_a")->Contains("k")); + EXPECT_FALSE(registry.Get("dram_b")->Contains("k")); +} + +TEST(PeerPool, ReclaimDemotesOnAWatermarkTierToo) { + // Reclaim demotes whenever the tier has a downstream, whatever its trigger. + constexpr uint64_t kPageSize = 64; + LocalCopyEngine engine; + BackendRegistry registry; + // One key stays far under the watermark, so only the explicit evict moves it. + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 16); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 16); + auto compiled = LogicalTierGraph::Compile(HotColdTiers(PoolOffloadTrigger::kWatermark), registry); + ASSERT_TRUE(compiled.ok()) << compiled.error; + PeerPool pool(®istry, MakeTieredPlacementPolicy(compiled.graph), &engine); + CommitKey(&pool, "k"); + + auto evicted = pool.Evict({PoolEvictRequest{"k", TierType::DRAM}}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().outcome, PoolEvictOutcome::kDemotionQueued); + // Runs although the tier is far under its own watermark. + ASSERT_TRUE(WaitForKeyGone(®istry, "hot", "k")); + EXPECT_TRUE(registry.Get("cold")->Contains("k")) + << "a watermark tier deleted instead of demoting"; +} + +TEST(PeerPool, DemotionWithNoRoomDownstreamDropsTheCopy) { + // Reclaim still frees the tier when the tier below is full (cold: one page). + constexpr uint64_t kPageSize = 4096; + LocalCopyEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kPageSize, 4); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kPageSize, 1); + auto compiled = LogicalTierGraph::Compile(HotColdTiers(PoolOffloadTrigger::kOnEvict), registry); + ASSERT_TRUE(compiled.ok()) << compiled.error; + PeerPool pool(®istry, MakeTieredPlacementPolicy(compiled.graph), &engine); + // Fill cold's only page directly. + auto* cold = registry.Get("cold"); + auto filler = cold->BatchAllocate({AllocateRequest{"filler", kPageSize}}).front(); + ASSERT_EQ(filler.outcome, AllocateOutcome::kSuccessAllocated); + ASSERT_TRUE(cold->BatchCommit({CommitRequest{filler.slot_id, "filler"}}).front().success); + CommitKey(&pool, "k"); + + // The worker finds no room downstream and drops the copy. + auto evicted = pool.Evict({PoolEvictRequest{"k", TierType::DRAM}}, PoolEvictMode::kReclaim); + EXPECT_EQ(evicted.front().outcome, PoolEvictOutcome::kDemotionQueued); + ASSERT_TRUE(WaitForKeyGone(®istry, "hot", "k")) << "a failed demotion left the copy in place"; + EXPECT_FALSE(cold->Contains("k")); + EXPECT_TRUE(cold->Contains("filler")); + EXPECT_FALSE(pool.PlacementBackend("k").has_value()); +} + TEST(PeerPool, WatermarkMigratesCommittedKey) { // Unlike the other tier tests, this one asserts on utilization, so the owned // buffer must be a whole number of host allocation granules: a smaller @@ -934,5 +1124,189 @@ TEST(PeerPool, ReadPromotesWithMoveDrainsSourceAndKeepsBytes) { payload); } +// ---- Local watermark eviction (frees through Evict(), as above) ---- + +bool WaitUntil(const std::function& done, + std::chrono::milliseconds budget = std::chrono::seconds(5)) { + const auto deadline = std::chrono::steady_clock::now() + budget; + while (std::chrono::steady_clock::now() < deadline) { + if (done()) return true; + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + return done(); +} + +// Host-page-sized pages, so a tier is exactly N pages; every key is one page. +constexpr uint64_t kHostPage = 4096; + +TEST(PeerPoolLocalEviction, DrainsAFullBackendColdestFirstToItsLowWatermark) { + LocalCopyEngine engine; + BackendRegistry registry; + // 1 ms lease: lapsed by the time the worker looks. + RegisterPageBackend(®istry, &engine, "dram", TierType::DRAM, kHostPage, 10, + std::chrono::milliseconds(1)); + PeerPool pool(®istry, MakeSingleBackendPolicy(), &engine); + // Enabled before any commit, as PoolClient does. + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(registry.Get("dram")), 0.9, 0.5)); + + for (int i = 0; i < 8; ++i) CommitKey(&pool, "k" + std::to_string(i)); + // Reading the oldest key renews its recency. + ASSERT_TRUE(pool.BatchResolve({"k0"}, false).front().resolved.found); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + CommitKey(&pool, "k8"); + + // 9 of 10 pages hits the high watermark; the worker frees exactly down to 5. + auto* dram = registry.Get("dram"); + ASSERT_TRUE(WaitUntil([&] { return dram->OwnedKeyCount() <= 5; })); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + EXPECT_EQ(dram->OwnedKeyCount(), 5u); + EXPECT_TRUE(dram->Contains("k0")) << "a just-read key was evicted ahead of colder ones"; + for (int i = 1; i <= 4; ++i) EXPECT_FALSE(dram->Contains("k" + std::to_string(i))) << i; + for (int i = 5; i <= 8; ++i) EXPECT_TRUE(dram->Contains("k" + std::to_string(i))) << i; + EXPECT_EQ(pool.PlacementCount(), 5u); + EXPECT_GE(pool.LocalEvictionStats().rounds, 1u); + EXPECT_EQ(pool.LocalEvictionStats().keys, 4u); +} + +TEST(PeerPoolLocalEviction, RankedCandidatesGoFirstAndTheMediumCoversTheRest) { + // The source (the master, in peer-driven mode) ranks differently from the + // medium's LRU, names a key this backend no longer holds, and reports value + // sizes rather than the whole pages the budget counts. + LocalCopyEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "dram", TierType::DRAM, kHostPage, 10, + std::chrono::milliseconds(1)); + PeerPool pool(®istry, MakeSingleBackendPolicy(), &engine); + std::atomic queries{0}; + std::atomic asked_bytes{0}; + pool.SetEvictionCandidateSource([&](TierType tier, const std::string&, uint64_t bytes) { + ++queries; + asked_bytes = tier == TierType::DRAM ? bytes : 0; + return std::optional>({{"k8", 64}, {"gone", 64}, {"k7", 64}}); + }); + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(registry.Get("dram")), 0.9, 0.5)); + + for (int i = 0; i < 9; ++i) CommitKey(&pool, "k" + std::to_string(i)); + auto* dram = registry.Get("dram"); + ASSERT_TRUE(WaitUntil([&] { return dram->OwnedKeyCount() <= 5; })); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + EXPECT_EQ(dram->OwnedKeyCount(), 5u) << "evicted past the overage"; + EXPECT_EQ(queries.load(), 1); + EXPECT_EQ(asked_bytes.load(), 4 * kHostPage); // 9 pages down to 5 + // Ranked k8, k7 first; then the medium's coldest, k0 and k1. + for (const char* gone : {"k8", "k7", "k0", "k1"}) EXPECT_FALSE(dram->Contains(gone)) << gone; + for (int i = 2; i <= 6; ++i) EXPECT_TRUE(dram->Contains("k" + std::to_string(i))) << i; + EXPECT_EQ(pool.LocalEvictionStats().master_keys, 2u); + + // A source slower than a pass waits for (a master with a large index): the + // medium's order covers the overage meanwhile. + std::atomic answered{false}; + pool.SetEvictionCandidateSource([&](TierType, const std::string&, uint64_t) { + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + answered = true; + return std::optional>({{"k2", 64}}); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); // past the query interval + for (int i = 9; i < 13; ++i) CommitKey(&pool, "k" + std::to_string(i)); + ASSERT_TRUE(WaitUntil([&] { return dram->OwnedKeyCount() <= 5; })); + EXPECT_FALSE(answered.load()) << "the pass waited on the source"; + EXPECT_EQ(pool.LocalEvictionStats().master_keys, 2u); +} + +TEST(PeerPoolLocalEviction, KeysUnderAReadLeaseAreNeverOffered) { + LocalCopyEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "dram", TierType::DRAM, kHostPage, 10); + PeerPool pool(®istry, MakeSingleBackendPolicy(), &engine); + std::vector keys; + for (int i = 0; i < 9; ++i) { + keys.push_back("k" + std::to_string(i)); + CommitKey(&pool, keys.back()); + } + // Every key is being read (the default lease outlives the test). + for (const auto& entry : pool.BatchResolve(keys, false)) ASSERT_TRUE(entry.resolved.found); + + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(registry.Get("dram")), 0.9, 0.5)); + EXPECT_EQ(pool.RunLocalEvictionOnceForTest(), 0u); + EXPECT_EQ(registry.Get("dram")->OwnedKeyCount(), 9u); + EXPECT_GE(pool.LocalEvictionStats().no_candidate, 1u); +} + +TEST(PeerPoolLocalEviction, ATierWithADownstreamDemotesJustTheOverage) { + // A tier with a downstream demotes, and passes that run while copies are + // held count them rather than queueing the whole tier. + GatedEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kHostPage, 10); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kHostPage, 20); + auto compiled = LogicalTierGraph::Compile(HotColdTiers(PoolOffloadTrigger::kOnEvict), registry); + ASSERT_TRUE(compiled.ok()) << compiled.error; + PeerPool pool(®istry, MakeTieredPlacementPolicy(compiled.graph), &engine); + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(registry.Get("hot")), 0.9, 0.5)); + + for (int i = 0; i < 9; ++i) CommitKey(&pool, "k" + std::to_string(i)); + auto* hot = registry.Get("hot"); + auto* cold = registry.Get("cold"); + ASSERT_TRUE(engine.WaitForHeld(1)) << "no demotion was queued"; + for (int i = 0; i < 3; ++i) pool.RunLocalEvictionOnceForTest(); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); // let polls run too + EXPECT_EQ(pool.LocalEvictionStats().keys, 4u) << "queued past the overage"; + EXPECT_EQ(hot->OwnedKeyCount(), 9u) << "a copy completed while held"; + + engine.Release(); + ASSERT_TRUE(WaitUntil([&] { return hot->OwnedKeyCount() <= 5; })); + for (int i = 0; i < 4; ++i) { + const std::string key = "k" + std::to_string(i); + EXPECT_FALSE(hot->Contains(key)) << key; + EXPECT_TRUE(cold->Contains(key)) << key << " was dropped, not demoted"; + EXPECT_TRUE(pool.BatchResolve({key}, false).front().resolved.found) << key; + } +} + +TEST(PeerPoolLocalEviction, NeverDeletesWhatAWatermarkOffloadWouldMove) { + // Regression: the old in-commit round deleted keys offload was about to move. + LocalCopyEngine engine; + BackendRegistry registry; + RegisterPageBackend(®istry, &engine, "hot", TierType::DRAM, kHostPage, 10); + RegisterPageBackend(®istry, &engine, "cold", TierType::SSD, kHostPage, 40); + auto tiers = HotColdTiers(PoolOffloadTrigger::kWatermark); + tiers.front().high_watermark = 0.9; + tiers.front().low_watermark = 0.5; + auto compiled = LogicalTierGraph::Compile(tiers, registry); + ASSERT_TRUE(compiled.ok()) << compiled.error; + PeerPool pool(®istry, MakeTieredPlacementPolicy(compiled.graph), &engine); + // The same watermarks as the offload: the case that used to lose keys. + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(registry.Get("hot")), 0.9, 0.5)); + + std::vector keys; + for (int i = 0; i < 20; ++i) { + keys.push_back("k" + std::to_string(i)); + CommitKey(&pool, keys.back()); + // Let the workers keep up, as a real put rate would. + ASSERT_TRUE(WaitUntil([&] { return registry.Get("hot")->OwnedKeyCount() < 9; })); + } + for (const auto& key : keys) { + EXPECT_TRUE(registry.Get("hot")->Contains(key) || registry.Get("cold")->Contains(key)) + << key << " was deleted although the cold tier had room for it"; + } +} + +TEST(PeerPoolLocalEviction, RejectsBadConfigAndLeavesAnUnorderedMediumAlone) { + BackendRegistry registry; + ASSERT_TRUE(registry.Register("dram", std::make_unique(TierType::DRAM))); + PeerPool pool(®istry, MakeSingleBackendPolicy()); + const uint32_t id = registry.BackendId(registry.Get("dram")); + EXPECT_FALSE(pool.EnableLocalEviction(id, 0.5, 0.7)); // low above high + EXPECT_FALSE(pool.EnableLocalEviction(id, 1.5, 0.7)); // high above 1 + EXPECT_FALSE(pool.EnableLocalEviction(id, 0.9, 0.0)); // low not positive + EXPECT_FALSE(pool.EnableLocalEviction(BackendRegistry::kMaxBackends - 1, 0.9, 0.7)); + + // MockBackend offers no candidates, so it must be left alone. + CommitKey(&pool, "k"); + ASSERT_TRUE(pool.EnableLocalEviction(id, 0.9, 0.5)); + pool.RunLocalEvictionOnceForTest(); + EXPECT_TRUE(registry.Get("dram")->Contains("k")); +} + } // namespace } // namespace mori::umbp diff --git a/tests/cpp/umbp/distributed/test_peer_service_dispatch.cpp b/tests/cpp/umbp/distributed/test_peer_service_dispatch.cpp index 80c33989d..078c874b5 100644 --- a/tests/cpp/umbp/distributed/test_peer_service_dispatch.cpp +++ b/tests/cpp/umbp/distributed/test_peer_service_dispatch.cpp @@ -420,22 +420,51 @@ TEST_F(PeerServiceDispatchTest, BatchResolveReportsTheServingTier) { EXPECT_EQ(resp.size(1), 55u); } -TEST_F(PeerServiceDispatchTest, EvictFansOutAcrossEveryMedium) { - // A key mirrored across media must disappear from ALL of them, and the freed - // bytes master sizes its next round from are the sum. +TEST_F(PeerServiceDispatchTest, EvictWithATierFreesOnlyThatMedium) { + // A mirrored key charged to the second medium loses only that copy. SeedKey(kFirstByTier, "mirrored", 111); SeedKey(kSecondByTier, "mirrored", 222); ::umbp::EvictKeyRequest req; req.add_keys("mirrored"); + req.add_tiers(Proto(kSecondByTier)); ::umbp::EvictKeyResponse resp; grpc::ClientContext ctx; ASSERT_TRUE(stub_->EvictKey(&ctx, req, &resp).ok()); ASSERT_EQ(resp.evicted_size(), 1); EXPECT_EQ(resp.evicted(0).key(), "mirrored"); - EXPECT_EQ(resp.evicted(0).bytes_freed(), 333u); - EXPECT_EQ(Backend(kFirstByTier)->OwnedKeyCount(), 0u); - EXPECT_EQ(Backend(kSecondByTier)->OwnedKeyCount(), 0u); + EXPECT_EQ(resp.evicted(0).bytes_freed(), 222u); + EXPECT_TRUE(Backend(kFirstByTier)->Contains("mirrored")); + EXPECT_FALSE(Backend(kSecondByTier)->Contains("mirrored")); +} + +TEST_F(PeerServiceDispatchTest, EvictWithoutUsableTiersFreesTheCopyThatHoldsTheKey) { + // No tiers (an older master), or a count that does not match `keys`: each + // key is freed from the one medium that holds it, never from every medium. + SeedKey(kFirstByTier, "mirrored", 111); + SeedKey(kSecondByTier, "mirrored", 222); + SeedKey(kSecondByTier, "b", 20); + + ::umbp::EvictKeyRequest req; + req.add_keys("mirrored"); + ::umbp::EvictKeyResponse resp; + grpc::ClientContext ctx; + ASSERT_TRUE(stub_->EvictKey(&ctx, req, &resp).ok()); + ASSERT_EQ(resp.evicted_size(), 1); + EXPECT_EQ(resp.evicted(0).bytes_freed(), 111u); // first owner in read order + EXPECT_FALSE(Backend(kFirstByTier)->Contains("mirrored")); + EXPECT_TRUE(Backend(kSecondByTier)->Contains("mirrored")); + + ::umbp::EvictKeyRequest mismatched; + mismatched.add_keys("mirrored"); + mismatched.add_keys("b"); + mismatched.add_tiers(Proto(kFirstByTier)); + ::umbp::EvictKeyResponse resp2; + grpc::ClientContext ctx2; + ASSERT_TRUE(stub_->EvictKey(&ctx2, mismatched, &resp2).ok()); + ASSERT_EQ(resp2.evicted_size(), 2); + EXPECT_EQ(resp2.evicted(0).bytes_freed(), 222u); // the tier was not misapplied + EXPECT_EQ(resp2.evicted(1).bytes_freed(), 20u); } TEST_F(PeerServiceDispatchTest, EvictOfAnAbsentKeyReportsZeroBytes) { diff --git a/tests/cpp/umbp/distributed/test_peer_ssd_eviction.cpp b/tests/cpp/umbp/distributed/test_peer_ssd_eviction.cpp index be19c8f93..d1e1a6c87 100644 --- a/tests/cpp/umbp/distributed/test_peer_ssd_eviction.cpp +++ b/tests/cpp/umbp/distributed/test_peer_ssd_eviction.cpp @@ -20,11 +20,8 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. // -// SSD local capacity management + eviction. Drives PeerSsdManager -// through a controllable in-memory TierBackend (the test-only constructor) so -// LRU ordering, watermark eviction, the in-flight-read guard, idempotent Write, -// backend-evict failure, concurrent eviction, and physical Clear are all -// deterministic without real disk IO. +// The SSD medium's Evict() and candidate order, over an in-memory TierBackend. +// Watermark eviction is PeerPool's, tested there. #include #include @@ -158,10 +155,10 @@ struct Harness { std::unique_ptr mgr; }; -Harness MakeHarness(size_t capacity, double high = 0.9, double low = 0.7) { +Harness MakeHarness(size_t capacity) { auto be = std::make_unique(capacity); FakeBackend* raw = be.get(); - return Harness{raw, std::make_unique(std::move(be), high, low)}; + return Harness{raw, std::make_unique(std::move(be))}; } int CountKind(const std::vector& events, KvEvent::Kind kind) { @@ -172,13 +169,6 @@ int CountKind(const std::vector& events, KvEvent::Kind kind) { return n; } -bool HasRemove(const std::vector& events, const std::string& key) { - for (const auto& e : events) { - if (e.kind == KvEvent::Kind::REMOVE && e.tier == TierType::SSD && e.key == key) return true; - } - return false; -} - // --------------------------------------------------------------------------- TEST(PeerSsdEviction, WriteAndPrepareReadRefreshLru) { @@ -187,63 +177,39 @@ TEST(PeerSsdEviction, WriteAndPrepareReadRefreshLru) { ASSERT_TRUE(h.mgr->Write("B", OneSeg("bbbb"), 4)); ASSERT_TRUE(h.mgr->Write("C", OneSeg("cccc"), 4)); - // LRU now (oldest->newest): A, B, C. Reading A promotes it to MRU, so the - // oldest becomes B and SelectVictims must pick B first (not the just-read A). + // LRU is A, B, C; reading A makes B the coldest. std::vector buf(4); auto out = h.mgr->PrepareRead("A", buf.data(), buf.size()); ASSERT_EQ(out.status, SsdReadStatus::kOk); EXPECT_EQ(std::string(buf.data(), out.size), "aaaa"); - auto victims = h.mgr->SelectVictims(/*bytes_to_free=*/1); - ASSERT_FALSE(victims.empty()); - EXPECT_EQ(victims.front(), "B"); - EXPECT_NE(victims.front(), "A"); + auto candidates = h.mgr->EvictionCandidates(/*max_count=*/1); + ASSERT_EQ(candidates.size(), 1u); + EXPECT_EQ(candidates.front().key, "B"); + EXPECT_EQ(candidates.front().bytes, 4u); + EXPECT_EQ(h.mgr->EvictionCandidates(100).size(), 3u); + EXPECT_TRUE(h.mgr->Exists("B")) << "asking for candidates freed one"; } -TEST(PeerSsdEviction, WatermarkTriggersEvictionDownToLow) { - // capacity 1000, high 0.9 (=>900), low 0.7 (=>700); 100-byte values. - auto h = MakeHarness(/*capacity=*/1000, /*high=*/0.9, /*low=*/0.7); +TEST(PeerSsdEviction, WritesNeverEvict) { + // Neither 90% usage nor ENOSPC makes a write evict earlier keys. + auto h = MakeHarness(/*capacity=*/1000); std::string val(100, 'x'); for (int i = 1; i <= 9; ++i) { ASSERT_TRUE(h.mgr->Write("k" + std::to_string(i), OneSeg(val), val.size())); } - // After k9: used hit 900 >= high -> evict oldest down to <= 700. - auto [used, total] = h.mgr->Capacity(); - EXPECT_EQ(total, 1000u); - EXPECT_LE(used, 700u); - - // Oldest (k1, k2) evicted first; newest still present. - EXPECT_FALSE(h.mgr->Exists("k1")); - EXPECT_FALSE(h.mgr->Exists("k2")); - EXPECT_TRUE(h.mgr->Exists("k9")); - - auto events = h.mgr->DrainPendingEvents(); - EXPECT_TRUE(HasRemove(events, "k1")); - EXPECT_TRUE(HasRemove(events, "k2")); - EXPECT_EQ(CountKind(events, KvEvent::Kind::REMOVE), 2); -} - -TEST(PeerSsdEviction, SuccessfulWriteDoesNotEvictItself) { - auto h = MakeHarness(/*capacity=*/100, /*high=*/0.9, /*low=*/0.7); - std::string value(90, 'x'); - ASSERT_TRUE(h.mgr->Write("only", OneSeg(value), value.size())); - EXPECT_TRUE(h.mgr->Exists("only")); -} - -TEST(PeerSsdEviction, EnospcTriggersEvictThenRetry) { - // Fill to 800/1000 (below the 0.9 high watermark, so no watermark eviction - // fires during the fill), then write a 300-byte value that overflows the - // device: backend Write -> ENOSPC -> one evict round (frees the oldest down - // to the 0.5 low watermark) -> retry succeeds. After the retry used is 800, - // still below high, so no second round disturbs the just-written key. - auto h = MakeHarness(/*capacity=*/1000, /*high=*/0.9, /*low=*/0.5); - for (int i = 1; i <= 8; ++i) { - ASSERT_TRUE(h.mgr->Write("k" + std::to_string(i), OneSeg(std::string(100, 'a')), 100)); - } - ASSERT_TRUE(h.mgr->Write("big", OneSeg(std::string(300, 'c')), 300)); - EXPECT_TRUE(h.mgr->Exists("big")); - EXPECT_FALSE(h.mgr->Exists("k1")); // oldest reclaimed to make room - EXPECT_LE(h.mgr->Capacity().first, 1000u); + EXPECT_EQ(h.mgr->Capacity().first, 900u); + EXPECT_FALSE(h.mgr->Write("big", OneSeg(std::string(300, 'c')), 300)); + EXPECT_FALSE(h.mgr->Exists("big")); + + // The batched path fails the oversized key alone. + const std::string big(300, 'c'); + const std::string small(100, 's'); + auto ok = + h.mgr->WriteBatch({"big", "small"}, {big.data(), small.data()}, {big.size(), small.size()}); + EXPECT_EQ(ok, (std::vector{false, true})); + for (int i = 1; i <= 9; ++i) EXPECT_TRUE(h.mgr->Exists("k" + std::to_string(i))); + EXPECT_EQ(CountKind(h.mgr->DrainPendingEvents(), KvEvent::Kind::REMOVE), 0); } TEST(PeerSsdEviction, InFlightReadIsNotEvicted) { @@ -257,9 +223,9 @@ TEST(PeerSsdEviction, InFlightReadIsNotEvicted) { std::thread reader([&] { out = h.mgr->PrepareRead("K", buf.data(), buf.size()); }); h.backend->WaitReadsStarted(1); // PrepareRead has marked K in-flight and is blocked in the read - // Eviction must skip a key that is being read. + // Eviction must skip a key that is being read, and must not offer it. EXPECT_FALSE(h.mgr->Evict("K")); - EXPECT_TRUE(h.mgr->SelectVictims(1'000'000).empty()); + EXPECT_TRUE(h.mgr->EvictionCandidates(100).empty()); EXPECT_TRUE(h.mgr->Exists("K")); h.backend->UnblockReads(); @@ -370,26 +336,19 @@ TEST(PeerSsdEviction, ClearLocalWaitsForInFlightRead) { EXPECT_FALSE(h.mgr->Exists("K")); } -TEST(PeerSsdEviction, InvalidWatermarksThrow) { - // low >= high, and high > 1 are both rejected (fail-fast, no silent clamp). - EXPECT_THROW(PeerSsdManager(std::make_unique(1000), 0.5, 0.7), std::runtime_error); - EXPECT_THROW(PeerSsdManager(std::make_unique(1000), 1.5, 0.7), std::runtime_error); - EXPECT_THROW(PeerSsdManager(std::make_unique(1000), 0.9, 0.0), std::runtime_error); -} - -TEST(PeerSsdEviction, SelectVictimsBoundaries) { +TEST(PeerSsdEviction, EvictionCandidatesBoundaries) { auto h = MakeHarness(/*capacity=*/1'000'000); ASSERT_TRUE(h.mgr->Write("K", OneSeg("data"), 4)); - EXPECT_TRUE(h.mgr->SelectVictims(0).empty()); // nothing to free + EXPECT_TRUE(h.mgr->EvictionCandidates(0).empty()); // nothing asked for - // All candidates in flight -> no victim, no spin. + // All candidates in flight -> none offered, no spin. h.backend->BlockReads(); std::vector buf(4); SsdReadOutcome out{}; std::thread reader([&] { out = h.mgr->PrepareRead("K", buf.data(), buf.size()); }); h.backend->WaitReadsStarted(1); - EXPECT_TRUE(h.mgr->SelectVictims(1'000'000).empty()); + EXPECT_TRUE(h.mgr->EvictionCandidates(100).empty()); h.backend->UnblockReads(); reader.join(); EXPECT_EQ(out.status, SsdReadStatus::kOk); @@ -402,7 +361,7 @@ TEST(PeerSsdEviction, DisabledManagerIsInert) { EXPECT_FALSE(mgr.Write("K", OneSeg("data"), 4)); EXPECT_FALSE(mgr.Evict("K")); - EXPECT_TRUE(mgr.SelectVictims(100).empty()); + EXPECT_TRUE(mgr.EvictionCandidates(100).empty()); std::vector buf(4); EXPECT_EQ(mgr.PrepareRead("K", buf.data(), buf.size()).status, SsdReadStatus::kNotFound); mgr.ClearLocal(); // no backend -> no crash, no-op diff --git a/tests/cpp/umbp/distributed/test_peer_ssd_single_flight.cpp b/tests/cpp/umbp/distributed/test_peer_ssd_single_flight.cpp index 20e382115..dca5075df 100644 --- a/tests/cpp/umbp/distributed/test_peer_ssd_single_flight.cpp +++ b/tests/cpp/umbp/distributed/test_peer_ssd_single_flight.cpp @@ -147,7 +147,7 @@ struct Harness { Harness MakeHarness(bool single_flight = true) { auto be = std::make_unique(1'000'000); GatedBackend* raw = be.get(); - return Harness{raw, std::make_unique(std::move(be), 0.9, 0.7, single_flight)}; + return Harness{raw, std::make_unique(std::move(be), single_flight)}; } std::vector> OneSeg(const std::string& s) { diff --git a/tests/cpp/umbp/distributed/test_ssd_backend.cpp b/tests/cpp/umbp/distributed/test_ssd_backend.cpp index f91edca33..8397c11ca 100644 --- a/tests/cpp/umbp/distributed/test_ssd_backend.cpp +++ b/tests/cpp/umbp/distributed/test_ssd_backend.cpp @@ -46,6 +46,7 @@ #include "umbp/common/fd_limit.h" #include "umbp/distributed/peer/backend/ssd_backend.h" +#include "umbp/distributed/pool/peer_pool.h" #include "umbp/distributed/transfer/composite_transfer_engine.h" #include "umbp/distributed/transfer/local_copy_engine.h" #ifdef UMBP_ENABLE_GDS @@ -556,6 +557,65 @@ TEST_F(SsdBackendTest, EvictResultsAreOnePerKeyInRequestOrder) { EXPECT_EQ(evicted[1].bytes_freed, 1024u); } +// ---- Local eviction: the backend offers order; the pool decides ---- + +TEST_F(SsdBackendTest, CandidatesAreColdestFirstAndSkipALeasedKey) { + SsdBackend* backend = Start(); + ASSERT_TRUE(Put(backend, "a", Payload(1024, 1))); + ASSERT_TRUE(Put(backend, "b", Payload(2048, 2))); + ASSERT_TRUE(Put(backend, "c", Payload(1024, 3))); + + auto offers = backend->EvictionCandidates(10); + ASSERT_EQ(offers.size(), 3u); + EXPECT_EQ(offers[0], (EvictionOffer{"a", 1024})); + EXPECT_EQ(offers[1], (EvictionOffer{"b", 2048})); + EXPECT_EQ(offers[2], (EvictionOffer{"c", 1024})); + + // A staged read's lease makes "b" unevictable, so it is not offered. + ASSERT_TRUE(backend->BatchResolve({"b"}, false)[0].found); + offers = backend->EvictionCandidates(10); + for (const auto& offer : offers) EXPECT_NE(offer.key, "b"); + EXPECT_EQ(offers.size(), 2u); +} + +TEST_F(SsdBackendTest, PoolLocalEvictionDrainsTheSsdMediumThroughEvict) { + // SSD's watermark eviction now runs in the pool and reaches it via Evict(). + constexpr uint64_t kCapacity = 1024 * 1024; + SsdBackend::Config cfg; + cfg.page_size = kPageSize; + cfg.staging_pages = 8; + cfg.read_lease_ttl = std::chrono::milliseconds{500}; + cfg.ssd.enabled = true; + cfg.ssd.ssd.enabled = true; + cfg.ssd.ssd.storage_dir = dir_.string(); + cfg.ssd.ssd.capacity_bytes = kCapacity; + cfg.ssd.ssd.io.backend = UMBPIoBackend::Posix; + auto owned = std::make_unique(std::move(cfg)); + ASSERT_TRUE(owned->Init(®istrar_)); + SsdBackend* ssd = owned.get(); + BackendRegistry registry; + ASSERT_TRUE(registry.Register("ssd", std::move(owned))); + PeerPool pool(®istry, MakeSingleBackendPolicy()); + + // Fill past the high watermark by measurement (record overhead varies). + std::vector keys; + const auto used = [&] { return ssd->Capacity().total_bytes - ssd->Capacity().available_bytes; }; + while (used() < kCapacity * 9 / 10) { + keys.push_back("k" + std::to_string(keys.size())); + ASSERT_TRUE(Put(ssd, keys.back(), Payload(32 * 1024, static_cast(keys.size())))) + << "filled to " << used() << " of " << kCapacity; + } + ASSERT_GE(keys.size(), 8u); + + ASSERT_TRUE(pool.EnableLocalEviction(registry.BackendId(ssd), 0.9, 0.5)); + pool.RunLocalEvictionOnceForTest(); + + EXPECT_LE(used(), kCapacity / 2) << "not drained to the low watermark"; + EXPECT_FALSE(ssd->Contains(keys.front())) << "the coldest key survived"; + EXPECT_TRUE(ssd->Contains(keys.back())) << "drained past the low watermark"; + EXPECT_GE(pool.LocalEvictionStats().keys, 1u); +} + TEST_F(SsdBackendTest, ClearLocalDropsEverythingAndGatesAllocation) { SsdBackend* backend = Start(); ASSERT_TRUE(Put(backend, "key-a", Payload(1024, 1))); diff --git a/tests/cpp/umbp/distributed/test_umbp_tier_benchmark.cpp b/tests/cpp/umbp/distributed/test_umbp_tier_benchmark.cpp index 5f1794219..c122b6647 100644 --- a/tests/cpp/umbp/distributed/test_umbp_tier_benchmark.cpp +++ b/tests/cpp/umbp/distributed/test_umbp_tier_benchmark.cpp @@ -199,6 +199,8 @@ class InProcessClusterTest : public ::testing::Test { config.cache_remote_fetches = false; config.placement_policy = PoolPlacementPolicy::WEIGHTED; config.backends = backends; + // Cases fill the pool on purpose; leave eviction to the master's rounds. + config.eviction_mode = PeerEvictionMode::kMaster; auto client = std::make_unique(std::move(config)); ASSERT_TRUE(client->Init()) << "failed to initialize PoolClient " << node_id;