From 6ec50c3bfe82ca829aa8cc4e42a433e33b3ba3ce Mon Sep 17 00:00:00 2001 From: 2xdevv <2xdevv@gmail.com> Date: Tue, 13 Oct 2026 19:59:50 +0900 Subject: [PATCH] [Store] Allow DFS-only replica placement --- .../api-reference/cpp/mooncake-store.md | 2 +- .../api-reference/python/mooncake-store.md | 7 +- .../mooncake-store-deployment-guide.md | 13 +- mooncake-store/src/master_service.cpp | 14 +- .../dfs_immutable_bucket_client_test.cpp | 54 +++++++ mooncake-store/tests/dfs_sync_client_test.cpp | 122 +++++++++++++++ mooncake-store/tests/master_service_test.cpp | 144 ++++++++++++++++++ 7 files changed, 338 insertions(+), 18 deletions(-) diff --git a/docs/source/api-reference/cpp/mooncake-store.md b/docs/source/api-reference/cpp/mooncake-store.md index 49d9ca4ed8..a2bb65cf4d 100644 --- a/docs/source/api-reference/cpp/mooncake-store.md +++ b/docs/source/api-reference/cpp/mooncake-store.md @@ -64,7 +64,7 @@ struct ReplicateConfig { }; ``` -`dfs_replica_num` may currently be `0` or `1`. When it is `1`, `replica_num >= 1` is required, so DFS-only placement is not supported. DFS replicas currently support only the `default` tenant and require the master and client DFS backends to be configured with the same shared root and shard layout. See the {ref}`DFS deployment documentation ` for setup and lifecycle limitations. Native C++ clients must initialize a `DistributedStorageBackend` and attach it with `SetDfsStorageBackend()` before issuing DFS reads or writes. DFS descriptors are carried by each `PutStart`, `UpsertStart`, or query response; there is no client-side descriptor cache. The Python/RealClient setup path attaches the backend through `FileStorage`. +`dfs_replica_num` may currently be `0` or `1`. DFS-only placement (`replica_num = 0`, `nof_replica_num = 0`, `dfs_replica_num = 1`) is supported; the DFS replica is then the object's only copy, so DFS eviction or removal deletes the object. DFS replicas currently support only the `default` tenant and require the master and client DFS backends to be configured with the same shared root and shard layout. See the {ref}`DFS deployment documentation ` for setup and lifecycle limitations. Native C++ clients must initialize a `DistributedStorageBackend` and attach it with `SetDfsStorageBackend()` before issuing DFS reads or writes. DFS descriptors are carried by each `PutStart`, `UpsertStart`, or query response; there is no client-side descriptor cache. The Python/RealClient setup path attaches the backend through `FileStorage`. Soft pinning starts when the first replica becomes readable and has a fixed lifetime: reads do not extend it. `PRESERVE` keeps the committed deadline on an diff --git a/docs/source/api-reference/python/mooncake-store.md b/docs/source/api-reference/python/mooncake-store.md index 4dc1e907ce..6da3ba545b 100644 --- a/docs/source/api-reference/python/mooncake-store.md +++ b/docs/source/api-reference/python/mooncake-store.md @@ -654,9 +654,10 @@ config.nof_replica_num = 1 **Default:** `0` **Status:** **Work in progress; development and evaluation only.** **Description:** Requests an additional replica in the configured shared -distributed filesystem. The supported values are currently `0` and `1`. When -set to `1`, `replica_num` must be at least `1`, so DFS-only placement is not -supported. DFS replicas currently support only the `default` tenant. +distributed filesystem. The supported values are currently `0` and `1`. +DFS-only placement (`replica_num = 0` with `dfs_replica_num = 1`) is supported; +the DFS replica is then the object's only copy, so DFS eviction or removal +deletes the object. DFS replicas currently support only the `default` tenant. ```python config = ReplicateConfig() diff --git a/docs/source/deployment/mooncake-store-deployment-guide.md b/docs/source/deployment/mooncake-store-deployment-guide.md index bf108dc531..2c0d99625c 100644 --- a/docs/source/deployment/mooncake-store-deployment-guide.md +++ b/docs/source/deployment/mooncake-store-deployment-guide.md @@ -1025,9 +1025,9 @@ config.dfs_replica_num = 1 store.put("key", b"value", config) ``` -`dfs_replica_num` may currently be `0` or `1`. A DFS replica must be requested -with at least one memory replica (`replica_num >= 1`), so DFS-only placement is -not supported. +`dfs_replica_num` may currently be `0` or `1`. DFS-only placement is supported +by setting `replica_num = 0` and `nof_replica_num = 0`; the object then has no +memory replica, and DFS eviction or removal deletes it. Allocation first tries the key's hash-selected DFS shard, then tries other ready shards if that shard has no suitable extent. This lets new shards accept @@ -1049,7 +1049,9 @@ failed key is revoked without downgrading successful keys. For a same-size `Upsert`, if either the existing object or the new request has a DFS replica, the requested memory, NoF, and DFS replica counts must match the existing topology. A different-size update releases the old placement and -allocates a new topology. +allocates a new topology. For example, after the memory replica of a +memory-plus-DFS object is evicted, a same-size `Upsert` must request the +DFS-only topology. On reads, the master returns the readable replica list through the normal query path, and the client selects the first complete replica. If it selects DFS, any @@ -1059,8 +1061,7 @@ reads for that descriptor. #### Current limitations - Only the `default` tenant is supported. -- `dfs_replica_num` must be `0` or `1`, and `replica_num >= 1` is required when - it is enabled. +- `dfs_replica_num` must be `0` or `1`. - C and Rust clients cannot currently request or access descriptor-based DFS: their replication configuration does not expose `dfs_replica_num`, and their setup API cannot initialize the distributed `FileStorage` backend. Use the diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index d3d9d57b47..be9164f097 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -5522,11 +5522,10 @@ auto MasterService::PutStart(const UUID& client_id, const std::string& key, << ", key_size=" << key.size() << ", error=invalid_params"; return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } - if (config.dfs_replica_num > 1 || - (config.dfs_replica_num > 0 && config.replica_num == 0)) { - LOG(ERROR) << "key=" << key << ", replica_num=" << config.replica_num + if (config.dfs_replica_num > 1) { + LOG(ERROR) << "key=" << key << ", dfs_replica_num=" << config.dfs_replica_num - << ", error=invalid_dfs_replica_config"; + << ", error=dfs_replica_num_must_be_0_or_1"; return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } if (config.dfs_replica_num > 0 && !object_id.tenant_id.IsDefault()) { @@ -6204,11 +6203,10 @@ auto MasterService::UpsertStart(const UUID& client_id, const std::string& key, << ", key_size=" << key.size() << ", error=invalid_params"; return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } - if (config.dfs_replica_num > 1 || - (config.dfs_replica_num > 0 && config.replica_num == 0)) { - LOG(ERROR) << "key=" << key << ", replica_num=" << config.replica_num + if (config.dfs_replica_num > 1) { + LOG(ERROR) << "key=" << key << ", dfs_replica_num=" << config.dfs_replica_num - << ", error=invalid_dfs_replica_config"; + << ", error=dfs_replica_num_must_be_0_or_1"; return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } if (config.dfs_replica_num > 0 && !object_id.tenant_id.IsDefault()) { diff --git a/mooncake-store/tests/dfs_immutable_bucket_client_test.cpp b/mooncake-store/tests/dfs_immutable_bucket_client_test.cpp index fd031b053d..22262b112d 100644 --- a/mooncake-store/tests/dfs_immutable_bucket_client_test.cpp +++ b/mooncake-store/tests/dfs_immutable_bucket_client_test.cpp @@ -242,6 +242,60 @@ TEST_F(DfsImmutableBucketClientTest, ExhaustionEvictsWholeBucketAndRetries) { EXPECT_EQ(third_query->replicas.size(), 2u); } +TEST_F(DfsImmutableBucketClientTest, DfsOnlyObjectsLiveAndDieWithTheirBucket) { + ReplicateConfig dfs_only; + dfs_only.replica_num = 0; + dfs_only.dfs_replica_num = 1; + auto expect_value = [this](const std::string& key, + const std::string& expected) { + auto query = writer_->Query(key); + ASSERT_TRUE(query) << key; + ASSERT_EQ(query->replicas.size(), 1u) << key; + ASSERT_TRUE(query->replicas[0].is_dfs_replica()) << key; + // The fixture's zero lease TTL makes Client::Get report LEASE_EXPIRED, + // so read the descriptor through the backend like the other tests. + std::string output(expected.size(), '\0'); + auto reads = + backend_->BatchRead({{key, + query->replicas[0].get_dfs_descriptor(), + {{output.data(), output.size()}}}}); + ASSERT_TRUE(reads[0]) << key; + EXPECT_EQ(output, expected) << key; + }; + + std::string first(4096, '1'); + std::string second(4096, '2'); + std::string third(4096, '3'); + auto first_slices = Slices(first); + auto second_slices = Slices(second); + auto third_slices = Slices(third); + ASSERT_TRUE(writer_->Put("first", first_slices, dfs_only)); + ASSERT_TRUE(writer_->Put("second", second_slices, dfs_only)); + // Both buckets are full: the third write forces a whole-bucket eviction. + // With no memory replica, the evicted object disappears entirely. + ASSERT_TRUE(writer_->Put("third", third_slices, dfs_only)); + auto first_query = writer_->Query("first"); + ASSERT_FALSE(first_query); + EXPECT_EQ(first_query.error(), ErrorCode::OBJECT_NOT_FOUND); + expect_value("third", third); + + adapter_->FailNextWrite(); + std::string failed(4096, 'F'); + auto failed_slices = Slices(failed); + auto failed_put = writer_->Put("failed", failed_slices, dfs_only); + ASSERT_FALSE(failed_put); + EXPECT_EQ(failed_put.error(), ErrorCode::FILE_WRITE_FAIL); + auto missing = writer_->Query("failed"); + ASSERT_FALSE(missing); + EXPECT_EQ(missing.error(), ErrorCode::OBJECT_NOT_FOUND); + + // Same-size upsert reallocates into a new bucket. + std::string replacement(4096, 'R'); + auto replacement_slices = Slices(replacement); + ASSERT_TRUE(writer_->Upsert("third", replacement_slices, dfs_only)); + expect_value("third", replacement); +} + TEST_F(DfsImmutableBucketLruClientTest, BatchQueryRefreshesDfsBucketLru) { std::string first(4096, '1'); std::string second(4096, '2'); diff --git a/mooncake-store/tests/dfs_sync_client_test.cpp b/mooncake-store/tests/dfs_sync_client_test.cpp index bb39afef04..513b051f10 100644 --- a/mooncake-store/tests/dfs_sync_client_test.cpp +++ b/mooncake-store/tests/dfs_sync_client_test.cpp @@ -127,6 +127,34 @@ class DfsSyncClientTest : public ::testing::Test { return config; } + static ReplicateConfig DfsOnlyConfig() { + ReplicateConfig config; + config.replica_num = 0; + config.dfs_replica_num = 1; + return config; + } + + // Reads through Query + Client::Get and checks that DFS is the only + // replica. + void ExpectDfsOnlyValue(const std::string& key, + const std::string& expected) { + auto query = writer_->Query(key); + ASSERT_TRUE(query.has_value()) << key; + last_read_lease_deadline_ = query->lease_timeout; + ASSERT_EQ(query->replicas.size(), 1) << key; + ASSERT_TRUE(query->replicas[0].is_dfs_replica()) << key; + std::string output(expected.size(), '\0'); + std::vector slices{{output.data(), output.size()}}; + ASSERT_TRUE(writer_->Get(key, *query, slices).has_value()) << key; + EXPECT_EQ(output, expected) << key; + } + + void ExpectNotFound(const std::string& key) { + auto query = writer_->Query(key); + ASSERT_FALSE(query.has_value()) << key; + EXPECT_EQ(query.error(), ErrorCode::OBJECT_NOT_FOUND) << key; + } + tl::expected QueryDfsOnly(const std::string& key) { auto query = writer_->Query(key); if (!query) { @@ -564,4 +592,98 @@ TEST_F(DfsSyncClientTest, DfsMetricsAppearInSerializedOutput) { EXPECT_NE(summary->find("=== DFS Metrics Summary ==="), std::string::npos); } +TEST_F(DfsSyncClientTest, DfsOnlyPutGetRemoveAndReput) { + const std::string key = "dfs_only_put"; + std::string value(4096, 'a'); + std::vector slices{{value.data(), value.size()}}; + ASSERT_TRUE(writer_->Put(key, slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, value); + + WaitForReadLeaseExpiry(); + ASSERT_TRUE(writer_->Remove(key).has_value()); + ExpectNotFound(key); + + std::string larger(8192, 'b'); + std::vector larger_slices{{larger.data(), larger.size()}}; + ASSERT_TRUE(writer_->Put(key, larger_slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, larger); +} + +TEST_F(DfsSyncClientTest, DfsOnlyBatchFailureOnlyRevokesFailedKey) { + std::vector keys{"dfs_only_ok", "dfs_only_fail"}; + std::vector values{std::string(4096, 'c'), + std::string(4096, 'd')}; + auto slices = MakeSlices(values); + adapter_->FailWriteCall(adapter_->WriteCalls() + 2); + + auto results = writer_->BatchPut(keys, slices, DfsOnlyConfig()); + ASSERT_EQ(results.size(), keys.size()); + EXPECT_TRUE(results[0].has_value()); + ASSERT_FALSE(results[1].has_value()); + EXPECT_EQ(results[1].error(), ErrorCode::FILE_WRITE_FAIL); + ExpectDfsOnlyValue(keys[0], values[0]); + ExpectNotFound(keys[1]); + + // The revoked key and its extent can be reused. + std::vector retry{{values[1].data(), values[1].size()}}; + ASSERT_TRUE(writer_->Put(keys[1], retry, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(keys[1], values[1]); +} + +TEST_F(DfsSyncClientTest, DfsOnlyUpsertSameSizeResizeAndBatch) { + const std::string key = "dfs_only_upsert"; + std::string initial(4096, 'e'); + std::vector initial_slices{{initial.data(), initial.size()}}; + ASSERT_TRUE( + writer_->Upsert(key, initial_slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, initial); + + WaitForReadLeaseExpiry(); + std::string same_size(4096, 'f'); + std::vector same_slices{{same_size.data(), same_size.size()}}; + ASSERT_TRUE(writer_->Upsert(key, same_slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, same_size); + + WaitForReadLeaseExpiry(); + std::string resized(12288, 'g'); + std::vector resized_slices{{resized.data(), resized.size()}}; + ASSERT_TRUE( + writer_->Upsert(key, resized_slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, resized); + + WaitForReadLeaseExpiry(); + std::vector keys{key, "dfs_only_upsert_new"}; + std::vector values{std::string(4096, 'h'), + std::string(4096, 'i')}; + auto batch_slices = MakeSlices(values); + auto results = writer_->BatchUpsert(keys, batch_slices, DfsOnlyConfig()); + ASSERT_EQ(results.size(), keys.size()); + ASSERT_TRUE(results[0].has_value()); + ASSERT_TRUE(results[1].has_value()); + ExpectDfsOnlyValue(keys[0], values[0]); + ExpectDfsOnlyValue(keys[1], values[1]); +} + +TEST_F(DfsSyncClientTest, DfsOnlyUpsertMatchesMemoryLostTopology) { + const std::string key = "dfs_only_memory_lost"; + std::string initial(4096, 'j'); + std::vector initial_slices{{initial.data(), initial.size()}}; + ASSERT_TRUE(writer_->Put(key, initial_slices, DfsConfig()).has_value()); + ASSERT_TRUE(provider_->UnmountSegment(segment_, segment_size_).has_value()); + ExpectDfsOnlyValue(key, initial); + + WaitForReadLeaseExpiry(); + std::string updated(4096, 'k'); + std::vector updated_slices{{updated.data(), updated.size()}}; + auto mismatched = writer_->Upsert(key, updated_slices, DfsConfig()); + ASSERT_FALSE(mismatched.has_value()); + EXPECT_EQ(mismatched.error(), ErrorCode::INVALID_PARAMS); + ExpectDfsOnlyValue(key, initial); + + WaitForReadLeaseExpiry(); + ASSERT_TRUE( + writer_->Upsert(key, updated_slices, DfsOnlyConfig()).has_value()); + ExpectDfsOnlyValue(key, updated); +} + } // namespace mooncake::test diff --git a/mooncake-store/tests/master_service_test.cpp b/mooncake-store/tests/master_service_test.cpp index ccccfe7180..9a63e98e2f 100644 --- a/mooncake-store/tests/master_service_test.cpp +++ b/mooncake-store/tests/master_service_test.cpp @@ -578,6 +578,150 @@ TEST_F(MasterServiceTest, DfsPutEndAllAndUpsertTopologyAreAtomic) { std::filesystem::remove_all(dfs_root, ec); } +// A one-object shard allocator: any DFS extent leaked by a lifecycle path makes +// the next DFS-only allocation fail. +class OneSlotDfsShardEnv { + public: + explicit OneSlotDfsShardEnv(const std::string& root) + : enable_dfs_("MOONCAKE_ENABLE_DFS", "1"), + fs_adapter_("MOONCAKE_DFS_FS_ADAPTER", "posix"), + root_dir_("MOONCAKE_DFS_ROOT_DIR", root.c_str()), + shard_count_("MOONCAKE_DFS_SHARD_COUNT", "1"), + shard_capacity_("MOONCAKE_DFS_SHARD_CAPACITY", "32768"), + alignment_("MOONCAKE_DFS_ALIGNMENT", "4096"), + eviction_("MOONCAKE_DFS_EVICTION_ENABLED", "0"), + deferred_free_("MOONCAKE_DFS_DEFERRED_FREE_SECONDS", "0"), + single_tenant_("MOONCAKE_DFS_SINGLE_TENANT", "true") {} + + static constexpr uint64_t kObjectSize = 20480; + + private: + ScopedEnvVar enable_dfs_, fs_adapter_, root_dir_, shard_count_, + shard_capacity_, alignment_, eviction_, deferred_free_, single_tenant_; +}; + +ReplicateConfig DfsOnlyConfig() { + ReplicateConfig config; + config.replica_num = 0; + config.dfs_replica_num = 1; + return config; +} + +TEST_F(MasterServiceTest, DfsOnlyLifecyclePathsReleaseTheExtent) { + const auto dfs_root = (std::filesystem::temp_directory_path() / + ("master_dfs_only_" + std::to_string(::getpid()))) + .string(); + std::filesystem::create_directories(dfs_root); + OneSlotDfsShardEnv env(dfs_root); + constexpr uint64_t kSize = OneSlotDfsShardEnv::kObjectSize; + + MasterServiceConfig master_config; + master_config.put_start_discard_timeout_sec = 1; + master_config.put_start_release_timeout_sec = 2; + MasterService service(master_config); + const UUID writer = generate_uuid(); + const UUID other = generate_uuid(); + + auto start = service.PutStart(writer, "a", TenantId::Default(), kSize, + DfsOnlyConfig()); + ASSERT_TRUE(start.has_value()) << toString(start.error()); + ASSERT_EQ(start->size(), 1u); + EXPECT_TRUE(start->front().is_dfs_replica()); + auto not_ready = service.GetReplicaList("a", TenantId::Default()); + ASSERT_FALSE(not_ready.has_value()); + EXPECT_EQ(not_ready.error(), ErrorCode::REPLICA_IS_NOT_READY); + auto full = service.PutStart(writer, "b", TenantId::Default(), kSize, + DfsOnlyConfig()); + ASSERT_FALSE(full.has_value()); + EXPECT_EQ(full.error(), ErrorCode::NO_AVAILABLE_HANDLE); + + // Revoke releases the extent. + ASSERT_TRUE( + service.PutRevoke(writer, "a", TenantId::Default(), ReplicaType::ALL)); + auto revoked = service.GetReplicaList("a", TenantId::Default()); + ASSERT_FALSE(revoked.has_value()); + EXPECT_EQ(revoked.error(), ErrorCode::OBJECT_NOT_FOUND); + + // Upsert preemption releases the in-flight writer's extent. + ASSERT_TRUE(service.PutStart(writer, "b", TenantId::Default(), kSize, + DfsOnlyConfig())); + ASSERT_TRUE(service.UpsertStart(other, "b", TenantId::Default(), kSize, + DfsOnlyConfig())); + auto stale_end = + service.PutEnd(writer, "b", TenantId::Default(), ReplicaType::ALL); + ASSERT_FALSE(stale_end.has_value()); + EXPECT_EQ(stale_end.error(), ErrorCode::ILLEGAL_CLIENT); + ASSERT_TRUE( + service.UpsertEnd(other, "b", TenantId::Default(), ReplicaType::ALL)); + auto complete = service.GetReplicaList("b", TenantId::Default()); + ASSERT_TRUE(complete.has_value()); + ASSERT_EQ(complete->replicas.size(), 1u); + EXPECT_EQ(complete->replicas[0].status, ReplicaStatus::COMPLETE); + + // Remove honours the read lease, then releases the extent. + auto leased = service.Remove("b", TenantId::Default()); + ASSERT_FALSE(leased.has_value()); + EXPECT_EQ(leased.error(), ErrorCode::OBJECT_HAS_LEASE); + ASSERT_TRUE(service.Remove("b", TenantId::Default(), /*force=*/true)); + + // An abandoned write is reaped after the release timeout. + ASSERT_TRUE(service.PutStart(writer, "c", TenantId::Default(), kSize, + DfsOnlyConfig())); + std::this_thread::sleep_for(std::chrono::seconds(4)); + auto reaped = service.GetReplicaList("c", TenantId::Default()); + ASSERT_FALSE(reaped.has_value()); + EXPECT_EQ(reaped.error(), ErrorCode::OBJECT_NOT_FOUND); + auto reuse = service.PutStart(other, "d", TenantId::Default(), kSize, + DfsOnlyConfig()); + ASSERT_TRUE(reuse.has_value()) << toString(reuse.error()); + + std::error_code ec; + std::filesystem::remove_all(dfs_root, ec); +} + +TEST_F(MasterServiceTest, DfsOnlyObjectsIgnoreMemoryEvictionAndSegmentLoss) { + const auto dfs_root = + (std::filesystem::temp_directory_path() / + ("master_dfs_only_eviction_" + std::to_string(::getpid()))) + .string(); + std::filesystem::create_directories(dfs_root); + OneSlotDfsShardEnv env(dfs_root); + + MasterServiceConfig master_config; + master_config.default_kv_lease_ttl = 1; + MasterService service(master_config); + const auto context = PrepareSimpleSegment(service); + const UUID writer = generate_uuid(); + + ASSERT_TRUE(service.PutStart(writer, "dfs_only", TenantId::Default(), + OneSlotDfsShardEnv::kObjectSize, + DfsOnlyConfig())); + ASSERT_TRUE(service.PutEnd(writer, "dfs_only", TenantId::Default(), + ReplicaType::ALL)); + ReplicateConfig memory_config; + memory_config.replica_num = 1; + for (int i = 0; i < 8; ++i) { + const auto key = "memory_" + std::to_string(i); + ASSERT_TRUE(service.PutStart(writer, key, TenantId::Default(), 1 << 20, + memory_config)); + ASSERT_TRUE(service.PutEnd(writer, key, TenantId::Default(), + ReplicaType::MEMORY)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + MasterServiceTestPeer(service).RunBatchEvictForTesting(1.0, 1.0); + + auto query = service.GetReplicaList("dfs_only", TenantId::Default()); + ASSERT_TRUE(query.has_value()) << toString(query.error()); + ASSERT_EQ(query->replicas.size(), 1u); + EXPECT_TRUE(query->replicas[0].is_dfs_replica()); + ASSERT_TRUE(service.UnmountSegment(context.segment_id, context.client_id)); + EXPECT_TRUE( + service.GetReplicaList("dfs_only", TenantId::Default()).has_value()); + + std::error_code ec; + std::filesystem::remove_all(dfs_root, ec); +} + TEST_F(MasterServiceTest, DfsBucketMemoryAllocationFailurePreservesBuckets) { const auto dfs_root = (std::filesystem::temp_directory_path() /