diff --git a/faiss/impl/residual_quantizer_encode_steps.cpp b/faiss/impl/residual_quantizer_encode_steps.cpp
index 8db6f9e5f7..802262969a 100644
--- a/faiss/impl/residual_quantizer_encode_steps.cpp
+++ b/faiss/impl/residual_quantizer_encode_steps.cpp
@@ -664,8 +664,6 @@ void refine_beam_mp(
     std::unique_ptr<Index> assign_index;
     if (rq.assign_index_factory) {
         assign_index.reset((*rq.assign_index_factory)(rq.d));
-    } else {
-        assign_index.reset(new IndexFlatL2(rq.d));
     }
 
     // main loop
@@ -701,7 +699,9 @@ void refine_beam_mp(
                 assign_index.get(),
                 rq.approx_topk_mode);
 
-        assign_index->reset();
+        if (assign_index != nullptr) {
+            assign_index->reset();
+        }
 
         std::swap(codes_ptr, new_codes_ptr);
         std::swap(residuals_ptr, new_residuals_ptr);