diff --git a/src/online2/online-ivector-feature.cc b/src/online2/online-ivector-feature.cc
index 2042fbb8b80..3642a9aeaff 100644
--- a/src/online2/online-ivector-feature.cc
+++ b/src/online2/online-ivector-feature.cc
@@ -195,6 +195,9 @@ void OnlineIvectorFeature::UpdateStatsForFrames(
   // Remove duplicates of frames.
   MergePairVectorSumming(&frame_weights);
 
+  if (frame_weights.empty())
+    return;
+
   int32 num_frames = static_cast<int32>(frame_weights.size());
   int32 feat_dim = lda_normalized_->Dim();
   Matrix<BaseFloat> feats(num_frames, feat_dim, kUndefined),