diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
index 4111abfaec0f7..897491b906aae 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieSortedMergeHandle.java
@@ -108,8 +108,9 @@ public void write(GenericRecord oldRecord) {
   @Override
   public List<WriteStatus> close() {
     // write out any pending records (this can happen when inserts are turned into updates)
-    newRecordKeysSorted.stream().forEach(key -> {
+    while (!newRecordKeysSorted.isEmpty()) {
       try {
+        String key = newRecordKeysSorted.poll();
         HoodieRecord<T> hoodieRecord = keyToNewRecords.get(key);
         if (!writtenRecordKeys.contains(hoodieRecord.getRecordKey())) {
           if (useWriterSchema) {
@@ -122,7 +123,7 @@ public List<WriteStatus> close() {
       } catch (IOException e) {
         throw new HoodieUpsertException("Failed to close UpdateHandle", e);
       }
-    });
+    }
     newRecordKeysSorted.clear();
     keyToNewRecords.clear();