diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
index 7e443bf2574bc..d7ecc16eafcd1 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregator.java
@@ -7,8 +7,6 @@
  */
 package org.elasticsearch.search.aggregations.bucket.nested;
 
-import com.carrotsearch.hppc.LongArrayList;
-
 import org.apache.lucene.index.IndexReaderContext;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.ReaderUtil;
@@ -35,6 +33,8 @@
 import org.elasticsearch.xcontent.ParseField;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
 public class NestedAggregator extends BucketsAggregator implements SingleBucketAggregator {
@@ -141,7 +141,7 @@ class BufferingNestedLeafBucketCollector extends LeafBucketCollectorBase {
         final BitSet parentDocs;
         final LeafBucketCollector sub;
         final DocIdSetIterator childDocs;
-        final LongArrayList bucketBuffer = new LongArrayList();
+        final List<Long> bucketBuffer = new ArrayList<>();
 
         Scorable scorer;
         int currentParentDoc = -1;
@@ -193,10 +193,8 @@ void processBufferedChildBuckets() throws IOException {
 
             for (; childDocId < currentParentDoc; childDocId = childDocs.nextDoc()) {
                 cachedScorer.doc = childDocId;
-                final long[] buffer = bucketBuffer.buffer;
-                final int size = bucketBuffer.size();
-                for (int i = 0; i < size; i++) {
-                    collectBucket(sub, childDocId, buffer[i]);
+                for (var bucket : bucketBuffer) {
+                    collectBucket(sub, childDocId, bucket);
                 }
             }
             bucketBuffer.clear();
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
index b0250c3df0427..5a213b55f4513 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregator.java
@@ -7,8 +7,6 @@
  */
 package org.elasticsearch.search.aggregations.bucket.nested;
 
-import com.carrotsearch.hppc.LongIntHashMap;
-
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
@@ -28,6 +26,7 @@
 import org.elasticsearch.xcontent.ParseField;
 
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.Map;
 
 public class ReverseNestedAggregator extends BucketsAggregator implements SingleBucketAggregator {
@@ -63,7 +62,7 @@ protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final Leaf
         if (parentDocs == null) {
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
-        final LongIntHashMap bucketOrdToLastCollectedParentDoc = new LongIntHashMap(32);
+        final Map<Long, Integer> bucketOrdToLastCollectedParentDoc = new HashMap<>(32);
         return new LeafBucketCollectorBase(sub, null) {
             @Override
             public void collect(int childDoc, long bucket) throws IOException {
@@ -71,16 +70,15 @@ public void collect(int childDoc, long bucket) throws IOException {
                 final int parentDoc = parentDocs.nextSetBit(childDoc);
                 assert childDoc <= parentDoc && parentDoc != DocIdSetIterator.NO_MORE_DOCS;
 
-                int keySlot = bucketOrdToLastCollectedParentDoc.indexOf(bucket);
-                if (bucketOrdToLastCollectedParentDoc.indexExists(keySlot)) {
-                    int lastCollectedParentDoc = bucketOrdToLastCollectedParentDoc.indexGet(keySlot);
+                Integer lastCollectedParentDoc = bucketOrdToLastCollectedParentDoc.get(bucket);
+                if (lastCollectedParentDoc != null) {
                     if (parentDoc > lastCollectedParentDoc) {
                         collectBucket(sub, parentDoc, bucket);
-                        bucketOrdToLastCollectedParentDoc.indexReplace(keySlot, parentDoc);
+                        bucketOrdToLastCollectedParentDoc.put(bucket, parentDoc);
                     }
                 } else {
                     collectBucket(sub, parentDoc, bucket);
-                    bucketOrdToLastCollectedParentDoc.indexInsert(keySlot, bucket, parentDoc);
+                    bucketOrdToLastCollectedParentDoc.put(bucket, parentDoc);
                 }
             }
         };