Skip to content

Commit b5cd364

Browse files
mkhludnevmkhludnev
committed
fix #18956 WIP
- I replaced RandomIndexWriter to plain IndexWriter to avoid unexpected segments in test which are unable to handle more than single segment - for other test I've added explicit segmenting. TODO It needs to be reviewed. - Perhaps StreamingTermsAggregator should reject second segment via assert or there should be a proper publish/reset flow. TODO, WIP so far this commit fixes the known reason for flackines (I belive) Signed-off-by: Mikhail Khludnev <[email protected]>
1 parent 1103b3f commit b5cd364

File tree

1 file changed

+27
-24
lines changed

1 file changed

+27
-24
lines changed

‎server/src/test/java/org/opensearch/search/aggregations/bucket/terms/StreamStringTermsAggregatorTests.java‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,10 @@
1111
import org.apache.lucene.document.Document;
1212
import org.apache.lucene.document.NumericDocValuesField;
1313
import org.apache.lucene.document.SortedSetDocValuesField;
14+
import org.apache.lucene.index.DirectoryReader;
1415
import org.apache.lucene.index.IndexReader;
16+
import org.apache.lucene.index.IndexWriter;
17+
import org.apache.lucene.index.IndexWriterConfig;
1518
import org.apache.lucene.search.IndexSearcher;
1619
import org.apache.lucene.search.MatchAllDocsQuery;
1720
import org.apache.lucene.store.Directory;
@@ -54,7 +57,7 @@
5457
public class StreamStringTermsAggregatorTests extends AggregatorTestCase {
5558
public void testBuildAggregationsBatchDirectBucketCreation() throws Exception {
5659
try (Directory directory = newDirectory()) {
57-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
60+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
5861
Document document = new Document();
5962
document.add(new SortedSetDocValuesField("field", new BytesRef("apple")));
6063
document.add(new SortedSetDocValuesField("field", new BytesRef("banana")));
@@ -69,7 +72,7 @@ public void testBuildAggregationsBatchDirectBucketCreation() throws Exception {
6972
document.add(new SortedSetDocValuesField("field", new BytesRef("banana")));
7073
indexWriter.addDocument(document);
7174

72-
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
75+
try (IndexReader indexReader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
7376
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
7477
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("field");
7578

@@ -151,14 +154,14 @@ public void testBuildAggregationsBatchEmptyResults() throws Exception {
151154

152155
public void testBuildAggregationsBatchWithSingleValuedOrds() throws Exception {
153156
try (Directory directory = newDirectory()) {
154-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
157+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
155158
for (int i = 0; i < 10; i++) {
156159
Document document = new Document();
157160
document.add(new SortedSetDocValuesField("field", new BytesRef("term_" + (i % 3))));
158161
indexWriter.addDocument(document);
159162
}
160163

161-
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
164+
try (IndexReader indexReader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
162165
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
163166
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("field");
164167

@@ -222,15 +225,15 @@ public void testBuildAggregationsBatchWithSingleValuedOrds() throws Exception {
222225

223226
public void testBuildAggregationsBatchWithSize() throws Exception {
224227
try (Directory directory = newDirectory()) {
225-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
228+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
226229
// Create fewer unique terms to test size parameter more meaningfully
227230
for (int i = 0; i < 20; i++) {
228231
Document document = new Document();
229232
document.add(new SortedSetDocValuesField("field", new BytesRef("term_" + (i % 10))));
230233
indexWriter.addDocument(document);
231234
}
232235

233-
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
236+
try (IndexReader indexReader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
234237
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
235238
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("field");
236239

@@ -271,7 +274,7 @@ public void testBuildAggregationsBatchWithSize() throws Exception {
271274

272275
public void testBuildAggregationsBatchWithCountOrder() throws Exception {
273276
try (Directory directory = newDirectory()) {
274-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
277+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
275278
for (int i = 0; i < 3; i++) {
276279
Document document = new Document();
277280
document.add(new SortedSetDocValuesField("field", new BytesRef("common")));
@@ -288,7 +291,7 @@ public void testBuildAggregationsBatchWithCountOrder() throws Exception {
288291
document.add(new SortedSetDocValuesField("field", new BytesRef("rare")));
289292
indexWriter.addDocument(document);
290293

291-
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
294+
try (IndexReader indexReader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
292295
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
293296
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("field");
294297

@@ -485,7 +488,7 @@ public void testSubAggregationWithMax() throws Exception {
485488

486489
public void testSubAggregationWithSum() throws Exception {
487490
try (Directory directory = newDirectory()) {
488-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
491+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
489492
Document document = new Document();
490493
document.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
491494
document.add(new NumericDocValuesField("sales", 1000));
@@ -501,7 +504,7 @@ public void testSubAggregationWithSum() throws Exception {
501504
document.add(new NumericDocValuesField("sales", 500));
502505
indexWriter.addDocument(document);
503506

504-
try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
507+
try (IndexReader indexReader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
505508
IndexSearcher indexSearcher = newIndexSearcher(indexReader);
506509
MappedFieldType categoryFieldType = new KeywordFieldMapper.KeywordFieldType("category");
507510
MappedFieldType salesFieldType = new NumberFieldMapper.NumberFieldType("sales", NumberFieldMapper.NumberType.LONG);
@@ -560,7 +563,7 @@ public void testSubAggregationWithAvg() throws Exception {
560563
document.add(new SortedSetDocValuesField("product", new BytesRef("laptop")));
561564
document.add(new NumericDocValuesField("rating", 4));
562565
indexWriter.addDocument(document);
563-
566+
indexWriter.commit();
564567
document = new Document();
565568
document.add(new SortedSetDocValuesField("product", new BytesRef("laptop")));
566569
document.add(new NumericDocValuesField("rating", 5));
@@ -630,7 +633,7 @@ public void testSubAggregationWithMinAndCount() throws Exception {
630633
document.add(new SortedSetDocValuesField("store", new BytesRef("store_a")));
631634
document.add(new NumericDocValuesField("inventory", 100));
632635
indexWriter.addDocument(document);
633-
636+
indexWriter.commit();
634637
document = new Document();
635638
document.add(new SortedSetDocValuesField("store", new BytesRef("store_a")));
636639
document.add(new NumericDocValuesField("inventory", 50));
@@ -715,7 +718,7 @@ public void testMultipleSubAggregations() throws Exception {
715718
document.add(new NumericDocValuesField("temperature", 25));
716719
document.add(new NumericDocValuesField("humidity", 60));
717720
indexWriter.addDocument(document);
718-
721+
indexWriter.commit();
719722
document = new Document();
720723
document.add(new SortedSetDocValuesField("region", new BytesRef("north")));
721724
document.add(new NumericDocValuesField("temperature", 30));
@@ -815,7 +818,7 @@ public void testReduceSimple() throws Exception {
815818
// Create first aggregation with some data
816819
List<InternalAggregation> aggs = new ArrayList<>();
817820

818-
try (RandomIndexWriter indexWriter1 = new RandomIndexWriter(random(), directory1)) {
821+
try (IndexWriter indexWriter1 = new IndexWriter(directory1, newIndexWriterConfig())) {
819822
Document doc = new Document();
820823
doc.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
821824
indexWriter1.addDocument(doc);
@@ -824,7 +827,7 @@ public void testReduceSimple() throws Exception {
824827
doc.add(new SortedSetDocValuesField("category", new BytesRef("books")));
825828
indexWriter1.addDocument(doc);
826829

827-
try (IndexReader reader1 = maybeWrapReaderEs(indexWriter1.getReader())) {
830+
try (IndexReader reader1 = maybeWrapReaderEs(DirectoryReader.open(indexWriter1))) {
828831
IndexSearcher searcher1 = newIndexSearcher(reader1);
829832
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("category");
830833
aggs.add(
@@ -834,7 +837,7 @@ public void testReduceSimple() throws Exception {
834837
}
835838

836839
// Create second aggregation with overlapping data
837-
try (RandomIndexWriter indexWriter2 = new RandomIndexWriter(random(), directory2)) {
840+
try (IndexWriter indexWriter2 = new IndexWriter(directory2, newIndexWriterConfig())) {
838841
Document doc = new Document();
839842
doc.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
840843
indexWriter2.addDocument(doc);
@@ -843,7 +846,7 @@ public void testReduceSimple() throws Exception {
843846
doc.add(new SortedSetDocValuesField("category", new BytesRef("clothing")));
844847
indexWriter2.addDocument(doc);
845848

846-
try (IndexReader reader2 = maybeWrapReaderEs(indexWriter2.getReader())) {
849+
try (IndexReader reader2 = maybeWrapReaderEs(DirectoryReader.open(indexWriter2))) {
847850
IndexSearcher searcher2 = newIndexSearcher(reader2);
848851
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("category");
849852
aggs.add(
@@ -904,7 +907,7 @@ public void testReduceWithSubAggregations() throws Exception {
904907
doc.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
905908
doc.add(new NumericDocValuesField("price", 100));
906909
indexWriter1.addDocument(doc);
907-
910+
indexWriter1.commit();
908911
doc = new Document();
909912
doc.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
910913
doc.add(new NumericDocValuesField("price", 200));
@@ -970,14 +973,14 @@ public void testReduceWithSizeLimit() throws Exception {
970973
List<InternalAggregation> aggs = new ArrayList<>();
971974

972975
// First aggregation with multiple terms
973-
try (RandomIndexWriter indexWriter1 = new RandomIndexWriter(random(), directory1)) {
976+
try (IndexWriter indexWriter1 = new IndexWriter(directory1, new IndexWriterConfig())) {
974977
for (int i = 0; i < 5; i++) {
975978
Document doc = new Document();
976979
doc.add(new SortedSetDocValuesField("category", new BytesRef("cat_" + i)));
977980
indexWriter1.addDocument(doc);
978981
}
979982

980-
try (IndexReader reader1 = maybeWrapReaderEs(indexWriter1.getReader())) {
983+
try (IndexReader reader1 = maybeWrapReaderEs(DirectoryReader.open(indexWriter1))) {
981984
IndexSearcher searcher1 = newIndexSearcher(reader1);
982985
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("category");
983986

@@ -988,14 +991,14 @@ public void testReduceWithSizeLimit() throws Exception {
988991
}
989992

990993
// Second aggregation with different terms
991-
try (RandomIndexWriter indexWriter2 = new RandomIndexWriter(random(), directory2)) {
994+
try (IndexWriter indexWriter2 = new IndexWriter(directory2, newIndexWriterConfig())) {
992995
for (int i = 3; i < 8; i++) {
993996
Document doc = new Document();
994997
doc.add(new SortedSetDocValuesField("category", new BytesRef("cat_" + i)));
995998
indexWriter2.addDocument(doc);
996999
}
9971000

998-
try (IndexReader reader2 = maybeWrapReaderEs(indexWriter2.getReader())) {
1001+
try (IndexReader reader2 = maybeWrapReaderEs(DirectoryReader.open(indexWriter2))) {
9991002
IndexSearcher searcher2 = newIndexSearcher(reader2);
10001003
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("category");
10011004

@@ -1034,7 +1037,7 @@ public void testReduceWithSizeLimit() throws Exception {
10341037

10351038
public void testReduceSingleAggregation() throws Exception {
10361039
try (Directory directory = newDirectory()) {
1037-
try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
1040+
try (IndexWriter indexWriter = new IndexWriter(directory, newIndexWriterConfig())) {
10381041
// Add multiple documents with different categories to test reduce logic properly
10391042
Document doc1 = new Document();
10401043
doc1.add(new SortedSetDocValuesField("category", new BytesRef("electronics")));
@@ -1058,7 +1061,7 @@ public void testReduceSingleAggregation() throws Exception {
10581061

10591062
indexWriter.commit(); // Ensure data is committed before reading
10601063

1061-
try (IndexReader reader = maybeWrapReaderEs(indexWriter.getReader())) {
1064+
try (IndexReader reader = maybeWrapReaderEs(DirectoryReader.open(indexWriter))) {
10621065
IndexSearcher searcher = newIndexSearcher(reader);
10631066
MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType("category");
10641067

0 commit comments

Comments
 (0)