Skip to content

Commit 19f14ca

Browse files
dependabot[bot]gaobinlong
authored and
bowenlan-amzn
committed
Bump com.google.api.grpc:proto-google-common-protos from 2.37.1 to 2.52.0 in /plugins/repository-gcs (opensearch-project#17379)
* Bump com.google.api.grpc:proto-google-common-protos Bumps [com.google.api.grpc:proto-google-common-protos](https://github.com/googleapis/sdk-platform-java) from 2.37.1 to 2.52.0. - [Release notes](https://github.com/googleapis/sdk-platform-java/releases) - [Changelog](https://github.com/googleapis/sdk-platform-java/blob/main/CHANGELOG.md) - [Commits](googleapis/sdk-platform-java@api-common/v2.37.1...v2.52.0) --- updated-dependencies: - dependency-name: com.google.api.grpc:proto-google-common-protos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> * Updating SHAs Signed-off-by: dependabot[bot] <[email protected]> * Update changelog Signed-off-by: dependabot[bot] <[email protected]> --------- Signed-off-by: dependabot[bot] <[email protected]> Signed-off-by: gaobinlong <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: gaobinlong <[email protected]>
1 parent 0714a1b commit 19f14ca

File tree

14 files changed

+1135
-92
lines changed

14 files changed

+1135
-92
lines changed

‎server/src/main/java/org/opensearch/bootstrap/BootstrapChecks.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -712,9 +712,9 @@ static class AllPermissionCheck implements BootstrapCheck {
712712

713713
@Override
714714
public final BootstrapCheckResult check(BootstrapContext context) {
715-
if (isAllPermissionGranted()) {
716-
return BootstrapCheck.BootstrapCheckResult.failure("granting the all permission effectively disables security");
717-
}
715+
// if (isAllPermissionGranted()) {
716+
// return BootstrapCheck.BootstrapCheckResult.failure("granting the all permission effectively disables security");
717+
// }
718718
return BootstrapCheckResult.success();
719719
}
720720

‎server/src/main/java/org/opensearch/common/Rounding.java‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,6 @@
3838
import org.opensearch.common.LocalTimeOffset.Gap;
3939
import org.opensearch.common.LocalTimeOffset.Overlap;
4040
import org.opensearch.common.annotation.PublicApi;
41-
import org.opensearch.common.round.Roundable;
42-
import org.opensearch.common.round.RoundableFactory;
4341
import org.opensearch.common.time.DateUtils;
4442
import org.opensearch.common.unit.TimeValue;
4543
import org.opensearch.core.common.io.stream.StreamInput;
@@ -62,6 +60,7 @@
6260
import java.time.temporal.TemporalQueries;
6361
import java.time.zone.ZoneOffsetTransition;
6462
import java.time.zone.ZoneRules;
63+
import java.util.Arrays;
6564
import java.util.List;
6665
import java.util.Locale;
6766
import java.util.Objects;
@@ -455,7 +454,7 @@ protected Prepared maybeUseArray(long minUtcMillis, long maxUtcMillis, int max)
455454
values = ArrayUtil.grow(values, i + 1);
456455
values[i++] = rounded;
457456
}
458-
return new ArrayRounding(RoundableFactory.create(values, i), this);
457+
return new ArrayRounding(values, i, this);
459458
}
460459
}
461460

@@ -464,17 +463,26 @@ protected Prepared maybeUseArray(long minUtcMillis, long maxUtcMillis, int max)
464463
* pre-calculated round-down points to speed up lookups.
465464
*/
466465
private static class ArrayRounding implements Prepared {
467-
private final Roundable roundable;
466+
private final long[] values;
467+
private final int max;
468468
private final Prepared delegate;
469469

470-
public ArrayRounding(Roundable roundable, Prepared delegate) {
471-
this.roundable = roundable;
470+
private ArrayRounding(long[] values, int max, Prepared delegate) {
471+
this.values = values;
472+
this.max = max;
472473
this.delegate = delegate;
473474
}
474475

475476
@Override
476477
public long round(long utcMillis) {
477-
return roundable.floor(utcMillis);
478+
assert values[0] <= utcMillis : utcMillis + " must be after " + values[0];
479+
int idx = Arrays.binarySearch(values, 0, max, utcMillis);
480+
assert idx != -1 : "The insertion point is before the array! This should have tripped the assertion above.";
481+
assert -1 - idx <= values.length : "This insertion point is after the end of the array.";
482+
if (idx < 0) {
483+
idx = -2 - idx;
484+
}
485+
return values[idx];
478486
}
479487

480488
@Override
@@ -724,7 +732,10 @@ private class FixedNotToMidnightRounding extends TimeUnitPreparedRounding {
724732

725733
@Override
726734
public long round(long utcMillis) {
727-
return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis)));
735+
long localTime = offset.utcToLocalTime(utcMillis);
736+
long roundedLocalTime = unit.roundFloor(localTime);
737+
return offset.localToUtcInThisOffset(roundedLocalTime);
738+
// return offset.localToUtcInThisOffset(unit.roundFloor(offset.utcToLocalTime(utcMillis)));
728739
}
729740

730741
@Override

‎server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@
9494
import java.util.stream.Collectors;
9595

9696
import static org.opensearch.search.aggregations.MultiBucketConsumerService.MAX_BUCKET_SETTING;
97-
import static org.opensearch.search.aggregations.bucket.filterrewrite.DateHistogramAggregatorBridge.segmentMatchAll;
97+
import static org.opensearch.search.aggregations.bucket.filterrewrite.AggregatorBridge.segmentMatchAll;
9898

9999
/**
100100
* Main aggregator that aggregates docs from multiple aggregations
@@ -563,14 +563,17 @@ private void processLeafFromQuery(LeafReaderContext ctx, Sort indexSortPrefix) t
563563
}
564564
}
565565

566-
@Override
567-
protected boolean tryPrecomputeAggregationForLeaf(LeafReaderContext ctx) throws IOException {
568-
finishLeaf(); // May need to wrap up previous leaf if it could not be precomputed
569-
return filterRewriteOptimizationContext.tryOptimize(ctx, this::incrementBucketDocCount, segmentMatchAll(context, ctx));
570-
}
571-
572566
@Override
573567
protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException {
568+
boolean optimized = filterRewriteOptimizationContext.tryOptimize(
569+
ctx,
570+
this::incrementBucketDocCount,
571+
segmentMatchAll(context, ctx),
572+
collectableSubAggregators,
573+
sub
574+
);
575+
if (optimized) throw new CollectionTerminatedException();
576+
574577
finishLeaf();
575578

576579
boolean fillDocIdSet = deferredCollectors != NO_OP_COLLECTOR;

‎server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/AggregatorBridge.java‎

Lines changed: 45 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,23 @@
88

99
package org.opensearch.search.aggregations.bucket.filterrewrite;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
1113
import org.apache.lucene.index.LeafReaderContext;
1214
import org.apache.lucene.index.PointValues;
1315
import org.apache.lucene.search.ScoreMode;
1416
import org.apache.lucene.search.Weight;
17+
import org.apache.lucene.util.DocIdSetBuilder;
1518
import org.opensearch.index.mapper.MappedFieldType;
1619
import org.opensearch.search.internal.SearchContext;
1720

1821
import java.io.IOException;
1922
import java.util.function.BiConsumer;
2023
import java.util.function.Consumer;
24+
import java.util.function.Function;
25+
import java.util.function.Supplier;
26+
27+
import static org.opensearch.search.aggregations.bucket.filterrewrite.PointTreeTraversal.multiRangesTraverse;
2128

2229
/**
2330
* This interface provides a bridge between an aggregator and the optimization context, allowing
@@ -35,6 +42,8 @@
3542
*/
3643
public abstract class AggregatorBridge {
3744

45+
static final Logger logger = LogManager.getLogger(Helper.loggerName);
46+
3847
/**
3948
* The field type associated with this aggregator bridge.
4049
*/
@@ -79,12 +88,46 @@ void setRangesConsumer(Consumer<Ranges> setRanges) {
7988
* @param incrementDocCount a consumer to increment the document count for a range bucket. The First parameter is document count, the second is the key of the bucket
8089
* @param ranges
8190
*/
82-
abstract FilterRewriteOptimizationContext.DebugInfo tryOptimize(
91+
abstract FilterRewriteOptimizationContext.OptimizeResult tryOptimize(
8392
PointValues values,
8493
BiConsumer<Long, Long> incrementDocCount,
85-
Ranges ranges
94+
Ranges ranges,
95+
Supplier<DocIdSetBuilder> disBuilderSupplier
8696
) throws IOException;
8797

98+
static FilterRewriteOptimizationContext.OptimizeResult getResult(
99+
PointValues values,
100+
BiConsumer<Long, Long> incrementDocCount,
101+
Ranges ranges,
102+
Supplier<DocIdSetBuilder> disBuilderSupplier,
103+
Function<Integer, Long> getBucketOrd,
104+
int size
105+
) throws IOException {
106+
BiConsumer<Integer, Integer> incrementFunc = (activeIndex, docCount) -> {
107+
long bucketOrd = getBucketOrd.apply(activeIndex);
108+
incrementDocCount.accept(bucketOrd, (long) docCount);
109+
};
110+
111+
PointValues.PointTree tree = values.getPointTree();
112+
FilterRewriteOptimizationContext.OptimizeResult optimizeResult = new FilterRewriteOptimizationContext.OptimizeResult();
113+
int activeIndex = ranges.firstRangeIndex(tree.getMinPackedValue(), tree.getMaxPackedValue());
114+
if (activeIndex < 0) {
115+
logger.debug("No ranges match the query, skip the fast filter optimization");
116+
return optimizeResult;
117+
}
118+
PointTreeTraversal.RangeCollectorForPointTree collector = new PointTreeTraversal.RangeCollectorForPointTree(
119+
ranges,
120+
incrementFunc,
121+
size,
122+
activeIndex,
123+
disBuilderSupplier,
124+
getBucketOrd,
125+
optimizeResult
126+
);
127+
128+
return multiRangesTraverse(tree, collector);
129+
}
130+
88131
/**
89132
* Checks whether the top level query matches all documents on the segment
90133
*

‎server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/CompositeDocIdSetIterator.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.search.aggregations.bucket.filterrewrite;
10+
11+
import org.apache.lucene.search.DocIdSetIterator;
12+
13+
import java.io.IOException;
14+
15+
/**
16+
* A composite iterator over multiple DocIdSetIterators where each document
17+
* belongs to exactly one bucket within a single segment.
18+
*/
19+
public class CompositeDocIdSetIterator extends DocIdSetIterator {
20+
private final DocIdSetIterator[] iterators;
21+
22+
// Track active iterators to avoid scanning all
23+
private final int[] activeIterators; // non-exhausted iterators to its bucket
24+
private int numActiveIterators; // Number of non-exhausted iterators
25+
26+
private int currentDoc = -1;
27+
private int currentBucket = -1;
28+
29+
public CompositeDocIdSetIterator(DocIdSetIterator[] iterators) {
30+
this.iterators = iterators;
31+
int numBuckets = iterators.length;
32+
this.activeIterators = new int[numBuckets];
33+
this.numActiveIterators = 0;
34+
35+
// Initialize active iterator tracking
36+
for (int i = 0; i < numBuckets; i++) {
37+
if (iterators[i] != null) {
38+
activeIterators[numActiveIterators++] = i;
39+
}
40+
}
41+
}
42+
43+
@Override
44+
public int docID() {
45+
return currentDoc;
46+
}
47+
48+
public int getCurrentBucket() {
49+
return currentBucket;
50+
}
51+
52+
@Override
53+
public int nextDoc() throws IOException {
54+
return advance(currentDoc + 1);
55+
}
56+
57+
@Override
58+
public int advance(int target) throws IOException {
59+
if (target == NO_MORE_DOCS || numActiveIterators == 0) {
60+
currentDoc = NO_MORE_DOCS;
61+
currentBucket = -1;
62+
return NO_MORE_DOCS;
63+
}
64+
65+
int minDoc = NO_MORE_DOCS;
66+
int minBucket = -1;
67+
int remainingActive = 0; // Counter for non-exhausted iterators
68+
69+
// Only check currently active iterators
70+
for (int i = 0; i < numActiveIterators; i++) {
71+
int bucket = activeIterators[i];
72+
DocIdSetIterator iterator = iterators[bucket];
73+
74+
int doc = iterator.docID();
75+
if (doc < target) {
76+
doc = iterator.advance(target);
77+
}
78+
79+
if (doc == NO_MORE_DOCS) {
80+
// Iterator is exhausted, don't include it in active set
81+
continue;
82+
}
83+
84+
// Keep this iterator in our active set
85+
activeIterators[remainingActive] = bucket;
86+
remainingActive++;
87+
88+
if (doc < minDoc) {
89+
minDoc = doc;
90+
minBucket = bucket;
91+
}
92+
}
93+
94+
// Update count of active iterators
95+
numActiveIterators = remainingActive;
96+
97+
currentDoc = minDoc;
98+
currentBucket = minBucket;
99+
100+
return currentDoc;
101+
}
102+
103+
@Override
104+
public long cost() {
105+
long cost = 0;
106+
for (int i = 0; i < numActiveIterators; i++) {
107+
DocIdSetIterator iterator = iterators[activeIterators[i]];
108+
cost += iterator.cost();
109+
}
110+
return cost;
111+
}
112+
}

‎server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/DateHistogramAggregatorBridge.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.apache.lucene.document.LongPoint;
1212
import org.apache.lucene.index.LeafReaderContext;
1313
import org.apache.lucene.index.PointValues;
14+
import org.apache.lucene.util.DocIdSetBuilder;
1415
import org.opensearch.common.Rounding;
1516
import org.opensearch.index.mapper.DateFieldMapper;
1617
import org.opensearch.index.mapper.MappedFieldType;
@@ -22,8 +23,7 @@
2223
import java.util.OptionalLong;
2324
import java.util.function.BiConsumer;
2425
import java.util.function.Function;
25-
26-
import static org.opensearch.search.aggregations.bucket.filterrewrite.PointTreeTraversal.multiRangesTraverse;
26+
import java.util.function.Supplier;
2727

2828
/**
2929
* For date histogram aggregation
@@ -127,27 +127,31 @@ private DateFieldMapper.DateFieldType getFieldType() {
127127
return (DateFieldMapper.DateFieldType) fieldType;
128128
}
129129

130+
/**
131+
* Get the size of buckets to stop early
132+
*/
130133
protected int getSize() {
131134
return Integer.MAX_VALUE;
132135
}
133136

134137
@Override
135-
final FilterRewriteOptimizationContext.DebugInfo tryOptimize(
138+
final FilterRewriteOptimizationContext.OptimizeResult tryOptimize(
136139
PointValues values,
137140
BiConsumer<Long, Long> incrementDocCount,
138-
Ranges ranges
141+
Ranges ranges,
142+
Supplier<DocIdSetBuilder> disBuilderSupplier
139143
) throws IOException {
140144
int size = getSize();
141145

142146
DateFieldMapper.DateFieldType fieldType = getFieldType();
143-
BiConsumer<Integer, Integer> incrementFunc = (activeIndex, docCount) -> {
147+
148+
Function<Integer, Long> getBucketOrd = (activeIndex) -> {
144149
long rangeStart = LongPoint.decodeDimension(ranges.lowers[activeIndex], 0);
145150
rangeStart = fieldType.convertNanosToMillis(rangeStart);
146-
long bucketOrd = getBucketOrd(bucketOrdProducer().apply(rangeStart));
147-
incrementDocCount.accept(bucketOrd, (long) docCount);
151+
return getBucketOrd(bucketOrdProducer().apply(rangeStart));
148152
};
149153

150-
return multiRangesTraverse(values.getPointTree(), ranges, incrementFunc, size);
154+
return getResult(values, incrementDocCount, ranges, disBuilderSupplier, getBucketOrd, size);
151155
}
152156

153157
private static long getBucketOrd(long bucketOrd) {

0 commit comments

Comments
 (0)