Update Apache Lucene to 9.12.0-snapshot-25253a1

Signed-off-by: Andriy Redko <[email protected]>

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.12.0-snapshot-847316d
+lucene            = 9.12.0-snapshot-25253a1
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.4+7

libs/core/licenses/lucene-core-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+440301d62a8dcef9451cc0f94659fe95d360cfc7

modules/lang-expression/licenses/lucene-expressions-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+cb83d5c129acf83116c2fba1cbde5437c41de080

plugins/analysis-icu/licenses/lucene-analysis-icu-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+af087a850103b268c21f5a9187aab6522ab71969

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+33e1e2d0208ee500a473a40bec7870b9ecfbf2d7

plugins/analysis-nori/licenses/lucene-analysis-nori-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+eb8499d06729c97cb8fa06138cdbc196003c4dce

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+190b139f55630a2e2c8f32827632bfa4710a4814

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+502562282159609956d03f276b8678330afbcb5b

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+3c0f7c95dd74d7c696cfd25a7a4bab3a12e84ef1

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+b82d2a8ca7b081f3771795255128c423f4444f31

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java

@@ -8,8 +8,8 @@
 
 package org.opensearch.plugin.correlation.core.index.codec;
 
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.CorrelationCodec;
 import org.opensearch.plugin.correlation.core.index.codec.correlation990.PerFieldCorrelationVectorsFormat;

server/licenses/lucene-analysis-common-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+b1f118320c1f730e59b7e4ccf32a63fd6765edde

server/licenses/lucene-backward-codecs-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+e30b13a395cc828ecfefd058c12b752726992c53

server/licenses/lucene-core-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+440301d62a8dcef9451cc0f94659fe95d360cfc7

server/licenses/lucene-grouping-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+ab357668bd4cd5fb9b0b0d6b65e96f368e2a6302

server/licenses/lucene-highlighter-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+d1d080787d38bdd04c1e79d7d0bbebfe583140dc

server/licenses/lucene-join-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+ce3e48336f80897c44c534904b588eb5e555a781

server/licenses/lucene-memory-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+f57dcd318d7d278f11c23411aa2fb075b6cce470

server/licenses/lucene-misc-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+ed21cecc264f55d262279a5bce3aba5bca5a14cc

server/licenses/lucene-queries-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+e6c6d94510adce557d66a16c24df354ae4f20624

server/licenses/lucene-queryparser-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+c5f8ac612d3a5d87df8b2281a592db75f21e772e

server/licenses/lucene-sandbox-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+ecea25f04b8ea21be6117beec0aa3c18fe015f6f

server/licenses/lucene-spatial-extras-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+d4d1424ddcbb65ae5d46925d7c242e3d25402144

server/licenses/lucene-spatial3d-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+bc9bde807fe1665f08fdf455fa41dcef5f5ac3d4

server/licenses/lucene-suggest-9.12.0-snapshot-25253a1.jar.sha1

@@ -0,0 +1 @@
+76b7d3fd8301ca404d3a27517640b48b9ac852ea

server/src/main/java/org/apache/lucene/util/CombinedBitSet.java

@@ -87,15 +87,7 @@ public int prevSetBit(int index) {
 
     @Override
     public int nextSetBit(int index) {
-        assert index >= 0 && index < length : "index=" + index + " numBits=" + length();
-        int next = first.nextSetBit(index);
-        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
-            if (next == length() - 1) {
-                return DocIdSetIterator.NO_MORE_DOCS;
-            }
-            next = first.nextSetBit(next + 1);
-        }
-        return next;
+        return nextSetBit(index, length() - 1);
     }
 
     @Override
@@ -132,4 +124,20 @@ public void clear(int startIndex, int endIndex) {
     public boolean getAndSet(int i) {
         throw new UnsupportedOperationException("not implemented");
     }
+
+    @Override
+    public int nextSetBit(int start, int end) {
+        assert start >= 0 && start < length() : "start=" + start + " numBits=" + length();
+        assert end >= start && end < length() : "end=" + end + " numBits=" + length();
+
+        int next = first.nextSetBit(start);
+        while (next != DocIdSetIterator.NO_MORE_DOCS && second.get(next) == false) {
+            if (next >= end) {
+                return DocIdSetIterator.NO_MORE_DOCS;
+            }
+            next = first.nextSetBit(next + 1);
+        }
+        return next;
+
+    }
 }

server/src/main/java/org/opensearch/common/lucene/Lucene.java

@@ -110,7 +110,7 @@
  * @opensearch.internal
  */
 public class Lucene {
-    public static final String LATEST_CODEC = "Lucene99";
+    public static final String LATEST_CODEC = "Lucene912";
 
     public static final String SOFT_DELETES_FIELD = "__soft_deletes";
 

server/src/main/java/org/opensearch/index/codec/CodecService.java

@@ -34,8 +34,8 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec.Mode;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec.Mode;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.index.IndexSettings;
@@ -70,10 +70,10 @@ public CodecService(@Nullable MapperService mapperService, IndexSettings indexSe
         final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
         assert null != indexSettings;
         if (mapperService == null) {
-            codecs.put(DEFAULT_CODEC, new Lucene99Codec());
-            codecs.put(LZ4, new Lucene99Codec());
-            codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Mode.BEST_COMPRESSION));
-            codecs.put(ZLIB, new Lucene99Codec(Mode.BEST_COMPRESSION));
+            codecs.put(DEFAULT_CODEC, new Lucene912Codec());
+            codecs.put(LZ4, new Lucene912Codec());
+            codecs.put(BEST_COMPRESSION_CODEC, new Lucene912Codec(Mode.BEST_COMPRESSION));
+            codecs.put(ZLIB, new Lucene912Codec(Mode.BEST_COMPRESSION));
         } else {
             // CompositeCodec still delegates to PerFieldMappingPostingFormatCodec
             // We can still support all the compression codecs when composite index is present

server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -37,7 +37,7 @@
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.index.codec.fuzzy.FuzzyFilterPostingsFormat;
 import org.opensearch.index.codec.fuzzy.FuzzySetFactory;
@@ -59,7 +59,7 @@
  *
  * @opensearch.internal
  */
-public class PerFieldMappingPostingFormatCodec extends Lucene99Codec {
+public class PerFieldMappingPostingFormatCodec extends Lucene912Codec {
     private final Logger logger;
     private final MapperService mapperService;
     private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();

server/src/main/java/org/opensearch/index/codec/composite/Composite912Codec.java

@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.composite;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
+import org.opensearch.index.mapper.MapperService;
+
+/**
+ *  Extends the Codec to support new file formats for composite indices eg: star tree index
+ *  based on the mappings.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class Composite912Codec extends FilterCodec {
+    public static final String COMPOSITE_INDEX_CODEC_NAME = "Composite912Codec";
+    private final MapperService mapperService;
+
+    // needed for SPI - this is used in reader path
+    public Composite912Codec() {
+        this(COMPOSITE_INDEX_CODEC_NAME, new Lucene912Codec(), null);
+    }
+
+    public Composite912Codec(Lucene912Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
+        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
+    }
+
+    /**
+     * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
+     * this ctor.
+     *
+     * @param name name of the codec
+     * @param delegate codec delegate
+     * @param mapperService mapper service instance
+     */
+    protected Composite912Codec(String name, Codec delegate, MapperService mapperService) {
+        super(name, delegate);
+        this.mapperService = mapperService;
+    }
+
+    @Override
+    public DocValuesFormat docValuesFormat() {
+        return new Composite99DocValuesFormat(mapperService);
+    }
+}

server/src/main/java/org/opensearch/index/codec/composite/Composite99Codec.java

@@ -9,10 +9,10 @@
 package org.opensearch.index.codec.composite;
 
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.FilterCodec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
 import org.opensearch.index.mapper.MapperService;
@@ -33,10 +33,6 @@ public Composite99Codec() {
         this(COMPOSITE_INDEX_CODEC_NAME, new Lucene99Codec(), null);
     }
 
-    public Composite99Codec(Lucene99Codec.Mode compressionMode, MapperService mapperService, Logger logger) {
-        this(COMPOSITE_INDEX_CODEC_NAME, new PerFieldMappingPostingFormatCodec(compressionMode, mapperService, logger), mapperService);
-    }
-
     /**
      * Sole constructor. When subclassing this codec, create a no-arg ctor and pass the delegate codec and a unique name to
      * this ctor.

server/src/main/java/org/opensearch/index/codec/composite/CompositeCodecFactory.java

@@ -10,7 +10,7 @@
 
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.index.mapper.MapperService;
 
@@ -33,10 +33,10 @@ public CompositeCodecFactory() {}
 
     public Map<String, Codec> getCompositeIndexCodecs(MapperService mapperService, Logger logger) {
         Map<String, Codec> codecs = new HashMap<>();
-        codecs.put(DEFAULT_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(LZ4, new Composite99Codec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger));
-        codecs.put(BEST_COMPRESSION_CODEC, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
-        codecs.put(ZLIB, new Composite99Codec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(DEFAULT_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(LZ4, new Composite912Codec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger));
+        codecs.put(BEST_COMPRESSION_CODEC, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
+        codecs.put(ZLIB, new Composite912Codec(Lucene912Codec.Mode.BEST_COMPRESSION, mapperService, logger));
         return codecs;
     }
 }

server/src/main/java/org/opensearch/search/internal/ContextIndexSearcher.java

@@ -299,7 +299,8 @@ protected void search(List<LeafReaderContext> leaves, Weight weight, Collector c
      * {@link LeafCollector#collect(int)} is called for every matching document in
      * the provided <code>ctx</code>.
      */
-    private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
+    @Override
+    protected void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException {
 
         // Check if at all we need to call this leaf for collecting results.
         if (canMatch(ctx) == false) {