diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index 1885498c6c4b7..1347796af5f4d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -210,6 +210,13 @@ public Settings getSettings() { return indexSettings.getSettings(); } + /** + * Returns the {@link IndexSettings} for this index + */ + public IndexSettings indexSettings() { + return indexSettings; + } + /** * Returns the index this module is associated with */ diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 0d50ad1ee36e5..1556410ca0c2c 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -66,6 +66,10 @@ public Status needsField(FieldInfo fieldInfo) { if (IgnoredFieldMapper.NAME.equals(fieldInfo.name)) { return Status.YES; } + // support _uid for loading older indices + if ("_uid".equals(fieldInfo.name)) { + return Status.YES; + } // All these fields are single-valued so we can stop when the set is // empty return requiredFields.isEmpty() ? Status.STOP : Status.NO; @@ -103,9 +107,18 @@ public void binaryField(FieldInfo fieldInfo, BytesRef value) { @Override public void stringField(FieldInfo fieldInfo, String value) { - assert IdFieldMapper.NAME.equals(fieldInfo.name) == false : "_id field must go through binaryField"; assert sourceFieldName.equals(fieldInfo.name) == false : "source field must go through binaryField"; - addValue(fieldInfo.name, value); + if ("_uid".equals(fieldInfo.name)) { + // 5.x-only + int delimiterIndex = value.indexOf('#'); // type is not allowed to have # in it..., ids can + // type = value.substring(0, delimiterIndex); + id = value.substring(delimiterIndex + 1); + } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) { + // only applies to 5.x indices that have single_type = true + id = value; + } else { + addValue(fieldInfo.name, value); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java index 1f79458cdfd7e..61cd167cd07b6 100644 --- a/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshot.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentFragment; @@ -41,6 +42,7 @@ public static class FileInfo implements Writeable { public static final String SERIALIZE_WRITER_UUID = "serialize_writer_uuid"; private final String name; + @Nullable private final ByteSizeValue partSize; private final long partBytes; private final int numberOfParts; @@ -53,7 +55,7 @@ public static class FileInfo implements Writeable { * @param metadata the files meta data * @param partSize size of the single chunk */ - public FileInfo(String name, StoreFileMetadata metadata, ByteSizeValue partSize) { + public FileInfo(String name, StoreFileMetadata metadata, @Nullable ByteSizeValue partSize) { this.name = Objects.requireNonNull(name); this.metadata = metadata; diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java index 1498893d4d489..74b8620cc7af9 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldLuceneVersions.java @@ -7,18 +7,10 @@ package org.elasticsearch.xpack.lucene.bwc; -import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.ChecksumIndexInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.StringHelper; -import org.apache.lucene.util.Version; -import org.elasticsearch.Build; +import org.elasticsearch.Version; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.index.IndexModule; @@ -28,6 +20,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.plugins.IndexStorePlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; import java.io.IOException; import java.io.UncheckedIOException; @@ -38,83 +31,69 @@ public class OldLuceneVersions extends Plugin implements IndexStorePlugin { @Override public void onIndexModule(IndexModule indexModule) { - if (Build.CURRENT.isSnapshot()) { + if (indexModule.indexSettings().getIndexVersionCreated().before(Version.CURRENT.minimumIndexCompatibilityVersion())) { indexModule.addIndexEventListener(new IndexEventListener() { @Override public void afterFilesRestoredFromRepository(IndexShard indexShard) { - maybeConvertToNewFormat(indexShard); + convertToNewFormat(indexShard); } }); } } - private static void maybeConvertToNewFormat(IndexShard indexShard) { + /** + * The trick used to allow newer Lucene versions to read older Lucene indices is to convert the old directory to a directory that new + * Lucene versions happily operate on. The way newer Lucene versions happily comply with reading older data is to put in place a + * segments file that the newer Lucene version can open, using codecs that allow reading everything from the old files, making it + * available under the newer interfaces. The way this works is to read in the old segments file using a special class + * {@link OldSegmentInfos} that supports reading older Lucene {@link SegmentInfos}, and then write out an updated segments file that + * newer Lucene versions can understand. + */ + private static void convertToNewFormat(IndexShard indexShard) { indexShard.store().incRef(); try { - try { - Version version = getLuceneVersion(indexShard.store().directory()); - // Lucene version in [7.0.0, 8.0.0) - if (version != null - && version.onOrAfter(Version.fromBits(7, 0, 0)) - && version.onOrAfter(Version.fromBits(8, 0, 0)) == false) { - final OldSegmentInfos oldSegmentInfos = OldSegmentInfos.readLatestCommit(indexShard.store().directory(), 7); - final SegmentInfos segmentInfos = convertLucene7x(oldSegmentInfos); - // write upgraded segments file - segmentInfos.commit(indexShard.store().directory()); + final OldSegmentInfos oldSegmentInfos = OldSegmentInfos.readLatestCommit(indexShard.store().directory(), 6); + final SegmentInfos segmentInfos = convertToNewerLuceneVersion(oldSegmentInfos); + // write upgraded segments file + segmentInfos.commit(indexShard.store().directory()); - // validate that what we have written can be read using standard path - // TODO: norelease: remove this when development completes - SegmentInfos segmentInfos1 = SegmentInfos.readLatestCommit(indexShard.store().directory()); + // what we have written can be read using standard path + assert SegmentInfos.readLatestCommit(indexShard.store().directory()) != null; - // clean older segments file - Lucene.pruneUnreferencedFiles(segmentInfos1.getSegmentsFileName(), indexShard.store().directory()); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + // clean older segments file + Lucene.pruneUnreferencedFiles(segmentInfos.getSegmentsFileName(), indexShard.store().directory()); + } catch (IOException e) { + throw new UncheckedIOException(e); } finally { indexShard.store().decRef(); } } - private static Version getLuceneVersion(Directory directory) throws IOException { - final String segmentFileName = SegmentInfos.getLastCommitSegmentsFileName(directory); - if (segmentFileName != null) { - long generation = SegmentInfos.generationFromSegmentsFileName(segmentFileName); - try (ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ)) { - CodecUtil.checkHeader(input, "segments", 0, Integer.MAX_VALUE); - byte[] id = new byte[StringHelper.ID_LENGTH]; - input.readBytes(id, 0, id.length); - CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX)); - - Version luceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt()); - int indexCreatedVersion = input.readVInt(); - return luceneVersion; - } catch (Exception e) { - // ignore - } - } - return null; - } - - private static SegmentInfos convertLucene7x(OldSegmentInfos oldSegmentInfos) { + private static SegmentInfos convertToNewerLuceneVersion(OldSegmentInfos oldSegmentInfos) { final SegmentInfos segmentInfos = new SegmentInfos(org.apache.lucene.util.Version.LATEST.major); segmentInfos.setNextWriteGeneration(oldSegmentInfos.getGeneration() + 1); final Map map = new HashMap<>(oldSegmentInfos.getUserData()); - map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); - map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); - map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); - map.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1"); + if (map.containsKey(Engine.HISTORY_UUID_KEY) == false) { + map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); + } + if (map.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) == false) { + map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); + } + if (map.containsKey(SequenceNumbers.MAX_SEQ_NO) == false) { + map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(SequenceNumbers.NO_OPS_PERFORMED)); + } + if (map.containsKey(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID) == false) { + map.put(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1"); + } segmentInfos.setUserData(map, true); for (SegmentCommitInfo infoPerCommit : oldSegmentInfos.asList()) { - SegmentInfo info = infoPerCommit.info; - SegmentInfo newInfo = wrap(info); + final SegmentInfo newInfo = BWCCodec.wrap(infoPerCommit.info); segmentInfos.add( new SegmentCommitInfo( newInfo, infoPerCommit.getDelCount(), - 0, + infoPerCommit.getSoftDelCount(), infoPerCommit.getDelGen(), infoPerCommit.getFieldInfosGen(), infoPerCommit.getDocValuesGen(), @@ -125,31 +104,6 @@ private static SegmentInfos convertLucene7x(OldSegmentInfos oldSegmentInfos) { return segmentInfos; } - static SegmentInfo wrap(SegmentInfo segmentInfo) { - // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) - // TODO: alternatively store the original version information in attributes? - byte[] id = segmentInfo.getId(); - if (id == null) { - id = StringHelper.randomId(); - } - Codec codec = segmentInfo.getCodec() instanceof Lucene70Codec ? new BWCLucene70Codec() : segmentInfo.getCodec(); - SegmentInfo segmentInfo1 = new SegmentInfo( - segmentInfo.dir, - org.apache.lucene.util.Version.LATEST, - org.apache.lucene.util.Version.LATEST, - segmentInfo.name, - segmentInfo.maxDoc(), - segmentInfo.getUseCompoundFile(), - codec, - segmentInfo.getDiagnostics(), - id, - segmentInfo.getAttributes(), - null - ); - segmentInfo1.setFiles(segmentInfo.files()); - return segmentInfo1; - } - @Override public Map getDirectoryFactories() { return Map.of(); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java index c62744cb66255..7ec783a0327cc 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/OldSegmentInfos.java @@ -14,6 +14,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. */ package org.elasticsearch.xpack.lucene.bwc; @@ -60,6 +62,12 @@ @SuppressWarnings("CheckStyle") @SuppressForbidden(reason = "Lucene class") public class OldSegmentInfos implements Cloneable, Iterable { + + /** + * Adds the {@link Version} that committed this segments_N file, as well as the {@link Version} + * of the oldest segment, since 5.3+ + */ + public static final int VERSION_53 = 6; /** * The version that added information about the Lucene version at the time when the index has been * created. @@ -209,13 +217,16 @@ static final OldSegmentInfos readCommit(Directory directory, ChecksumIndexInput if (magic != CodecUtil.CODEC_MAGIC) { throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC); } - format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_70, VERSION_CURRENT); + format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_53, VERSION_CURRENT); byte[] id = new byte[StringHelper.ID_LENGTH]; input.readBytes(id, 0, id.length); CodecUtil.checkIndexHeaderSuffix(input, Long.toString(generation, Character.MAX_RADIX)); Version luceneVersion = Version.fromBits(input.readVInt(), input.readVInt(), input.readVInt()); - int indexCreatedVersion = input.readVInt(); + int indexCreatedVersion = 6; + if (format >= VERSION_70) { + indexCreatedVersion = input.readVInt(); + } if (luceneVersion.major < indexCreatedVersion) { throw new CorruptIndexException( "Creation version [" @@ -252,7 +263,7 @@ static final OldSegmentInfos readCommit(Directory directory, ChecksumIndexInput } catch (Throwable t) { priorE = t; } finally { - if (format >= VERSION_70) { // oldest supported version + if (format >= VERSION_53) { // oldest supported version CodecUtil.checkFooter(input, priorE); } else { throw IOUtils.rethrowAlways(priorE); @@ -283,6 +294,14 @@ private static void parseSegmentInfos(Directory directory, DataInput input, OldS long totalDocs = 0; for (int seg = 0; seg < numSegments; seg++) { String segName = input.readString(); + if (format < VERSION_70) { + byte hasID = input.readByte(); + if (hasID == 0) { + throw new IndexFormatTooOldException(input, "Segment is from Lucene 4.x"); + } else if (hasID != 1) { + throw new CorruptIndexException("invalid hasID byte, got: " + hasID, input); + } + } byte[] segmentID = new byte[StringHelper.ID_LENGTH]; input.readBytes(segmentID, 0, segmentID.length); Codec codec = readCodec(input); diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java similarity index 67% rename from x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/BWCLucene70Codec.java rename to x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java index 5bb62a595cd10..7d25366b20114 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/BWCLucene70Codec.java +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/BWCCodec.java @@ -5,27 +5,20 @@ * 2.0. */ -package org.elasticsearch.xpack.lucene.bwc; +package org.elasticsearch.xpack.lucene.bwc.codecs; -import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; -import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; -import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; -import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70Codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.CompoundFormat; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.LiveDocsFormat; import org.apache.lucene.codecs.NormsFormat; import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.SegmentInfoFormat; -import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.TermVectorsFormat; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; @@ -38,44 +31,22 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -public class BWCLucene70Codec extends Codec { - - private final FieldInfosFormat fieldInfosFormat; - private final SegmentInfoFormat segmentInfosFormat; - private final LiveDocsFormat liveDocsFormat; - private final CompoundFormat compoundFormat; - private final PostingsFormat postingsFormat; - private final StoredFieldsFormat storedFieldsFormat; - - public BWCLucene70Codec() { - super("BWCLucene70Codec"); - fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); - segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); - liveDocsFormat = new Lucene50LiveDocsFormat(); - compoundFormat = new Lucene50CompoundFormat(); - postingsFormat = new EmptyPostingsFormat(); - storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); - } - - @Override - public FieldInfosFormat fieldInfosFormat() { - return fieldInfosFormat; - } +/** + * Base class for older BWC codecs + */ +public abstract class BWCCodec extends Codec { - @Override - public SegmentInfoFormat segmentInfoFormat() { - return segmentInfosFormat; - } + private final PostingsFormat postingsFormat = new EmptyPostingsFormat(); - @Override - public NormsFormat normsFormat() { - throw new UnsupportedOperationException(); + protected BWCCodec(String name) { + super(name); } @Override @@ -84,13 +55,13 @@ public PostingsFormat postingsFormat() { } @Override - public DocValuesFormat docValuesFormat() { + public NormsFormat normsFormat() { throw new UnsupportedOperationException(); } @Override - public StoredFieldsFormat storedFieldsFormat() { - return storedFieldsFormat; + public DocValuesFormat docValuesFormat() { + throw new UnsupportedOperationException(); } @Override @@ -98,16 +69,6 @@ public TermVectorsFormat termVectorsFormat() { throw new UnsupportedOperationException(); } - @Override - public LiveDocsFormat liveDocsFormat() { - return liveDocsFormat; - } - - @Override - public CompoundFormat compoundFormat() { - return compoundFormat; - } - @Override public PointsFormat pointsFormat() { throw new UnsupportedOperationException(); @@ -118,11 +79,67 @@ public KnnVectorsFormat knnVectorsFormat() { throw new UnsupportedOperationException(); } - private static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { + /** + * In-memory postings format that shows no postings available. + * TODO: Remove once https://issues.apache.org/jira/browse/LUCENE-10291 is fixed. + */ + static class EmptyPostingsFormat extends PostingsFormat { + + protected EmptyPostingsFormat() { + super("EmptyPostingsFormat"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) { + return new FieldsConsumer() { + @Override + public void write(Fields fields, NormsProducer norms) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + + } + }; + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) { + return new FieldsProducer() { + @Override + public void close() { + + } + + @Override + public void checkIntegrity() { + + } + + @Override + public Iterator iterator() { + return null; + } + + @Override + public Terms terms(String field) { + return null; + } + + @Override + public int size() { + return 0; + } + }; + } + } + + protected static SegmentInfoFormat wrap(SegmentInfoFormat wrapped) { return new SegmentInfoFormat() { @Override public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException { - return adaptVersion(wrapped.read(directory, segmentName, segmentID, context)); + return wrap(wrapped.read(directory, segmentName, segmentID, context)); } @Override @@ -132,7 +149,7 @@ public void write(Directory dir, SegmentInfo info, IOContext ioContext) throws I }; } - private static FieldInfosFormat wrap(FieldInfosFormat wrapped) { + protected static FieldInfosFormat wrap(FieldInfosFormat wrapped) { return new FieldInfosFormat() { @Override public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) @@ -176,59 +193,27 @@ private static FieldInfos filterFields(FieldInfos fieldInfos) { return newFieldInfos; } - private static SegmentInfo adaptVersion(SegmentInfo segmentInfo) { - return OldLuceneVersions.wrap(segmentInfo); + public static SegmentInfo wrap(SegmentInfo segmentInfo) { + // special handling for Lucene70Codec (which is currently bundled with Lucene) + // Use BWCLucene70Codec instead as that one extends BWCCodec (similar to all other older codecs) + final Codec codec = segmentInfo.getCodec() instanceof Lucene70Codec ? new BWCLucene70Codec() : segmentInfo.getCodec(); + final SegmentInfo segmentInfo1 = new SegmentInfo( + segmentInfo.dir, + // Use Version.LATEST instead of original version, otherwise SegmentCommitInfo will bark when processing (N-1 limitation) + // TODO: perhaps store the original version information in attributes so that we can retrieve it later when needed? + org.apache.lucene.util.Version.LATEST, + org.apache.lucene.util.Version.LATEST, + segmentInfo.name, + segmentInfo.maxDoc(), + segmentInfo.getUseCompoundFile(), + codec, + segmentInfo.getDiagnostics(), + segmentInfo.getId(), + segmentInfo.getAttributes(), + segmentInfo.getIndexSort() + ); + segmentInfo1.setFiles(segmentInfo.files()); + return segmentInfo1; } - static class EmptyPostingsFormat extends PostingsFormat { - - protected EmptyPostingsFormat() { - super("EmptyPostingsFormat"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new FieldsConsumer() { - @Override - public void write(Fields fields, NormsProducer norms) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public void close() throws IOException { - - } - }; - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new FieldsProducer() { - @Override - public void close() throws IOException { - - } - - @Override - public void checkIntegrity() throws IOException { - - } - - @Override - public Iterator iterator() { - return null; - } - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public int size() { - return 0; - } - }; - } - } } diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java new file mode 100644 index 0000000000000..49e449d50c9f7 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50FieldInfosFormat.java @@ -0,0 +1,183 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene50; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +/** + * Lucene 5.0 Field Infos format. + * @deprecated + */ +@Deprecated +public final class Lucene50FieldInfosFormat extends FieldInfosFormat { + + public Lucene50FieldInfosFormat() {} + + @Override + public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(directory, fileName, context)) { + Throwable priorE = null; + FieldInfo infos[] = null; + try { + CodecUtil.checkIndexHeader( + input, + Lucene50FieldInfosFormat.CODEC_NAME, + Lucene50FieldInfosFormat.FORMAT_START, + Lucene50FieldInfosFormat.FORMAT_CURRENT, + segmentInfo.getId(), + segmentSuffix + ); + + final int size = input.readVInt(); // read in the size + infos = new FieldInfo[size]; + + // previous field's attribute map, we share when possible: + Map lastAttributes = Collections.emptyMap(); + + for (int i = 0; i < size; i++) { + String name = input.readString(); + final int fieldNumber = input.readVInt(); + if (fieldNumber < 0) { + throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input); + } + byte bits = input.readByte(); + boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; + boolean omitNorms = (bits & OMIT_NORMS) != 0; + boolean storePayloads = (bits & STORE_PAYLOADS) != 0; + + final IndexOptions indexOptions = getIndexOptions(input, input.readByte()); + + // DV Types are packed in one byte + final DocValuesType docValuesType = getDocValuesType(input, input.readByte()); + final long dvGen = input.readLong(); + Map attributes = input.readMapOfStrings(); + + // just use the last field's map if its the same + if (attributes.equals(lastAttributes)) { + attributes = lastAttributes; + } + lastAttributes = attributes; + try { + infos[i] = new FieldInfo( + name, + fieldNumber, + storeTermVector, + omitNorms, + storePayloads, + indexOptions, + docValuesType, + dvGen, + attributes, + 0, + 0, + 0, + 0, + VectorSimilarityFunction.EUCLIDEAN, + false + ); + infos[i].checkConsistency(); + } catch (IllegalStateException e) { + throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e); + } + } + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(input, priorE); + } + return new FieldInfos(infos); + } + } + + private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException { + switch (b) { + case 0: + return DocValuesType.NONE; + case 1: + return DocValuesType.NUMERIC; + case 2: + return DocValuesType.BINARY; + case 3: + return DocValuesType.SORTED; + case 4: + return DocValuesType.SORTED_SET; + case 5: + return DocValuesType.SORTED_NUMERIC; + default: + throw new CorruptIndexException("invalid docvalues byte: " + b, input); + } + } + + private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException { + switch (b) { + case 0: + return IndexOptions.NONE; + case 1: + return IndexOptions.DOCS; + case 2: + return IndexOptions.DOCS_AND_FREQS; + case 3: + return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + case 4: + return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; + default: + // BUG + throw new CorruptIndexException("invalid IndexOptions byte: " + b, input); + } + } + + @Override + public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) { + throw new UnsupportedOperationException(); + } + + /** Extension of field infos */ + static final String EXTENSION = "fnm"; + + // Codec header + static final String CODEC_NAME = "Lucene50FieldInfos"; + static final int FORMAT_SAFE_MAPS = 1; + static final int FORMAT_START = FORMAT_SAFE_MAPS; + static final int FORMAT_CURRENT = FORMAT_SAFE_MAPS; + + // Field flags + static final byte STORE_TERMVECTOR = 0x1; + static final byte OMIT_NORMS = 0x2; + static final byte STORE_PAYLOADS = 0x4; +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java new file mode 100644 index 0000000000000..cf4437a230c0d --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene50/Lucene50SegmentInfoFormat.java @@ -0,0 +1,95 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene50; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +/** + * Lucene 5.0 Segment info format. + * @deprecated Only for reading old 5.0-6.0 segments + */ +@Deprecated +public class Lucene50SegmentInfoFormat extends SegmentInfoFormat { + + public Lucene50SegmentInfoFormat() {} + + @Override + public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene50SegmentInfoFormat.SI_EXTENSION); + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) { + Throwable priorE = null; + SegmentInfo si = null; + try { + CodecUtil.checkIndexHeader( + input, + Lucene50SegmentInfoFormat.CODEC_NAME, + Lucene50SegmentInfoFormat.VERSION_START, + Lucene50SegmentInfoFormat.VERSION_CURRENT, + segmentID, + "" + ); + final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt()); + + final int docCount = input.readInt(); + if (docCount < 0) { + throw new CorruptIndexException("invalid docCount: " + docCount, input); + } + final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; + + final Map diagnostics = input.readMapOfStrings(); + final Set files = input.readSetOfStrings(); + final Map attributes = input.readMapOfStrings(); + + si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, null); + si.setFiles(files); + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(input, priorE); + } + return si; + } + } + + @Override + public void write(Directory dir, SegmentInfo si, IOContext ioContext) { + throw new UnsupportedOperationException("this codec can only be used for reading"); + } + + /** File extension used to store {@link SegmentInfo}. */ + public static final String SI_EXTENSION = "si"; + static final String CODEC_NAME = "Lucene50SegmentInfo"; + static final int VERSION_SAFE_MAPS = 1; + static final int VERSION_START = VERSION_SAFE_MAPS; + static final int VERSION_CURRENT = VERSION_SAFE_MAPS; +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java new file mode 100644 index 0000000000000..3599b97ec16f2 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene60/Lucene60Codec.java @@ -0,0 +1,92 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene60; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; +import org.elasticsearch.xpack.lucene.bwc.codecs.lucene50.Lucene50SegmentInfoFormat; + +import java.util.Objects; + +/** + * Implements the Lucene 6.0 index format. + * + * @deprecated Only for 6.0 back compat + */ +@Deprecated +public class Lucene60Codec extends BWCCodec { + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene50SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Instantiates a new codec. + */ + public Lucene60Codec() { + this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + /** + * Instantiates a new codec, specifying the stored fields compression + * mode to use. + * @param mode stored fields compression mode to use for newly + * flushed/merged segments. + */ + public Lucene60Codec(Lucene50StoredFieldsFormat.Mode mode) { + super("Lucene60"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public final CompoundFormat compoundFormat() { + return compoundFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java new file mode 100644 index 0000000000000..804128dfe97ae --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62Codec.java @@ -0,0 +1,81 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene62; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +import java.util.Objects; + +/** + * Implements the Lucene 6.2 index format. + * + * @deprecated Only for 6.2 back compat + */ +@Deprecated +public class Lucene62Codec extends BWCCodec { + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene62SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final StoredFieldsFormat storedFieldsFormat; + + public Lucene62Codec() { + this(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + public Lucene62Codec(Lucene50StoredFieldsFormat.Mode mode) { + super("Lucene62"); + this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode)); + } + + @Override + public final StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public final LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public final CompoundFormat compoundFormat() { + return compoundFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java new file mode 100644 index 0000000000000..d2cef04d59113 --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene62/Lucene62SegmentInfoFormat.java @@ -0,0 +1,251 @@ +/* + * @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene62; + +import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.Version; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +/** + * Lucene 6.2 Segment info format. + * @deprecated Only for reading old 6.2+ segments + */ +@Deprecated +public class Lucene62SegmentInfoFormat extends SegmentInfoFormat { + + public Lucene62SegmentInfoFormat() {} + + @Override + public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException { + final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene62SegmentInfoFormat.SI_EXTENSION); + try (ChecksumIndexInput input = EndiannessReverserUtil.openChecksumInput(dir, fileName, context)) { + Throwable priorE = null; + SegmentInfo si = null; + try { + int format = CodecUtil.checkIndexHeader( + input, + Lucene62SegmentInfoFormat.CODEC_NAME, + Lucene62SegmentInfoFormat.VERSION_START, + Lucene62SegmentInfoFormat.VERSION_CURRENT, + segmentID, + "" + ); + final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt()); + + final int docCount = input.readInt(); + if (docCount < 0) { + throw new CorruptIndexException("invalid docCount: " + docCount, input); + } + final boolean isCompoundFile = input.readByte() == SegmentInfo.YES; + + final Map diagnostics = input.readMapOfStrings(); + final Set files = input.readSetOfStrings(); + final Map attributes = input.readMapOfStrings(); + + int numSortFields = input.readVInt(); + Sort indexSort; + if (numSortFields > 0) { + SortField[] sortFields = new SortField[numSortFields]; + for (int i = 0; i < numSortFields; i++) { + String fieldName = input.readString(); + int sortTypeID = input.readVInt(); + SortField.Type sortType; + SortedSetSelector.Type sortedSetSelector = null; + SortedNumericSelector.Type sortedNumericSelector = null; + switch (sortTypeID) { + case 0: + sortType = SortField.Type.STRING; + break; + case 1: + sortType = SortField.Type.LONG; + break; + case 2: + sortType = SortField.Type.INT; + break; + case 3: + sortType = SortField.Type.DOUBLE; + break; + case 4: + sortType = SortField.Type.FLOAT; + break; + case 5: + sortType = SortField.Type.STRING; + byte selector = input.readByte(); + if (selector == 0) { + sortedSetSelector = SortedSetSelector.Type.MIN; + } else if (selector == 1) { + sortedSetSelector = SortedSetSelector.Type.MAX; + } else if (selector == 2) { + sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN; + } else if (selector == 3) { + sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX; + } else { + throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input); + } + break; + case 6: + byte type = input.readByte(); + if (type == 0) { + sortType = SortField.Type.LONG; + } else if (type == 1) { + sortType = SortField.Type.INT; + } else if (type == 2) { + sortType = SortField.Type.DOUBLE; + } else if (type == 3) { + sortType = SortField.Type.FLOAT; + } else { + throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input); + } + byte numericSelector = input.readByte(); + if (numericSelector == 0) { + sortedNumericSelector = SortedNumericSelector.Type.MIN; + } else if (numericSelector == 1) { + sortedNumericSelector = SortedNumericSelector.Type.MAX; + } else { + throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input); + } + break; + default: + throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input); + } + byte b = input.readByte(); + boolean reverse; + if (b == 0) { + reverse = true; + } else if (b == 1) { + reverse = false; + } else { + throw new CorruptIndexException("invalid index sort reverse: " + b, input); + } + + if (sortedSetSelector != null) { + sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector); + } else if (sortedNumericSelector != null) { + sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector); + } else { + sortFields[i] = new SortField(fieldName, sortType, reverse); + } + + Object missingValue; + b = input.readByte(); + if (b == 0) { + missingValue = null; + } else { + switch (sortType) { + case STRING: + if (b == 1) { + missingValue = SortField.STRING_LAST; + } else if (b == 2) { + missingValue = SortField.STRING_FIRST; + } else { + throw new CorruptIndexException("invalid missing value flag: " + b, input); + } + break; + case LONG: + if (b != 1) { + throw new CorruptIndexException("invalid missing value flag: " + b, input); + } + missingValue = input.readLong(); + break; + case INT: + if (b != 1) { + throw new CorruptIndexException("invalid missing value flag: " + b, input); + } + missingValue = input.readInt(); + break; + case DOUBLE: + if (b != 1) { + throw new CorruptIndexException("invalid missing value flag: " + b, input); + } + missingValue = Double.longBitsToDouble(input.readLong()); + break; + case FLOAT: + if (b != 1) { + throw new CorruptIndexException("invalid missing value flag: " + b, input); + } + missingValue = Float.intBitsToFloat(input.readInt()); + break; + default: + throw new AssertionError("unhandled sortType=" + sortType); + } + } + if (missingValue != null) { + sortFields[i].setMissingValue(missingValue); + } + } + indexSort = new Sort(sortFields); + } else if (numSortFields < 0) { + throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input); + } else { + indexSort = null; + } + + si = new SegmentInfo( + dir, + version, + null, + segment, + docCount, + isCompoundFile, + null, + diagnostics, + segmentID, + attributes, + indexSort + ); + si.setFiles(files); + } catch (Throwable exception) { + priorE = exception; + } finally { + CodecUtil.checkFooter(input, priorE); + } + return si; + } + } + + @Override + public void write(Directory dir, SegmentInfo info, IOContext ioContext) { + throw new UnsupportedOperationException("This format can only be used for reading"); + } + + /** File extension used to store {@link SegmentInfo}. */ + public static final String SI_EXTENSION = "si"; + static final String CODEC_NAME = "Lucene62SegmentInfo"; + static final int VERSION_START = 0; + static final int VERSION_MULTI_VALUED_SORT = 1; + static final int VERSION_CURRENT = VERSION_MULTI_VALUED_SORT; +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java new file mode 100644 index 0000000000000..a69622928518c --- /dev/null +++ b/x-pack/plugin/old-lucene-versions/src/main/java/org/elasticsearch/xpack/lucene/bwc/codecs/lucene70/BWCLucene70Codec.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + * + * Modifications copyright (C) 2021 Elasticsearch B.V. + */ + +package org.elasticsearch.xpack.lucene.bwc.codecs.lucene70; + +import org.apache.lucene.backward_codecs.lucene50.Lucene50CompoundFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50LiveDocsFormat; +import org.apache.lucene.backward_codecs.lucene50.Lucene50StoredFieldsFormat; +import org.apache.lucene.backward_codecs.lucene60.Lucene60FieldInfosFormat; +import org.apache.lucene.backward_codecs.lucene70.Lucene70SegmentInfoFormat; +import org.apache.lucene.codecs.CompoundFormat; +import org.apache.lucene.codecs.FieldInfosFormat; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.elasticsearch.xpack.lucene.bwc.codecs.BWCCodec; + +public class BWCLucene70Codec extends BWCCodec { + + private final FieldInfosFormat fieldInfosFormat = wrap(new Lucene60FieldInfosFormat()); + private final SegmentInfoFormat segmentInfosFormat = wrap(new Lucene70SegmentInfoFormat()); + private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat(); + private final CompoundFormat compoundFormat = new Lucene50CompoundFormat(); + private final StoredFieldsFormat storedFieldsFormat; + + public BWCLucene70Codec() { + super("BWCLucene70Codec"); + storedFieldsFormat = new Lucene50StoredFieldsFormat(Lucene50StoredFieldsFormat.Mode.BEST_SPEED); + } + + @Override + public FieldInfosFormat fieldInfosFormat() { + return fieldInfosFormat; + } + + @Override + public SegmentInfoFormat segmentInfoFormat() { + return segmentInfosFormat; + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public LiveDocsFormat liveDocsFormat() { + return liveDocsFormat; + } + + @Override + public CompoundFormat compoundFormat() { + return compoundFormat; + } +} diff --git a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 587144572c025..6e5205d664f2d 100644 --- a/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/x-pack/plugin/old-lucene-versions/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -5,4 +5,6 @@ # 2.0. # -org.elasticsearch.xpack.lucene.bwc.BWCLucene70Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene70.BWCLucene70Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene62.Lucene62Codec +org.elasticsearch.xpack.lucene.bwc.codecs.lucene60.Lucene60Codec diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java index 1bf7001cb846b..2be08e7c36766 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/BaseSearchableSnapshotIndexInput.java @@ -210,7 +210,7 @@ protected InputStream openSlice(int slice) throws IOException { */ private int getPartNumberForPosition(long position) { ensureValidPosition(position); - final int part = Math.toIntExact(position / fileInfo.partSize().getBytes()); + final int part = fileInfo.numberOfParts() == 1 ? 0 : Math.toIntExact(position / fileInfo.partSize().getBytes()); assert part <= fileInfo.numberOfParts() : "part number [" + part + "] exceeds number of parts: " + fileInfo.numberOfParts(); assert part >= 0 : "part number [" + part + "] is negative"; return part; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java index 4680cbb6a5879..c0f112accfd6b 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/store/input/CachedBlobContainerIndexInput.java @@ -102,8 +102,8 @@ private CachedBlobContainerIndexInput( @Override protected long getDefaultRangeSize() { - return (context != CACHE_WARMING_CONTEXT) - ? (directory.isRecoveryFinalized() ? defaultRangeSize : recoveryRangeSize) + return (context != CACHE_WARMING_CONTEXT) ? (directory.isRecoveryFinalized() ? defaultRangeSize : recoveryRangeSize) + : fileInfo.numberOfParts() == 1 ? Long.MAX_VALUE : fileInfo.partSize().getBytes(); } diff --git a/x-pack/qa/repository-old-versions/build.gradle b/x-pack/qa/repository-old-versions/build.gradle index 76d999923f258..fce544839e21a 100644 --- a/x-pack/qa/repository-old-versions/build.gradle +++ b/x-pack/qa/repository-old-versions/build.gradle @@ -71,11 +71,15 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { def testClusterProvider = testClusters.register(clusterName) { testDistribution = 'DEFAULT' + setting 'path.repo', repoLocation setting 'xpack.license.self_generated.type', 'trial' setting 'xpack.security.enabled', 'true' user username: 'admin', password: 'admin-password', role: 'superuser' + + setting 'xpack.searchable.snapshot.shared_cache.size', '16MB' + setting 'xpack.searchable.snapshot.shared_cache.region_size', '256KB' } TaskProvider fixture = tasks.register("oldES${versionNoDots}Fixture", AntFixture) { diff --git a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java index 6f547bb0829df..5e6b44b0243df 100644 --- a/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java +++ b/x-pack/qa/repository-old-versions/src/test/java/org/elasticsearch/oldrepos/OldRepositoryAccessIT.java @@ -163,8 +163,7 @@ public void testOldRepoAccess() throws IOException { assertThat(snapshotStatus.getStats().getTotalSize(), greaterThan(0L)); assertThat(snapshotStatus.getStats().getTotalFileCount(), greaterThan(0)); - // so far only added basic infrastructure for reading 6.0.0+ indices - if (Build.CURRENT.isSnapshot() && oldVersion.onOrAfter(Version.fromString("6.0.0"))) { + if (Build.CURRENT.isSnapshot()) { // restore / mount and check whether searches work restoreMountAndVerify(numDocs, expectedIds, client, numberOfShards); @@ -173,7 +172,14 @@ public void testOldRepoAccess() throws IOException { client.indices().close(new CloseIndexRequest("restored_test"), RequestOptions.DEFAULT).isShardsAcknowledged() ); assertTrue( - client.indices().close(new CloseIndexRequest("mounted_test"), RequestOptions.DEFAULT).isShardsAcknowledged() + client.indices() + .close(new CloseIndexRequest("mounted_full_copy_test"), RequestOptions.DEFAULT) + .isShardsAcknowledged() + ); + assertTrue( + client.indices() + .close(new CloseIndexRequest("mounted_shared_cache_test"), RequestOptions.DEFAULT) + .isShardsAcknowledged() ); // restore / mount again @@ -208,11 +214,26 @@ private void restoreMountAndVerify(int numDocs, Set expectedIds, RestHig // run a search against the index assertDocs("restored_test", numDocs, expectedIds, client); - // mount as searchable snapshot + // mount as full copy searchable snapshot RestoreSnapshotResponse mountSnapshotResponse = client.searchableSnapshots() .mountSnapshot( new MountSnapshotRequest("testrepo", "snap1", "test").storage(MountSnapshotRequest.Storage.FULL_COPY) - .renamedIndex("mounted_test") + .renamedIndex("mounted_full_copy_test") + .waitForCompletion(true), + RequestOptions.DEFAULT + ); + assertNotNull(mountSnapshotResponse.getRestoreInfo()); + assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().totalShards()); + assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards()); + + // run a search against the index + assertDocs("mounted_full_copy_test", numDocs, expectedIds, client); + + // mount as shared cache searchable snapshot + mountSnapshotResponse = client.searchableSnapshots() + .mountSnapshot( + new MountSnapshotRequest("testrepo", "snap1", "test").storage(MountSnapshotRequest.Storage.SHARED_CACHE) + .renamedIndex("mounted_shared_cache_test") .waitForCompletion(true), RequestOptions.DEFAULT ); @@ -221,7 +242,7 @@ private void restoreMountAndVerify(int numDocs, Set expectedIds, RestHig assertEquals(numberOfShards, mountSnapshotResponse.getRestoreInfo().successfulShards()); // run a search against the index - assertDocs("mounted_test", numDocs, expectedIds, client); + assertDocs("mounted_shared_cache_test", numDocs, expectedIds, client); } @SuppressWarnings("removal")