Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LUCENE-9955: Reduced state of stored fields readers. #137

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ public final class Lucene50CompressingStoredFieldsReader extends StoredFieldsRea
private final int chunkSize;
private final int packedIntsVersion;
private final CompressionMode compressionMode;
private final Decompressor decompressor;
private final int numDocs;
private final boolean merging;
private final BlockState state;
Expand All @@ -114,12 +113,11 @@ private Lucene50CompressingStoredFieldsReader(
this.chunkSize = reader.chunkSize;
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = reader.decompressor.clone();
this.numDocs = reader.numDocs;
this.numDirtyChunks = reader.numDirtyChunks;
this.numDirtyDocs = reader.numDirtyDocs;
this.merging = merging;
this.state = new BlockState();
this.state = merging ? new BlockState() : null;
this.closed = false;
}

Expand Down Expand Up @@ -171,9 +169,8 @@ public Lucene50CompressingStoredFieldsReader(
packedIntsVersion = fieldsStream.readVInt();
}

decompressor = compressionMode.newDecompressor();
this.merging = false;
this.state = new BlockState();
this.state = null;

// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
Expand Down Expand Up @@ -445,17 +442,10 @@ private class BlockState {
// the start pointer at which you can read the compressed documents
private long startPointer;

private final BytesRef spare;
private final BytesRef bytes;
private final BytesRef spare = new BytesRef();
private final BytesRef bytes = new BytesRef();

BlockState() {
if (merging) {
spare = new BytesRef();
bytes = new BytesRef();
} else {
spare = bytes = null;
}
}
private final Decompressor decompressor = compressionMode.newDecompressor();

boolean contains(int docID) {
return docID >= docBase && docID < docBase + chunkDocs;
Expand Down Expand Up @@ -609,13 +599,6 @@ SerializedDocument document(int docID) throws IOException {
final int totalLength = Math.toIntExact(offsets[chunkDocs]);
final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);

final BytesRef bytes;
if (merging) {
bytes = this.bytes;
} else {
bytes = new BytesRef();
}

final DataInput documentInput;
if (length == 0) {
// empty
Expand Down Expand Up @@ -690,6 +673,11 @@ public void skipBytes(long numBytes) throws IOException {
}

SerializedDocument document(int docID) throws IOException {
assert merging == (state != null);
BlockState state = this.state;
if (state == null) {
state = new BlockState();
}
if (state.contains(docID) == false) {
fieldsStream.seek(indexReader.getStartPointer(docID));
state.reset(docID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ public final class Lucene50CompressingTermVectorsReader extends TermVectorsReade
private final int version;
private final int packedIntsVersion;
private final CompressionMode compressionMode;
private final Decompressor decompressor;
private final int chunkSize;
private final int numDocs;
private boolean closed;
Expand All @@ -102,7 +101,6 @@ private Lucene50CompressingTermVectorsReader(Lucene50CompressingTermVectorsReade
this.indexReader = reader.indexReader.clone();
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = reader.decompressor.clone();
this.chunkSize = reader.chunkSize;
this.numDocs = reader.numDocs;
this.reader =
Expand Down Expand Up @@ -235,7 +233,6 @@ public Lucene50CompressingTermVectorsReader(
numDirtyChunks = numDirtyDocs = -1;
}

decompressor = compressionMode.newDecompressor();
this.reader =
new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);

Expand Down Expand Up @@ -689,6 +686,7 @@ public Fields get(int doc) throws IOException {

// decompress data
final BytesRef suffixBytes = new BytesRef();
final Decompressor decompressor = compressionMode.newDecompressor();
decompressor.decompress(
vectorsStream,
totalLen + totalPayloadLength,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.apache.lucene.util.BytesRef;

/** A decompressor. */
public abstract class Decompressor implements Cloneable {
public abstract class Decompressor {

/** Sole constructor, typically called from sub-classes. */
protected Decompressor() {}
Expand All @@ -41,7 +41,4 @@ protected Decompressor() {}
*/
public abstract void decompress(
DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException;

@Override
public abstract Decompressor clone();
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ public final class Lucene90CompressingStoredFieldsReader extends StoredFieldsRea
private final IndexInput fieldsStream;
private final int chunkSize;
private final CompressionMode compressionMode;
private final Decompressor decompressor;
private final int numDocs;
private final boolean merging;
private final BlockState state;
Expand All @@ -97,13 +96,12 @@ private Lucene90CompressingStoredFieldsReader(
this.maxPointer = reader.maxPointer;
this.chunkSize = reader.chunkSize;
this.compressionMode = reader.compressionMode;
this.decompressor = reader.decompressor.clone();
this.numDocs = reader.numDocs;
this.numChunks = reader.numChunks;
this.numDirtyChunks = reader.numDirtyChunks;
this.numDirtyDocs = reader.numDirtyDocs;
this.merging = merging;
this.state = new BlockState();
this.state = merging ? new BlockState() : null;
this.closed = false;
}

Expand Down Expand Up @@ -148,9 +146,8 @@ public Lucene90CompressingStoredFieldsReader(

chunkSize = metaIn.readVInt();

decompressor = compressionMode.newDecompressor();
this.merging = false;
this.state = new BlockState();
this.state = null;

// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
Expand Down Expand Up @@ -401,17 +398,10 @@ private class BlockState {
// the start pointer at which you can read the compressed documents
private long startPointer;

private final BytesRef spare;
private final BytesRef bytes;
private final BytesRef spare = new BytesRef();
private final BytesRef bytes = new BytesRef();

BlockState() {
if (merging) {
spare = new BytesRef();
bytes = new BytesRef();
} else {
spare = bytes = null;
}
}
private final Decompressor decompressor = compressionMode.newDecompressor();

boolean contains(int docID) {
return docID >= docBase && docID < docBase + chunkDocs;
Expand Down Expand Up @@ -522,13 +512,6 @@ SerializedDocument document(int docID) throws IOException {
final int totalLength = Math.toIntExact(offsets[chunkDocs]);
final int numStoredFields = Math.toIntExact(this.numStoredFields[index]);

final BytesRef bytes;
if (merging) {
bytes = this.bytes;
} else {
bytes = new BytesRef();
}

final DataInput documentInput;
if (length == 0) {
// empty
Expand Down Expand Up @@ -602,6 +585,11 @@ public void skipBytes(long numBytes) throws IOException {
}

SerializedDocument document(int docID) throws IOException {
assert merging == (state != null);
BlockState state = this.state;
if (state == null) {
state = new BlockState();
}
if (state.contains(docID) == false) {
fieldsStream.seek(indexReader.getStartPointer(docID));
state.reset(docID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ public final class Lucene90CompressingTermVectorsReader extends TermVectorsReade
private final int version;
private final int packedIntsVersion;
private final CompressionMode compressionMode;
private final Decompressor decompressor;
private final int chunkSize;
private final int numDocs;
private boolean closed;
Expand All @@ -99,7 +98,6 @@ private Lucene90CompressingTermVectorsReader(Lucene90CompressingTermVectorsReade
this.indexReader = reader.indexReader.clone();
this.packedIntsVersion = reader.packedIntsVersion;
this.compressionMode = reader.compressionMode;
this.decompressor = reader.decompressor.clone();
this.chunkSize = reader.chunkSize;
this.numDocs = reader.numDocs;
this.reader =
Expand Down Expand Up @@ -202,7 +200,6 @@ public Lucene90CompressingTermVectorsReader(
metaIn);
}

decompressor = compressionMode.newDecompressor();
this.reader =
new BlockPackedReaderIterator(vectorsStream, packedIntsVersion, PACKED_BLOCK_SIZE, 0);

Expand Down Expand Up @@ -651,6 +648,7 @@ public Fields get(int doc) throws IOException {

// decompress data
final BytesRef suffixBytes = new BytesRef();
final Decompressor decompressor = compressionMode.newDecompressor();
decompressor.decompress(
vectorsStream,
totalLen + totalPayloadLength,
Expand Down