Skip to content

Get from translog fails with large dense_vector #104700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jimczi merged 2 commits into from
Jan 30, 2024
Merged

Get from translog fails with large dense_vector #104700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d026341
Get from translog fails with large dense_vector
jimczi Jan 24, 2024
b7c3629
Merge branch 'main' into get_vector_from_translog
elasticmachine Jan 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,7 +811,7 @@ private GetResult getFromTranslog(
index,
mappingLookup,
documentParser,
config().getAnalyzer(),
config(),
translogInMemorySegmentsCount::incrementAndGet
);
final Engine.Searcher searcher = new Engine.Searcher(
Expand Down
16 changes: 9 additions & 7 deletions server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

package org.elasticsearch.index.engine;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.ByteVectorValues;
Expand Down Expand Up @@ -83,10 +82,10 @@ final class TranslogDirectoryReader extends DirectoryReader {
Translog.Index operation,
MappingLookup mappingLookup,
DocumentParser documentParser,
Analyzer analyzer,
EngineConfig engineConfig,
Runnable onSegmentCreated
) throws IOException {
this(new TranslogLeafReader(shardId, operation, mappingLookup, documentParser, analyzer, onSegmentCreated));
this(new TranslogLeafReader(shardId, operation, mappingLookup, documentParser, engineConfig, onSegmentCreated));
}

private TranslogDirectoryReader(TranslogLeafReader leafReader) throws IOException {
Expand Down Expand Up @@ -205,7 +204,7 @@ private static class TranslogLeafReader extends LeafReader {
private final Translog.Index operation;
private final MappingLookup mappingLookup;
private final DocumentParser documentParser;
private final Analyzer analyzer;
private final EngineConfig engineConfig;
private final Directory directory;
private final Runnable onSegmentCreated;

Expand All @@ -217,14 +216,14 @@ private static class TranslogLeafReader extends LeafReader {
Translog.Index operation,
MappingLookup mappingLookup,
DocumentParser documentParser,
Analyzer analyzer,
EngineConfig engineConfig,
Runnable onSegmentCreated
) {
this.shardId = shardId;
this.operation = operation;
this.mappingLookup = mappingLookup;
this.documentParser = documentParser;
this.analyzer = analyzer;
this.engineConfig = engineConfig;
this.onSegmentCreated = onSegmentCreated;
this.directory = new ByteBuffersDirectory();
this.uid = Uid.encodeId(operation.id());
Expand Down Expand Up @@ -264,7 +263,10 @@ private LeafReader createInMemoryLeafReader() {

parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm());
parsedDocs.version().setLongValue(operation.version());
final IndexWriterConfig writeConfig = new IndexWriterConfig(analyzer).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
// To guarantee indexability, we configure the analyzer and codec using the main engine configuration
final IndexWriterConfig writeConfig = new IndexWriterConfig(engineConfig.getAnalyzer()).setOpenMode(
IndexWriterConfig.OpenMode.CREATE
).setCodec(engineConfig.getCodec());
try (IndexWriter writer = new IndexWriter(directory, writeConfig)) {
writer.addDocument(parsedDocs.rootDoc());
final DirectoryReader reader = open(writer);
Expand Down
66 changes: 53 additions & 13 deletions server/src/test/java/org/elasticsearch/index/shard/ShardGetServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.elasticsearch.xcontent.XContentType;

import java.io.IOException;
import java.util.Arrays;
import java.util.function.LongSupplier;

import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
Expand Down Expand Up @@ -114,6 +115,20 @@ public void testGetFromTranslogWithSyntheticSource() throws IOException {
runGetFromTranslogWithOptions(docToIndex, sourceOptions, expectedFetchedSource, "\"long\"", 7L, true);
}

public void testGetFromTranslogWithDenseVector() throws IOException {
float[] vector = new float[2048];
for (int i = 0; i < vector.length; i++) {
vector[i] = randomFloat();
}
String docToIndex = Strings.format("""
{
"bar": %s,
"foo": "foo"
}
""", Arrays.toString(vector));
runGetFromTranslogWithOptions(docToIndex, "\"enabled\": true", docToIndex, "\"text\"", "foo", "\"dense_vector\"", false);
Copy link
Copy Markdown
Contributor

@henningandersen henningandersen Jan 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be nice to also validate that we can read out the right dense vector value, though with the way it is implemented, it should hardly be an issue.

Copy link
Copy Markdown
Contributor Author

@jimczi jimczi Jan 26, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test already checks that the retrieved source is identical: source
Do you think it's not enough?

}

private void runGetFromTranslogWithOptions(
String docToIndex,
String sourceOptions,
Expand All @@ -122,23 +137,48 @@ private void runGetFromTranslogWithOptions(
Object expectedFooVal,
boolean sourceOnlyFetchCreatesInMemoryReader
) throws IOException {
IndexMetadata metadata = IndexMetadata.builder("test").putMapping(Strings.format("""
{
"properties": {
"foo": {
"type": %s,
"store": true
},
"bar": { "type": %s }
},
"_source": { %s }
}
}""", fieldType, fieldType, sourceOptions)).settings(indexSettings(IndexVersion.current(), 1, 1)).primaryTerm(0, 1).build();
runGetFromTranslogWithOptions(
docToIndex,
sourceOptions,
expectedResult,
fieldType,
expectedFooVal,
fieldType,
sourceOnlyFetchCreatesInMemoryReader
);
}

private void runGetFromTranslogWithOptions(
String docToIndex,
String sourceOptions,
String expectedResult,
String fieldTypeFoo,
Object expectedFooVal,
String fieldTypeBar,
boolean sourceOnlyFetchCreatesInMemoryReader
) throws IOException {
IndexMetadata metadata = IndexMetadata.builder("test")
.putMapping(Strings.format("""
{
"properties": {
"foo": {
"type": %s,
"store": true
},
"bar": { "type": %s }
},
"_source": { %s }
}
}""", fieldTypeFoo, fieldTypeBar, sourceOptions))
.settings(indexSettings(IndexVersion.current(), 1, 1))
.primaryTerm(0, 1)
.build();
IndexShard primary = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, EngineTestCase.randomReaderWrapper());
recoverShardFromStore(primary);
LongSupplier translogInMemorySegmentCount = ((InternalEngine) primary.getEngine()).translogInMemorySegmentsCount::get;
long translogInMemorySegmentCountExpected = 0;
indexDoc(primary, "test", "0", docToIndex);
Engine.IndexResult res = indexDoc(primary, "test", "0", docToIndex);
assertTrue(res.isCreated());
assertTrue(primary.getEngine().refreshNeeded());
GetResult testGet = primary.getService().getForUpdate("0", UNASSIGNED_SEQ_NO, UNASSIGNED_PRIMARY_TERM);
assertFalse(testGet.getFields().containsKey(RoutingFieldMapper.NAME));
Expand Down