Skip to content

Adding support for composite index #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 11, 2025
Merged

Adding support for composite index #263

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
999e87c
Adding support for composite index
bharath-techie Jul 7, 2025
361acb6
fetching default codec
bharath-techie Jul 7, 2025
30e5f34
Addressing comment
bharath-techie Jul 9, 2025
11a5206
addressing comment
bharath-techie Jul 10, 2025
5fdb203
Switch main to 3.2.0-SNAPSHOT
reta Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import java.util.Arrays;
import java.util.Map;
import java.util.function.Supplier;
import java.util.stream.Stream;

import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING;
Expand Down Expand Up @@ -52,6 +53,8 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
super(mapperService, indexSettings, logger);
int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING);
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
// Delegate to OpenSearch "default" codec
Supplier<Codec> defaultCodecSupplier = () -> super.defaultCodec();
Copy link
Collaborator

@reta reta Jul 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe you don't need defaultCodecSupplier anymore, just use this::defaultCodec

if (mapperService == null) {
codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel));
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel));
Expand All @@ -69,18 +72,28 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin
);
}
} else {
codecs.put(ZSTD_CODEC, new Zstd101Codec(mapperService, logger, compressionLevel));
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(mapperService, logger, compressionLevel));
codecs.put(ZSTD_CODEC, new Zstd101Codec(compressionLevel, defaultCodecSupplier));
codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict101Codec(compressionLevel, defaultCodecSupplier));
if (QatZipperFactory.isQatAvailable()) {
codecs.put(QAT_LZ4_CODEC, new QatLz4101Codec(mapperService, logger, compressionLevel, () -> {
return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
}));
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(mapperService, logger, compressionLevel, () -> {
return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
}));
codecs.put(QAT_ZSTD_CODEC, new QatZstd101Codec(mapperService, logger, compressionLevel, () -> {
codecs.put(
QAT_LZ4_CODEC,
new QatLz4101Codec(
compressionLevel,
() -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); },
defaultCodecSupplier
)
);
codecs.put(QAT_DEFLATE_CODEC, new QatDeflate101Codec(compressionLevel, () -> {
return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING);
}));
}, defaultCodecSupplier));
codecs.put(
QAT_ZSTD_CODEC,
new QatZstd101Codec(
compressionLevel,
() -> { return indexSettings.getValue(INDEX_CODEC_QAT_MODE_SETTING); },
defaultCodecSupplier
)
);
}
}
this.codecs = codecs.immutableMap();
Expand Down
12 changes: 5 additions & 7 deletions src/main/java/org/opensearch/index/codec/customcodecs/Lucene101CustomCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;

import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL;

Expand Down Expand Up @@ -94,11 +93,10 @@ public Lucene101CustomCodec(Mode mode, int compressionLevel) {
*
* @param mode The compression codec (ZSTD or ZSTDNODICT).
* @param compressionLevel The compression level.
* @param mapperService The mapper service.
* @param logger The logger.
* @param defaultCodecSupplier Default codec supplier
*/
public Lucene101CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
public Lucene101CustomCodec(Mode mode, int compressionLevel, Supplier<Codec> defaultCodecSupplier) {
super(mode.getCodec(), defaultCodecSupplier.get());
this.storedFieldsFormat = new Lucene101CustomStoredFieldsFormat(mode, compressionLevel);
}

Expand Down
24 changes: 7 additions & 17 deletions src/main/java/org/opensearch/index/codec/customcodecs/Lucene101QatCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;
Expand Down Expand Up @@ -105,12 +103,11 @@ public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier<QatZipper.Mod
* lucene_default, and best_compression.
*
* @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD).
* @param defaultCodecSupplier default opensearch codec supplier
* @param compressionLevel The compression level.
* @param mapperService The mapper service.
* @param logger The logger.
*/
public Lucene101QatCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) {
super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
public Lucene101QatCodec(Mode mode, Supplier<Codec> defaultCodecSupplier, int compressionLevel) {
super(mode.getCodec(), defaultCodecSupplier.get());
this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel);
}

Expand All @@ -121,18 +118,11 @@ public Lucene101QatCodec(Mode mode, int compressionLevel, MapperService mapperSe
*
* @param mode The compression codec (QAT_LZ4, QAT_DEFLATE, or QAT_ZSTD).
* @param compressionLevel The compression level.
* @param mapperService The mapper service.
* @param logger The logger.
* @param supplier supplier for QAT mode.
* @param defaultCodecSupplier default opensearch codec supplier
*/
public Lucene101QatCodec(
Mode mode,
int compressionLevel,
MapperService mapperService,
Logger logger,
Supplier<QatZipper.Mode> supplier
) {
super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene101Codec.Mode.BEST_SPEED, mapperService, logger));
public Lucene101QatCodec(Mode mode, int compressionLevel, Supplier<QatZipper.Mode> supplier, Supplier<Codec> defaultCodecSupplier) {
super(mode.getCodec(), defaultCodecSupplier.get());
this.storedFieldsFormat = new Lucene101QatStoredFieldsFormat(mode, compressionLevel, supplier);
}

Expand Down
16 changes: 6 additions & 10 deletions src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate101Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.opensearch.common.settings.Setting;
import org.opensearch.index.codec.CodecAliases;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;
Expand Down Expand Up @@ -55,24 +54,21 @@ public QatDeflate101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplie
/**
* Creates a new QatDeflate101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param defaultCodecSupplier default opensearch codec supplier
* @param compressionLevel The compression level.
*/
public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel) {
super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger);
public QatDeflate101Codec(Supplier<Codec> defaultCodecSupplier, int compressionLevel) {
super(Mode.QAT_DEFLATE, defaultCodecSupplier, compressionLevel);
}

/**
* Creates a new QatDeflate101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param compressionLevel The compression level.
* @param supplier supplier for QAT acceleration mode.
*/
public QatDeflate101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier<QatZipper.Mode> supplier) {
super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier);
public QatDeflate101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplier, Supplier<Codec> defaultCodecSupplier) {
super(Mode.QAT_DEFLATE, compressionLevel, supplier, defaultCodecSupplier);
}

/** The name for this codec. */
Expand Down
17 changes: 7 additions & 10 deletions src/main/java/org/opensearch/index/codec/customcodecs/QatLz4101Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.opensearch.common.settings.Setting;
import org.opensearch.index.codec.CodecAliases;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;
Expand Down Expand Up @@ -55,24 +54,22 @@ public QatLz4101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplier) {
/**
* Creates a new QatLz4101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param defaultCodecSupplier default opensearch codec supplier
* @param compressionLevel The compression level.
*/
public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel) {
super(Mode.QAT_LZ4, compressionLevel, mapperService, logger);
public QatLz4101Codec(Supplier<Codec> defaultCodecSupplier, int compressionLevel) {
super(Mode.QAT_LZ4, defaultCodecSupplier, compressionLevel);
}

/**
* Creates a new QatLz4101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param compressionLevel The compression level.
* @param supplier supplier for QAT acceleration mode.
* @param defaultCodecSupplier default opensearch codec supplier
*/
public QatLz4101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier<QatZipper.Mode> supplier) {
super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier);
public QatLz4101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplier, Supplier<Codec> defaultCodecSupplier) {
super(Mode.QAT_LZ4, compressionLevel, supplier, defaultCodecSupplier);
}

/** The name for this codec. */
Expand Down
17 changes: 7 additions & 10 deletions src/main/java/org/opensearch/index/codec/customcodecs/QatZstd101Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.opensearch.common.settings.Setting;
import org.opensearch.index.codec.CodecAliases;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;
Expand Down Expand Up @@ -55,24 +54,22 @@ public QatZstd101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplier)
/**
* Creates a new QatZstd101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param defaultCodecSupplier default opensearch codec supplier
* @param compressionLevel The compression level.
*/
public QatZstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) {
super(Mode.QAT_ZSTD, compressionLevel, mapperService, logger);
public QatZstd101Codec(Supplier<Codec> defaultCodecSupplier, int compressionLevel) {
super(Mode.QAT_ZSTD, defaultCodecSupplier, compressionLevel);
}

/**
* Creates a new QatZstd101Codec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param compressionLevel The compression level.
* @param supplier supplier for QAT acceleration mode.
* @param defaultCodecSupplier default opensearch codec supplier
*/
public QatZstd101Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier<QatZipper.Mode> supplier) {
super(Mode.QAT_ZSTD, compressionLevel, mapperService, logger, supplier);
public QatZstd101Codec(int compressionLevel, Supplier<QatZipper.Mode> supplier, Supplier<Codec> defaultCodecSupplier) {
super(Mode.QAT_ZSTD, compressionLevel, supplier, defaultCodecSupplier);
}

/** The name for this codec. */
Expand Down
11 changes: 5 additions & 6 deletions src/main/java/org/opensearch/index/codec/customcodecs/Zstd101Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.opensearch.common.settings.Setting;
import org.opensearch.index.codec.CodecAliases;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;

import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL;

Expand All @@ -42,12 +42,11 @@ public Zstd101Codec(int compressionLevel) {
/**
* Creates a new ZstdCodec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param compressionLevel The compression level.
*
*/
public Zstd101Codec(MapperService mapperService, Logger logger, int compressionLevel) {
super(Mode.ZSTD, compressionLevel, mapperService, logger);
public Zstd101Codec(int compressionLevel, Supplier<Codec> defaultCodecSupplier) {
super(Mode.ZSTD, compressionLevel, defaultCodecSupplier);
}

/** The name for this codec. */
Expand Down
11 changes: 5 additions & 6 deletions src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict101Codec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@

package org.opensearch.index.codec.customcodecs;

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.opensearch.common.settings.Setting;
import org.opensearch.index.codec.CodecAliases;
import org.opensearch.index.codec.CodecSettings;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.mapper.MapperService;

import java.util.Set;
import java.util.function.Supplier;

import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL;

Expand All @@ -39,12 +39,11 @@ public ZstdNoDict101Codec(int compressionLevel) {
/**
* Creates a new ZstdNoDictCodec instance.
*
* @param mapperService The mapper service.
* @param logger The logger.
* @param compressionLevel The compression level.
* @param defaultCodecSupplier default opensearch codec supplier
*/
public ZstdNoDict101Codec(MapperService mapperService, Logger logger, int compressionLevel) {
super(Mode.ZSTD_NO_DICT, compressionLevel, mapperService, logger);
public ZstdNoDict101Codec(int compressionLevel, Supplier<Codec> defaultCodecSupplier) {
super(Mode.ZSTD_NO_DICT, compressionLevel, defaultCodecSupplier);
}

/** The name for this codec. */
Expand Down
Loading
Loading