Skip to content

Add translog encryption #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Aug 28, 2025
Merged

Add translog encryption #39

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ebfafec
Add translog encryption
Jul 27, 2025
be4f23d
Cleanup unused functions
Aug 5, 2025
acbea36
Minor changes
Aug 6, 2025
5983d6f
Use imports rather than classnames
Aug 11, 2025
98f2a35
Fix forbidden api checks
Aug 12, 2025
3092dab
Use imports for constants
Aug 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 2 additions & 42 deletions src/main/java/org/opensearch/index/store/CryptoEngineFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.apache.logging.log4j.Logger;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.opensearch.index.codec.CodecService;
import org.opensearch.index.engine.Engine;
import org.opensearch.index.engine.EngineConfig;
import org.opensearch.index.engine.EngineFactory;
Expand Down Expand Up @@ -47,37 +46,9 @@ public Engine newReadWriteEngine(EngineConfig config) {

// Create new engine config by copying all fields from existing config
// but replace the translog factory with our crypto version
EngineConfig cryptoConfig = new EngineConfig.Builder()
.shardId(config.getShardId())
.threadPool(config.getThreadPool())
.indexSettings(config.getIndexSettings())
.warmer(config.getWarmer())
.store(config.getStore())
.mergePolicy(config.getMergePolicy())
.analyzer(config.getAnalyzer())
.similarity(config.getSimilarity())
.codecService(getCodecService(config))
.eventListener(config.getEventListener())
.queryCache(config.getQueryCache())
.queryCachingPolicy(config.getQueryCachingPolicy())
.translogConfig(config.getTranslogConfig())
.translogDeletionPolicyFactory(config.getCustomTranslogDeletionPolicyFactory())
.flushMergesAfter(config.getFlushMergesAfter())
.externalRefreshListener(config.getExternalRefreshListener())
.internalRefreshListener(config.getInternalRefreshListener())
.indexSort(config.getIndexSort())
.circuitBreakerService(config.getCircuitBreakerService())
.globalCheckpointSupplier(config.getGlobalCheckpointSupplier())
.retentionLeasesSupplier(config.retentionLeasesSupplier())
.primaryTermSupplier(config.getPrimaryTermSupplier())
.tombstoneDocSupplier(config.getTombstoneDocSupplier())
.readOnlyReplica(config.isReadOnlyReplica())
.startedPrimarySupplier(config.getStartedPrimarySupplier())
EngineConfig cryptoConfig = config
.toBuilder()
Comment on lines +49 to +50
Copy link
Collaborator

@kumargu kumargu Aug 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

super!

.translogFactory(cryptoTranslogFactory) // <- Replace with our crypto factory
.leafSorter(config.getLeafSorter())
.documentMapperForTypeSupplier(config.getDocumentMapperForTypeSupplier())
.indexReaderWarmer(config.getIndexReaderWarmer())
.clusterApplierService(config.getClusterApplierService())
.build();

// Return the default engine with crypto-enabled translog
Expand Down Expand Up @@ -108,15 +79,4 @@ private KeyIvResolver createTranslogKeyIvResolver(EngineConfig config) throws IO
);
}

/**
* Helper method to create a CodecService from existing EngineConfig.
* Since EngineConfig doesn't expose CodecService directly, we create a new one
* using the same IndexSettings.
*/
private CodecService getCodecService(EngineConfig config) {
// Create a CodecService using the same IndexSettings as the original config
// We pass null for MapperService and use a simple logger since we're just
// preserving the existing codec behavior
return new CodecService(null, config.getIndexSettings(), logger);
}
}
13 changes: 3 additions & 10 deletions src/main/java/org/opensearch/index/translog/TranslogChunkManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,6 @@ public class TranslogChunkManager {
// Header size - calculated exactly using TranslogHeader.headerSizeInBytes()
private volatile int actualHeaderSize = -1;

// TranslogHeader constants replicated to avoid cross-classloader access
private static final String TRANSLOG_CODEC = "translog";
private static final int VERSION_PRIMARY_TERM = 3;
private static final int CURRENT_VERSION = VERSION_PRIMARY_TERM;

/**
* Helper class for chunk position mapping
*/
Expand Down Expand Up @@ -118,15 +113,13 @@ public int determineHeaderSize() {
* @return the header size in bytes
*/
private static int calculateTranslogHeaderSize(String translogUUID) {
// Replicate: headerSizeInBytes(CURRENT_VERSION, new BytesRef(translogUUID).length)
int uuidLength = translogUUID.getBytes(StandardCharsets.UTF_8).length;

// Replicate the internal calculation
int size = CodecUtil.headerLength(TRANSLOG_CODEC); // Lucene codec header
// Calculate header size using official TranslogHeader constants
int size = CodecUtil.headerLength(TranslogHeader.TRANSLOG_CODEC); // Lucene codec header
size += Integer.BYTES + uuidLength; // uuid length field + uuid bytes

// VERSION_PRIMARY_TERM = 3, CURRENT_VERSION = 3
if (CURRENT_VERSION >= VERSION_PRIMARY_TERM) {
if (TranslogHeader.CURRENT_VERSION >= TranslogHeader.VERSION_PRIMARY_TERM) {
size += Long.BYTES; // primary term
size += Integer.BYTES; // checksum
}
Expand Down