Add translog encryption

build.gradle

@@ -11,7 +11,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.2.0-SNAPSHOT")
     }
 
     repositories {

src/internalClusterTest/java/org/opensearch/index/store/niofs/CryptoDirectoryIntegTestCases.java

@@ -4,10 +4,11 @@
  */
 package org.opensearch.index.store.niofs;
 
+import static org.hamcrest.Matchers.is;
+
 import java.util.Arrays;
 import java.util.Collection;
 
-import static org.hamcrest.Matchers.is;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
@@ -21,7 +22,6 @@
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
-
 public class CryptoDirectoryIntegTestCases extends OpenSearchIntegTestCase {
 
     @Override

src/main/java/org/opensearch/index/store/CryptoDirectoryPlugin.java

@@ -5,17 +5,23 @@
 package org.opensearch.index.store;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import org.opensearch.common.settings.Setting;
+import org.opensearch.index.IndexModule;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.engine.EngineFactory;
+import org.opensearch.plugins.EnginePlugin;
 import org.opensearch.plugins.IndexStorePlugin;
 import org.opensearch.plugins.Plugin;
 
 /**
  * A plugin that enables index level encryption and decryption.
  */
-public class CryptoDirectoryPlugin extends Plugin implements IndexStorePlugin {
+public class CryptoDirectoryPlugin extends Plugin implements IndexStorePlugin, EnginePlugin {
 
     /**
      * The default constructor.
@@ -37,6 +43,18 @@ public List<Setting<?>> getSettings() {
      */
     @Override
     public Map<String, DirectoryFactory> getDirectoryFactories() {
-        return java.util.Collections.singletonMap("cryptofs", new CryptoDirectoryFactory());
+        return Collections.singletonMap("cryptofs", new CryptoDirectoryFactory());
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
+        // Only provide our custom engine factory for cryptofs indices
+        if ("cryptofs".equals(indexSettings.getValue(IndexModule.INDEX_STORE_TYPE_SETTING))) {
+            return Optional.of(new CryptoEngineFactory());
+        }
+        return Optional.empty();
     }
 }

src/main/java/org/opensearch/index/store/CryptoEngineFactory.java

@@ -0,0 +1,122 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.index.store;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.engine.Engine;
+import org.opensearch.index.engine.EngineConfig;
+import org.opensearch.index.engine.EngineFactory;
+import org.opensearch.index.engine.InternalEngine;
+import org.opensearch.index.store.iv.DefaultKeyIvResolver;
+import org.opensearch.index.store.iv.KeyIvResolver;
+import org.opensearch.index.translog.CryptoTranslogFactory;
+
+/**
+ * A factory that creates engines with crypto-enabled translogs for cryptofs indices.
+ */
+public class CryptoEngineFactory implements EngineFactory {
+
+    private static final Logger logger = LogManager.getLogger(CryptoEngineFactory.class);
+
+    /**
+     * Default constructor.
+     */
+    public CryptoEngineFactory() {}
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public Engine newReadWriteEngine(EngineConfig config) {
+
+        try {
+            // Create a separate KeyIvResolver for translog encryption
+            KeyIvResolver keyIvResolver = createTranslogKeyIvResolver(config);
+
+            // Create the crypto translog factory using the same KeyIvResolver as the directory
+            CryptoTranslogFactory cryptoTranslogFactory = new CryptoTranslogFactory(keyIvResolver);
+
+            // Create new engine config by copying all fields from existing config
+            // but replace the translog factory with our crypto version
+            EngineConfig cryptoConfig = new EngineConfig.Builder()
+                .shardId(config.getShardId())
+                .threadPool(config.getThreadPool())
+                .indexSettings(config.getIndexSettings())
+                .warmer(config.getWarmer())
+                .store(config.getStore())
+                .mergePolicy(config.getMergePolicy())
+                .analyzer(config.getAnalyzer())
+                .similarity(config.getSimilarity())
+                .codecService(getCodecService(config))
+                .eventListener(config.getEventListener())
+                .queryCache(config.getQueryCache())
+                .queryCachingPolicy(config.getQueryCachingPolicy())
+                .translogConfig(config.getTranslogConfig())
+                .translogDeletionPolicyFactory(config.getCustomTranslogDeletionPolicyFactory())
+                .flushMergesAfter(config.getFlushMergesAfter())
+                .externalRefreshListener(config.getExternalRefreshListener())
+                .internalRefreshListener(config.getInternalRefreshListener())
+                .indexSort(config.getIndexSort())
+                .circuitBreakerService(config.getCircuitBreakerService())
+                .globalCheckpointSupplier(config.getGlobalCheckpointSupplier())
+                .retentionLeasesSupplier(config.retentionLeasesSupplier())
+                .primaryTermSupplier(config.getPrimaryTermSupplier())
+                .tombstoneDocSupplier(config.getTombstoneDocSupplier())
+                .readOnlyReplica(config.isReadOnlyReplica())
+                .startedPrimarySupplier(config.getStartedPrimarySupplier())
+                .translogFactory(cryptoTranslogFactory)  // <- Replace with our crypto factory
+                .leafSorter(config.getLeafSorter())
+                .documentMapperForTypeSupplier(config.getDocumentMapperForTypeSupplier())
+                .indexReaderWarmer(config.getIndexReaderWarmer())
+                .clusterApplierService(config.getClusterApplierService())
+                .build();
+
+            // Return the default engine with crypto-enabled translog
+            return new InternalEngine(cryptoConfig);
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to create crypto engine", e);
+        }
+    }
+
+    /**
+     * Create a separate KeyIvResolver for translog encryption.
+     */
+    private KeyIvResolver createTranslogKeyIvResolver(EngineConfig config) throws IOException {
+        // Create a separate key resolver for translog files
+
+        // Use the translog location for key storage
+        Path translogPath = config.getTranslogConfig().getTranslogPath();
+        Directory keyDirectory = FSDirectory.open(translogPath);
+
+        // Create crypto directory factory to get the key provider
+        CryptoDirectoryFactory directoryFactory = new CryptoDirectoryFactory();
+
+        // Create a dedicated key resolver for translog
+        return new DefaultKeyIvResolver(
+            keyDirectory,
+            config.getIndexSettings().getValue(CryptoDirectoryFactory.INDEX_CRYPTO_PROVIDER_SETTING),
+            directoryFactory.getKeyProvider(config.getIndexSettings())
+        );
+    }
+
+    /**
+     * Helper method to create a CodecService from existing EngineConfig.
+     * Since EngineConfig doesn't expose CodecService directly, we create a new one
+     * using the same IndexSettings.
+     */
+    private CodecService getCodecService(EngineConfig config) {
+        // Create a CodecService using the same IndexSettings as the original config
+        // We pass null for MapperService and use a simple logger since we're just
+        // preserving the existing codec behavior
+        return new CodecService(null, config.getIndexSettings(), logger);
+    }
+}

src/main/java/org/opensearch/index/store/cipher/AesGcmCipherFactory.java

@@ -1,22 +1,19 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
  */
-
 package org.opensearch.index.store.cipher;
 
-import javax.crypto.Cipher;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.spec.GCMParameterSpec;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.Key;
 import java.security.NoSuchAlgorithmException;
 import java.security.Provider;
 
+import javax.crypto.Cipher;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.spec.GCMParameterSpec;
+
 public class AesGcmCipherFactory {
 
     public static final int GCM_TAG_LENGTH = 16;
@@ -126,7 +123,7 @@ public static byte[] encryptWithTag(Key key, byte[] iv, byte[] input, int length
             System.arraycopy(iv, 0, gcmIv, 0, 12);
             GCMParameterSpec spec = new GCMParameterSpec(128, gcmIv);
             cipher.init(Cipher.ENCRYPT_MODE, key, spec);
-            
+
             return cipher.doFinal(input, 0, length);
         } catch (Exception e) {
             throw new JavaCryptoException("GCM encryption with tag failed", e);
@@ -153,7 +150,7 @@ public static byte[] decryptWithTag(Key key, byte[] iv, byte[] ciphertext) throw
             System.arraycopy(iv, 0, gcmIv, 0, 12);
             GCMParameterSpec spec = new GCMParameterSpec(128, gcmIv);
             cipher.init(Cipher.DECRYPT_MODE, key, spec);
-            
+
             return cipher.doFinal(ciphertext);
         } catch (Exception e) {
             throw new JavaCryptoException("GCM decryption with tag verification failed", e);

src/main/java/org/opensearch/index/store/cipher/OpenSslNativeCipher.java

@@ -4,6 +4,8 @@
  */
 package org.opensearch.index.store.cipher;
 
+import static org.opensearch.index.store.cipher.AesCipherFactory.computeOffsetIVForAesGcmEncrypted;
+
 import java.lang.foreign.Arena;
 import java.lang.foreign.FunctionDescriptor;
 import java.lang.foreign.Linker;
@@ -18,7 +20,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.SuppressForbidden;
-import static org.opensearch.index.store.cipher.AesCipherFactory.computeOffsetIVForAesGcmEncrypted;
 
 /**
  * Provides native bindings to OpenSSL EVP_aes_256_ctr using the Java Panama FFI.

src/main/java/org/opensearch/index/store/niofs/CryptoOutputStreamIndexOutput.java

@@ -8,14 +8,14 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.file.Path;
+import java.security.Key;
+
+import javax.crypto.Cipher;
 
 import org.apache.lucene.store.OutputStreamIndexOutput;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.index.store.cipher.AesGcmCipherFactory;
 
-import javax.crypto.Cipher;
-import java.security.Key;
-
 /**
  * An IndexOutput implementation that encrypts data before writing using native
  * OpenSSL AES-CTR.
@@ -40,7 +40,8 @@ public final class CryptoOutputStreamIndexOutput extends OutputStreamIndexOutput
      * @throws IOException If there is an I/O error
      * @throws IllegalArgumentException If key or iv lengths are invalid
      */
-    public CryptoOutputStreamIndexOutput(String name, Path path, OutputStream os, Key key, byte[] iv, java.security.Provider provider) throws IOException {
+    public CryptoOutputStreamIndexOutput(String name, Path path, OutputStream os, Key key, byte[] iv, java.security.Provider provider)
+        throws IOException {
         super("FSIndexOutput(path=\"" + path + "\")", name, new EncryptedOutputStream(os, key, iv, provider), CHUNK_SIZE);
     }
 
@@ -106,7 +107,8 @@ private void flushBuffer() throws IOException {
 
         private void processAndWrite(byte[] data, int offset, int length) throws IOException {
             try {
-                byte[] encrypted = org.opensearch.index.store.cipher.AesGcmCipherFactory.encryptWithoutTag(streamOffset, cipher, slice(data, offset, length), length);
+                byte[] encrypted = org.opensearch.index.store.cipher.AesGcmCipherFactory
+                    .encryptWithoutTag(streamOffset, cipher, slice(data, offset, length), length);
                 out.write(encrypted);
                 streamOffset += length;
             } catch (Throwable t) {

src/main/java/org/opensearch/index/translog/CryptoChannelFactory.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.opensearch.index.translog;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.OpenOption;
+import java.nio.file.Path;
+import java.util.Set;
+
+import org.opensearch.index.store.iv.KeyIvResolver;
+
+/**
+ * A ChannelFactory implementation that creates FileChannels with transparent
+ * AES-GCM encryption/decryption for translog files.
+ *
+ * This factory determines whether to apply encryption based on the file extension:
+ * - .tlog files: Encrypted using AES-GCM with 8KB authenticated chunks
+ * - .ckp files: Not encrypted (checkpoint metadata)
+ *
+ * Updated to use unified KeyIvResolver (same as index files) for consistent
+ * key management across all encrypted components.
+ *
+ * @opensearch.internal
+ */
+public class CryptoChannelFactory implements ChannelFactory {
+
+    private final KeyIvResolver keyIvResolver;
+    private final String translogUUID;
+
+    /**
+     * Creates a new CryptoChannelFactory.
+     *
+     * @param keyIvResolver the key and IV resolver for encryption keys (unified with index files)
+     * @param translogUUID the translog UUID for exact header size calculation
+     */
+    public CryptoChannelFactory(KeyIvResolver keyIvResolver, String translogUUID) {
+        if (translogUUID == null) {
+            throw new IllegalArgumentException("translogUUID is required for exact header size calculation");
+        }
+        this.keyIvResolver = keyIvResolver;
+        this.translogUUID = translogUUID;
+    }
+
+    @Override
+    public FileChannel open(Path path, OpenOption... options) throws IOException {
+        FileChannel baseChannel = FileChannel.open(path, options);
+
+        if (!path.getFileName().toString().endsWith(".tlog")) {
+            return baseChannel;
+        }
+
+        Set<OpenOption> optionsSet = Set.of(options);
+        return new CryptoFileChannelWrapper(baseChannel, keyIvResolver, path, optionsSet, translogUUID);
+    }
+}