Merge branch 'main' of https://github.com/opensearch-project/OpenSearch…

… into listsegmentsfinal

.ci/bwcVersions

@@ -49,4 +49,5 @@ BWC_VERSION:
   - "2.1.1"
   - "2.2.0"
   - "2.2.1"
+  - "2.2.2"
   - "2.3.0"

.github/workflows/changelog_verifier.yml

@@ -14,3 +14,5 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
 
       - uses: dangoslen/changelog-enforcer@v3
+        with:
+          skipLabels: "autocut"

.github/workflows/version.yml

@@ -5,7 +5,7 @@ on:
     tags:
       - '*.*.*'
 
-jobs:  
+jobs:
   build:
     runs-on: ubuntu-latest
     steps:
@@ -61,6 +61,8 @@ jobs:
           commit-message: Incremented version to ${{ env.NEXT_VERSION }}
           signoff: true
           delete-branch: true
+          labels: |
+            autocut
           title: '[AUTO] Incremented version to ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and incremented the version from ${{ env.CURRENT_VERSION }} to ${{ env.NEXT_VERSION }}.
@@ -86,6 +88,8 @@ jobs:
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
           signoff: true
           delete-branch: true
+          labels: |
+            autocut
           title: '[AUTO] [${{ env.BASE_X }}] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.
@@ -111,6 +115,8 @@ jobs:
           commit-message: Added bwc version ${{ env.NEXT_VERSION }}
           signoff: true
           delete-branch: true
+          labels: |
+            autocut
           title: '[AUTO] [main] Added bwc version ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.

CHANGELOG.md

@@ -11,11 +11,18 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added release notes for 2.2.1 ([#4344](https://github.com/opensearch-project/OpenSearch/pull/4344))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
 - Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
+- BWC version 2.2.2 ([#4383](https://github.com/opensearch-project/OpenSearch/pull/4383))
+- Support for labels on version bump PRs, skip label support for changelog verifier ([#4391](https://github.com/opensearch-project/OpenSearch/pull/4391))
+
+### Dependencies
+- Bumps `com.diffplug.spotless` from 6.9.1 to 6.10.0
+- Bumps `xmlbeans` from 5.1.0 to 5.1.1
 
 ### Changed
 - Dependency updates (httpcore, mockito, slf4j, httpasyncclient, commons-codec) ([#4308](https://github.com/opensearch-project/OpenSearch/pull/4308))
 - Use RemoteSegmentStoreDirectory instead of RemoteDirectory ([#4240](https://github.com/opensearch-project/OpenSearch/pull/4240))
 - Plugin ZIP publication groupId value is configurable ([#4156](https://github.com/opensearch-project/OpenSearch/pull/4156))
+- Update to Netty 4.1.80.Final ([#4359](https://github.com/opensearch-project/OpenSearch/pull/4359))
 
 ### Deprecated
 
@@ -30,7 +37,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add timeout on Mockito.verify to reduce flakyness in testReplicationOnDone test([#4314](https://github.com/opensearch-project/OpenSearch/pull/4314))
 - Commit workflow for dependabot changelog helper ([#4331](https://github.com/opensearch-project/OpenSearch/pull/4331))
 - Fixed cancellation of segment replication events ([#4225](https://github.com/opensearch-project/OpenSearch/pull/4225))
-- Bugs for dependabot changelog verifier workflow ([#4364](https://github.com/opensearch-project/OpenSearch/pull/4364)) 
+- [Segment Replication] Bump segment infos counter before commit during replica promotion ([#4365](https://github.com/opensearch-project/OpenSearch/pull/4365))
+- Bugs for dependabot changelog verifier workflow ([#4364](https://github.com/opensearch-project/OpenSearch/pull/4364))
+- Fix flaky random test `NRTReplicationEngineTests.testUpdateSegments` ([#4352](https://github.com/opensearch-project/OpenSearch/pull/4352))
+- [Segment Replication] Extend FileChunkWriter to allow cancel on transport client ([#4386](https://github.com/opensearch-project/OpenSearch/pull/4386))
+- [Segment Replication] Add check to cancel ongoing replication with old primary on onNewCheckpoint on replica ([#4363](https://github.com/opensearch-project/OpenSearch/pull/4363))
 
 ### Security
 - CVE-2022-25857 org.yaml:snakeyaml DOS vulnerability ([#4341](https://github.com/opensearch-project/OpenSearch/pull/4341))
@@ -39,6 +50,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
 - Label configuration for dependabot PRs ([#4348](https://github.com/opensearch-project/OpenSearch/pull/4348))
+- Added RestLayer Changes for PIT stats ([#4217](https://github.com/opensearch-project/OpenSearch/pull/4217))
 
 ### Changed
 

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.9.1" apply false
+  id "com.diffplug.spotless" version "6.10.0" apply false
   id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/version.properties

@@ -21,7 +21,7 @@ asm               = 9.3
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.79.Final
+netty             = 4.1.80.Final
 joda              = 2.10.13
 
 # client dependencies

modules/transport-netty4/build.gradle

@@ -144,6 +144,14 @@ thirdPartyAudit {
     'org.apache.log4j.Level',
     'org.apache.log4j.Logger',
 
+    // from io.netty.handler.ssl.OpenSslEngine (netty)
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
+
     // from io.netty.handler.ssl.OpenSslEngine (netty)
     'io.netty.internal.tcnative.Buffer',
     'io.netty.internal.tcnative.CertificateCompressionAlgo',

modules/transport-netty4/licenses/netty-buffer-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+a087321a63d9991e25f7b7d24ef53edcbcb954ff

modules/transport-netty4/licenses/netty-codec-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+4941821a158d16311665d8606aefa610ecf0f64c

modules/transport-netty4/licenses/netty-codec-http-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+efb23f9d5187d2f733595ef7930137f0cb2cec48

modules/transport-netty4/licenses/netty-codec-http2-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+bf7b66834188ef1a6f6095291c6b81a1880798ba

modules/transport-netty4/licenses/netty-common-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+3d43ce22863bc590e4e33fbdabbb58dc05f4c43d

modules/transport-netty4/licenses/netty-handler-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+cf7029d2f9bc4eeae8ff15af7a528d06b518a017

modules/transport-netty4/licenses/netty-resolver-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+3bbb0d4bfbbab867e5b757b97a6e5e0d1348d94c

modules/transport-netty4/licenses/netty-transport-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+57fcace7a1b8567aa39921c915d1b1ba78fd4d2d

modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+da3d7da1a8d317ae2c82b400fd255fe610c43ebe

plugins/ingest-attachment/build.gradle

@@ -79,7 +79,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.1.0'
+  api 'org.apache.xmlbeans:xmlbeans:5.1.1'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

plugins/ingest-attachment/licenses/xmlbeans-5.1.1.jar.sha1

@@ -0,0 +1 @@
+48a369df0eccb509d46203104e4df9cb00f0f68b

plugins/repository-azure/licenses/netty-codec-dns-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+6926d2ea779f41071ecb1948d880dfbb3a6ee126

plugins/repository-azure/licenses/netty-codec-http2-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+bf7b66834188ef1a6f6095291c6b81a1880798ba

plugins/repository-azure/licenses/netty-codec-socks-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+00025b767be3425f3b31a34ee095c85619169f17

plugins/repository-azure/licenses/netty-handler-proxy-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+9b3b42ff805723fb98120f5ab2019c53e71da91b

plugins/repository-azure/licenses/netty-resolver-dns-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+6b1602f80b6235b0b7d53bc5e9c1a6cd11c1b804

plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+da3d7da1a8d317ae2c82b400fd255fe610c43ebe

plugins/repository-hdfs/licenses/netty-all-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+39e73b76a3ec65df731b371179e15f2c3e4e7575

plugins/transport-nio/build.gradle

@@ -83,6 +83,12 @@ thirdPartyAudit {
     'org.bouncycastle.cert.X509v3CertificateBuilder',
     'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
     'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
 
     // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
     'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',

plugins/transport-nio/licenses/netty-buffer-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+a087321a63d9991e25f7b7d24ef53edcbcb954ff

plugins/transport-nio/licenses/netty-codec-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+4941821a158d16311665d8606aefa610ecf0f64c

plugins/transport-nio/licenses/netty-codec-http-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+efb23f9d5187d2f733595ef7930137f0cb2cec48

plugins/transport-nio/licenses/netty-common-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+3d43ce22863bc590e4e33fbdabbb58dc05f4c43d

plugins/transport-nio/licenses/netty-handler-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+cf7029d2f9bc4eeae8ff15af7a528d06b518a017

plugins/transport-nio/licenses/netty-resolver-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+3bbb0d4bfbbab867e5b757b97a6e5e0d1348d94c

plugins/transport-nio/licenses/netty-transport-4.1.80.Final.jar.sha1

@@ -0,0 +1 @@
+57fcace7a1b8567aa39921c915d1b1ba78fd4d2d

rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -67,6 +67,9 @@
                     search.scroll_current            .+   \n
                     search.scroll_time               .+   \n
                     search.scroll_total              .+   \n
+                    search.point_in_time_current     .+   \n
+                    search.point_in_time_time        .+   \n
+                    search.point_in_time_total       .+   \n
                     segments.count                   .+   \n
                     segments.memory                  .+   \n
                     segments.index_writer_memory     .+   \n

server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java

@@ -33,17 +33,23 @@
 import org.opensearch.index.engine.Segment;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.recovery.FileChunkRequest;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.plugins.Plugin;
 import org.opensearch.test.BackgroundIndexer;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.transport.MockTransportService;
+import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -65,6 +71,11 @@ public static void assumeFeatureFlag() {
         assumeTrue("Segment replication Feature flag is enabled", Boolean.parseBoolean(System.getProperty(FeatureFlags.REPLICATION_TYPE)));
     }
 
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Arrays.asList(MockTransportService.TestPlugin.class);
+    }
+
     @Override
     public Settings indexSettings() {
         return Settings.builder()
@@ -318,6 +329,65 @@ public void testReplicationAfterForceMerge() throws Exception {
         }
     }
 
+    public void testCancellation() throws Exception {
+        final String primaryNode = internalCluster().startNode();
+        createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
+        ensureYellow(INDEX_NAME);
+
+        final String replicaNode = internalCluster().startNode();
+
+        final SegmentReplicationSourceService segmentReplicationSourceService = internalCluster().getInstance(
+            SegmentReplicationSourceService.class,
+            primaryNode
+        );
+        final IndexShard primaryShard = getIndexShard(primaryNode);
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(
+            TransportService.class,
+            primaryNode
+        ));
+        mockTransportService.addSendBehavior(
+            internalCluster().getInstance(TransportService.class, replicaNode),
+            (connection, requestId, action, request, options) -> {
+                if (action.equals(SegmentReplicationTargetService.Actions.FILE_CHUNK)) {
+                    FileChunkRequest req = (FileChunkRequest) request;
+                    logger.debug("file chunk [{}] lastChunk: {}", req, req.lastChunk());
+                    if (req.name().endsWith("cfs") && req.lastChunk()) {
+                        try {
+                            latch.await();
+                        } catch (InterruptedException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+                }
+                connection.sendRequest(requestId, action, request, options);
+            }
+        );
+
+        final int docCount = scaledRandomIntBetween(0, 200);
+        try (
+            BackgroundIndexer indexer = new BackgroundIndexer(
+                INDEX_NAME,
+                "_doc",
+                client(),
+                -1,
+                RandomizedTest.scaledRandomIntBetween(2, 5),
+                false,
+                random()
+            )
+        ) {
+            indexer.start(docCount);
+            waitForDocs(docCount, indexer);
+
+            flush(INDEX_NAME);
+        }
+        segmentReplicationSourceService.beforeIndexShardClosed(primaryShard.shardId(), primaryShard, indexSettings());
+        latch.countDown();
+        assertDocCounts(docCount, primaryNode);
+    }
+
     public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception {
         final String primaryNode = internalCluster().startNode();
         createIndex(INDEX_NAME, Settings.builder().put(indexSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build());