Failure during rolling-upgrade-multi-cluster:v6.7.0#leader#twoThirdsUpgradedTestCluster due to AssertionError #40373
Description
Failure: https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+7.x+periodic/246/console
This one didn't come with a reproduce line and isn't associated with a single test as far as I can tell, it happens in:
:x-pack:qa:rolling-upgrade-multi-cluster:v6.7.0#leader#twoThirdsUpgradedTestCluster#wait
Where getting http://[::1]:35123/_cluster/health?wait_for_nodes=>=1&wait_for_status=yellow fails. The most relevant part of the stack trace appears to be tripping a real assertion:
\| [2019-03-22T15:38:28,583][ERROR][o.e.b.ElasticsearchUncaughtExceptionHandler] [upgraded-node-leader-1] fatal error in thread [Thread-5], exiting
--
\| java.lang.AssertionError: shard [leader_index3][0], node[Tx87M0psSTKfxswOcbbGWg], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=RKhAp1RsT7Sm0xjVE8WfKQ], unassigned_info[[reason=NODE_LEFT], at[2019-03-22T15:37:55.999Z], delayed=true, details[node_left [Tx87M0psSTKfxswOcbbGWg]], allocation_status[fetching_shard_data]] is not a primary shard in primary mode
\| at org.elasticsearch.index.shard.IndexShard.assertPrimaryMode(IndexShard.java:1581) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.shard.IndexShard.renewRetentionLease(IndexShard.java:1979) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$Renew$TransportAction.doRetentionLeaseAction(RetentionLeaseActions.java:217) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$Renew$TransportAction.doRetentionLeaseAction(RetentionLeaseActions.java:193) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction$1.onResponse(RetentionLeaseActions.java:103) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction$1.onResponse(RetentionLeaseActions.java:98) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:269) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.shard.IndexShardOperationPermits.acquire(IndexShardOperationPermits.java:236) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationPermit(IndexShard.java:2512) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction.asyncShardOperation(RetentionLeaseActions.java:97) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.index.seqno.RetentionLeaseActions$TransportRetentionLeaseAction.asyncShardOperation(RetentionLeaseActions.java:59) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$ShardTransportHandler.messageReceived(TransportSingleShardAction.java:299) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.action.support.single.shard.TransportSingleShardAction$ShardTransportHandler.messageReceived(TransportSingleShardAction.java:292) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1071) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.TcpTransport.handleRequest(TcpTransport.java:1030) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:916) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:751) ~[elasticsearch-7.1.0-SNAPSHOT.jar:7.1.0-SNAPSHOT]
\| at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
\| at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) ~[?:?]
\| at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
\| at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
\| at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
\| at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
\| at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965) ~[?:?]
\| at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
\| at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:656) ~[?:?]
\| at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:556) ~[?:?]
\| at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:510) ~[?:?]
\| at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:470) ~[?:?]
\| at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:909) ~[?:?]
\| at java.lang.Thread.run(Thread.java:748) [?:1.8.0_202]