Exception=redis.clients.jedis.exceptions.JedisConnectionException: java.net.SocketTimeoutException: Read timed out #2105
Description
Sometimes I am getting these exceptions on my application connecting to Redis.
I am running a service with 40 containers (spring-boot)
Env: AWS
Redis: AWS managed
Jedis version: 3.1.0
connect.timeout: 700
write.timeout: 1000
maxIdle: 20
minIdle: 20
maxActive: 20
Node type:cache.r4.large
Engine Version: 4.0.10
Java version 1.8
Traffic on the service: 5-7000 Transactions per second.
Issue frequency: Sometimes in 10 days sometimes once a week, No consistency observed. No changes in the incoming traffic pattern at that time.
Observations: New connections at Redis side spikes to 25-30k and the whole system becomes unstable. Huge spike in network packets in during the issue.
Need help in debugging the issue, This causes a live site the service availability going down.
@configuration
@ConfigurationProperties(prefix = "redis")
public class CacheConfiguration {
Map<String, String> hostname = new HashMap<>();
@Value("${redis.port:6379}")
int redisPort;
@Value("${redis.connect.timeout:1000}")
int connectTimeOut;
@Value("${redis.write.timeout:1000}")
int writeTimeOut;
@Value("${redis.enabled:true}")
boolean isRedisEnabled;
@Value("${redis.maxIdle:5}")
int maxIdle;
@Value("${redis.minIdle:1}")
int minIdle;
@Value("${redis.maxActive:10}")
int maxActive;
public Map<String, String> getHostname() {
return hostname;
}
/**
* Bean for elasticache.
*
* @return configured jedisPool
*/
@Bean
public JedisPool getRedisConfiguration() {
JedisPool jedisPool = null;
if (isRedisEnabled) {
final JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
jedisPoolConfig.setMaxWaitMillis(writeTimeOut);
jedisPoolConfig.setMaxTotal(maxActive);
jedisPoolConfig.setMaxIdle(maxIdle);
jedisPoolConfig.setMinIdle(minIdle);
final Region region = (Regions.getCurrentRegion() != null) ? Regions.getCurrentRegion() : Region.getRegion(Regions.US_WEST_2);
jedisPool = new JedisPool(jedisPoolConfig, hostname.get(region.getName()), redisPort, connectTimeOut);
}
return jedisPool;
}
}