Improve Hazelcast health indicator to check that Hazelcast is running

This commit improves the Hazelcast health indicator to fail if the
Lifecycle manager indicates that the instance is no longer running.

This is needed because when the default Out-Of-Memory handler in
Hazelcast detects a OOM error, and terminates Hazelcast, it does so on
the node but not via the Hazelcast lifecycle manager - and the results
is that it is still possible to successfully execute transactions in
Hazelcast after this termination. The health indicator should not report
UP after Hazelcast has detected an OOM and terminated itself.

See gh-46877

Signed-off-by: Tommy Karlsson <[email protected]>

Author: tommyk-gears

spring-boot-project/spring-boot-actuator/src/main/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicator.java

@@ -28,6 +28,7 @@
  *
  * @author Dmytro Nosan
  * @author Stephane Nicoll
+ * @author Tommy Karlsson
  * @since 2.2.0
  */
 public class HazelcastHealthIndicator extends AbstractHealthIndicator {
@@ -42,6 +43,10 @@ public HazelcastHealthIndicator(HazelcastInstance hazelcast) {
 
 	@Override
 	protected void doHealthCheck(Health.Builder builder) {
+		if (!this.hazelcast.getLifecycleService().isRunning()) {
+			builder.down();
+			return;
+		}
 		this.hazelcast.executeTransaction((context) -> {
 			String uuid = this.hazelcast.getLocalEndpoint().getUuid().toString();
 			builder.up().withDetail("name", this.hazelcast.getName()).withDetail("uuid", uuid);

spring-boot-project/spring-boot-actuator/src/test/java/org/springframework/boot/actuate/hazelcast/HazelcastHealthIndicatorTests.java

@@ -18,6 +18,8 @@
 
 import com.hazelcast.core.HazelcastException;
 import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.instance.impl.HazelcastInstanceProxy;
+import com.hazelcast.instance.impl.OutOfMemoryHandlerHelper;
 import org.junit.jupiter.api.Test;
 
 import org.springframework.boot.actuate.health.Health;
@@ -37,25 +39,26 @@
  *
  * @author Dmytro Nosan
  * @author Stephane Nicoll
+ * @author Tommy Karlsson
  */
+@WithResource(name = "hazelcast.xml", content = """
+		<hazelcast xmlns="http://www.hazelcast.com/schema/config"
+				   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+				   xsi:schemaLocation="http://www.hazelcast.com/schema/config
+		           http://www.hazelcast.com/schema/config/hazelcast-config-5.0.xsd">
+			<instance-name>actuator-hazelcast</instance-name>
+			<map name="defaultCache" />
+			<network>
+				<join>
+					<auto-detection enabled="false"/>
+					<multicast enabled="false"/>
+				</join>
+			</network>
+		</hazelcast>
+		""")
 class HazelcastHealthIndicatorTests {
 
 	@Test
-	@WithResource(name = "hazelcast.xml", content = """
-			<hazelcast xmlns="http://www.hazelcast.com/schema/config"
-					   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-					   xsi:schemaLocation="http://www.hazelcast.com/schema/config
-			           http://www.hazelcast.com/schema/config/hazelcast-config-5.0.xsd">
-				<instance-name>actuator-hazelcast</instance-name>
-				<map name="defaultCache" />
-				<network>
-					<join>
-						<auto-detection enabled="false"/>
-						<multicast enabled="false"/>
-					</join>
-				</network>
-			</hazelcast>
-			""")
 	void hazelcastUp() {
 		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
 			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
@@ -69,6 +72,32 @@ void hazelcastUp() {
 			});
 	}
 
+	@Test
+	void hazelcastShutdown() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
+			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
+			.run((context) -> {
+				HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class);
+				hazelcast.shutdown();
+				Health health = new HazelcastHealthIndicator(hazelcast).health();
+				assertThat(health.getStatus()).isEqualTo(Status.DOWN);
+			});
+	}
+
+	@Test
+	void hazelcastOOMShutdown() {
+		new ApplicationContextRunner().withConfiguration(AutoConfigurations.of(HazelcastAutoConfiguration.class))
+			.withPropertyValues("spring.hazelcast.config=hazelcast.xml")
+			.run((context) -> {
+				HazelcastInstance hazelcast = context.getBean(HazelcastInstance.class);
+				HazelcastInstance original = ((HazelcastInstanceProxy) hazelcast).getOriginal();
+				OutOfMemoryHandlerHelper.tryCloseConnections(original);
+				OutOfMemoryHandlerHelper.tryShutdown(original);
+				Health health = new HazelcastHealthIndicator(hazelcast).health();
+				assertThat(health.getStatus()).isEqualTo(Status.DOWN);
+			});
+	}
+
 	@Test
 	void hazelcastDown() {
 		HazelcastInstance hazelcast = mock(HazelcastInstance.class);