From 2fd9c1b1a238d63d40dbe28eb9be29410c116578 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Wed, 4 Jun 2025 10:18:05 +0530 Subject: [PATCH 01/10] Add FS Health Check Failure Metric Signed-off-by: Bhumika Sharma --- .../monitor/fs/FsHealthService.java | 25 ++++++- .../main/java/org/opensearch/node/Node.java | 3 +- .../monitor/fs/FsHealthServiceTests.java | 65 +++++++++++++++---- 3 files changed, 79 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index 4b0a79783885f..1a7674da7dbfc 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; @@ -46,6 +47,7 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.metrics.tags.Tags; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -55,6 +57,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -74,6 +77,7 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH private static final Logger logger = LogManager.getLogger(FsHealthService.class); private final ThreadPool threadPool; + private final ClusterManagerMetrics clusterManagerMetrics; private volatile boolean enabled; private volatile boolean brokenLock; private final TimeValue refreshInterval; @@ -115,7 +119,13 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH Setting.Property.Dynamic ); - public FsHealthService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool, NodeEnvironment nodeEnv) { + public FsHealthService( + Settings settings, + ClusterSettings clusterSettings, + ThreadPool threadPool, + NodeEnvironment nodeEnv, + ClusterManagerMetrics clusterManagerMetrics + ) { this.threadPool = threadPool; this.enabled = ENABLED_SETTING.get(settings); this.refreshInterval = REFRESH_INTERVAL_SETTING.get(settings); @@ -123,6 +133,7 @@ public FsHealthService(Settings settings, ClusterSettings clusterSettings, Threa this.currentTimeMillisSupplier = threadPool::relativeTimeInMillis; this.healthyTimeoutThreshold = HEALTHY_TIMEOUT_SETTING.get(settings); this.nodeEnv = nodeEnv; + this.clusterManagerMetrics = clusterManagerMetrics; clusterSettings.addSettingsUpdateConsumer(SLOW_PATH_LOGGING_THRESHOLD_SETTING, this::setSlowPathLoggingThreshold); clusterSettings.addSettingsUpdateConsumer(HEALTHY_TIMEOUT_SETTING, this::setHealthyTimeoutThreshold); clusterSettings.addSettingsUpdateConsumer(ENABLED_SETTING, this::setEnabled); @@ -198,6 +209,7 @@ public void run() { } catch (Exception e) { logger.error("health check failed", e); } finally { + emitMetric(); if (checkEnabled) { boolean completed = checkInProgress.compareAndSet(true, false); assert completed; @@ -205,6 +217,17 @@ public void run() { } } + private void emitMetric() { + StatusInfo healthStatus = getHealth(); + if (healthStatus.getStatus() == UNHEALTHY) { + clusterManagerMetrics.incrementCounter( + clusterManagerMetrics.fsHealthFailCounter, + 1.0, + Optional.ofNullable(Tags.create().addTag("node_id", nodeEnv.nodeId())) + ); + } + } + private void monitorFSHealth() { Set currentUnhealthyPaths = null; Path[] paths = null; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 95424cc9f32f1..ca1b2d6534e72 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -738,7 +738,8 @@ protected Node(final Environment initialEnvironment, Collection clas settings, clusterService.getClusterSettings(), threadPool, - nodeEnvironment + nodeEnvironment, + clusterManagerMetrics ); final SetOnce rerouteServiceReference = new SetOnce<>(); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index 48b2941fe3b7e..66b9d48ae952f 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.tests.mockfile.FilterFileChannel; import org.apache.lucene.tests.mockfile.FilterFileSystemProvider; +import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.coordination.DeterministicTaskQueue; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; @@ -43,6 +44,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -57,6 +59,7 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; +import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -71,11 +74,15 @@ public class FsHealthServiceTests extends OpenSearchTestCase { private DeterministicTaskQueue deterministicTaskQueue; + private TestInMemoryMetricsRegistry metricsRegistry; + private ClusterManagerMetrics clusterManagerMetrics; @Before public void createObjects() { Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); deterministicTaskQueue = new DeterministicTaskQueue(settings, random()); + metricsRegistry = new TestInMemoryMetricsRegistry(); + clusterManagerMetrics = new ClusterManagerMetrics(metricsRegistry); } public void testSchedulesHealthCheckAtRefreshIntervals() throws Exception { @@ -83,7 +90,13 @@ public void testSchedulesHealthCheckAtRefreshIntervals() throws Exception { final Settings settings = Settings.builder().put(FsHealthService.REFRESH_INTERVAL_SETTING.getKey(), refreshInterval + "ms").build(); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, deterministicTaskQueue.getThreadPool(), env); + FsHealthService fsHealthService = new FsHealthService( + settings, + clusterSettings, + deterministicTaskQueue.getThreadPool(), + env, + clusterManagerMetrics + ); final long startTimeMillis = deterministicTaskQueue.getCurrentTimeMillis(); fsHealthService.doStart(); assertFalse(deterministicTaskQueue.hasRunnableTasks()); @@ -117,7 +130,7 @@ public void testFailsHealthOnIOException() throws IOException { final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -125,9 +138,16 @@ public void testFailsHealthOnIOException() throws IOException { // disrupt file system disruptFileSystemProvider.restrictPathPrefix(""); // disrupt all paths disruptFileSystemProvider.injectIOException.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); + assertEquals( + Double.valueOf(1), + metricsRegistry.getCounterStore() + .get("fsHealth.failure.count") + .getCounterValueForTags() + .get((Map.of("node_id", env.nodeId()))) + ); for (Path path : env.nodeDataPaths()) { assertTrue(fsHealthService.getHealth().getInfo().contains(path.toString())); } @@ -160,7 +180,7 @@ public void testLoggingOnHungIO() throws Exception { MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(FsHealthService.class)); NodeEnvironment env = newNodeEnvironment() ) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); int counter = 0; for (Path path : env.nodeDataPaths()) { mockAppender.addExpectation( @@ -202,7 +222,7 @@ public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { PathUtilsForTesting.installMock(fileSystem); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); logger.info("--> Initial health status prior to the first monitor run"); StatusInfo fsHealth = fsHealthService.getHealth(); assertEquals(HEALTHY, fsHealth.getStatus()); @@ -214,7 +234,7 @@ public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { assertEquals("health check passed", fsHealth.getInfo()); logger.info("--> Disrupt file system"); disruptFileSystemProvider.injectIODelay.set(true); - final FsHealthService fsHealthSrvc = new FsHealthService(settings, clusterSettings, testThreadPool, env); + final FsHealthService fsHealthSrvc = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthSrvc.doStart(); waitUntil( () -> fsHealthSrvc.getHealth().getStatus() == UNHEALTHY, @@ -254,7 +274,7 @@ public void testFailsHealthOnSinglePathFsyncFailure() throws IOException { TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { Path[] paths = env.nodeDataPaths(); - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -263,9 +283,16 @@ public void testFailsHealthOnSinglePathFsyncFailure() throws IOException { disruptFsyncFileSystemProvider.injectIOException.set(true); String disruptedPath = randomFrom(paths).toString(); disruptFsyncFileSystemProvider.restrictPathPrefix(disruptedPath); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); + assertEquals( + Double.valueOf(1), + metricsRegistry.getCounterStore() + .get("fsHealth.failure.count") + .getCounterValueForTags() + .get((Map.of("node_id", env.nodeId()))) + ); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptFsyncFileSystemProvider.getInjectedPathCount()); } finally { @@ -285,7 +312,7 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { Path[] paths = env.nodeDataPaths(); - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -294,9 +321,16 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { String disruptedPath = randomFrom(paths).toString(); disruptWritesFileSystemProvider.restrictPathPrefix(disruptedPath); disruptWritesFileSystemProvider.injectIOException.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); + assertEquals( + Double.valueOf(1), + metricsRegistry.getCounterStore() + .get("fsHealth.failure.count") + .getCounterValueForTags() + .get((Map.of("node_id", env.nodeId()))) + ); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptWritesFileSystemProvider.getInjectedPathCount()); } finally { @@ -319,7 +353,7 @@ public void testFailsHealthOnUnexpectedLockFileSize() throws IOException { PathUtilsForTesting.installMock(fileSystem); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -327,9 +361,16 @@ public void testFailsHealthOnUnexpectedLockFileSize() throws IOException { // enabling unexpected file size injection unexpectedLockFileSizeFileSystemProvider.injectUnexpectedFileSize.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env); + fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); + assertEquals( + Double.valueOf(1), + metricsRegistry.getCounterStore() + .get("fsHealth.failure.count") + .getCounterValueForTags() + .get((Map.of("node_id", env.nodeId()))) + ); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed due to broken node lock")); assertEquals(1, unexpectedLockFileSizeFileSystemProvider.getInjectedPathCount()); } finally { From 672bf19da22ff47faa1c7358961e86c4f710bf07 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Sat, 7 Jun 2025 15:23:26 +0530 Subject: [PATCH 02/10] Fix: Moves FSHealthFailure metric to FsHealth class Signed-off-by: Bhumika Sharma --- .../monitor/fs/FsHealthService.java | 28 +++++--- .../main/java/org/opensearch/node/Node.java | 2 +- .../monitor/fs/FsHealthServiceTests.java | 69 +++++-------------- 3 files changed, 37 insertions(+), 62 deletions(-) diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index 1a7674da7dbfc..b95c89af44521 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -35,7 +35,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.common.Nullable; import org.opensearch.common.UUIDs; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; @@ -47,7 +46,8 @@ import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.NodeHealthService; import org.opensearch.monitor.StatusInfo; -import org.opensearch.telemetry.metrics.tags.Tags; +import org.opensearch.telemetry.metrics.Counter; +import org.opensearch.telemetry.metrics.MetricsRegistry; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; @@ -57,7 +57,6 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.HashSet; -import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; @@ -77,7 +76,7 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH private static final Logger logger = LogManager.getLogger(FsHealthService.class); private final ThreadPool threadPool; - private final ClusterManagerMetrics clusterManagerMetrics; + private final MetricsRegistry metricsRegistry; private volatile boolean enabled; private volatile boolean brokenLock; private final TimeValue refreshInterval; @@ -88,6 +87,8 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH private volatile TimeValue healthyTimeoutThreshold; private final AtomicLong lastRunStartTimeMillis = new AtomicLong(Long.MIN_VALUE); private final AtomicBoolean checkInProgress = new AtomicBoolean(); + public Counter fsHealthFailCounter; + private static final String COUNTER_METRICS_UNIT = "1"; @Nullable private volatile Set unhealthyPaths; @@ -124,7 +125,7 @@ public FsHealthService( ClusterSettings clusterSettings, ThreadPool threadPool, NodeEnvironment nodeEnv, - ClusterManagerMetrics clusterManagerMetrics + MetricsRegistry metricsRegistry ) { this.threadPool = threadPool; this.enabled = ENABLED_SETTING.get(settings); @@ -133,7 +134,12 @@ public FsHealthService( this.currentTimeMillisSupplier = threadPool::relativeTimeInMillis; this.healthyTimeoutThreshold = HEALTHY_TIMEOUT_SETTING.get(settings); this.nodeEnv = nodeEnv; - this.clusterManagerMetrics = clusterManagerMetrics; + this.metricsRegistry = metricsRegistry; + fsHealthFailCounter = metricsRegistry.createCounter( + "fsHealth.failure.count", + "Counter for number of times FS health check has failed", + COUNTER_METRICS_UNIT + ); clusterSettings.addSettingsUpdateConsumer(SLOW_PATH_LOGGING_THRESHOLD_SETTING, this::setSlowPathLoggingThreshold); clusterSettings.addSettingsUpdateConsumer(HEALTHY_TIMEOUT_SETTING, this::setHealthyTimeoutThreshold); clusterSettings.addSettingsUpdateConsumer(ENABLED_SETTING, this::setEnabled); @@ -220,14 +226,14 @@ public void run() { private void emitMetric() { StatusInfo healthStatus = getHealth(); if (healthStatus.getStatus() == UNHEALTHY) { - clusterManagerMetrics.incrementCounter( - clusterManagerMetrics.fsHealthFailCounter, - 1.0, - Optional.ofNullable(Tags.create().addTag("node_id", nodeEnv.nodeId())) - ); + incrementCounter(fsHealthFailCounter, 1.0); } } + public void incrementCounter(Counter counter, Double value) { + counter.add(value); + } + private void monitorFSHealth() { Set currentUnhealthyPaths = null; Path[] paths = null; diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index ca1b2d6534e72..024581122c8a9 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -739,7 +739,7 @@ protected Node(final Environment initialEnvironment, Collection clas clusterService.getClusterSettings(), threadPool, nodeEnvironment, - clusterManagerMetrics + metricsRegistry ); final SetOnce rerouteServiceReference = new SetOnce<>(); final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index 66b9d48ae952f..a7f444c5e7202 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -36,7 +36,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.tests.mockfile.FilterFileChannel; import org.apache.lucene.tests.mockfile.FilterFileSystemProvider; -import org.opensearch.cluster.ClusterManagerMetrics; import org.opensearch.cluster.coordination.DeterministicTaskQueue; import org.opensearch.common.io.PathUtils; import org.opensearch.common.io.PathUtilsForTesting; @@ -44,6 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.StatusInfo; +import org.opensearch.telemetry.TestInMemoryCounter; import org.opensearch.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; @@ -59,8 +59,8 @@ import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; -import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -75,14 +75,12 @@ public class FsHealthServiceTests extends OpenSearchTestCase { private DeterministicTaskQueue deterministicTaskQueue; private TestInMemoryMetricsRegistry metricsRegistry; - private ClusterManagerMetrics clusterManagerMetrics; @Before public void createObjects() { Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build(); deterministicTaskQueue = new DeterministicTaskQueue(settings, random()); metricsRegistry = new TestInMemoryMetricsRegistry(); - clusterManagerMetrics = new ClusterManagerMetrics(metricsRegistry); } public void testSchedulesHealthCheckAtRefreshIntervals() throws Exception { @@ -95,7 +93,7 @@ public void testSchedulesHealthCheckAtRefreshIntervals() throws Exception { clusterSettings, deterministicTaskQueue.getThreadPool(), env, - clusterManagerMetrics + metricsRegistry ); final long startTimeMillis = deterministicTaskQueue.getCurrentTimeMillis(); fsHealthService.doStart(); @@ -130,7 +128,7 @@ public void testFailsHealthOnIOException() throws IOException { final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -138,16 +136,9 @@ public void testFailsHealthOnIOException() throws IOException { // disrupt file system disruptFileSystemProvider.restrictPathPrefix(""); // disrupt all paths disruptFileSystemProvider.injectIOException.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals( - Double.valueOf(1), - metricsRegistry.getCounterStore() - .get("fsHealth.failure.count") - .getCounterValueForTags() - .get((Map.of("node_id", env.nodeId()))) - ); + assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); for (Path path : env.nodeDataPaths()) { assertTrue(fsHealthService.getHealth().getInfo().contains(path.toString())); } @@ -180,7 +171,7 @@ public void testLoggingOnHungIO() throws Exception { MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(FsHealthService.class)); NodeEnvironment env = newNodeEnvironment() ) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); int counter = 0; for (Path path : env.nodeDataPaths()) { mockAppender.addExpectation( @@ -222,7 +213,7 @@ public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { PathUtilsForTesting.installMock(fileSystem); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); logger.info("--> Initial health status prior to the first monitor run"); StatusInfo fsHealth = fsHealthService.getHealth(); assertEquals(HEALTHY, fsHealth.getStatus()); @@ -234,14 +225,13 @@ public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { assertEquals("health check passed", fsHealth.getInfo()); logger.info("--> Disrupt file system"); disruptFileSystemProvider.injectIODelay.set(true); - final FsHealthService fsHealthSrvc = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); - fsHealthSrvc.doStart(); + fsHealthService.doStart(); waitUntil( - () -> fsHealthSrvc.getHealth().getStatus() == UNHEALTHY, + () -> fsHealthService.getHealth().getStatus() == UNHEALTHY, healthyTimeoutThreshold + (2 * refreshInterval), TimeUnit.MILLISECONDS ); - fsHealth = fsHealthSrvc.getHealth(); + fsHealth = fsHealthService.getHealth(); assertEquals(UNHEALTHY, fsHealth.getStatus()); assertEquals("healthy threshold breached", fsHealth.getInfo()); int disruptedPathCount = disruptFileSystemProvider.getInjectedPathCount(); @@ -249,15 +239,15 @@ public void testFailsHealthOnHungIOBeyondHealthyTimeout() throws Exception { logger.info("--> Fix file system disruption"); disruptFileSystemProvider.injectIODelay.set(false); waitUntil( - () -> fsHealthSrvc.getHealth().getStatus() == HEALTHY, + () -> fsHealthService.getHealth().getStatus() == HEALTHY, delayBetweenChecks + (4 * refreshInterval), TimeUnit.MILLISECONDS ); - fsHealth = fsHealthSrvc.getHealth(); + fsHealth = fsHealthService.getHealth(); assertEquals(HEALTHY, fsHealth.getStatus()); assertEquals("health check passed", fsHealth.getInfo()); assertEquals(disruptedPathCount, disruptFileSystemProvider.getInjectedPathCount()); - fsHealthSrvc.doStop(); + fsHealthService.doStop(); } finally { PathUtilsForTesting.teardown(); ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS); @@ -274,7 +264,7 @@ public void testFailsHealthOnSinglePathFsyncFailure() throws IOException { TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { Path[] paths = env.nodeDataPaths(); - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -283,16 +273,9 @@ public void testFailsHealthOnSinglePathFsyncFailure() throws IOException { disruptFsyncFileSystemProvider.injectIOException.set(true); String disruptedPath = randomFrom(paths).toString(); disruptFsyncFileSystemProvider.restrictPathPrefix(disruptedPath); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals( - Double.valueOf(1), - metricsRegistry.getCounterStore() - .get("fsHealth.failure.count") - .getCounterValueForTags() - .get((Map.of("node_id", env.nodeId()))) - ); + assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptFsyncFileSystemProvider.getInjectedPathCount()); } finally { @@ -312,7 +295,7 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { TestThreadPool testThreadPool = new TestThreadPool(getClass().getName(), settings); try (NodeEnvironment env = newNodeEnvironment()) { Path[] paths = env.nodeDataPaths(); - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -321,16 +304,9 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { String disruptedPath = randomFrom(paths).toString(); disruptWritesFileSystemProvider.restrictPathPrefix(disruptedPath); disruptWritesFileSystemProvider.injectIOException.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals( - Double.valueOf(1), - metricsRegistry.getCounterStore() - .get("fsHealth.failure.count") - .getCounterValueForTags() - .get((Map.of("node_id", env.nodeId()))) - ); + assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptWritesFileSystemProvider.getInjectedPathCount()); } finally { @@ -353,7 +329,7 @@ public void testFailsHealthOnUnexpectedLockFileSize() throws IOException { PathUtilsForTesting.installMock(fileSystem); final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); try (NodeEnvironment env = newNodeEnvironment()) { - FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); + FsHealthService fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, metricsRegistry); fsHealthService.new FsHealthMonitor().run(); assertEquals(HEALTHY, fsHealthService.getHealth().getStatus()); assertEquals("health check passed", fsHealthService.getHealth().getInfo()); @@ -361,16 +337,9 @@ public void testFailsHealthOnUnexpectedLockFileSize() throws IOException { // enabling unexpected file size injection unexpectedLockFileSizeFileSystemProvider.injectUnexpectedFileSize.set(true); - fsHealthService = new FsHealthService(settings, clusterSettings, testThreadPool, env, clusterManagerMetrics); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals( - Double.valueOf(1), - metricsRegistry.getCounterStore() - .get("fsHealth.failure.count") - .getCounterValueForTags() - .get((Map.of("node_id", env.nodeId()))) - ); + assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed due to broken node lock")); assertEquals(1, unexpectedLockFileSizeFileSystemProvider.getInjectedPathCount()); } finally { From 3c2641f5e45490d97af7014453a697e0c9e93940 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Sun, 8 Jun 2025 14:23:27 +0530 Subject: [PATCH 03/10] Fix: make method and variable private Signed-off-by: Bhumika Sharma --- .../main/java/org/opensearch/monitor/fs/FsHealthService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index b95c89af44521..b291fbc4bcb0a 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -87,7 +87,7 @@ public class FsHealthService extends AbstractLifecycleComponent implements NodeH private volatile TimeValue healthyTimeoutThreshold; private final AtomicLong lastRunStartTimeMillis = new AtomicLong(Long.MIN_VALUE); private final AtomicBoolean checkInProgress = new AtomicBoolean(); - public Counter fsHealthFailCounter; + private final Counter fsHealthFailCounter; private static final String COUNTER_METRICS_UNIT = "1"; @Nullable @@ -230,7 +230,7 @@ private void emitMetric() { } } - public void incrementCounter(Counter counter, Double value) { + private void incrementCounter(Counter counter, Double value) { counter.add(value); } From 9f8a274faf51227f89b7160f3bf7426192ada41b Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Sun, 8 Jun 2025 16:46:18 +0530 Subject: [PATCH 04/10] Fix: removes increment counter method Signed-off-by: Bhumika Sharma --- .../java/org/opensearch/monitor/fs/FsHealthService.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index b291fbc4bcb0a..c63b7b7867846 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -63,8 +63,7 @@ import java.util.function.LongSupplier; import java.util.stream.Collectors; -import static org.opensearch.monitor.StatusInfo.Status.HEALTHY; -import static org.opensearch.monitor.StatusInfo.Status.UNHEALTHY; +import static org.opensearch.monitor.StatusInfo.Status.*; /** * Runs periodically and attempts to create a temp file to see if the filesystem is writable. If not then it marks the @@ -226,14 +225,10 @@ public void run() { private void emitMetric() { StatusInfo healthStatus = getHealth(); if (healthStatus.getStatus() == UNHEALTHY) { - incrementCounter(fsHealthFailCounter, 1.0); + fsHealthFailCounter.add(1.0); } } - private void incrementCounter(Counter counter, Double value) { - counter.add(value); - } - private void monitorFSHealth() { Set currentUnhealthyPaths = null; Path[] paths = null; From 67ad3f45126e54ebeb9db10b13941bcd931f6972 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Mon, 9 Jun 2025 13:09:17 +0530 Subject: [PATCH 05/10] Fix: applying spotless changes Signed-off-by: Bhumika Sharma --- .../main/java/org/opensearch/monitor/fs/FsHealthService.java | 3 ++- .../java/org/opensearch/monitor/fs/FsHealthServiceTests.java | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java index c63b7b7867846..557e590a456a8 100644 --- a/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java +++ b/server/src/main/java/org/opensearch/monitor/fs/FsHealthService.java @@ -63,7 +63,8 @@ import java.util.function.LongSupplier; import java.util.stream.Collectors; -import static org.opensearch.monitor.StatusInfo.Status.*; +import static org.opensearch.monitor.StatusInfo.Status.HEALTHY; +import static org.opensearch.monitor.StatusInfo.Status.UNHEALTHY; /** * Runs periodically and attempts to create a temp file to see if the filesystem is writable. If not then it marks the diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index a7f444c5e7202..c0db7281c7908 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -43,7 +43,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.StatusInfo; -import org.opensearch.telemetry.TestInMemoryCounter; import org.opensearch.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; @@ -60,7 +59,6 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; From dd147f152676570819c48708179e6fb353acf78d Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Mon, 9 Jun 2025 22:23:52 +0530 Subject: [PATCH 06/10] Fix: UTs of follower and leader checker Signed-off-by: Bhumika Sharma --- CHANGELOG.md | 1 + .../opensearch/monitor/fs/FsHealthServiceTests.java | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56df4629bb1fc..c12a0e07d9cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added node-left metric to cluster manager ([#18421](https://github.com/opensearch-project/OpenSearch/pull/18421)) - [Star tree] Remove star tree feature flag and add index setting to configure star tree search on index basis ([#18070](https://github.com/opensearch-project/OpenSearch/pull/18070)) - Approximation Framework Enhancement: Update the BKD traversal logic to improve the performance on skewed data ([#18439](https://github.com/opensearch-project/OpenSearch/issues/18439)) +- Added FS Health Check Failure metric ([#18435](https://github.com/opensearch-project/OpenSearch/pull/18435)) ### Changed - Create generic DocRequest to better categorize ActionRequests ([#18269](https://github.com/opensearch-project/OpenSearch/pull/18269))) diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index c0db7281c7908..34c0fadda2950 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -43,7 +43,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.StatusInfo; -import org.opensearch.telemetry.TestInMemoryMetricsRegistry; +import org.opensearch.test.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; @@ -136,7 +136,7 @@ public void testFailsHealthOnIOException() throws IOException { disruptFileSystemProvider.injectIOException.set(true); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); + assertEquals(Integer.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); for (Path path : env.nodeDataPaths()) { assertTrue(fsHealthService.getHealth().getInfo().contains(path.toString())); } @@ -273,7 +273,7 @@ public void testFailsHealthOnSinglePathFsyncFailure() throws IOException { disruptFsyncFileSystemProvider.restrictPathPrefix(disruptedPath); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); + assertEquals(Integer.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptFsyncFileSystemProvider.getInjectedPathCount()); } finally { @@ -304,7 +304,7 @@ public void testFailsHealthOnSinglePathWriteFailure() throws IOException { disruptWritesFileSystemProvider.injectIOException.set(true); fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); + assertEquals(Integer.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed on [" + disruptedPath + "]")); assertEquals(1, disruptWritesFileSystemProvider.getInjectedPathCount()); } finally { @@ -337,7 +337,7 @@ public void testFailsHealthOnUnexpectedLockFileSize() throws IOException { fsHealthService.new FsHealthMonitor().run(); assertEquals(UNHEALTHY, fsHealthService.getHealth().getStatus()); - assertEquals(Double.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); + assertEquals(Integer.valueOf(1), metricsRegistry.getCounterStore().get("fsHealth.failure.count").getCounterValue()); assertThat(fsHealthService.getHealth().getInfo(), is("health check failed due to broken node lock")); assertEquals(1, unexpectedLockFileSizeFileSystemProvider.getInjectedPathCount()); } finally { From 6e531e5d9ff9c1a4170e37e8da5bcb871a80d2c3 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Wed, 11 Jun 2025 14:08:33 +0530 Subject: [PATCH 07/10] Fix: spotless apply Signed-off-by: Bhumika Sharma --- .../java/org/opensearch/monitor/fs/FsHealthServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java index 34c0fadda2950..f529294273f45 100644 --- a/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java +++ b/server/src/test/java/org/opensearch/monitor/fs/FsHealthServiceTests.java @@ -43,10 +43,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.env.NodeEnvironment; import org.opensearch.monitor.StatusInfo; -import org.opensearch.test.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.test.MockLogAppender; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.junit.annotations.TestLogging; +import org.opensearch.test.telemetry.TestInMemoryMetricsRegistry; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; import org.junit.Before; From 90b52c0cd234875d702366b30f42fb8fa407f1c9 Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Wed, 11 Jun 2025 15:30:35 +0530 Subject: [PATCH 08/10] Trigger Build Signed-off-by: Bhumika Sharma From 3b9fb4f398be50f8d531696acf3303a3258d92db Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Wed, 11 Jun 2025 15:57:20 +0530 Subject: [PATCH 09/10] Trigger Build Signed-off-by: Bhumika Sharma From 0f5062004b22c7b6d7c819985438e7aee9c9a0ae Mon Sep 17 00:00:00 2001 From: Bhumika Sharma Date: Wed, 11 Jun 2025 16:35:56 +0530 Subject: [PATCH 10/10] Trigger Build Signed-off-by: Bhumika Sharma