Add Disk IO time metric for Windows in ResourceMonitoring (#6338)

Author: makazeu

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Linux/Network/LinuxNetworkUtilizationParser.cs

@@ -18,12 +18,12 @@ internal sealed class LinuxNetworkUtilizationParser
     private static readonly ObjectPool<BufferWriter<char>> _sharedBufferWriterPool = BufferWriterPool.CreateBufferWriterPool<char>();
 
     /// <remarks>
-    /// File that provide information about currently active TCP_IPv4 connections.
+    /// File that provides information about currently active TCP_IPv4 connections.
     /// </remarks>
     private static readonly FileInfo _tcp = new("/proc/net/tcp");
 
     /// <remarks>
-    /// File that provide information about currently active TCP_IPv6 connections.
+    /// File that provides information about currently active TCP_IPv6 connections.
     /// </remarks>
     private static readonly FileInfo _tcp6 = new("/proc/net/tcp6");
 

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Disk/WindowsDiskIoRatePerfCounter.cs

@@ -17,7 +17,7 @@ internal sealed class WindowsDiskIoRatePerfCounter
     private readonly string _categoryName;
     private readonly string _counterName;
     private readonly string[] _instanceNames;
-    private long _lastTimestamp;
+    private long _lastTimeTicks;
 
     internal WindowsDiskIoRatePerfCounter(
         IPerformanceCounterFactory performanceCounterFactory,
@@ -43,15 +43,9 @@ internal void InitializeDiskCounters()
     {
         foreach (string instanceName in _instanceNames)
         {
-            // Skip the total instance
-            if (instanceName.Equals("_Total", StringComparison.OrdinalIgnoreCase))
-            {
-                continue;
-            }
-
             // Create counters for each disk
             _counters.Add(_performanceCounterFactory.Create(_categoryName, _counterName, instanceName));
-            TotalCountDict.Add(instanceName, 0);
+            TotalCountDict[instanceName] = 0L;
         }
 
         // Initialize the counters to get the first value
@@ -60,24 +54,25 @@ internal void InitializeDiskCounters()
             _ = counter.NextValue();
         }
 
-        _lastTimestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
+        _lastTimeTicks = _timeProvider.GetUtcNow().Ticks;
     }
 
     internal void UpdateDiskCounters()
     {
-        long currentTimestamp = _timeProvider.GetUtcNow().ToUnixTimeMilliseconds();
-        double elapsedSeconds = (currentTimestamp - _lastTimestamp) / 1000.0; // Convert to seconds
+        long currentTimeTicks = _timeProvider.GetUtcNow().Ticks;
+        long elapsedTimeTicks = currentTimeTicks - _lastTimeTicks;
 
         // For the kind of "rate" perf counters, this algorithm calculates the total value over a time interval
         // by multiplying the per-second rate (e.g., Disk Bytes/sec) by the time interval between two samples.
         // This effectively reverses the per-second rate calculation to a total amount (e.g., total bytes transferred) during that period.
+        // See https://learn.microsoft.com/zh-cn/archive/blogs/askcore/windows-performance-monitor-disk-counters-explained#windows-performance-monitor-disk-counters-explained
         foreach (IPerformanceCounter counter in _counters)
         {
             // total value = per-second rate * elapsed seconds
-            double value = counter.NextValue() * elapsedSeconds;
-            TotalCountDict[counter.InstanceName] += (long)value;
+            double value = counter.NextValue() * (double)elapsedTimeTicks;
+            TotalCountDict[counter.InstanceName] += (long)value / TimeSpan.TicksPerSecond;
         }
 
-        _lastTimestamp = currentTimestamp;
+        _lastTimeTicks = currentTimeTicks;
     }
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Disk/WindowsDiskIoTimePerfCounter.cs

@@ -0,0 +1,76 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Windows.Disk;
+
+internal sealed class WindowsDiskIoTimePerfCounter
+{
+    private readonly List<IPerformanceCounter> _counters = [];
+    private readonly IPerformanceCounterFactory _performanceCounterFactory;
+    private readonly TimeProvider _timeProvider;
+    private readonly string _categoryName;
+    private readonly string _counterName;
+    private readonly string[] _instanceNames;
+    private long _lastTimeTicks;
+
+    internal WindowsDiskIoTimePerfCounter(
+        IPerformanceCounterFactory performanceCounterFactory,
+        TimeProvider timeProvider,
+        string categoryName,
+        string counterName,
+        string[] instanceNames)
+    {
+        _performanceCounterFactory = performanceCounterFactory;
+        _timeProvider = timeProvider;
+        _categoryName = categoryName;
+        _counterName = counterName;
+        _instanceNames = instanceNames;
+    }
+
+    /// <summary>
+    /// Gets the disk time measurements.
+    /// Key: Disk name, Value: Real elapsed time used in busy state.
+    /// </summary>
+    internal IDictionary<string, double> TotalSeconds { get; } = new ConcurrentDictionary<string, double>();
+
+    internal void InitializeDiskCounters()
+    {
+        foreach (string instanceName in _instanceNames)
+        {
+            // Create counters for each disk
+            _counters.Add(_performanceCounterFactory.Create(_categoryName, _counterName, instanceName));
+            TotalSeconds[instanceName] = 0f;
+        }
+
+        // Initialize the counters to get the first value
+        foreach (IPerformanceCounter counter in _counters)
+        {
+            _ = counter.NextValue();
+        }
+
+        _lastTimeTicks = _timeProvider.GetUtcNow().Ticks;
+    }
+
+    internal void UpdateDiskCounters()
+    {
+        long currentTimeTicks = _timeProvider.GetUtcNow().Ticks;
+        long elapsedTimeTicks = currentTimeTicks - _lastTimeTicks;
+
+        // The real elapsed time ("wall clock") used in the I/O path (time from operations running in parallel are not counted).
+        // Measured as the complement of "Disk\% Idle Time" performance counter: uptime * (100 - "Disk\% Idle Time") / 100
+        // See https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskio_time
+        foreach (IPerformanceCounter counter in _counters)
+        {
+            // io busy time = (1 - (% idle time / 100)) * elapsed seconds
+            float idleTimePercentage = Math.Min(counter.NextValue(), 100f);
+            double busyTimeTicks = (1 - (idleTimePercentage / 100f)) * (double)elapsedTimeTicks;
+            TotalSeconds[counter.InstanceName] += busyTimeTicks / TimeSpan.TicksPerSecond;
+        }
+
+        _lastTimeTicks = currentTimeTicks;
+    }
+}

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Disk/WindowsDiskMetrics.cs

@@ -5,7 +5,10 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
+using System.Linq;
 using System.Runtime.Versioning;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using Microsoft.Shared.Instruments;
 
@@ -19,14 +22,18 @@ internal sealed class WindowsDiskMetrics
 
     private static readonly KeyValuePair<string, object?> _directionReadTag = new(DirectionKey, "read");
     private static readonly KeyValuePair<string, object?> _directionWriteTag = new(DirectionKey, "write");
+    private readonly ILogger<WindowsDiskMetrics> _logger;
     private readonly Dictionary<string, WindowsDiskIoRatePerfCounter> _diskIoRateCounters = new();
+    private WindowsDiskIoTimePerfCounter? _diskIoTimePerfCounter;
 
     public WindowsDiskMetrics(
+        ILogger<WindowsDiskMetrics>? logger,
         IMeterFactory meterFactory,
         IPerformanceCounterFactory performanceCounterFactory,
         TimeProvider timeProvider,
         IOptions<ResourceMonitoringOptions> options)
     {
+        _logger = logger ?? NullLogger<WindowsDiskMetrics>.Instance;
         if (!options.Value.EnableDiskIoMetrics)
         {
             return;
@@ -42,39 +49,69 @@ public WindowsDiskMetrics(
         InitializeDiskCounters(performanceCounterFactory, timeProvider);
 
         // The metric is aligned with
-        // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/system/system-metrics.md#metric-systemdiskio
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskio
         _ = meter.CreateObservableCounter(
             ResourceUtilizationInstruments.SystemDiskIo,
             GetDiskIoMeasurements,
             unit: "By",
             description: "Disk bytes transferred");
 
         // The metric is aligned with
-        // https://github.com/open-telemetry/semantic-conventions/blob/main/docs/system/system-metrics.md#metric-systemdiskoperations
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskoperations
         _ = meter.CreateObservableCounter(
             ResourceUtilizationInstruments.SystemDiskOperations,
             GetDiskOperationMeasurements,
             unit: "{operation}",
             description: "Disk operations");
+
+        // The metric is aligned with
+        // https://opentelemetry.io/docs/specs/semconv/system/system-metrics/#metric-systemdiskio_time
+        _ = meter.CreateObservableCounter(
+            ResourceUtilizationInstruments.SystemDiskIoTime,
+            GetDiskIoTimeMeasurements,
+            unit: "s",
+            description: "Time disk spent activated");
     }
 
     private void InitializeDiskCounters(IPerformanceCounterFactory performanceCounterFactory, TimeProvider timeProvider)
     {
         const string DiskCategoryName = "LogicalDisk";
-        string[] instanceNames = performanceCounterFactory.GetCategoryInstances(DiskCategoryName);
+        string[] instanceNames = performanceCounterFactory.GetCategoryInstances(DiskCategoryName)
+            .Where(instanceName => !instanceName.Equals("_Total", StringComparison.OrdinalIgnoreCase))
+            .ToArray();
         if (instanceNames.Length == 0)
         {
             return;
         }
 
-        List<string> diskIoRatePerformanceCounters =
+        // Initialize disk performance counters for "system.disk.io_time" metric
+        try
+        {
+            var ioTimePerfCounter = new WindowsDiskIoTimePerfCounter(
+                performanceCounterFactory,
+                timeProvider,
+                DiskCategoryName,
+                WindowsDiskPerfCounterNames.DiskIdleTimeCounter,
+                instanceNames);
+            ioTimePerfCounter.InitializeDiskCounters();
+            _diskIoTimePerfCounter = ioTimePerfCounter;
+        }
+#pragma warning disable CA1031
+        catch (Exception ex)
+#pragma warning restore CA1031
+        {
+            Log.DiskIoPerfCounterException(_logger, WindowsDiskPerfCounterNames.DiskIdleTimeCounter, ex.Message);
+        }
+
+        // Initialize disk performance counters for "system.disk.io" and "system.disk.operations" metrics
+        List<string> diskIoRatePerfCounterNames =
         [
             WindowsDiskPerfCounterNames.DiskWriteBytesCounter,
             WindowsDiskPerfCounterNames.DiskReadBytesCounter,
             WindowsDiskPerfCounterNames.DiskWritesCounter,
             WindowsDiskPerfCounterNames.DiskReadsCounter,
         ];
-        foreach (string counterName in diskIoRatePerformanceCounters)
+        foreach (string counterName in diskIoRatePerfCounterNames)
         {
             try
             {
@@ -91,7 +128,7 @@ private void InitializeDiskCounters(IPerformanceCounterFactory performanceCounte
             catch (Exception ex)
 #pragma warning restore CA1031
             {
-                Debug.WriteLine("Error initializing disk performance counter: " + ex.Message);
+                Log.DiskIoPerfCounterException(_logger, counterName, ex.Message);
             }
         }
     }
@@ -145,4 +182,19 @@ private IEnumerable<Measurement<long>> GetDiskOperationMeasurements()
 
         return measurements;
     }
+
+    private IEnumerable<Measurement<double>> GetDiskIoTimeMeasurements()
+    {
+        List<Measurement<double>> measurements = [];
+        if (_diskIoTimePerfCounter != null)
+        {
+            _diskIoTimePerfCounter.UpdateDiskCounters();
+            foreach (KeyValuePair<string, double> pair in _diskIoTimePerfCounter.TotalSeconds)
+            {
+                measurements.Add(new Measurement<double>(pair.Value, new TagList { new(DeviceKey, pair.Key) }));
+            }
+        }
+
+        return measurements;
+    }
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Disk/WindowsDiskPerfCounterNames.cs

@@ -9,4 +9,5 @@ internal static class WindowsDiskPerfCounterNames
     internal const string DiskReadBytesCounter = "Disk Read Bytes/sec";
     internal const string DiskWritesCounter = "Disk Writes/sec";
     internal const string DiskReadsCounter = "Disk Reads/sec";
+    internal const string DiskIdleTimeCounter = "% Idle Time";
 }

src/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring/Windows/Log.cs

@@ -45,4 +45,8 @@ public static partial void CpuContainerUsageData(ILogger logger,
     [LoggerMessage(6, LogLevel.Debug,
         "System resources information: CpuLimit = {cpuLimit}, CpuRequest = {cpuRequest}, MemoryLimit = {memoryLimit}, MemoryRequest = {memoryRequest}.")]
     public static partial void SystemResourcesInfo(ILogger logger, double cpuLimit, double cpuRequest, ulong memoryLimit, ulong memoryRequest);
+
+    [LoggerMessage(7, LogLevel.Warning,
+        "Error initializing disk io perf counter: PerfCounter={counterName}, Error={errorMessage}")]
+    public static partial void DiskIoPerfCounterException(ILogger logger, string counterName, string errorMessage);
 }

src/Shared/Instruments/ResourceUtilizationInstruments.cs

@@ -66,6 +66,14 @@ internal static class ResourceUtilizationInstruments
     /// </remarks>
     public const string SystemDiskIo = "system.disk.io";
 
+    /// <summary>
+    /// The name of an instrument to retrieve the time disk spent activated.
+    /// </summary>
+    /// <remarks>
+    /// The type of an instrument is <see cref="System.Diagnostics.Metrics.ObservableCounter{T}"/>.
+    /// </remarks>
+    public const string SystemDiskIoTime = "system.disk.io_time";
+
     /// <summary>
     /// The name of an instrument to retrieve disk operations.
     /// </summary>

test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/Disk/WindowsDiskIoRatePerfCounterTests.cs

@@ -29,7 +29,7 @@ public void DiskReadsPerfCounter_Per60Seconds()
             fakeTimeProvider,
             CategoryName,
             CounterName,
-            instanceNames: ["C:", "D:", "_Total"]);
+            instanceNames: ["C:", "D:"]);
 
         // Set up
         var counterC = new FakePerformanceCounter("C:", [0, 1, 1.5f, 2, 2.5f]);
@@ -75,7 +75,7 @@ public void DiskWriteBytesPerfCounter_Per30Seconds()
             fakeTimeProvider,
             CategoryName,
             counterName: CounterName,
-            instanceNames: ["C:", "D:", "_Total"]);
+            instanceNames: ["C:", "D:"]);
 
         // Set up
         var counterC = new FakePerformanceCounter("C:", [0, 100, 150.5f, 20, 3.1416f]);

test/Libraries/Microsoft.Extensions.Diagnostics.ResourceMonitoring.Tests/Windows/Disk/WindowsDiskIoTimePerfCounterTests.cs

@@ -0,0 +1,61 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime.Versioning;
+using Microsoft.Extensions.Diagnostics.ResourceMonitoring.Windows.Test;
+using Microsoft.Extensions.Time.Testing;
+using Microsoft.TestUtilities;
+using Moq;
+using Xunit;
+
+namespace Microsoft.Extensions.Diagnostics.ResourceMonitoring.Windows.Disk.Test;
+
+[SupportedOSPlatform("windows")]
+[OSSkipCondition(OperatingSystems.Linux | OperatingSystems.MacOSX, SkipReason = "Windows specific.")]
+public class WindowsDiskIoTimePerfCounterTests
+{
+    private const string CategoryName = "LogicalDisk";
+
+    [ConditionalFact]
+    public void DiskReadsPerfCounter_Per60Seconds()
+    {
+        const string CounterName = WindowsDiskPerfCounterNames.DiskReadsCounter;
+        var performanceCounterFactory = new Mock<IPerformanceCounterFactory>();
+        var fakeTimeProvider = new FakeTimeProvider { AutoAdvanceAmount = TimeSpan.FromSeconds(60) };
+
+        var perfCounters = new WindowsDiskIoTimePerfCounter(
+            performanceCounterFactory.Object,
+            fakeTimeProvider,
+            CategoryName,
+            CounterName,
+            instanceNames: ["C:", "D:"]);
+
+        // Set up
+        var counterC = new FakePerformanceCounter("C:", [100, 100, 0, 99.5f]);
+        var counterD = new FakePerformanceCounter("D:", [100, 99.9f, 88.8f, 66.6f]);
+        performanceCounterFactory.Setup(x => x.Create(CategoryName, CounterName, "C:")).Returns(counterC);
+        performanceCounterFactory.Setup(x => x.Create(CategoryName, CounterName, "D:")).Returns(counterD);
+
+        // Initialize the counters
+        perfCounters.InitializeDiskCounters();
+        Assert.Equal(2, perfCounters.TotalSeconds.Count);
+        Assert.Equal(0, perfCounters.TotalSeconds["C:"]);
+        Assert.Equal(0, perfCounters.TotalSeconds["D:"]);
+
+        // Simulate the first tick
+        perfCounters.UpdateDiskCounters();
+        Assert.Equal(0, perfCounters.TotalSeconds["C:"], precision: 2); // (100-100)% * 60 = 0
+        Assert.Equal(0.06, perfCounters.TotalSeconds["D:"], precision: 2); // (100-99.9)% * 60 = 0.06
+
+        // Simulate the second tick
+        perfCounters.UpdateDiskCounters();
+        Assert.Equal(60, perfCounters.TotalSeconds["C:"], precision: 2); // 0 + (100-0)% * 60 = 60
+        Assert.Equal(6.78, perfCounters.TotalSeconds["D:"], precision: 2); // 0.06 + (100-88.8)% * 60 = 6.78
+
+        // Simulate the third tick
+        perfCounters.UpdateDiskCounters();
+        Assert.Equal(60.3, perfCounters.TotalSeconds["C:"], precision: 2); // 60 + (100-99.5)% * 60 = 60.3
+        Assert.Equal(26.82, perfCounters.TotalSeconds["D:"], precision: 2); // 6.78 + (100-66.6)% * 60 = 6.78 + 20.04 = 26.82
+    }
+}