Skip to content

Commit 5d2d178

Browse files
shyamnamboodiripadshyamnamboodiripad
committed
Introduce telemetry in aieval dotnet tool
1 parent ee9b1d3 commit 5d2d178

File tree

10 files changed

+1163
-145
lines changed

10 files changed

+1163
-145
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,46 +2,66 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33

44
using System;
5+
using System.Collections.Generic;
56
using System.IO;
67
using System.Threading;
78
using System.Threading.Tasks;
89
using Azure.Identity;
910
using Azure.Storage.Files.DataLake;
11+
using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
1012
using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
1113
using Microsoft.Extensions.AI.Evaluation.Reporting;
1214
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
1315
using Microsoft.Extensions.Logging;
1416

1517
namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
1618

17-
internal sealed class CleanCacheCommand(ILogger logger)
19+
internal sealed class CleanCacheCommand(ILogger logger, TelemetryHelper telemetryHelper)
1820
{
19-
internal async Task<int> InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default)
21+
internal async Task<int> InvokeAsync(
22+
DirectoryInfo? storageRootDir,
23+
Uri? endpointUri,
24+
CancellationToken cancellationToken = default)
2025
{
21-
IEvaluationResponseCacheProvider cacheProvider;
22-
23-
if (storageRootDir is not null)
24-
{
25-
string storageRootPath = storageRootDir.FullName;
26-
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
27-
logger.LogInformation("Deleting expired cache entries...");
28-
29-
cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
30-
}
31-
else if (endpointUri is not null)
32-
{
33-
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
34-
35-
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
36-
cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
37-
}
38-
else
39-
{
40-
throw new InvalidOperationException("Either --path or --endpoint must be specified");
41-
}
26+
var telemetryProperties = new Dictionary<string, string>();
4227

4328
await logger.ExecuteWithCatchAsync(
44-
() => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);
29+
operation: () =>
30+
telemetryHelper.ReportOperationAsync(
31+
operationName: TelemetryConstants.EventNames.CleanCacheCommand,
32+
operation: () =>
33+
{
34+
IEvaluationResponseCacheProvider cacheProvider;
35+
36+
if (storageRootDir is not null)
37+
{
38+
string storageRootPath = storageRootDir.FullName;
39+
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
40+
logger.LogInformation("Deleting expired cache entries...");
41+
42+
cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
43+
44+
telemetryProperties[TelemetryConstants.PropertyNames.StorageType] =
45+
TelemetryConstants.PropertyValues.StorageTypeDisk;
46+
}
47+
else if (endpointUri is not null)
48+
{
49+
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
50+
51+
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
52+
cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
53+
54+
telemetryProperties[TelemetryConstants.PropertyNames.StorageType] =
55+
TelemetryConstants.PropertyValues.StorageTypeAzure;
56+
}
57+
else
58+
{
59+
throw new InvalidOperationException("Either --path or --endpoint must be specified");
60+
}
61+
62+
return cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken);
63+
},
64+
properties: telemetryProperties)).ConfigureAwait(false);
4565

4666
return 0;
4767
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs‎

Lines changed: 67 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -8,76 +8,98 @@
88
using System.Threading.Tasks;
99
using Azure.Identity;
1010
using Azure.Storage.Files.DataLake;
11+
using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
1112
using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
1213
using Microsoft.Extensions.AI.Evaluation.Reporting;
1314
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
1415
using Microsoft.Extensions.Logging;
1516

1617
namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
1718

18-
internal sealed class CleanResultsCommand(ILogger logger)
19+
internal sealed class CleanResultsCommand(ILogger logger, TelemetryHelper telemetryHelper)
1920
{
2021
internal async Task<int> InvokeAsync(
2122
DirectoryInfo? storageRootDir,
2223
Uri? endpointUri,
2324
int lastN,
2425
CancellationToken cancellationToken = default)
2526
{
26-
IEvaluationResultStore resultStore;
27-
28-
if (storageRootDir is not null)
29-
{
30-
string storageRootPath = storageRootDir.FullName;
31-
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
27+
var telemetryProperties =
28+
new Dictionary<string, string>
29+
{
30+
[TelemetryConstants.PropertyNames.LastN] = lastN.ToTelemetryPropertyValue()
31+
};
3232

33-
resultStore = new DiskBasedResultStore(storageRootPath);
34-
}
35-
else if (endpointUri is not null)
36-
{
37-
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
33+
await logger.ExecuteWithCatchAsync(
34+
operation: () =>
35+
telemetryHelper.ReportOperationAsync(
36+
operationName: TelemetryConstants.EventNames.CleanResultsCommand,
37+
operation: async ValueTask () =>
38+
{
39+
IEvaluationResultStore resultStore;
3840

39-
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
40-
resultStore = new AzureStorageResultStore(fsClient);
41-
}
42-
else
43-
{
44-
throw new InvalidOperationException("Either --path or --endpoint must be specified");
45-
}
41+
if (storageRootDir is not null)
42+
{
43+
string storageRootPath = storageRootDir.FullName;
44+
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
4645

47-
await logger.ExecuteWithCatchAsync(
48-
async ValueTask () =>
49-
{
50-
if (lastN is 0)
51-
{
52-
logger.LogInformation("Deleting all results...");
46+
resultStore = new DiskBasedResultStore(storageRootPath);
5347

54-
await resultStore.DeleteResultsAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
55-
}
56-
else
57-
{
58-
logger.LogInformation("Deleting all results except the {lastN} most recent ones...", lastN);
48+
telemetryProperties[TelemetryConstants.PropertyNames.StorageType] =
49+
TelemetryConstants.PropertyValues.StorageTypeDisk;
50+
}
51+
else if (endpointUri is not null)
52+
{
53+
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
5954

60-
HashSet<string> toPreserve = [];
55+
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
56+
resultStore = new AzureStorageResultStore(fsClient);
6157

62-
await foreach (string executionName in
63-
resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
64-
{
65-
_ = toPreserve.Add(executionName);
66-
}
58+
telemetryProperties[TelemetryConstants.PropertyNames.StorageType] =
59+
TelemetryConstants.PropertyValues.StorageTypeAzure;
60+
}
61+
else
62+
{
63+
throw new InvalidOperationException("Either --path or --endpoint must be specified");
64+
}
6765

68-
await foreach (string executionName in
69-
resultStore.GetLatestExecutionNamesAsync(
70-
cancellationToken: cancellationToken).ConfigureAwait(false))
71-
{
72-
if (!toPreserve.Contains(executionName))
66+
if (lastN is 0)
7367
{
68+
logger.LogInformation("Deleting all results...");
69+
7470
await resultStore.DeleteResultsAsync(
75-
executionName,
7671
cancellationToken: cancellationToken).ConfigureAwait(false);
7772
}
78-
}
79-
}
80-
}).ConfigureAwait(false);
73+
else
74+
{
75+
logger.LogInformation(
76+
"Deleting all results except the {lastN} most recent ones...",
77+
lastN);
78+
79+
HashSet<string> toPreserve = [];
80+
81+
await foreach (string executionName in
82+
resultStore.GetLatestExecutionNamesAsync(
83+
lastN,
84+
cancellationToken).ConfigureAwait(false))
85+
{
86+
_ = toPreserve.Add(executionName);
87+
}
88+
89+
await foreach (string executionName in
90+
resultStore.GetLatestExecutionNamesAsync(
91+
cancellationToken: cancellationToken).ConfigureAwait(false))
92+
{
93+
if (!toPreserve.Contains(executionName))
94+
{
95+
await resultStore.DeleteResultsAsync(
96+
executionName,
97+
cancellationToken: cancellationToken).ConfigureAwait(false);
98+
}
99+
}
100+
}
101+
},
102+
properties: telemetryProperties)).ConfigureAwait(false);
81103

82104
return 0;
83105
}

0 commit comments

Comments
 (0)