Add support for new Azure AI Foundry project type for Safety evals (#6621)

Fixes #6592

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs

@@ -35,27 +35,38 @@ public ContentSafetyChatClient(
 
         ChatClientMetadata? originalMetadata = _originalChatClient?.GetService<ChatClientMetadata>();
 
-        string providerName =
-            $"{Moniker} (" +
-            $"Subscription: {contentSafetyServiceConfiguration.SubscriptionId}, " +
-            $"Resource Group: {contentSafetyServiceConfiguration.ResourceGroupName}, " +
-            $"Project: {contentSafetyServiceConfiguration.ProjectName})";
+        string providerName;
+        Uri? providerUri = originalMetadata?.ProviderUri;
+
+        if (contentSafetyServiceConfiguration.IsHubBasedProject)
+        {
+            providerName =
+                $"{Moniker} (" +
+                $"Subscription: {contentSafetyServiceConfiguration.SubscriptionId}, " +
+                $"Resource Group: {contentSafetyServiceConfiguration.ResourceGroupName}, " +
+                $"Project: {contentSafetyServiceConfiguration.ProjectName})";
+        }
+        else
+        {
+            providerName = $"{Moniker} (Endpoint: {contentSafetyServiceConfiguration.Endpoint})";
+            providerUri = contentSafetyServiceConfiguration.Endpoint;
+        }
 
         if (originalMetadata?.ProviderName is string originalProviderName &&
             !string.IsNullOrWhiteSpace(originalProviderName))
         {
-            providerName = $"{originalProviderName}; {providerName}";
+            providerName = $"{providerName}; {originalProviderName}";
         }
 
         string modelId = Moniker;
 
         if (originalMetadata?.DefaultModelId is string originalModelId &&
             !string.IsNullOrWhiteSpace(originalModelId))
         {
-            modelId = $"{originalModelId}; {modelId}";
+            modelId = $"{modelId}; {originalModelId}";
         }
 
-        _metadata = new ChatClientMetadata(providerName, originalMetadata?.ProviderUri, modelId);
+        _metadata = new ChatClientMetadata(providerName, providerUri, modelId);
     }
 
     public async Task<ChatResponse> GetResponseAsync(

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.UrlCacheKey.cs

@@ -26,11 +26,14 @@ public bool Equals(UrlCacheKey? other)
             }
             else
             {
+#pragma warning disable S1067 // Expressions should not be too complex
                 return
                     other.Configuration.SubscriptionId == Configuration.SubscriptionId &&
                     other.Configuration.ResourceGroupName == Configuration.ResourceGroupName &&
                     other.Configuration.ProjectName == Configuration.ProjectName &&
+                    other.Configuration.Endpoint == Configuration.Endpoint &&
                     other.AnnotationTask == AnnotationTask;
+#pragma warning restore S1067
             }
         }
 
@@ -42,6 +45,7 @@ public override int GetHashCode() =>
                 Configuration.SubscriptionId,
                 Configuration.ResourceGroupName,
                 Configuration.ProjectName,
+                Configuration.Endpoint,
                 AnnotationTask);
     }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs

@@ -22,6 +22,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety;
 
 internal sealed partial class ContentSafetyService(ContentSafetyServiceConfiguration serviceConfiguration)
 {
+    private const string APIVersionForServiceDiscoveryInHubBasedProjects = "?api-version=2023-08-01-preview";
+    private const string APIVersionForNonHubBasedProjects = "?api-version=2025-05-15-preview";
+
     private static HttpClient? _sharedHttpClient;
     private static HttpClient SharedHttpClient
     {
@@ -168,20 +171,27 @@ private async ValueTask<string> GetServiceUrlAsync(
             return _serviceUrl;
         }
 
-        string discoveryUrl =
-            await GetServiceDiscoveryUrlAsync(evaluatorName, cancellationToken).ConfigureAwait(false);
-
-        serviceUrl =
-            $"{discoveryUrl}/raisvc/v1.0" +
-            $"/subscriptions/{serviceConfiguration.SubscriptionId}" +
-            $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" +
-            $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}";
+        if (serviceConfiguration.IsHubBasedProject)
+        {
+            string discoveryUrl =
+                await GetServiceDiscoveryUrlAsync(evaluatorName, cancellationToken).ConfigureAwait(false);
+
+            serviceUrl =
+                $"{discoveryUrl}/raisvc/v1.0" +
+                $"/subscriptions/{serviceConfiguration.SubscriptionId}" +
+                $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" +
+                $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}";
+        }
+        else
+        {
+            serviceUrl = $"{serviceConfiguration.Endpoint.AbsoluteUri}/evaluations";
+        }
 
         await EnsureServiceAvailabilityAsync(
-            serviceUrl,
-            capability: annotationTask,
-            evaluatorName,
-            cancellationToken).ConfigureAwait(false);
+                serviceUrl,
+                capability: annotationTask,
+                evaluatorName,
+                cancellationToken).ConfigureAwait(false);
 
         _ = _serviceUrlCache.TryAdd(key, serviceUrl);
         _serviceUrl = serviceUrl;
@@ -196,7 +206,7 @@ private async ValueTask<string> GetServiceDiscoveryUrlAsync(
             $"https://management.azure.com/subscriptions/{serviceConfiguration.SubscriptionId}" +
             $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" +
             $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}" +
-            $"?api-version=2023-08-01-preview";
+            $"{APIVersionForServiceDiscoveryInHubBasedProjects}";
 
         HttpResponseMessage response =
             await GetResponseAsync(
@@ -244,7 +254,10 @@ private async ValueTask EnsureServiceAvailabilityAsync(
         string evaluatorName,
         CancellationToken cancellationToken)
     {
-        string serviceAvailabilityUrl = $"{serviceUrl}/checkannotation";
+        string serviceAvailabilityUrl =
+            serviceConfiguration.IsHubBasedProject
+                ? $"{serviceUrl}/checkannotation"
+                : $"{serviceUrl}/checkannotation{APIVersionForNonHubBasedProjects}";
 
         HttpResponseMessage response =
             await GetResponseAsync(
@@ -297,7 +310,10 @@ private async ValueTask<string> SubmitAnnotationRequestAsync(
         string evaluatorName,
         CancellationToken cancellationToken)
     {
-        string annotationUrl = $"{serviceUrl}/submitannotation";
+        string annotationUrl =
+            serviceConfiguration.IsHubBasedProject
+                ? $"{serviceUrl}/submitannotation"
+                : $"{serviceUrl}/submitannotation{APIVersionForNonHubBasedProjects}";
 
         HttpResponseMessage response =
             await GetResponseAsync(
@@ -426,10 +442,13 @@ private async ValueTask AddHeadersAsync(
 
         httpRequestMessage.Headers.Add("User-Agent", userAgent);
 
+        TokenRequestContext context =
+            serviceConfiguration.IsHubBasedProject
+                ? new TokenRequestContext(scopes: ["https://management.azure.com/.default"])
+                : new TokenRequestContext(scopes: ["https://ai.azure.com/.default"]);
+
         AccessToken token =
-            await serviceConfiguration.Credential.GetTokenAsync(
-                new TokenRequestContext(scopes: ["https://management.azure.com/.default"]),
-                cancellationToken).ConfigureAwait(false);
+            await serviceConfiguration.Credential.GetTokenAsync(context, cancellationToken).ConfigureAwait(false);
 
         httpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token.Token);