Resolve key vault references concurrently

src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationKeyVaultOptions.cs

@@ -31,6 +31,13 @@ public class AzureAppConfigurationKeyVaultOptions
         internal TimeSpan? DefaultSecretRefreshInterval = null;
         internal bool IsKeyVaultRefreshConfigured = false;
 
+        /// <summary>
+        /// Specifies whether Key Vault references should be resolved in parallel.
+        /// Default value is false. Enabling this can reduce the time required to resolve Key Vault references
+        /// when many references are loaded from Azure App Configuration.
+        /// </summary>
+        public bool ParallelSecretResolutionEnabled { get; set; }
+
         /// <summary>
         /// Sets the credentials used to authenticate to key vaults that have no registered <see cref="SecretClient"/>.
         /// </summary>

src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs

@@ -144,6 +144,11 @@ internal IEnumerable<IKeyValueAdapter> Adapters
         /// </summary>
         internal bool IsKeyVaultRefreshConfigured { get; private set; } = false;
 
+        /// <summary>
+        /// Flag to indicate whether Key Vault references should be resolved in parallel.
+        /// </summary>
+        internal bool IsParallelSecretResolutionEnabled { get; private set; } = false;
+
         /// <summary>
         /// Indicates all feature flag features used by the application.
         /// </summary>
@@ -520,6 +525,7 @@ public AzureAppConfigurationOptions ConfigureKeyVault(Action<AzureAppConfigurati
             _adapters.Add(new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider(keyVaultOptions)));
 
             IsKeyVaultRefreshConfigured = keyVaultOptions.IsKeyVaultRefreshConfigured;
+            IsParallelSecretResolutionEnabled = keyVaultOptions.ParallelSecretResolutionEnabled;
             IsKeyVaultConfigured = true;
             return this;
         }

src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs

@@ -15,6 +15,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Net.Mime;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
@@ -625,17 +626,44 @@ private async Task<Dictionary<string, string>> PrepareData(Dictionary<string, Co
                 _requestTracingOptions.ResetAiConfigurationTracing();
             }
 
+            bool parallelSecretResolution = _options.IsParallelSecretResolutionEnabled;
+
+            // Only Key Vault references perform network I/O during adapter processing; other
+            // adapters complete synchronously. When parallel resolution is enabled, Key Vault
+            // references are dispatched concurrently while non-Key Vault settings are processed
+            // inline. Results are merged once at the end.
+            var results = new List<IEnumerable<KeyValuePair<string, string>>>(data.Count);
+            List<Task<IEnumerable<KeyValuePair<string, string>>>> pendingKeyVaultTasks = parallelSecretResolution
+                ? new List<Task<IEnumerable<KeyValuePair<string, string>>>>()
+                : null;
+
             foreach (KeyValuePair<string, ConfigurationSetting> kvp in data)
             {
-                IEnumerable<KeyValuePair<string, string>> keyValuePairs = null;
-
                 if (_requestTracingEnabled && _requestTracingOptions != null)
                 {
                     _requestTracingOptions.UpdateAiConfigurationTracing(kvp.Value.ContentType);
                 }
 
-                keyValuePairs = await ProcessAdapters(kvp.Value, cancellationToken).ConfigureAwait(false);
+                if (parallelSecretResolution && IsKeyVaultReference(kvp.Value))
+                {
+                    pendingKeyVaultTasks.Add(ProcessAdapters(kvp.Value, cancellationToken));
+                }
+                else
+                {
+                    results.Add(await ProcessAdapters(kvp.Value, cancellationToken).ConfigureAwait(false));
+                }
+            }
 
+            if (pendingKeyVaultTasks?.Count > 0)
+            {
+                IEnumerable<KeyValuePair<string, string>>[] keyVaultResults =
+                    await Task.WhenAll(pendingKeyVaultTasks).ConfigureAwait(false);
+
+                results.AddRange(keyVaultResults);
+            }
+
+            foreach (IEnumerable<KeyValuePair<string, string>> keyValuePairs in results)
+            {
                 foreach (KeyValuePair<string, string> kv in keyValuePairs)
                 {
                     string key = kv.Key;
@@ -656,6 +684,12 @@ private async Task<Dictionary<string, string>> PrepareData(Dictionary<string, Co
             return applicationData;
         }
 
+        private static bool IsKeyVaultReference(ConfigurationSetting setting)
+        {
+            return setting.ContentType.TryParseContentType(out ContentType contentType)
+                && contentType.IsKeyVaultReference();
+        }
+
         private async Task LoadAsync(bool ignoreFailures, CancellationToken cancellationToken)
         {
             var startupStopwatch = Stopwatch.StartNew();

src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureKeyVaultReference/AzureKeyVaultSecretProvider.cs

@@ -4,8 +4,8 @@
 using Azure.Security.KeyVault.Secrets;
 using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -14,16 +14,14 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.AzureKeyVault
     internal class AzureKeyVaultSecretProvider
     {
         private readonly AzureAppConfigurationKeyVaultOptions _keyVaultOptions;
-        private readonly IDictionary<string, SecretClient> _secretClients;
-        private readonly Dictionary<Uri, CachedKeyVaultSecret> _cachedKeyVaultSecrets;
-        private Uri _nextRefreshSourceId;
-        private DateTimeOffset? _nextRefreshTime;
+        private readonly ConcurrentDictionary<string, SecretClient> _secretClients;
+        private readonly ConcurrentDictionary<Uri, CachedKeyVaultSecret> _cachedKeyVaultSecrets;
 
         public AzureKeyVaultSecretProvider(AzureAppConfigurationKeyVaultOptions keyVaultOptions = null)
         {
             _keyVaultOptions = keyVaultOptions ?? new AzureAppConfigurationKeyVaultOptions();
-            _cachedKeyVaultSecrets = new Dictionary<Uri, CachedKeyVaultSecret>();
-            _secretClients = new Dictionary<string, SecretClient>(StringComparer.OrdinalIgnoreCase);
+            _cachedKeyVaultSecrets = new ConcurrentDictionary<Uri, CachedKeyVaultSecret>();
+            _secretClients = new ConcurrentDictionary<string, SecretClient>(StringComparer.OrdinalIgnoreCase);
 
             if (_keyVaultOptions.SecretClients != null)
             {
@@ -39,10 +37,10 @@ public async Task<string> GetSecretValue(KeyVaultSecretIdentifier secretIdentifi
         {
             string secretValue = null;
 
-            if (_cachedKeyVaultSecrets.TryGetValue(secretIdentifier.SourceId, out CachedKeyVaultSecret cachedSecret) &&
-                    (!cachedSecret.RefreshAt.HasValue || DateTimeOffset.UtcNow < cachedSecret.RefreshAt.Value))
+            if (_cachedKeyVaultSecrets.TryGetValue(secretIdentifier.SourceId, out CachedKeyVaultSecret cachedHit) &&
+                    (!cachedHit.RefreshAt.HasValue || DateTimeOffset.UtcNow < cachedHit.RefreshAt.Value))
             {
-                return cachedSecret.SecretValue;
+                return cachedHit.SecretValue;
             }
 
             SecretClient client = GetSecretClient(secretIdentifier.SourceId);
@@ -52,6 +50,7 @@ public async Task<string> GetSecretValue(KeyVaultSecretIdentifier secretIdentifi
                 throw new UnauthorizedAccessException("No key vault credential or secret resolver callback configured, and no matching secret client could be found.");
             }
 
+            CachedKeyVaultSecret cachedSecret = null;
             bool success = false;
 
             try
@@ -81,42 +80,35 @@ public async Task<string> GetSecretValue(KeyVaultSecretIdentifier secretIdentifi
 
         public bool ShouldRefreshKeyVaultSecrets()
         {
-            return _nextRefreshTime.HasValue && _nextRefreshTime.Value < DateTimeOffset.UtcNow;
+            DateTimeOffset utcNow = DateTimeOffset.UtcNow;
+
+            foreach (KeyValuePair<Uri, CachedKeyVaultSecret> secret in _cachedKeyVaultSecrets)
+            {
+                if (secret.Value.RefreshAt.HasValue && secret.Value.RefreshAt.Value < utcNow)
+                {
+                    return true;
+                }
+            }
+
+            return false;
         }
 
         public void ClearCache()
         {
-            var sourceIdsToRemove = new List<Uri>();
-
-            var utcNow = DateTimeOffset.UtcNow;
+            DateTimeOffset utcNow = DateTimeOffset.UtcNow;
 
             foreach (KeyValuePair<Uri, CachedKeyVaultSecret> secret in _cachedKeyVaultSecrets)
             {
                 if (secret.Value.LastRefreshTime + RefreshConstants.MinimumSecretRefreshInterval < utcNow)
                 {
-                    sourceIdsToRemove.Add(secret.Key);
+                    _cachedKeyVaultSecrets.TryRemove(secret.Key, out _);
                 }
             }
-
-            foreach (Uri sourceId in sourceIdsToRemove)
-            {
-                _cachedKeyVaultSecrets.Remove(sourceId);
-            }
-
-            if (_cachedKeyVaultSecrets.Any())
-            {
-                UpdateNextRefreshableSecretFromCache();
-            }
         }
 
         public void RemoveSecretFromCache(Uri sourceId)
         {
-            _cachedKeyVaultSecrets.Remove(sourceId);
-
-            if (sourceId == _nextRefreshSourceId)
-            {
-                UpdateNextRefreshableSecretFromCache();
-            }
+            _cachedKeyVaultSecrets.TryRemove(sourceId, out _);
         }
 
         private SecretClient GetSecretClient(Uri secretUri)
@@ -133,14 +125,12 @@ private SecretClient GetSecretClient(Uri secretUri)
                 return null;
             }
 
-            client = new SecretClient(
-                new Uri(secretUri.GetLeftPart(UriPartial.Authority)),
-                _keyVaultOptions.Credential,
-                _keyVaultOptions.ClientOptions);
-
-            _secretClients.Add(keyVaultId, client);
-
-            return client;
+            return _secretClients.GetOrAdd(
+                keyVaultId,
+                _ => new SecretClient(
+                    new Uri(secretUri.GetLeftPart(UriPartial.Authority)),
+                    _keyVaultOptions.Credential,
+                    _keyVaultOptions.ClientOptions));
         }
 
         private void SetSecretInCache(Uri sourceId, string key, CachedKeyVaultSecret cachedSecret, bool success = true)
@@ -152,37 +142,6 @@ private void SetSecretInCache(Uri sourceId, string key, CachedKeyVaultSecret cac
 
             UpdateCacheExpirationTimeForSecret(key, cachedSecret, success);
             _cachedKeyVaultSecrets[sourceId] = cachedSecret;
-
-            if (sourceId == _nextRefreshSourceId)
-            {
-                UpdateNextRefreshableSecretFromCache();
-            }
-            else if ((cachedSecret.RefreshAt.HasValue && _nextRefreshTime.HasValue && cachedSecret.RefreshAt.Value < _nextRefreshTime.Value)
-                    || (cachedSecret.RefreshAt.HasValue && !_nextRefreshTime.HasValue))
-            {
-                _nextRefreshSourceId = sourceId;
-                _nextRefreshTime = cachedSecret.RefreshAt.Value;
-            }
-        }
-
-        private void UpdateNextRefreshableSecretFromCache()
-        {
-            _nextRefreshSourceId = null;
-            _nextRefreshTime = DateTimeOffset.MaxValue;
-
-            foreach (KeyValuePair<Uri, CachedKeyVaultSecret> secret in _cachedKeyVaultSecrets)
-            {
-                if (secret.Value.RefreshAt.HasValue && secret.Value.RefreshAt.Value < _nextRefreshTime)
-                {
-                    _nextRefreshTime = secret.Value.RefreshAt;
-                    _nextRefreshSourceId = secret.Key;
-                }
-            }
-
-            if (_nextRefreshTime == DateTimeOffset.MaxValue)
-            {
-                _nextRefreshTime = null;
-            }
         }
 
         private void UpdateCacheExpirationTimeForSecret(string key, CachedKeyVaultSecret cachedSecret, bool success)