Use forTls parameter to enable TLS-compatible certificate key types while maintaining platform compatibility (#119265)

* Initial plan

* Add CRYPTO_TESTS preprocessor directive and differentiate RSA key sizes

Co-authored-by: rzikm <[email protected]>

* Add comprehensive tests to validate different RSA key sizes in crypto vs networking tests

Co-authored-by: rzikm <[email protected]>

* Switch to NETWORKING_TESTS directive and remove test files per feedback

Co-authored-by: rzikm <[email protected]>

* Remove explicit keyFactory parameter to use conditional RSA key sizes

Co-authored-by: rzikm <[email protected]>

* Add NETWORKING_TESTS directive to all networking test projects that use CertificateAuthority

Co-authored-by: rzikm <[email protected]>

* Implement forTls parameter approach for TLS-compatible certificate key type selection

Co-authored-by: bartonjs <[email protected]>

* Revert all .csproj changes as they are no longer needed

Co-authored-by: bartonjs <[email protected]>

* Add debug

* Disable SLH-DSA on linux

* Revert "Add debug"

This reverts commit 11d2ce2.

* Force RSA certificates for MsQuic on non-windows platform

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: rzikm <[email protected]>
Co-authored-by: bartonjs <[email protected]>
Co-authored-by: Radek Zikmund <[email protected]>

Author: 3 people

src/libraries/Common/tests/System/Net/Configuration.Certificates.Dynamic.cs

@@ -162,9 +162,8 @@ public void Dispose()
                 }
             }
 
-            internal static PkiHolder GenerateCertificates(string targetName, [CallerMemberName] string? testName = null, bool longChain = false, bool serverCertificate = true, bool ephemeralKey = false)
+            internal static PkiHolder GenerateCertificates(string targetName, [CallerMemberName] string? testName = null, bool longChain = false, bool serverCertificate = true, bool ephemeralKey = false, bool forceRsaCertificate = false)
             {
-                const int keySize = 2048;
                 if (PlatformDetection.IsWindows && testName != null)
                 {
                     CleanupCertificates(testName);
@@ -182,7 +181,9 @@ internal static PkiHolder GenerateCertificates(string targetName, [CallerMemberN
                     intermediateAuthorityCount: longChain ? 3 : 1,
                     subjectName: targetName,
                     testName: testName,
-                    keyFactory: CertificateAuthority.KeyFactory.RSASize(keySize),
+                    forTls: true,
+                    // [ActiveIssue("https://github.com/dotnet/runtime/issues/119641")]
+                    keyFactory: !forceRsaCertificate ? null : CertificateAuthority.KeyFactory.RSASize(2048),
                     extensions: extensions);
 
                 if (!ephemeralKey && PlatformDetection.IsWindows)

src/libraries/Common/tests/System/Security/Cryptography/X509Certificates/CertificateAuthority.cs

@@ -48,6 +48,7 @@ internal sealed class CertificateAuthority : IDisposable
         private static readonly Asn1Tag s_context1 = new Asn1Tag(TagClass.ContextSpecific, 1);
         private static readonly Asn1Tag s_context2 = new Asn1Tag(TagClass.ContextSpecific, 2);
         private static readonly KeyFactory[] s_variantKeyFactories = KeyFactory.BuildVariantFactories();
+        private static readonly KeyFactory[] s_tlsVariantKeyFactories = KeyFactory.BuildTlsVariantFactories();
 
         private static readonly X500DistinguishedName s_nonParticipatingName =
             new X500DistinguishedName("CN=The Ghost in the Machine");
@@ -804,6 +805,7 @@ internal static void BuildPrivatePki(
             bool pkiOptionsInSubject = false,
             string subjectName = null,
             KeyFactory keyFactory = null,
+            bool forTls = false,
             X509ExtensionCollection extensions = null)
         {
             bool rootDistributionViaHttp = !pkiOptions.HasFlag(PkiOptions.NoRootCertDistributionUri);
@@ -842,9 +844,10 @@ internal static void BuildPrivatePki(
                     int written = hasher.GetCurrentHash(hash);
                     Debug.Assert(written == hash.Length);
 
-                    // Using mod here will create an imbalance any time s_variantKeyFactories isn't a power of 2,
+                    // Using mod here will create an imbalance any time the key factories array isn't a power of 2,
                     // but that's OK.
-                    keyFactory = s_variantKeyFactories[hash[0] % s_variantKeyFactories.Length];
+                    KeyFactory[] keyFactories = forTls ? s_tlsVariantKeyFactories : s_variantKeyFactories;
+                    keyFactory = keyFactories[hash[0] % keyFactories.Length];
                 }
             }
 
@@ -946,6 +949,7 @@ internal static void BuildPrivatePki(
             bool pkiOptionsInSubject = false,
             string subjectName = null,
             KeyFactory keyFactory = null,
+            bool forTls = false,
             X509ExtensionCollection extensions = null)
         {
             BuildPrivatePki(
@@ -960,6 +964,7 @@ internal static void BuildPrivatePki(
                 pkiOptionsInSubject: pkiOptionsInSubject,
                 subjectName: subjectName,
                 keyFactory: keyFactory,
+                forTls: forTls,
                 extensions: extensions);
 
             intermediateAuthority = intermediateAuthorities.Single();
@@ -1052,6 +1057,29 @@ internal static KeyFactory[] BuildVariantFactories()
 
                 return factories.ToArray();
             }
+
+            internal static KeyFactory[] BuildTlsVariantFactories()
+            {
+                List<KeyFactory> factories = [RSASize(2048), ECDsa];
+
+                if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+                {
+                    if (Cryptography.MLDsa.IsSupported)
+                    {
+                        factories.Add(MLDsa);
+                    }
+
+                    // OpenSSL default provider does not advertise SLH-DSA in TLS-SIGALG capability,
+                    // causing it to not recognize SLH-DSA certificates for use in TLS connections
+                    // [ActiveIssue("https://github.com/dotnet/runtime/issues/119573")]
+                    if (!PlatformDetection.IsOpenSslSupported && Cryptography.SlhDsa.IsSupported)
+                    {
+                        factories.Add(SlhDsa);
+                    }
+                }
+
+                return factories.ToArray();
+            }
         }
 
         private sealed class KeyHolder : IDisposable

src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs

@@ -34,7 +34,9 @@ public class CertificateSetup : IDisposable
 
         public CertificateSetup()
         {
-            _pkiHolder = Configuration.Certificates.GenerateCertificates("localhost", nameof(MsQuicTests), longChain: true);
+            _pkiHolder = Configuration.Certificates.GenerateCertificates("localhost", nameof(MsQuicTests), longChain: true,
+                // [ActiveIssue("https://github.com/dotnet/runtime/issues/119641")]
+                forceRsaCertificate: !PlatformDetection.IsWindows);
         }
 
         public SslStreamCertificateContext CreateSslStreamCertificateContext() => _pkiHolder.CreateSslStreamCertificateContext();
@@ -572,7 +574,9 @@ public async Task ConnectWithCertificateForLoopbackIP_IndicatesExpectedError(str
                 throw new SkipTestException("IPv6 is not available on this platform");
             }
 
-            using Configuration.Certificates.PkiHolder pkiHolder = Configuration.Certificates.GenerateCertificates(expectsError ? "badhost" : "localhost");
+            using Configuration.Certificates.PkiHolder pkiHolder = Configuration.Certificates.GenerateCertificates(expectsError ? "badhost" : "localhost",
+                // [ActiveIssue("https://github.com/dotnet/runtime/issues/119641")]
+                forceRsaCertificate: !PlatformDetection.IsWindows);
             X509Certificate2 certificate = pkiHolder.EndEntity;
 
             var listenerOptions = new QuicListenerOptions()