Set destination length correctly

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs

@@ -34,22 +34,25 @@ private static extern int CryptoNative_RsaDecrypt(
             int sourceLength,
             RSAEncryptionPaddingMode paddingMode,
             IntPtr digestAlgorithm,
-            ref byte destination);
+            ref byte destination,
+            int destinationLength);
 
         internal static int RsaDecrypt(
             SafeEvpPKeyHandle pkey,
             ReadOnlySpan<byte> source,
             RSAEncryptionPaddingMode paddingMode,
             IntPtr digestAlgorithm,
-            Span<byte> destination)
+            Span<byte> destination,
+            int destinationLength)
         {
             int written = CryptoNative_RsaDecrypt(
                 pkey,
                 ref MemoryMarshal.GetReference(source),
                 source.Length,
                 paddingMode,
                 digestAlgorithm,
-                ref MemoryMarshal.GetReference(destination));
+                ref MemoryMarshal.GetReference(destination),
+                destinationLength);
 
             if (written < 0)
             {

src/libraries/Common/src/System/Security/Cryptography/RSAOpenSsl.cs

@@ -119,8 +119,9 @@ public override bool TryDecrypt(
             SafeEvpPKeyHandle key = GetPKey();
             int keySizeBytes = Interop.Crypto.EvpPKeySize(key);
 
-            // OpenSSL does not take a length value for the destination, so it can write out of bounds.
-            // To prevent the OOB write, decrypt into a temporary buffer.
+            // OpenSSL requires that the destination buffer be at least as large as EVP_PKEY_size.
+            // If the destination is too small, allocate a temporary buffer that is large enough
+            // since the decrypted contents may actually be smaller than the key size.
             if (destination.Length < keySizeBytes)
             {
                 // RSA up through 4096 use a stackalloc
@@ -202,7 +203,8 @@ private static int Decrypt(
                 data,
                 padding.Mode,
                 hashAlgorithm,
-                destination);
+                destination,
+                destination.Length);
         }
 
         public override byte[] Encrypt(byte[] data, RSAEncryptionPadding padding)

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.c

@@ -40,7 +40,8 @@ int32_t CryptoNative_RsaDecrypt(EVP_PKEY* pkey,
                                 int32_t sourceLen,
                                 RsaPaddingMode padding,
                                 const EVP_MD* digest,
-                                uint8_t* destination)
+                                uint8_t* destination,
+                                int32_t destinationLen)
 {
     assert(pkey != NULL);
     assert(source != NULL);
@@ -94,7 +95,9 @@ int32_t CryptoNative_RsaDecrypt(EVP_PKEY* pkey,
         }
     }
 
-    size_t written;
+    // EVP_PKEY_decrypt uses the outlen parameter as input to check the destination
+    // is large enough, and set it to the actual amount written when successful.
+    size_t written = Int32ToSizeT(destinationLen);
 
     if (EVP_PKEY_decrypt(ctx, destination, &written, source, Int32ToSizeT(sourceLen)) > 0)
     {

src/libraries/Native/Unix/System.Security.Cryptography.Native/pal_evp_pkey_rsa.h

@@ -30,7 +30,8 @@ PALEXPORT int32_t CryptoNative_RsaDecrypt(EVP_PKEY* pkey,
                                           int32_t sourceLen,
                                           RsaPaddingMode padding,
                                           const EVP_MD* digest,
-                                          uint8_t* destination);
+                                          uint8_t* destination,
+                                          int32_t destinationLen);
 
 /*
 Shims the EVP_PKEY_get1_RSA method.