[release/8.0-staging] Permit MD5 regardless of FIPS configuration for…

… Linux (dotnet#94979)

* Re-enable MD5 for FIPS with OpenSSL 3

* Permit MD5 on OpenSSL 1.1.1 regardless of FIPS

* Fix build

---------

Co-authored-by: Kevin Jones <[email protected]>

src/native/libs/System.Security.Cryptography.Native/opensslshim.h

@@ -347,6 +347,8 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len);
     REQUIRED_FUNCTION(EVP_MD_CTX_copy_ex) \
     RENAMED_FUNCTION(EVP_MD_CTX_free, EVP_MD_CTX_destroy) \
     RENAMED_FUNCTION(EVP_MD_CTX_new, EVP_MD_CTX_create) \
+    REQUIRED_FUNCTION(EVP_MD_CTX_set_flags) \
+    LIGHTUP_FUNCTION(EVP_MD_fetch) \
     RENAMED_FUNCTION(EVP_MD_get_size, EVP_MD_size) \
     REQUIRED_FUNCTION(EVP_PKCS82PKEY) \
     REQUIRED_FUNCTION(EVP_PKEY2PKCS8) \
@@ -842,6 +844,8 @@ FOR_ALL_OPENSSL_FUNCTIONS
 #define EVP_MD_CTX_copy_ex EVP_MD_CTX_copy_ex_ptr
 #define EVP_MD_CTX_free EVP_MD_CTX_free_ptr
 #define EVP_MD_CTX_new EVP_MD_CTX_new_ptr
+#define EVP_MD_CTX_set_flags EVP_MD_CTX_set_flags_ptr
+#define EVP_MD_fetch EVP_MD_fetch_ptr
 #define EVP_MD_get_size EVP_MD_get_size_ptr
 #define EVP_PKCS82PKEY EVP_PKCS82PKEY_ptr
 #define EVP_PKEY2PKCS8 EVP_PKEY2PKCS8_ptr

src/native/libs/System.Security.Cryptography.Native/osslcompat_30.h

@@ -19,6 +19,7 @@ void ERR_new(void);
 void ERR_set_debug(const char *file, int line, const char *func);
 void ERR_set_error(int lib, int reason, const char *fmt, ...);
 int EVP_CIPHER_get_nid(const EVP_CIPHER *e);
+EVP_MD* EVP_MD_fetch(OSSL_LIB_CTX *ctx, const char *algorithm, const char *properties);
 int EVP_MD_get_size(const EVP_MD* md);
 int EVP_PKEY_CTX_set_rsa_keygen_bits(EVP_PKEY_CTX* ctx, int bits);
 int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX* ctx, const EVP_MD* md);

src/native/libs/System.Security.Cryptography.Native/pal_evp.c

@@ -1,13 +1,41 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+#include "openssl.h"
 #include "pal_evp.h"
 #include "pal_utilities.h"
 
 #include <assert.h>
+#include <pthread.h>
 
 #define SUCCESS 1
 
+static const EVP_MD* g_evpFetchMd5 = NULL;
+static pthread_once_t g_evpFetch = PTHREAD_ONCE_INIT;
+
+static void EnsureFetchEvpMdAlgorithms(void)
+{
+    // This is called from a pthread_once - this method should not be called directly.
+
+#ifdef NEED_OPENSSL_3_0
+    if (API_EXISTS(EVP_MD_fetch))
+    {
+        ERR_clear_error();
+
+        // Try to fetch an MD5 implementation that will work regardless if
+        // FIPS is enforced or not.
+        g_evpFetchMd5 = EVP_MD_fetch(NULL, "MD5", "-fips");
+    }
+#endif
+
+    // No error queue impact.
+    // If EVP_MD_fetch is unavailable, use the implicit loader. If it failed, use the implicit loader as a last resort.
+    if (g_evpFetchMd5 == NULL)
+    {
+        g_evpFetchMd5 = EVP_md5();
+    }
+}
+
 EVP_MD_CTX* CryptoNative_EvpMdCtxCreate(const EVP_MD* type)
 {
     ERR_clear_error();
@@ -23,6 +51,13 @@ EVP_MD_CTX* CryptoNative_EvpMdCtxCreate(const EVP_MD* type)
         return NULL;
     }
 
+    // For OpenSSL 1.x, set the non-FIPS allow flag for MD5. OpenSSL 3 does this differently with EVP_MD_fetch
+    // and no longer has this flag.
+    if (CryptoNative_OpenSslVersionNumber() < OPENSSL_VERSION_3_0_RTM && type == EVP_md5())
+    {
+        EVP_MD_CTX_set_flags(ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+    }
+
     int ret = EVP_DigestInit_ex(ctx, type, NULL);
     if (!ret)
     {
@@ -230,8 +265,8 @@ int32_t CryptoNative_EvpMdSize(const EVP_MD* md)
 
 const EVP_MD* CryptoNative_EvpMd5(void)
 {
-    // No error queue impact.
-    return EVP_md5();
+    pthread_once(&g_evpFetch, EnsureFetchEvpMdAlgorithms);
+    return g_evpFetchMd5;
 }
 
 const EVP_MD* CryptoNative_EvpSha1(void)