crypto/rsa: check hash message length first in SignPKCS1v15

This restores the error checking behavior from Go 1.23.
In particular, the boringcrypto code path now contains this
check again.

Change-Id: Ie38c68c27bc3a95c6a0b5a78fc4196694527cd6b
Reviewed-on: https://go-review.googlesource.com/c/go/+/632495
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Roland Shoemaker <[email protected]>

src/crypto/rsa/fips.go

@@ -278,6 +278,14 @@ func decryptOAEP(hash, mgfHash hash.Hash, priv *PrivateKey, ciphertext []byte, l
 // messages to signatures and identify the signed messages. As ever,
 // signatures provide authenticity, not confidentiality.
 func SignPKCS1v15(random io.Reader, priv *PrivateKey, hash crypto.Hash, hashed []byte) ([]byte, error) {
+	var hashName string
+	if hash != crypto.Hash(0) {
+		if len(hashed) != hash.Size() {
+			return nil, errors.New("crypto/rsa: input must be hashed message")
+		}
+		hashName = hash.String()
+	}
+
 	if err := checkPublicKeySize(&priv.PublicKey); err != nil {
 		return nil, err
 	}
@@ -300,13 +308,6 @@ func SignPKCS1v15(random io.Reader, priv *PrivateKey, hash crypto.Hash, hashed [
 	if err != nil {
 		return nil, err
 	}
-	var hashName string
-	if hash != crypto.Hash(0) {
-		if len(hashed) != hash.Size() {
-			return nil, errors.New("crypto/rsa: input must be hashed message")
-		}
-		hashName = hash.String()
-	}
 	return fipsError2(rsa.SignPKCS1v15(k, hashName, hashed))
 }
 

src/crypto/rsa/rsa_test.go

@@ -248,8 +248,14 @@ func testEverything(t *testing.T, priv *PrivateKey) {
 		}
 	}
 
+	const hashMsg = "crypto/rsa: input must be hashed message"
+	sig, err := SignPKCS1v15(nil, priv, crypto.SHA256, msg)
+	if err == nil || err.Error() != hashMsg {
+		t.Errorf("SignPKCS1v15 with bad hash: err = %q, want %q", err, hashMsg)
+	}
+
 	hash := sha256.Sum256(msg)
-	sig, err := SignPKCS1v15(nil, priv, crypto.SHA256, hash[:])
+	sig, err = SignPKCS1v15(nil, priv, crypto.SHA256, hash[:])
 	if err == ErrMessageTooLong {
 		t.Log("key too small for SignPKCS1v15")
 	} else if err != nil {