diff --git a/CHANGELOG.md b/CHANGELOG.md
index eb5ec41595c..bc2c26ced10 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -193,6 +193,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed `ax` plotting logging in `MetricCollection ([#1783](https://github.com/Lightning-AI/torchmetrics/pull/1783))
 
 
+- Fixed lookup for punkt sources being downloaded in `RougeScore` [#1789](https://github.com/Lightning-AI/torchmetrics/pull/1789)
+
+
 ## [0.11.4] - 2023-03-10
 
 ### Fixed
diff --git a/requirements/nominal_test.txt b/requirements/nominal_test.txt
index df025808422..f4e41f7151f 100644
--- a/requirements/nominal_test.txt
+++ b/requirements/nominal_test.txt
@@ -2,5 +2,5 @@
 #  in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment
 
 pandas >1.0.0, <=2.0.1 # cannot pin version due to numpy version incompatibility
-dython <=0.7.3 # todo: pin version, but some version resolution issue
+dython <=0.7.4
 scipy >1.0.0, <1.11.0 # cannot pin version due to some version conflicts with `oldest` CI configuration
diff --git a/src/torchmetrics/functional/text/rouge.py b/src/torchmetrics/functional/text/rouge.py
index cfeee0d5874..015d7014ff6 100644
--- a/src/torchmetrics/functional/text/rouge.py
+++ b/src/torchmetrics/functional/text/rouge.py
@@ -49,7 +49,7 @@ def _ensure_nltk_punkt_is_downloaded() -> None:
     import nltk
 
     try:
-        nltk.data.find("tokenizers/punkt.zip")
+        nltk.data.find("tokenizers/punkt")
     except LookupError:
         try:
             nltk.download("punkt", quiet=True, force=False, halt_on_error=False, raise_on_error=True)