diff --git a/utils/notification_service.py b/utils/notification_service.py
index 44675ef97827..29c5649d2e2d 100644
--- a/utils/notification_service.py
+++ b/utils/notification_service.py
@@ -732,6 +732,9 @@ def add_path(self, path: str, gpu: str = None):
                             artifact_path["gpu"]
                         ] += f"*{line}*\n_{stacktraces.pop(0)}_\n\n"
 
+    # To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)`
+    pr_number_re = re.compile(r"\(#(\d+)\)$")
+
     title = f"🤗 Results of the {ci_event} tests."
     # Add PR title with a link for push CI
     ci_title = os.environ.get("CI_TITLE")
@@ -739,6 +742,13 @@ def add_path(self, path: str, gpu: str = None):
     if ci_title is not None:
         assert commit_url is not None
         ci_title = ci_title.strip().split("\n")[0].strip()
+
+        # Find the PR number (if any) and change the url to the actual PR page.
+        numbers = pr_number_re.findall(ci_title)
+        if len(numbers) > 0:
+            pr_number = numbers[0]
+            commit_url = f"https://github.com/huggingface/transformers/pull/{pr_number}"
+
         ci_title = f"<{commit_url}|{ci_title}>"
     else:
         ci_title = ""