fix: Need to linkify during Markdown rendering (#5066)

* fix: Need to linkify during Markdown rendering

* Don't depend on mdx_linkify

* Also linkify IETF docs as part of the Markdown conversion

* Add test case

* Disable automatic links via angle brackets for email addresses

* Inline the markdown test files

ietf/utils/markdown.py

@@ -6,22 +6,46 @@
 the datatracker.
 """
 import markdown as python_markdown
+from markdown.extensions import Extension
+from markdown.postprocessors import Postprocessor
 
 from django.utils.safestring import mark_safe
 
 from ietf.doc.templatetags.ietf_filters import urlize_ietf_docs
 from ietf.utils.text import bleach_cleaner, bleach_linker
 
 
+class LinkifyExtension(Extension):
+    """
+    Simple Markdown extension inspired by https://github.com/daGrevis/mdx_linkify,
+    but using our bleach_linker directly. Doing the linkification on the converted
+    Markdown output introduces artifacts.
+    """
+
+    def extendMarkdown(self, md):
+        md.postprocessors.register(LinkifyPostprocessor(md), "linkify", 50)
+        # disable automatic links via angle brackets for email addresses
+        md.inlinePatterns.deregister("automail")
+        # "autolink" for URLs does not seem to cause issues, so leave it on
+
+
+class LinkifyPostprocessor(Postprocessor):
+    def run(self, text):
+        return urlize_ietf_docs(bleach_linker.linkify(text))
+
+
 def markdown(text):
     return mark_safe(
-        bleach_linker.linkify(
-            urlize_ietf_docs(
-                bleach_cleaner.clean(
-                    python_markdown.markdown(
-                        text, extensions=["extra", "nl2br", "sane_lists", "toc"]
-                    )
-                )
+        bleach_cleaner.clean(
+            python_markdown.markdown(
+                text,
+                extensions=[
+                    "extra",
+                    "nl2br",
+                    "sane_lists",
+                    "toc",
+                    LinkifyExtension(),
+                ],
             )
         )
     )

ietf/utils/tests_markdown.py

@@ -0,0 +1,60 @@
+# Copyright The IETF Trust 2023, All Rights Reserved
+"""Markdown API utilities tests"""
+
+from textwrap import dedent
+
+from ietf.utils.tests import TestCase
+from ietf.utils.markdown import markdown
+
+
+class MarkdownTests(TestCase):
+    SAMPLE_MARKDOWN = dedent(
+        """
+        # IETF Markdown Test File
+
+        This file contains a bunch of constructs to test our markdown converter in
+        `ietf/utils/markdown.py`.
+
+        ## Links
+
+        * https://example.com
+        * <https://example.com>
+        * [Example](https://example.com)
+        * [email protected]
+        * <[email protected]>
+        * [User](mailto:[email protected])
+        * RFC2119
+        * BCP 3
+        * STD  1
+        * FYI2
+        * draft-ietf-opsec-indicators-of-compromise
+        * draft-ietf-opsec-indicators-of-compromise-01
+        """
+    )
+
+    SAMPLE_MARKDOWN_OUTPUT = dedent(
+        """
+        <h1 id="ietf-markdown-test-file">IETF Markdown Test File</h1>
+        <p>This file contains a bunch of constructs to test our markdown converter in<br>
+        <code>ietf/utils/<a href="http://markdown.py">markdown.py</a></code>.</p>
+        <h2 id="links">Links</h2>
+        <ul>
+        <li><a href="https://example.com">https://example.com</a></li>
+        <li><a href="https://example.com">https://example.com</a></li>
+        <li><a href="https://example.com">Example</a></li>
+        <li><a href="mailto:[email protected]">[email protected]</a></li>
+        <li>&lt;<a href="mailto:[email protected]">[email protected]</a>&gt;</li>
+        <li><a href="mailto:[email protected]">User</a></li>
+        <li>RFC2119</li>
+        <li>BCP 3</li>
+        <li>STD  1</li>
+        <li>FYI2</li>
+        <li>draft-ietf-opsec-indicators-of-compromise</li>
+        <li>draft-ietf-opsec-indicators-of-compromise-01</li>
+        </ul>
+        """
+    ).strip()
+
+    def test_markdown(self):
+        result = markdown(self.SAMPLE_MARKDOWN)
+        self.assertEqual(result, self.SAMPLE_MARKDOWN_OUTPUT)