Scan every comment on a line, and read a verbatim string correctly

scripts/prose_lint.py

@@ -190,23 +190,29 @@ class Syntax(TypedDict):
     block: tuple[tuple[str, str], ...]
     doc: tuple[str, ...]
     quotes: str
-
-
-HASH: Syntax = {'line': ('#',), 'block': (), 'doc': (), 'quotes': '"\''}
-C_LIKE: Syntax = {'line': ('//',), 'block': (('/*', '*/'),), 'doc': ('///', '/**'), 'quotes': '"\''}
-XML_LIKE: Syntax = {'line': (), 'block': (('<!--', '-->'),), 'doc': (), 'quotes': '"'}
-POWERSHELL: Syntax = {'line': ('#',), 'block': (('<#', '#>'),), 'doc': (), 'quotes': '"\''}
-INI: Syntax = {'line': ('#', ';'), 'block': (), 'doc': (), 'quotes': '"\''}
-LISP_LIKE: Syntax = {'line': ('#',), 'block': (), 'doc': (), 'quotes': '"'}
+    verbatim: bool
+
+
+HASH: Syntax = {'line': ('#',), 'block': (), 'doc': (), 'quotes': '"\'', 'verbatim': False}
+C_LIKE: Syntax = {'line': ('//',), 'block': (('/*', '*/'),), 'doc': ('///', '/**'),
+                  'quotes': '"\'', 'verbatim': False}
+# C# alone carries the verbatim string, where a backslash is ordinary and a doubled quote escapes.
+CSHARP: Syntax = {**C_LIKE, 'verbatim': True}
+XML_LIKE: Syntax = {'line': (), 'block': (('<!--', '-->'),), 'doc': (), 'quotes': '"',
+                    'verbatim': False}
+POWERSHELL: Syntax = {'line': ('#',), 'block': (('<#', '#>'),), 'doc': (), 'quotes': '"\'',
+                      'verbatim': False}
+INI: Syntax = {'line': ('#', ';'), 'block': (), 'doc': (), 'quotes': '"\'', 'verbatim': False}
+LISP_LIKE: Syntax = {'line': ('#',), 'block': (), 'doc': (), 'quotes': '"', 'verbatim': False}
 # CSS has block comments only, so a `//` in it is the scheme separator of a URL.
-CSS: Syntax = {'line': (), 'block': (('/*', '*/'),), 'doc': (), 'quotes': '"\''}
+CSS: Syntax = {'line': (), 'block': (('/*', '*/'),), 'doc': (), 'quotes': '"\'', 'verbatim': False}
 
 SYNTAX: dict[str, Syntax] = {
     # Python, shell, and the hash-commented configs
     '.py': HASH, '.sh': HASH, '.bash': HASH, '.yml': HASH, '.yaml': HASH,
     '.toml': HASH, '.tf': HASH, '.gitattributes': HASH, '.gitignore': HASH,
     # C#, C, and C++
-    '.cs': C_LIKE, '.c': C_LIKE, '.cpp': C_LIKE, '.cc': C_LIKE, '.cxx': C_LIKE,
+    '.cs': CSHARP, '.c': C_LIKE, '.cpp': C_LIKE, '.cc': C_LIKE, '.cxx': C_LIKE,
     '.h': C_LIKE, '.hpp': C_LIKE, '.jsonc': C_LIKE, '.json5': C_LIKE,
     '.js': C_LIKE, '.ts': C_LIKE, '.css': CSS, '.scss': CSS,
     # JSON carries comments in practice, which is what JSONC names.
@@ -244,29 +250,48 @@ def syntax_for(path: Path) -> Syntax | None:
     return HASH if not suffix else None
 
 
-def strip_strings(line: str, quotes: str) -> str:
+def strip_strings(line: str, quotes: str, verbatim: bool = False) -> str:
     """Blank quoted spans so a comment marker inside a string is not read as one.
 
     Length-preserving, so an offset into the result is an offset into the line.
+    A verbatim string takes its own rules where the syntax has one.
+    There the backslash is an ordinary character and a doubled quote is the escape, so reading a
+    backslash as an escape consumes the closing quote and blanks the rest of the line.
     """
     out = list(line)
     quote = ''
+    inside_verbatim = False
     escaped = False
-    for i, ch in enumerate(line):
+    i = 0
+    while i < len(line):
+        ch = line[i]
         if escaped:
             escaped = False
             out[i] = ' '
-            continue
-        if ch == '\\' and quote:
+        elif inside_verbatim:
+            if ch == quote and line[i + 1:i + 2] == quote:   # a doubled quote is one character
+                out[i] = out[i + 1] = ' '
+                i += 2
+                continue
+            out[i] = ' ' if ch != quote else ch
+            if ch == quote:
+                quote, inside_verbatim = '', False
+        elif ch == '\\' and quote:
             escaped = True
             out[i] = ' '
-            continue
-        if quote:
+        elif quote:
             out[i] = ' ' if ch != quote else ch
             if ch == quote:
                 quote = ''
         elif ch in quotes:
             quote = ch
+            # An interpolated one is spelled either way round, so read the whole prefix.
+            # Only the double-quoted form has a verbatim spelling, so a char literal is ordinary.
+            start = i
+            while start > 0 and line[start - 1] in '@$':
+                start -= 1
+            inside_verbatim = verbatim and ch == '"' and '@' in line[start:i]
+        i += 1
     return ''.join(out)
 
 
@@ -373,48 +398,65 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo
         return []
     out: list[tuple[int, str, bool]] = []
     closing = ''
+    doc_closing = ''
     for n, raw in enumerate(lines, 1):
         line = raw.rstrip('\r')
-        if closing:                                  # inside a block comment
+        masked = strip_strings(line, spec['quotes'], spec['verbatim'])
+        pos = 0
+        if doc_closing:                              # CODESTYLE owns every line until it closes
+            end = line.find(doc_closing)
+            if end < 0:
+                continue
+            pos, doc_closing = end + len(doc_closing), ''
+        elif closing:                                # carried in from an unclosed block
             end = line.find(closing)
             body = (line if end < 0 else line[:end]).strip().lstrip('*').strip()
             if body:
                 out.append((n, body, True))
-            closing = '' if end >= 0 else closing
-            continue
-        masked = strip_strings(line, spec['quotes'])
-        # A line comment runs to end of line, so a block opener after one is text.
-        # Left unbounded it opens a block that swallows the code lines below.
-        # A documentation comment bounds it too, being exempt from linting rather than from here.
-        line_at = len(line)
-        for marker in spec['line']:
-            at = masked.find(marker)
-            if 0 <= at < line_at:
-                line_at = at
-        cut = len(line)
-        leading = True
-        for opener, closer in spec['block']:
-            at = masked.find(opener)
-            if 0 <= at < min(cut, line_at):
-                if any(line[at:].startswith(d) for d in spec['doc']):
-                    continue
-                cut, leading = at, not line[:at].strip()
-                end = masked.find(closer, at + len(opener))
-                body = (line[at + len(opener):end if end >= 0 else None]).strip().lstrip('*').strip()
+            if end < 0:
+                continue
+            pos, closing = end + len(closing), ''
+        # Scan left to right and take whichever marker comes first.
+        # A ceiling can only describe the first comment, so a later one was unreachable.
+        while pos < len(line):
+            found: str | tuple[str, str] | None = None
+            at = len(line)
+            for marker in spec['line']:
+                where = masked.find(marker, pos)
+                if 0 <= where < at:
+                    at, found = where, marker
+            for opener, closer in spec['block']:
+                where = masked.find(opener, pos)
+                if 0 <= where < at:
+                    at, found = where, (opener, closer)
+            if found is None:
+                break
+            # CODESTYLE owns a documentation comment, so this rule skips over it.
+            # A line one runs to end of line, while a closed block one gives the rest back.
+            if any(line[at:].startswith(d) for d in spec['doc']):
+                if isinstance(found, str):
+                    break
+                end = masked.find(found[1], at + len(found[0]))
+                if end < 0:
+                    doc_closing = found[1]               # it carries on into the lines below
+                    break
+                pos = end + len(found[1])
+                continue
+            leading = not line[:at].strip()
+            if isinstance(found, str):               # a line comment runs to end of line
+                body = line[at + len(found):].strip()
                 if body:
                     out.append((n, body, leading))
-                closing = '' if end >= 0 else closer
-        if closing:
-            continue
-        for marker in spec['line']:
-            at = masked.find(marker)
-            if 0 <= at < cut:
-                if any(line[at:].startswith(d) for d in spec['doc']):
-                    continue
-                body = line[at + len(marker):].strip()
-                if body:
-                    out.append((n, body, not line[:at].strip()))
-                cut = at
+                break
+            opener, closer = found
+            end = masked.find(closer, at + len(opener))
+            body = (line[at + len(opener):end if end >= 0 else None]).strip().lstrip('*').strip()
+            if body:
+                out.append((n, body, leading))
+            if end < 0:
+                closing = closer
+                break
+            pos = end + len(closer)
     return out
 
 

scripts/test_prose_lint.py

@@ -298,6 +298,35 @@ def test_a_documentation_comment_is_left_to_codestyle(self) -> None:
         self.assertEqual([], self.flag('a.cs', f'/// <summary>{self.RUN_ON}</summary>\n'))
         self.assertEqual([], self.flag('a.py', f'"""{self.RUN_ON}"""\n'))
 
+    def test_a_closed_block_doc_gives_the_rest_of_the_line_back(self) -> None:
+        """CODESTYLE owns the documentation comment, not the line it happens to sit on.
+
+        A line doc comment does run to end of line, so only the block form gives anything back.
+        """
+        self.assertEqual([], self.flag('a.cs', f'/** {self.RUN_ON} */\n'))
+        self.assertEqual([], self.flag('a.cs', f'/// {self.RUN_ON} // and more\n'))
+        self.assertEqual(['comment-wrap'],
+                         self.flag('a.cs', '/** Docs. */ // Two things. Here.\n'))
+
+    def test_a_multi_line_doc_block_owns_every_line_until_it_closes(self) -> None:
+        """A marker in documentation text is prose, so scanning those lines invents comments.
+
+        The closing line still gives back what follows the closer, which is the one finding here.
+        """
+        self.assertEqual(['comment-wrap'], self.flag('a.cs', '/** Docs start\n'
+                                                             ' * // Two things. Here.\n'
+                                                             ' * /* not an opener\n'
+                                                             ' */ // Two things. Here.\n'))
+
+    def test_verbatim_rules_apply_to_the_double_quoted_form_only(self) -> None:
+        """C# spells a verbatim string with double quotes, so `@` on a char literal is ordinary.
+
+        Under verbatim rules the doubled quote is one escaped character and both are blanked,
+        so counting what survives tells the two readings apart.
+        """
+        masked = prose_lint.strip_strings("var c = @'a''b'; // t", '"\'', True)
+        self.assertEqual(4, masked.count("'"))
+
     def test_a_format_with_no_comment_syntax_is_skipped(self) -> None:
         for name in ('a.lock', 'a.csv', 'a.txt'):
             with self.subTest(file=name):
@@ -380,6 +409,47 @@ def test_a_block_opener_inside_a_line_comment_is_text(self) -> None:
             with self.subTest(file=name, line=text.split('\n')[0]):
                 self.assertEqual(['comment-wrap'], self.flag(name, text))
 
+    def test_every_comment_on_a_line_is_read_not_just_the_first(self) -> None:
+        """A ceiling can only describe the first comment, so a later one was unreachable.
+
+        Each case puts the offending sentence in the second comment, which a scan that stops at
+        the first reports as clean.
+        """
+        for name, text in (
+            ('a.cs', 'var x = 1; /* Note. */ // Two things. Here.\n'),
+            ('a.cs', '/* Note. */ /* Two things. Here. */\n'),
+            ('a.cs', '/* Start here.\n   Still going. */ // Two things. Here.\n'),
+        ):
+            with self.subTest(line=text.split('\n')[0]):
+                self.assertEqual(['comment-wrap'], self.flag(name, text))
+
+    def test_a_verbatim_string_keeps_its_own_closing_quote(self) -> None:
+        """A backslash is ordinary inside one and a doubled quote is the escape.
+
+        Read with C escape rules the string never closes, so the masker blanks the rest of the
+        line and the trailing comment goes unseen.
+        """
+        # Ending in a backslash, the string swallows its closing quote and hides a real comment.
+        self.assertEqual(['comment-wrap'],
+                         self.flag('a.cs', 'var p = @"C:\\tmp\\"; // Two things. Here.\n'))
+        # Reading a doubled quote as a close then a reopen puts string content outside the string.
+        self.assertEqual([],
+                         self.flag('a.cs', 'var s = @"a""// One thing. Another thing.""b"; // ok\n'))
+        # An interpolated one is spelled either way round, and only one of them abuts the quote.
+        for text in ('var s = $@"C:\\tmp\\"; // Two things. Here.\n',
+                     'var s = @$"C:\\tmp\\"; // Two things. Here.\n'):
+            with self.subTest(line=text.strip()):
+                self.assertEqual(['comment-wrap'], self.flag('a.cs', text))
+
+    def test_only_the_syntax_that_has_verbatim_strings_gets_them(self) -> None:
+        """C shares the C-like spec without the form, so `@` there is an ordinary character."""
+        self.assertTrue(prose_lint.SYNTAX['.cs']['verbatim'])
+        self.assertFalse(prose_lint.SYNTAX['.c']['verbatim'])
+        self.assertFalse(prose_lint.SYNTAX['.json']['verbatim'])
+        # The C escape still hides a marker, which is what the verbatim rule must not undo.
+        self.assertEqual(['comment-wrap'],
+                         self.flag('a.cs', 'var s = "a\\"b"; // Two things. Here.\n'))
+
     def test_css_has_block_comments_only(self) -> None:
         """A `//` in CSS is the scheme separator of a URL, not a comment marker."""
         self.assertEqual([], self.flag('a.css', 'a { background: url(http://x/y. Z); }\n'))