From f56626bbc463d8f30d1373214c3556b03fc25c2f Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 30 May 2025 11:17:47 -0700 Subject: [PATCH 1/5] Update tokenize.py --- src/blib2to3/pgen2/tokenize.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 5cbfd5148d8..46c43195a6f 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -113,7 +113,17 @@ def transform_whitespace( and prev_token.type not in (TokenType.nl, TokenType.newline) ): token_str = source[token.start_index : token.end_index] - if token_str.startswith("\\\n"): + if token_str.startswith("\\\r\n"): + return pytokens.Token( + TokenType.nl, + token.start_index, + token.start_index + 3, + token.start_line, + token.start_col, + token.start_line, + token.start_col + 3, + ) + elif token_str.startswith("\\\n") or token_str.startswith("\\\r"): return pytokens.Token( TokenType.nl, token.start_index, From 5b8859d1a2980fc8815e7abd032e4276310d20e5 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 30 May 2025 11:21:57 -0700 Subject: [PATCH 2/5] Update CHANGES.md --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index cf415f15fc3..ae8bb78b6ad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ - Handle `# fmt: skip` followed by a comment at the end of file (#4635) - Fix crash when a tuple appears in the `as` clause of a `with` statement (#4634) - Fix crash when tuple is used as a context manager inside a `with` statement (#4646) +- Fix crash on a `\\r\n` (#4673) ### Preview style From d6ac14b5e2d3baf68860a99c55eb3cff778f18f5 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 30 May 2025 23:54:41 -0700 Subject: [PATCH 3/5] Update test_black.py --- tests/test_black.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_black.py b/tests/test_black.py index ee026f312de..82679607d4c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2065,6 +2065,17 @@ def test_lines_with_leading_tabs_expanded(self) -> None: assert lines_with_leading_tabs_expanded("\t\tx") == [f"{tab}{tab}x"] assert lines_with_leading_tabs_expanded("\tx\n y") == [f"{tab}x", " y"] + def test_backslash_carriage_return(self) -> None: + # These tests are here instead of in the normal cases because + # of git's newline normalization and because it's hard to + # get `\r` vs `\r\n` vs `\n` to display properly in editors + assert black.format_str("x=\\r\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\r1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("class A\\r\n:...", mode=black.FileMode()) == "class A: ...\n" + assert black.format_str("class A\\n:...", mode=black.FileMode()) == "class A: ...\n" + assert black.format_str("class A\\r:...", mode=black.FileMode()) == "class A: ...\n" + class TestCaching: def test_get_cache_dir( From 2bb7e620fe4c1e353ac1b0641661487e9063a932 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Fri, 30 May 2025 23:56:30 -0700 Subject: [PATCH 4/5] Update test_black.py --- tests/test_black.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 82679607d4c..90de24b681d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2072,9 +2072,18 @@ def test_backslash_carriage_return(self) -> None: assert black.format_str("x=\\r\n1", mode=black.FileMode()) == "x = 1\n" assert black.format_str("x=\\n1", mode=black.FileMode()) == "x = 1\n" assert black.format_str("x=\\r1", mode=black.FileMode()) == "x = 1\n" - assert black.format_str("class A\\r\n:...", mode=black.FileMode()) == "class A: ...\n" - assert black.format_str("class A\\n:...", mode=black.FileMode()) == "class A: ...\n" - assert black.format_str("class A\\r:...", mode=black.FileMode()) == "class A: ...\n" + assert ( + black.format_str("class A\\r\n:...", mode=black.FileMode()) + == "class A: ...\n" + ) + assert ( + black.format_str("class A\\n:...", mode=black.FileMode()) + == "class A: ...\n" + ) + assert ( + black.format_str("class A\\r:...", mode=black.FileMode()) + == "class A: ...\n" + ) class TestCaching: From 5eeb3d663ee5131da786a10aae2d2271e6569533 Mon Sep 17 00:00:00 2001 From: GiGaGon <107241144+MeGaGiGaGon@users.noreply.github.com> Date: Sat, 31 May 2025 00:06:40 -0700 Subject: [PATCH 5/5] Update test_black.py --- tests/test_black.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_black.py b/tests/test_black.py index 90de24b681d..4588addab4b 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2069,19 +2069,19 @@ def test_backslash_carriage_return(self) -> None: # These tests are here instead of in the normal cases because # of git's newline normalization and because it's hard to # get `\r` vs `\r\n` vs `\n` to display properly in editors - assert black.format_str("x=\\r\n1", mode=black.FileMode()) == "x = 1\n" - assert black.format_str("x=\\n1", mode=black.FileMode()) == "x = 1\n" - assert black.format_str("x=\\r1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\\r\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\\n1", mode=black.FileMode()) == "x = 1\n" + assert black.format_str("x=\\\r1", mode=black.FileMode()) == "x = 1\n" assert ( - black.format_str("class A\\r\n:...", mode=black.FileMode()) + black.format_str("class A\\\r\n:...", mode=black.FileMode()) == "class A: ...\n" ) assert ( - black.format_str("class A\\n:...", mode=black.FileMode()) + black.format_str("class A\\\n:...", mode=black.FileMode()) == "class A: ...\n" ) assert ( - black.format_str("class A\\r:...", mode=black.FileMode()) + black.format_str("class A\\\r:...", mode=black.FileMode()) == "class A: ...\n" )