Fix decoding issue while formatting SyntaxErrors during collection

This happens only in Python 2, as in Python 3 we receive
the "badline" in the exception is already properly encoded

Fix pytest-dev#578

CHANGELOG.rst

@@ -10,9 +10,20 @@
 
 *
 
+* Fix (`#578 <https://github.com/pytest-dev/pytest/issues/578>`_): SyntaxErrors
+  containing non-ascii lines at the point of failure generated an internal
+  py.test error.
+  Thanks `@asottile`_ for the report and `@nicoddemus`_ for the PR.
+
+*
+
+*
+
 .. _#469: https://github.com/pytest-dev/pytest/issues/469
 .. _#1431: https://github.com/pytest-dev/pytest/pull/1431
 
+.. _@asottile: https://github.com/asottile
+
 
 2.9.0
 =====

_pytest/_code/_py2traceback.py

@@ -47,7 +47,9 @@ def format_exception_only(etype, value):
         filename = filename or "<string>"
         lines.append('  File "%s", line %d\n' % (filename, lineno))
         if badline is not None:
-            lines.append('    %s\n' % badline.strip())
+            if isinstance(badline, bytes):  # python 2 only
+                badline = badline.decode('utf-8', 'replace')
+            lines.append(u'    %s\n' % badline.strip())
             if offset is not None:
                 caretspace = badline.rstrip('\n')[:offset].lstrip()
                 # non-space whitespace (likes tabs) must be kept for alignment

testing/code/test_code.py

@@ -93,6 +93,17 @@ def f():
     if sys.version_info[0] < 3:
         unicode(excinfo)
 
+
+@pytest.mark.skipif(sys.version_info[0] >= 3, reason='python 2 only issue')
+def test_unicode_handling_syntax_error():
+    value = py.builtin._totext('\xc4\x85\xc4\x87\n', 'utf-8').encode('utf8')
+    def f():
+        raise SyntaxError('invalid syntax', (None, 1, 3, value))
+    excinfo = pytest.raises(Exception, f)
+    str(excinfo)
+    if sys.version_info[0] < 3:
+        unicode(excinfo)
+
 def test_code_getargs():
     def f1(x):
         pass

testing/python/collect.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 import sys
 from textwrap import dedent
 
@@ -1181,3 +1182,19 @@ def test_injection(self):
     result = testdir.runpytest()
     assert "RuntimeError: dictionary changed size during iteration" not in result.stdout.str()
     result.stdout.fnmatch_lines(['*1 passed*'])
+
+
+def test_syntax_error_with_non_ascii_chars(testdir):
+    """Fix decoding issue while formatting SyntaxErrors during collection (#578)
+    """
+    testdir.makepyfile(u"""
+    # -*- coding: UTF-8 -*-
+
+    ☃
+    """)
+    result = testdir.runpytest()
+    result.stdout.fnmatch_lines([
+        '*ERROR collecting*',
+        '*SyntaxError*',
+        '*1 error in*',
+    ])