BUG: Disable multichar/regex sep for Python engine in read_csv

Closes gh-13374.

[ci skip]

Author: gfyoung

doc/source/whatsnew/v0.20.0.txt

@@ -48,6 +48,8 @@ Backwards incompatible API changes
 Other API Changes
 ^^^^^^^^^^^^^^^^^
 
+- ``pd.read_csv()`` will raise a ValueError for the Python engine if a multi-character or regex separator is used with quoted data (:issue:`13374`)
+
 .. _whatsnew_0200.deprecations:
 
 Deprecations

pandas/io/parsers.py

@@ -784,7 +784,9 @@ def _clean_options(self, options, engine):
                                   " skipfooter"
                 engine = 'python'
 
+        quoting = options['quoting']
         encoding = sys.getfilesystemencoding() or 'utf-8'
+
         if sep is None and not delim_whitespace:
             if engine == 'c':
                 fallback_reason = "the 'c' engine does not support"\
@@ -801,6 +803,15 @@ def _clean_options(self, options, engine):
                                   " different from '\s+' are"\
                                   " interpreted as regex)"
                 engine = 'python'
+            elif quoting != csv.QUOTE_NONE:
+                # gh-13374: The Python engine breaks with quotation
+                # marks because we do not properly handle quoted
+                # fields with multi-char / regex separators.
+                msg = ("the %r engine does not support regex separators "
+                       "(separators > 1 char and different from '\s+' are "
+                       "interpreted as regex) with quoted fields") % engine
+                raise ValueError(msg)
+
         elif delim_whitespace:
             if 'python' in engine:
                 result['delimiter'] = '\s+'

pandas/io/tests/parser/common.py

@@ -836,7 +836,8 @@ def test_integer_overflow_bug(self):
         result = self.read_csv(StringIO(data), header=None, sep=' ')
         self.assertTrue(result[0].dtype == np.float64)
 
-        result = self.read_csv(StringIO(data), header=None, sep='\s+')
+        result = self.read_csv(StringIO(data), header=None,
+                               quoting=csv.QUOTE_NONE, sep='\s+')
         self.assertTrue(result[0].dtype == np.float64)
 
     def test_catch_too_many_names(self):
@@ -852,7 +853,8 @@ def test_catch_too_many_names(self):
     def test_ignore_leading_whitespace(self):
         # see gh-3374, gh-6607
         data = ' a b c\n 1 2 3\n 4 5 6\n 7 8 9'
-        result = self.read_table(StringIO(data), sep='\s+')
+        result = self.read_table(StringIO(data), sep='\s+',
+                                 quoting=csv.QUOTE_NONE)
         expected = DataFrame({'a': [1, 4, 7], 'b': [2, 5, 8], 'c': [3, 6, 9]})
         tm.assert_frame_equal(result, expected)
 
@@ -1157,7 +1159,8 @@ def test_empty_lines(self):
                              [-70., .4, 1.]])
         df = self.read_csv(StringIO(data))
         tm.assert_numpy_array_equal(df.values, expected)
-        df = self.read_csv(StringIO(data.replace(',', '  ')), sep='\s+')
+        df = self.read_csv(StringIO(data.replace(',', '  ')),
+                           quoting=csv.QUOTE_NONE, sep='\s+')
         tm.assert_numpy_array_equal(df.values, expected)
         expected = np.array([[1., 2., 4.],
                              [np.nan, np.nan, np.nan],
@@ -1189,14 +1192,16 @@ def test_regex_separator(self):
 b   1   2   3   4
 c   1   2   3   4
 """
-        df = self.read_table(StringIO(data), sep='\s+')
+        df = self.read_table(StringIO(data), sep='\s+',
+                             quoting=csv.QUOTE_NONE)
         expected = self.read_csv(StringIO(re.sub('[ ]+', ',', data)),
                                  index_col=0)
         self.assertIsNone(expected.index.name)
         tm.assert_frame_equal(df, expected)
 
         data = '    a b c\n1 2 3 \n4 5  6\n 7 8 9'
-        result = self.read_table(StringIO(data), sep='\s+')
+        result = self.read_table(StringIO(data), sep='\s+',
+                                 quoting=csv.QUOTE_NONE)
         expected = DataFrame([[1, 2, 3], [4, 5, 6], [7, 8, 9]],
                              columns=['a', 'b', 'c'])
         tm.assert_frame_equal(result, expected)
@@ -1580,7 +1585,8 @@ def test_temporary_file(self):
         new_file.flush()
         new_file.seek(0)
 
-        result = self.read_csv(new_file, sep='\s+', header=None)
+        result = self.read_csv(new_file, sep='\s+', header=None,
+                               quoting=csv.QUOTE_NONE)
         new_file.close()
         expected = DataFrame([[0, 0]])
         tm.assert_frame_equal(result, expected)

pandas/io/tests/parser/python_parser_only.py

@@ -7,6 +7,7 @@
 arguments when parsing.
 """
 
+import csv
 import sys
 import nose
 
@@ -81,7 +82,8 @@ def test_BytesIO_input(self):
                 "Bytes-related test - only needs to work on Python 3")
 
         data = BytesIO("שלום::1234\n562::123".encode('cp1255'))
-        result = self.read_table(data, sep="::", encoding='cp1255')
+        result = self.read_table(data, sep="::", quoting=csv.QUOTE_NONE,
+                                 encoding='cp1255')
         expected = DataFrame([[562, 123]], columns=["שלום", "1234"])
         tm.assert_frame_equal(result, expected)
 
@@ -140,15 +142,17 @@ def test_decompression_regex_sep(self):
             tmp.write(data)
             tmp.close()
 
-            result = self.read_csv(path, sep='::', compression='gzip')
+            result = self.read_csv(path, sep='::', quoting=csv.QUOTE_NONE,
+                                   compression='gzip')
             tm.assert_frame_equal(result, expected)
 
         with tm.ensure_clean() as path:
             tmp = bz2.BZ2File(path, mode='wb')
             tmp.write(data)
             tmp.close()
 
-            result = self.read_csv(path, sep='::', compression='bz2')
+            result = self.read_csv(path, sep='::', quoting=csv.QUOTE_NONE,
+                                   compression='bz2')
             tm.assert_frame_equal(result, expected)
 
             self.assertRaises(ValueError, self.read_csv,
@@ -162,15 +166,17 @@ def test_read_table_buglet_4x_multiindex(self):
 a   q   20      4     0.4473  1.4152  0.2834  1.00661  0.1744
 x   q   30      3    -0.6662 -0.5243 -0.3580  0.89145  2.5838"""
 
-        df = self.read_table(StringIO(text), sep='\s+')
+        df = self.read_table(StringIO(text), sep='\s+',
+                             quoting=csv.QUOTE_NONE)
         self.assertEqual(df.index.names, ('one', 'two', 'three', 'four'))
 
         # see gh-6893
         data = '      A B C\na b c\n1 3 7 0 3 6\n3 1 4 1 5 9'
         expected = DataFrame.from_records(
             [(1, 3, 7, 0, 3, 6), (3, 1, 4, 1, 5, 9)],
             columns=list('abcABC'), index=list('abc'))
-        actual = self.read_table(StringIO(data), sep='\s+')
+        actual = self.read_table(StringIO(data), sep='\s+',
+                                 quoting=csv.QUOTE_NONE)
         tm.assert_frame_equal(actual, expected)
 
     def test_skipfooter_with_decimal(self):
@@ -201,6 +207,6 @@ def test_encoding_non_utf8_multichar_sep(self):
                              'utf-32', 'cp037']:
                 encoded_data = data.encode(encoding)
                 result = self.read_csv(BytesIO(encoded_data),
-                                       sep=sep, names=['a', 'b'],
-                                       encoding=encoding)
+                                       sep=sep, quoting=csv.QUOTE_NONE,
+                                       names=['a', 'b'], encoding=encoding)
                 tm.assert_frame_equal(result, expected)

pandas/io/tests/parser/test_read_fwf.py

@@ -8,6 +8,7 @@
 
 from datetime import datetime
 
+import csv
 import nose
 import numpy as np
 import pandas as pd
@@ -327,9 +328,10 @@ def test_multiple_delimiters(self):
 '''.strip('\r\n')
         colspecs = ((0, 4), (7, 13), (15, 19), (21, 41))
         expected = read_fwf(StringIO(test), colspecs=colspecs,
-                            delimiter=' +~.\\')
+                            delimiter=' +~.\\', quoting=csv.QUOTE_NONE)
         tm.assert_frame_equal(expected, read_fwf(StringIO(test),
-                                                 delimiter=' +~.\\'))
+                                                 delimiter=' +~.\\',
+                                                 quoting=csv.QUOTE_NONE))
 
     def test_variable_width_unicode(self):
         if not compat.PY3:

pandas/io/tests/parser/test_unsupported.py

@@ -9,6 +9,7 @@
 test suite as new feature support is added to the parsers.
 """
 
+import csv
 import nose
 
 import pandas.io.parsers as parsers
@@ -117,6 +118,12 @@ def test_python_engine(self):
                 with tm.assertRaisesRegexp(ValueError, msg):
                     read_csv(StringIO(data), engine=engine, **kwargs)
 
+            msg = "the %r engine does not support " % engine
+            kwargs = {'sep': '\s+', 'quoting': csv.QUOTE_MINIMAL}
+
+            with tm.assertRaisesRegexp(ValueError, msg):
+                read_csv(StringIO(data), engine=engine, **kwargs)
+
 
 class TestDeprecatedFeatures(tm.TestCase):
     def test_deprecated_args(self):

pandas/io/tests/parser/usecols.py

@@ -5,6 +5,7 @@
 for all of the parsers defined in parsers.py
 """
 
+import csv
 import nose
 
 import numpy as np
@@ -139,7 +140,8 @@ def test_usecols_regex_sep(self):
         # see gh-2733
         data = 'a  b  c\n4  apple  bat  5.7\n8  orange  cow  10'
 
-        df = self.read_csv(StringIO(data), sep='\s+', usecols=('a', 'b'))
+        df = self.read_csv(StringIO(data), sep='\s+', usecols=('a', 'b'),
+                           quoting=csv.QUOTE_NONE)
 
         expected = DataFrame({'a': ['apple', 'orange'],
                               'b': ['bat', 'cow']}, index=[4, 8])

pandas/tests/test_multilevel.py

@@ -3,6 +3,7 @@
 import datetime
 import itertools
 import nose
+import csv
 
 from numpy.random import randn
 import numpy as np
@@ -554,7 +555,8 @@ def test_xs_level_multiple(self):
 a   q   20      4     0.4473  1.4152  0.2834  1.00661  0.1744
 x   q   30      3    -0.6662 -0.5243 -0.3580  0.89145  2.5838"""
 
-        df = read_table(StringIO(text), sep='\s+', engine='python')
+        df = read_table(StringIO(text), sep='\s+', engine='python',
+                        quoting=csv.QUOTE_NONE)
 
         result = df.xs(('a', 4), level=['one', 'four'])
         expected = df.xs('a').xs(4, level='four')
@@ -588,7 +590,8 @@ def test_xs_level0(self):
 a   q   20      4     0.4473  1.4152  0.2834  1.00661  0.1744
 x   q   30      3    -0.6662 -0.5243 -0.3580  0.89145  2.5838"""
 
-        df = read_table(StringIO(text), sep='\s+', engine='python')
+        df = read_table(StringIO(text), sep='\s+', engine='python',
+                        quoting=csv.QUOTE_NONE)
 
         result = df.xs('a', level=0)
         expected = df.xs('a')