revert #16663, which was a revert of #16039 (#16675)

* Revert "BUG: Revert gh-16039 (#16663)"

This reverts commit c550372.

* always treat files as binary to cope with windows and EOF.

on windows, EOF can appear "in band" if the file is considered text. when
moving from fread() to read(), i lost the "b" part of the mode. at the
time i believed this was a nop, since unix doesnt treat files differently
based on that flag.

this adds O_BINARY to the flags to open to restore the behaviour lost
when taking "b" away from fopen. if a platform doesn't provide O_BINARY,
this defines it to 0 so it can still be used without effect later on in
the code.

* dont leak the fd in new_file_source() if buffer allocation fails.

* reapply the test for EOF in the middle of a stream.

part of c550372

* pass rb to _get_handle on python 3, otherwise stick to r.

part of c550372

* replace goto with inline unwinding of state.

requested by @jreback in #16675 feedback.

* describe the fixes to the read_csv() backend and issue numbers.

requested by @jreback in feedback on #16675

Author: dgwynne

doc/source/whatsnew/v0.20.3.txt

@@ -55,8 +55,7 @@ Indexing
 I/O
 ^^^
 
-- Bug in ``pd.read_csv()`` in which files containing EOF characters mid-field could fail with the C engine on Windows (:issue:`16039`, :issue:`16559`)
-
+-- Bug in ``pd.read_csv()`` in which files weren't opened as binary files by the C engine on Windows, causing EOF characters mid-field, which would fail (:issue:`16039`, :issue:`16559`, :issue`16675`)
 
 Plotting
 ^^^^^^^^

pandas/_libs/src/parser/io.c

@@ -9,31 +9,41 @@ The full license is in the LICENSE file, distributed with this software.
 
 #include "io.h"
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif /* O_BINARY */
+
 /*
   On-disk FILE, uncompressed
 */
 
 void *new_file_source(char *fname, size_t buffer_size) {
     file_source *fs = (file_source *)malloc(sizeof(file_source));
-    fs->fp = fopen(fname, "rb");
+    if (fs == NULL) {
+        return NULL;
+    }
 
-    if (fs->fp == NULL) {
+    fs->fd = open(fname, O_RDONLY | O_BINARY);
+    if (fs->fd == -1) {
         free(fs);
         return NULL;
     }
-    setbuf(fs->fp, NULL);
-
-    fs->initial_file_pos = ftell(fs->fp);
 
     // Only allocate this heap memory if we are not memory-mapping the file
     fs->buffer = (char *)malloc((buffer_size + 1) * sizeof(char));
 
     if (fs->buffer == NULL) {
+        close(fs->fd);
+        free(fs);
         return NULL;
     }
 
-    memset(fs->buffer, 0, buffer_size + 1);
-    fs->buffer[buffer_size] = '\0';
+    memset(fs->buffer, '\0', buffer_size + 1);
+    fs->size = buffer_size;
 
     return (void *)fs;
 }
@@ -56,12 +66,12 @@ void *new_rd_source(PyObject *obj) {
 
  */
 
-int del_file_source(void *fs) {
+int del_file_source(void *ptr) {
+    file_source *fs = ptr;
     if (fs == NULL) return 0;
 
-    /* allocated on the heap */
-    free(FS(fs)->buffer);
-    fclose(FS(fs)->fp);
+    free(fs->buffer);
+    close(fs->fd);
     free(fs);
 
     return 0;
@@ -83,17 +93,31 @@ int del_rd_source(void *rds) {
 
 void *buffer_file_bytes(void *source, size_t nbytes, size_t *bytes_read,
                         int *status) {
-    file_source *src = FS(source);
+    file_source *fs = FS(source);
+    ssize_t rv;
 
-    *bytes_read = fread((void *)src->buffer, sizeof(char), nbytes, src->fp);
+    if (nbytes > fs->size) {
+        nbytes = fs->size;
+    }
 
-    if (*bytes_read == 0) {
+    rv = read(fs->fd, fs->buffer, nbytes);
+    switch (rv) {
+    case -1:
+        *status = CALLING_READ_FAILED;
+        *bytes_read = 0;
+        return NULL;
+    case 0:
         *status = REACHED_EOF;
-    } else {
+        *bytes_read = 0;
+        return NULL;
+    default:
         *status = 0;
+        *bytes_read = rv;
+        fs->buffer[rv] = '\0';
+        break;
     }
 
-    return (void *)src->buffer;
+    return (void *)fs->buffer;
 }
 
 void *buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read,
@@ -152,80 +176,84 @@ void *buffer_rd_bytes(void *source, size_t nbytes, size_t *bytes_read,
 #ifdef HAVE_MMAP
 
 #include <sys/mman.h>
-#include <sys/stat.h>
 
 void *new_mmap(char *fname) {
-    struct stat buf;
-    int fd;
     memory_map *mm;
-    off_t filesize;
+    struct stat stat;
+    size_t filesize;
 
     mm = (memory_map *)malloc(sizeof(memory_map));
-    mm->fp = fopen(fname, "rb");
-
-    fd = fileno(mm->fp);
-    if (fstat(fd, &buf) == -1) {
-        fprintf(stderr, "new_file_buffer: fstat() failed. errno =%d\n", errno);
-        return NULL;
-    }
-    filesize = buf.st_size; /* XXX This might be 32 bits. */
-
     if (mm == NULL) {
-        /* XXX Eventually remove this print statement. */
         fprintf(stderr, "new_file_buffer: malloc() failed.\n");
+        return (NULL);
+    }
+    mm->fd = open(fname, O_RDONLY | O_BINARY);
+    if (mm->fd == -1) {
+        fprintf(stderr, "new_file_buffer: open(%s) failed. errno =%d\n",
+          fname, errno);
+        free(mm);
         return NULL;
     }
-    mm->size = (off_t)filesize;
-    mm->line_number = 0;
 
-    mm->fileno = fd;
-    mm->position = ftell(mm->fp);
-    mm->last_pos = (off_t)filesize;
+    if (fstat(mm->fd, &stat) == -1) {
+        fprintf(stderr, "new_file_buffer: fstat() failed. errno =%d\n",
+          errno);
+        close(mm->fd);
+        free(mm);
+        return NULL;
+    }
+    filesize = stat.st_size; /* XXX This might be 32 bits. */
 
-    mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, fd, 0);
-    if (mm->memmap == NULL) {
+    mm->memmap = mmap(NULL, filesize, PROT_READ, MAP_SHARED, mm->fd, 0);
+    if (mm->memmap == MAP_FAILED) {
         /* XXX Eventually remove this print statement. */
         fprintf(stderr, "new_file_buffer: mmap() failed.\n");
+        close(mm->fd);
         free(mm);
-        mm = NULL;
+        return NULL;
     }
 
-    return (void *)mm;
+    mm->size = (off_t)filesize;
+    mm->position = 0;
+
+    return mm;
 }
 
-int del_mmap(void *src) {
-    munmap(MM(src)->memmap, MM(src)->size);
+int del_mmap(void *ptr) {
+    memory_map *mm = ptr;
+
+    if (mm == NULL) return 0;
 
-    fclose(MM(src)->fp);
-    free(src);
+    munmap(mm->memmap, mm->size);
+    close(mm->fd);
+    free(mm);
 
     return 0;
 }
 
 void *buffer_mmap_bytes(void *source, size_t nbytes, size_t *bytes_read,
                         int *status) {
     void *retval;
-    memory_map *src = MM(source);
+    memory_map *src = source;
+    size_t remaining = src->size - src->position;
 
-    if (src->position == src->last_pos) {
+    if (remaining == 0) {
         *bytes_read = 0;
         *status = REACHED_EOF;
         return NULL;
     }
 
-    retval = src->memmap + src->position;
-
-    if (src->position + (off_t)nbytes > src->last_pos) {
-        // fewer than nbytes remaining
-        *bytes_read = src->last_pos - src->position;
-    } else {
-        *bytes_read = nbytes;
+    if (nbytes > remaining) {
+        nbytes = remaining;
     }
 
-    *status = 0;
+    retval = src->memmap + src->position;
 
     /* advance position in mmap data structure */
-    src->position += *bytes_read;
+    src->position += nbytes;
+
+    *bytes_read = nbytes;
+    *status = 0;
 
     return retval;
 }

pandas/_libs/src/parser/io.h

@@ -15,19 +15,10 @@ The full license is in the LICENSE file, distributed with this software.
 
 typedef struct _file_source {
     /* The file being read. */
-    FILE *fp;
+    int fd;
 
     char *buffer;
-
-    /* file position when the file_buffer was created. */
-    off_t initial_file_pos;
-
-    /* Offset in the file of the data currently in the buffer. */
-    off_t buffer_file_pos;
-
-    /* Actual number of bytes in the current buffer. (Can be less than
-     * buffer_size.) */
-    off_t last_pos;
+    size_t size;
 } file_source;
 
 #define FS(source) ((file_source *)source)
@@ -37,20 +28,13 @@ typedef struct _file_source {
 #endif
 
 typedef struct _memory_map {
-    FILE *fp;
+    int fd;
 
     /* Size of the file, in bytes. */
-    off_t size;
-
-    /* file position when the file_buffer was created. */
-    off_t initial_file_pos;
-
-    int line_number;
-
-    int fileno;
-    off_t position;
-    off_t last_pos;
     char *memmap;
+    size_t size;
+
+    size_t position;
 } memory_map;
 
 #define MM(src) ((memory_map *)src)