Open Windows standard file descriptors before initializing Ruby

Author: jonisavo

binding/binding-mri-win32.h

@@ -0,0 +1,42 @@
+#ifndef BINDING_MRI_WIN32_H
+#define BINDING_MRI_WIN32_H
+
+#include <windows.h>
+#include "util/win-consoleutils.h"
+
+// Attempts to set $stdout and $stdin accordingly on Windows. Only
+// called when debug mode is on, since that's when the console
+// should be active.
+void configureWindowsStreams() {
+    const int stdoutFD = getStdFD(STD_OUTPUT_HANDLE);
+
+    // Configure $stdout
+    if (stdoutFD >= 0) {
+        VALUE winStdout = rb_funcall(rb_cIO, rb_intern("new"), 2,
+            INT2NUM(stdoutFD), rb_str_new_cstr("w+"));
+
+        rb_gv_set("stdout", winStdout);
+    }
+
+    const int stdinFD = getStdFD(STD_INPUT_HANDLE);
+
+    // Configure $stdin
+    if (stdinFD >= 0) {
+        VALUE winStdin = rb_funcall(rb_cIO, rb_intern("new"), 2,
+            INT2NUM(stdinFD), rb_str_new_cstr("r"));
+
+        rb_gv_set("stdin", winStdin);
+    }
+
+    const int stderrFD = getStdFD(STD_ERROR_HANDLE);
+
+    // Configure $stderr
+    if (stderrFD >= 0) {
+        VALUE winStderr = rb_funcall(rb_cIO, rb_intern("new"), 2,
+            INT2NUM(stderrFD), rb_str_new_cstr("w+"));
+
+        rb_gv_set("stderr", winStderr);
+    }
+}
+
+#endif

binding/binding-mri.cpp

@@ -50,8 +50,7 @@ extern "C" {
 }
 
 #ifdef __WIN32__
-#include <fcntl.h>
-#include "util/win-consoleutils.h"
+#include "binding-mri-win32.h"
 #endif
 
 #include <assert.h>
@@ -72,9 +71,6 @@ extern const char module_rpg3[];
 static void mriBindingExecute();
 static void mriBindingTerminate();
 static void mriBindingReset();
-#ifdef __WIN32__
-static void configureWindowsStreams();
-#endif
 
 ScriptBinding scriptBindingImpl = {mriBindingExecute, mriBindingTerminate,
     mriBindingReset};
@@ -274,10 +270,7 @@ static void mriBindingInit() {
     // Set $stdout and its ilk accordingly on Windows
 #ifdef __WIN32__
     if (shState->config().editor.debug)
-    {
-        reopenWindowsStreams();
         configureWindowsStreams();
-    }
 #endif
 
     // Load zlib, if it's present. Requires --with-static-linked-ext or zlib.so.
@@ -947,53 +940,6 @@ static void runRMXPScripts(BacktraceData &btData) {
     }
 }
 
-// Attempts to set $stdout and $stdin accordingly on Windows. Only
-// called when debug mode is on, since that's when the console
-// should be active.
-#ifdef __WIN32__
-static void configureWindowsStreams() {
-    #define HANDLE_VALID(handle) handle && handle != INVALID_HANDLE_VALUE
-
-    const HANDLE outputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
-
-    // Configure $stdout
-    if (HANDLE_VALID(outputHandle)) {
-        const int stdoutFD = _open_osfhandle((intptr_t)outputHandle, _O_TEXT);
-
-        VALUE winStdout = rb_funcall(rb_cIO, rb_intern("new"), 2,
-            INT2NUM(stdoutFD), rb_str_new_cstr("w+"));
-
-        rb_gv_set("stdout", winStdout);
-    }
-
-    const HANDLE inputHandle = GetStdHandle(STD_INPUT_HANDLE);
-
-    // Configure $stdin
-    if (HANDLE_VALID(inputHandle)) {
-        const int stdinFD = _open_osfhandle((intptr_t)inputHandle, _O_TEXT);
-
-        VALUE winStdin = rb_funcall(rb_cIO, rb_intern("new"), 2,
-            INT2NUM(stdinFD), rb_str_new_cstr("r"));
-
-        rb_gv_set("stdin", winStdin);
-    }
-
-    const HANDLE errorHandle = GetStdHandle(STD_ERROR_HANDLE);
-
-    // Configure $stderr
-    if (HANDLE_VALID(errorHandle)) {
-        const int stderrFD = _open_osfhandle((intptr_t)errorHandle, _O_TEXT);
-
-        VALUE winStderr = rb_funcall(rb_cIO, rb_intern("new"), 2,
-            INT2NUM(stderrFD), rb_str_new_cstr("w+"));
-
-        rb_gv_set("stderr", winStderr);
-    }
-
-    #undef HANDLE_VALID
-}
-#endif // #ifdef __WIN32__
-
 static void showExc(VALUE exc, const BacktraceData &btData) {
     VALUE bt = rb_funcall2(exc, rb_intern("backtrace"), 0, NULL);
     VALUE msg = rb_funcall2(exc, rb_intern("message"), 0, NULL);

src/main.cpp

@@ -296,7 +296,9 @@ int main(int argc, char *argv[]) {
 
     // Create a debug console in debug mode
     if (conf.editor.debug) {
-      if (!setupWindowsConsole()) {
+      if (setupWindowsConsole()) {
+        reopenWindowsStreams();
+      } else {
         char buf[200];
         snprintf(buf, sizeof(buf), "Error allocating console: %lu",
                 GetLastError());

src/util/win-consoleutils.cpp

@@ -13,9 +13,15 @@ bool setupWindowsConsole()
     return (handle != NULL && handle != INVALID_HANDLE_VALUE);
 }
 
-FILE *outStream;
-FILE *inStream;
-FILE *errStream;
+static FILE *outStream;
+static FILE *inStream;
+static FILE *errStream;
+
+static int stdoutFD = -1;
+static int stderrFD = -1;
+static int stdinFD  = -1;
+
+static int openStdHandle(const DWORD &nStdHandle);
 
 // Reopens the file streams. This should be done after successfully
 // setting up the console.
@@ -28,4 +34,33 @@ void reopenWindowsStreams()
     std::clog.clear();
     std::cerr.clear();
     std::cin.clear();
+
+    stdoutFD = openStdHandle(STD_OUTPUT_HANDLE);
+    stdinFD = openStdHandle(STD_INPUT_HANDLE);
+    stderrFD = openStdHandle(STD_ERROR_HANDLE);
+}
+
+int getStdFD(const DWORD &nStdHandle)
+{
+    switch (nStdHandle)
+    {
+        case STD_OUTPUT_HANDLE:
+            return stdoutFD;
+        case STD_INPUT_HANDLE:
+            return stdinFD;
+        case STD_ERROR_HANDLE:
+            return stderrFD;
+        default:
+            return -1;
+    }
+}
+
+static int openStdHandle(const DWORD &nStdHandle)
+{
+    const HANDLE handle = GetStdHandle(nStdHandle);
+
+    if (!handle || handle == INVALID_HANDLE_VALUE)
+        return -1;
+
+    return _open_osfhandle((intptr_t)handle, _O_TEXT);
 }

src/util/win-consoleutils.h

@@ -3,9 +3,13 @@
 
 #include <iostream>
 #include <windows.h>
+#include <fcntl.h>
+#include <io.h>
 
 bool setupWindowsConsole();
 
 void reopenWindowsStreams();
 
+int getStdFD(const DWORD &nStdHandle);
+
 #endif