Solve filename and command line encoding issues on Windows

src/Makefile.test.include

@@ -109,6 +109,7 @@ BITCOIN_TESTS =\
   test/evo_deterministicmns_tests.cpp \
   test/evo_specialtx_tests.cpp \
   test/flatfile_tests.cpp \
+  test/fs_tests.cpp \
   test/getarg_tests.cpp \
   test/hash_tests.cpp \
   test/key_tests.cpp \

src/fs.cpp

@@ -5,12 +5,221 @@
 
 #include "fs.h"
 
+#ifndef WIN32
+#include <fcntl.h>
+#else
+#define NOMINMAX
+#include <codecvt>
+#include <windows.h>
+#endif
+
 namespace fsbridge {
 
 FILE *fopen(const fs::path& p, const char *mode)
 {
+#ifndef WIN32
     return ::fopen(p.string().c_str(), mode);
+#else
+    std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t> utf8_cvt;
+    return ::_wfopen(p.wstring().c_str(), utf8_cvt.from_bytes(mode).c_str());
+#endif
+}
+
+
+#ifndef WIN32
+
+static std::string GetErrorReason() {
+    return std::strerror(errno);
+}
+
+FileLock::FileLock(const fs::path& file)
+{
+    fd = open(file.string().c_str(), O_RDWR);
+    if (fd == -1) {
+        reason = GetErrorReason();
+    }
+}
+
+FileLock::~FileLock()
+{
+    if (fd != -1) {
+        close(fd);
+    }
+}
+
+bool FileLock::TryLock()
+{
+    if (fd == -1) {
+        return false;
+    }
+    struct flock lock;
+    lock.l_type = F_WRLCK;
+    lock.l_whence = SEEK_SET;
+    lock.l_start = 0;
+    lock.l_len = 0;
+    if (fcntl(fd, F_SETLK, &lock) == -1) {
+        reason = GetErrorReason();
+        return false;
+    }
+    return true;
+}
+#else
+
+static std::string GetErrorReason() {
+    wchar_t* err;
+    FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+        nullptr, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), reinterpret_cast<WCHAR*>(&err), 0, nullptr);
+    std::wstring err_str(err);
+    LocalFree(err);
+    return std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>>().to_bytes(err_str);
+}
+
+FileLock::FileLock(const fs::path& file)
+{
+    hFile = CreateFileW(file.wstring().c_str(),  GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+        nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr);
+    if (hFile == INVALID_HANDLE_VALUE) {
+        reason = GetErrorReason();
+    }
+}
+
+FileLock::~FileLock()
+{
+    if (hFile != INVALID_HANDLE_VALUE) {
+        CloseHandle(hFile);
+    }
 }
 
+bool FileLock::TryLock()
+{
+    if (hFile == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+    _OVERLAPPED overlapped = {0};
+    if (!LockFileEx(hFile, LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY, 0, std::numeric_limits<DWORD>::max(), std::numeric_limits<DWORD>::max(), &overlapped)) {
+        reason = GetErrorReason();
+        return false;
+    }
+    return true;
+}
+#endif
+
+std::string get_filesystem_error_message(const fs::filesystem_error& e)
+{
+#ifndef WIN32
+    return e.what();
+#else
+    // Convert from Multi Byte to utf-16
+    std::string mb_string(e.what());
+    int size = MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), nullptr, 0);
+
+    std::wstring utf16_string(size, L'\0');
+    MultiByteToWideChar(CP_ACP, 0, mb_string.c_str(), mb_string.size(), &*utf16_string.begin(), size);
+    // Convert from utf-16 to utf-8
+    return std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t>().to_bytes(utf16_string);
+#endif
+}
+
+#ifdef WIN32
+#ifdef __GLIBCXX__
+
+// reference: https://github.com/gcc-mirror/gcc/blob/gcc-7_3_0-release/libstdc%2B%2B-v3/include/std/fstream#L270
+
+static std::string openmodeToStr(std::ios_base::openmode mode)
+{
+    switch (mode & ~std::ios_base::ate) {
+    case std::ios_base::out:
+    case std::ios_base::out | std::ios_base::trunc:
+        return "w";
+    case std::ios_base::out | std::ios_base::app:
+    case std::ios_base::app:
+        return "a";
+    case std::ios_base::in:
+        return "r";
+    case std::ios_base::in | std::ios_base::out:
+        return "r+";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::trunc:
+        return "w+";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::app:
+    case std::ios_base::in | std::ios_base::app:
+        return "a+";
+    case std::ios_base::out | std::ios_base::binary:
+    case std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
+        return "wb";
+    case std::ios_base::out | std::ios_base::app | std::ios_base::binary:
+    case std::ios_base::app | std::ios_base::binary:
+        return "ab";
+    case std::ios_base::in | std::ios_base::binary:
+        return "rb";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::binary:
+        return "r+b";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::trunc | std::ios_base::binary:
+        return "w+b";
+    case std::ios_base::in | std::ios_base::out | std::ios_base::app | std::ios_base::binary:
+    case std::ios_base::in | std::ios_base::app | std::ios_base::binary:
+        return "a+b";
+    default:
+        return std::string();
+    }
+}
+
+void ifstream::open(const fs::path& p, std::ios_base::openmode mode)
+{
+    close();
+    m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
+    if (m_file == nullptr) {
+        return;
+    }
+    m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
+    rdbuf(&m_filebuf);
+    if (mode & std::ios_base::ate) {
+        seekg(0, std::ios_base::end);
+    }
+}
+
+void ifstream::close()
+{
+    if (m_file != nullptr) {
+        m_filebuf.close();
+        fclose(m_file);
+    }
+    m_file = nullptr;
+}
+
+void ofstream::open(const fs::path& p, std::ios_base::openmode mode)
+{
+    close();
+    m_file = fsbridge::fopen(p, openmodeToStr(mode).c_str());
+    if (m_file == nullptr) {
+        return;
+    }
+    m_filebuf = __gnu_cxx::stdio_filebuf<char>(m_file, mode);
+    rdbuf(&m_filebuf);
+    if (mode & std::ios_base::ate) {
+        seekp(0, std::ios_base::end);
+    }
+}
+
+void ofstream::close()
+{
+    if (m_file != nullptr) {
+        m_filebuf.close();
+        fclose(m_file);
+    }
+    m_file = nullptr;
+}
+#else // __GLIBCXX__
+
+static_assert(sizeof(*fs::path().BOOST_FILESYSTEM_C_STR) == sizeof(wchar_t),
+    "Warning: This build is using boost::filesystem ofstream and ifstream "
+    "implementations which will fail to open paths containing multibyte "
+    "characters. You should delete this static_assert to ignore this warning, "
+    "or switch to a different C++ standard library like the Microsoft C++ "
+    "Standard Library (where boost uses non-standard extensions to construct "
+    "stream objects with wide filenames), or the GNU libstdc++ library (where "
+    "a more complicated workaround has been implemented above).");
+
+#endif // __GLIBCXX__
+#endif // WIN32
 
 } // fsbridge

src/fs.h

@@ -8,6 +8,9 @@
 
 #include <stdio.h>
 #include <string>
+#if defined WIN32 && defined __GLIBCXX__
+#include <ext/stdio_filebuf.h>
+#endif
 
 #define BOOST_FILESYSTEM_NO_DEPRECATED
 #include <boost/filesystem.hpp>
@@ -19,6 +22,76 @@ namespace fs = boost::filesystem;
 /** Bridge operations to C stdio */
 namespace fsbridge {
     FILE *fopen(const fs::path& p, const char *mode);
-};
 
+    class FileLock
+    {
+    public:
+        FileLock() = delete;
+        FileLock(const FileLock&) = delete;
+        FileLock(FileLock&&) = delete;
+        explicit FileLock(const fs::path& file);
+        ~FileLock();
+        bool TryLock();
+        std::string GetReason() { return reason; }
+
+    private:
+        std::string reason;
+#ifndef WIN32
+        int fd = -1;
+#else
+        void* hFile = (void*)-1; // INVALID_HANDLE_VALUE
 #endif
+    };
+
+    std::string get_filesystem_error_message(const fs::filesystem_error& e);
+
+    // GNU libstdc++ specific workaround for opening UTF-8 paths on Windows.
+    //
+    // On Windows, it is only possible to reliably access multibyte file paths through
+    // `wchar_t` APIs, not `char` APIs. But because the C++ standard doesn't
+    // require ifstream/ofstream `wchar_t` constructors, and the GNU library doesn't
+    // provide them (in contrast to the Microsoft C++ library, see
+    // https://stackoverflow.com/questions/821873/how-to-open-an-stdfstream-ofstream-or-ifstream-with-a-unicode-filename/822032#822032),
+    // Boost is forced to fall back to `char` constructors which may not work properly.
+    //
+    // Work around this issue by creating stream objects with `_wfopen` in
+    // combination with `__gnu_cxx::stdio_filebuf`. This workaround can be removed
+    // with an upgrade to C++17, where streams can be constructed directly from
+    // `std::filesystem::path` objects.
+
+#if defined WIN32 && defined __GLIBCXX__
+    class ifstream : public std::istream
+    {
+    public:
+        ifstream() = default;
+        explicit ifstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in) { open(p, mode); }
+        ~ifstream() { close(); }
+        void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::in);
+        bool is_open() { return m_filebuf.is_open(); }
+        void close();
+
+    private:
+        __gnu_cxx::stdio_filebuf<char> m_filebuf;
+        FILE* m_file = nullptr;
+    };
+    class ofstream : public std::ostream
+    {
+    public:
+        ofstream() = default;
+        explicit ofstream(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out) { open(p, mode); }
+        ~ofstream() { close(); }
+        void open(const fs::path& p, std::ios_base::openmode mode = std::ios_base::out);
+        bool is_open() { return m_filebuf.is_open(); }
+        void close();
+
+    private:
+        __gnu_cxx::stdio_filebuf<char> m_filebuf;
+        FILE* m_file = nullptr;
+    };
+#else  // !(WIN32 && __GLIBCXX__)
+    typedef fs::ifstream ifstream;
+    typedef fs::ofstream ofstream;
+#endif // WIN32 && __GLIBCXX__
+};
+
+#endif // BITCOIN_FS_H

src/init.cpp

@@ -75,12 +75,12 @@
 
 #ifndef WIN32
 #include <signal.h>
+#include <sys/stat.h>
 #endif
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/algorithm/string/replace.hpp>
-#include <boost/interprocess/sync/file_lock.hpp>
 #include <boost/thread.hpp>
 
 #if ENABLE_ZMQ
@@ -1190,6 +1190,9 @@ static bool LockDataDirectory(bool probeOnly)
 {
     // Make sure only a single PIVX process is using the data directory.
     fs::path datadir = GetDataDir();
+    if (!DirIsWritable(datadir)) {
+        return UIError(strprintf(_("Cannot write to data directory '%s'; check permissions."), datadir.string()));
+    }
     if (!LockDirectory(datadir, ".lock", probeOnly)) {
         return UIError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), datadir.string(), _(PACKAGE_NAME)));
     }

src/logging.h

@@ -149,9 +149,9 @@ template<typename... Args> std::string FormatStringFromLogArgs(const char *fmt,
         std::string _log_msg_; /* Unlikely name to avoid shadowing variables */     \
         try {                                                                       \
             _log_msg_ = tfm::format(__VA_ARGS__);                                   \
-        } catch (tinyformat::format_error &e) {                                     \
+        } catch (tinyformat::format_error &fmterr) {                                     \
             /* Original format string will have newline so don't add one here */    \
-            _log_msg_ = "Error \"" + std::string(e.what()) +                        \
+            _log_msg_ = "Error \"" + std::string(fmterr.what()) +                        \
                         "\" while formatting log message: " +                       \
                         FormatStringFromLogArgs(__VA_ARGS__);                       \
         }                                                                           \

src/masternodeconfig.cpp

@@ -37,7 +37,7 @@ bool CMasternodeConfig::read(std::string& strErr)
     LOCK(cs_entries);
     int linenumber = 1;
     fs::path pathMasternodeConfigFile = GetMasternodeConfigFile();
-    fs::ifstream streamConfig(pathMasternodeConfigFile);
+    fsbridge::ifstream streamConfig(pathMasternodeConfigFile);
 
     if (!streamConfig.good()) {
         FILE* configFile = fsbridge::fopen(pathMasternodeConfigFile, "a");

src/netbase.cpp

@@ -18,6 +18,8 @@
 
 #ifndef WIN32
 #include <fcntl.h>
+#else
+#include <codecvt>
 #endif
 
 #if !defined(HAVE_MSG_NOSIGNAL) && !defined(MSG_NOSIGNAL)
@@ -703,12 +705,14 @@ bool LookupSubNet(const char* pszName, CSubNet& ret)
 #ifdef WIN32
 std::string NetworkErrorString(int err)
 {
-    char buf[256];
+    wchar_t buf[256];
     buf[0] = 0;
-    if (FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK,
-            NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
-            buf, sizeof(buf), NULL)) {
-        return strprintf("%s (%d)", buf, err);
+    if(FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_MAX_WIDTH_MASK,
+            nullptr, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+            buf, ARRAYSIZE(buf), nullptr))
+    {
+        const auto& bufConvert = std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>,wchar_t>().to_bytes(buf);
+        return strprintf("%s (%d)", bufConvert, err);
     } else {
         return strprintf("Unknown error (%d)", err);
     }