diff --git a/cmake/external/firestore.patch.txt b/cmake/external/firestore.patch.txt index fa029b4ac1..1055b3b575 100644 --- a/cmake/external/firestore.patch.txt +++ b/cmake/external/firestore.patch.txt @@ -11,7 +11,10 @@ index 920bf2928..c5c9cc7ee 100644 +# in the firebase-cpp-sdk. If this version ever changes then make sure +# to update leveldb.cmake in the firebase-cpp-sdk accordingly. +set(version 1.23) - ++ ++# Patch LevelDB with support for Windows Unicode paths. ++set(patch_file ${CMAKE_SOURCE_DIR}/scripts/git/patches/leveldb/0001-windows-unicode-support.patch) + ExternalProject_Get_property(snappy SOURCE_DIR) set(snappy_source_dir "${SOURCE_DIR}") @@ -39,7 +42,7 @@ ExternalProject_Add( @@ -20,6 +23,7 @@ index 920bf2928..c5c9cc7ee 100644 URL https://github.com/google/leveldb/archive/${version}.tar.gz - URL_HASH SHA256=55423cac9e3306f4a9502c738a001e4a339d1a38ffbee7572d4a07d5d63949b2 + URL_HASH SHA256=9a37f8a6174f09bd622bc723b55881dc541cd50747cbd08831c2a82d620f6d76 ++ PATCH_COMMAND git apply ${patch_file} && git gc --aggressive PREFIX ${PROJECT_BINARY_DIR} diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 866e82ac6e..b5bc099e64 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -23,6 +23,9 @@ endif() # firestore.patch.txt accordingly. set(version 1.23) +# Patch LevelDB with support for Windows Unicode paths. +set(patch_file ${CMAKE_CURRENT_LIST_DIR}/../../scripts/git/patches/leveldb/0001-windows-unicode-support.patch) + ExternalProject_Add( leveldb @@ -30,6 +33,7 @@ ExternalProject_Add( DOWNLOAD_NAME leveldb-${version}.tar.gz URL https://github.com/google/leveldb/archive/${version}.tar.gz URL_HASH SHA256=9a37f8a6174f09bd622bc723b55881dc541cd50747cbd08831c2a82d620f6d76 + PATCH_COMMAND patch -p1 < ${patch_file} PREFIX ${PROJECT_BINARY_DIR} diff --git a/scripts/git/patches/leveldb/0001-windows-unicode-support.patch b/scripts/git/patches/leveldb/0001-windows-unicode-support.patch new file mode 100644 index 0000000000..9fb84817d4 --- /dev/null +++ b/scripts/git/patches/leveldb/0001-windows-unicode-support.patch @@ -0,0 +1,308 @@ +diff --git a/util/env_windows.cc b/util/env_windows.cc +index 449f564..deaaab0 100644 +--- a/util/env_windows.cc ++++ b/util/env_windows.cc +@@ -375,8 +375,9 @@ class WindowsEnv : public Env { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; +- ScopedHandle handle = ::CreateFileA( +- filename.c_str(), desired_access, share_mode, ++ auto wFilename = toUtf16(filename); ++ ScopedHandle handle = ::CreateFileW( ++ wFilename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { +@@ -392,8 +393,9 @@ class WindowsEnv : public Env { + *result = nullptr; + DWORD desired_access = GENERIC_READ; + DWORD share_mode = FILE_SHARE_READ; ++ auto wFilename = toUtf16(filename); + ScopedHandle handle = +- ::CreateFileA(filename.c_str(), desired_access, share_mode, ++ ::CreateFileW(wFilename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_EXISTING, + FILE_ATTRIBUTE_READONLY, + /*hTemplateFile=*/nullptr); +@@ -413,11 +415,12 @@ class WindowsEnv : public Env { + } + + ScopedHandle mapping = +- ::CreateFileMappingA(handle.get(), +- /*security attributes=*/nullptr, PAGE_READONLY, +- /*dwMaximumSizeHigh=*/0, +- /*dwMaximumSizeLow=*/0, +- /*lpName=*/nullptr); ++ ::CreateFileMappingW(handle.get(), ++ /*security attributes=*/nullptr, ++ PAGE_READONLY, ++ /*dwMaximumSizeHigh=*/0, ++ /*dwMaximumSizeLow=*/0, ++ /*lpName=*/nullptr); + if (mapping.is_valid()) { + void* mmap_base = ::MapViewOfFile(mapping.get(), FILE_MAP_READ, + /*dwFileOffsetHigh=*/0, +@@ -438,8 +441,9 @@ class WindowsEnv : public Env { + WritableFile** result) override { + DWORD desired_access = GENERIC_WRITE; + DWORD share_mode = 0; // Exclusive access. +- ScopedHandle handle = ::CreateFileA( +- filename.c_str(), desired_access, share_mode, ++ auto wFilename = toUtf16(filename); ++ ScopedHandle handle = ::CreateFileW( ++ wFilename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { +@@ -455,8 +459,9 @@ class WindowsEnv : public Env { + WritableFile** result) override { + DWORD desired_access = FILE_APPEND_DATA; + DWORD share_mode = 0; // Exclusive access. +- ScopedHandle handle = ::CreateFileA( +- filename.c_str(), desired_access, share_mode, ++ auto wFilename = toUtf16(filename); ++ ScopedHandle handle = ::CreateFileW( ++ wFilename.c_str(), desired_access, share_mode, + /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + /*hTemplateFile=*/nullptr); + if (!handle.is_valid()) { +@@ -469,14 +474,16 @@ class WindowsEnv : public Env { + } + + bool FileExists(const std::string& filename) override { +- return GetFileAttributesA(filename.c_str()) != INVALID_FILE_ATTRIBUTES; ++ auto wFilename = toUtf16(filename); ++ return GetFileAttributesW(wFilename.c_str()) != INVALID_FILE_ATTRIBUTES; + } + + Status GetChildren(const std::string& directory_path, + std::vector* result) override { + const std::string find_pattern = directory_path + "\\*"; +- WIN32_FIND_DATAA find_data; +- HANDLE dir_handle = ::FindFirstFileA(find_pattern.c_str(), &find_data); ++ WIN32_FIND_DATAW find_data; ++ auto wFind_pattern = toUtf16(find_pattern); ++ HANDLE dir_handle = ::FindFirstFileW(wFind_pattern.c_str(), &find_data); + if (dir_handle == INVALID_HANDLE_VALUE) { + DWORD last_error = ::GetLastError(); + if (last_error == ERROR_FILE_NOT_FOUND) { +@@ -488,11 +495,12 @@ class WindowsEnv : public Env { + char base_name[_MAX_FNAME]; + char ext[_MAX_EXT]; + +- if (!_splitpath_s(find_data.cFileName, nullptr, 0, nullptr, 0, base_name, ++ auto find_data_filename = toUtf8(find_data.cFileName); ++ if (!_splitpath_s(find_data_filename.c_str(), nullptr, 0, nullptr, 0, base_name, + ARRAYSIZE(base_name), ext, ARRAYSIZE(ext))) { + result->emplace_back(std::string(base_name) + ext); + } +- } while (::FindNextFileA(dir_handle, &find_data)); ++ } while (::FindNextFileW(dir_handle, &find_data)); + DWORD last_error = ::GetLastError(); + ::FindClose(dir_handle); + if (last_error != ERROR_NO_MORE_FILES) { +@@ -501,15 +509,17 @@ class WindowsEnv : public Env { + return Status::OK(); + } + +- Status RemoveFile(const std::string& filename) override { +- if (!::DeleteFileA(filename.c_str())) { ++ Status DeleteFile(const std::string& filename) override { ++ auto wFilename = toUtf16(filename); ++ if (!::DeleteFileW(wFilename.c_str())) { + return WindowsError(filename, ::GetLastError()); + } + return Status::OK(); + } + + Status CreateDir(const std::string& dirname) override { +- if (!::CreateDirectoryA(dirname.c_str(), nullptr)) { ++ auto wDirname = toUtf16(dirname); ++ if (!::CreateDirectoryW(wDirname.c_str(), nullptr)) { + return WindowsError(dirname, ::GetLastError()); + } + return Status::OK(); +@@ -517,6 +527,9 @@ class WindowsEnv : public Env { + + Status RemoveDir(const std::string& dirname) override { + if (!::RemoveDirectoryA(dirname.c_str())) { ++ Status DeleteDir(const std::string& dirname) override { ++ auto wDirname = toUtf16(dirname); ++ if (!::RemoveDirectoryW(wDirname.c_str())) { + return WindowsError(dirname, ::GetLastError()); + } + return Status::OK(); +@@ -524,7 +537,8 @@ class WindowsEnv : public Env { + + Status GetFileSize(const std::string& filename, uint64_t* size) override { + WIN32_FILE_ATTRIBUTE_DATA file_attributes; +- if (!::GetFileAttributesExA(filename.c_str(), GetFileExInfoStandard, ++ auto wFilename = toUtf16(filename); ++ if (!::GetFileAttributesExW(wFilename.c_str(), GetFileExInfoStandard, + &file_attributes)) { + return WindowsError(filename, ::GetLastError()); + } +@@ -538,7 +552,9 @@ class WindowsEnv : public Env { + Status RenameFile(const std::string& from, const std::string& to) override { + // Try a simple move first. It will only succeed when |to| doesn't already + // exist. +- if (::MoveFileA(from.c_str(), to.c_str())) { ++ auto wFrom = toUtf16(from); ++ auto wTo = toUtf16(to); ++ if (::MoveFileW(wFrom.c_str(), wTo.c_str())) { + return Status::OK(); + } + DWORD move_error = ::GetLastError(); +@@ -547,7 +563,7 @@ class WindowsEnv : public Env { + // succeed when |to| does exist. When writing to a network share, we may not + // be able to change the ACLs. Ignore ACL errors then + // (REPLACEFILE_IGNORE_MERGE_ERRORS). +- if (::ReplaceFileA(to.c_str(), from.c_str(), /*lpBackupFileName=*/nullptr, ++ if (::ReplaceFileW(wTo.c_str(), wFrom.c_str(), /*lpBackupFileName=*/nullptr, + REPLACEFILE_IGNORE_MERGE_ERRORS, + /*lpExclude=*/nullptr, /*lpReserved=*/nullptr)) { + return Status::OK(); +@@ -567,8 +583,9 @@ class WindowsEnv : public Env { + Status LockFile(const std::string& filename, FileLock** lock) override { + *lock = nullptr; + Status result; +- ScopedHandle handle = ::CreateFileA( +- filename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, ++ auto wFilename = toUtf16(filename); ++ ScopedHandle handle = ::CreateFileW( ++ wFilename.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, + /*lpSecurityAttributes=*/nullptr, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, + nullptr); + if (!handle.is_valid()) { +@@ -608,10 +625,11 @@ class WindowsEnv : public Env { + return Status::OK(); + } + +- char tmp_path[MAX_PATH]; +- if (!GetTempPathA(ARRAYSIZE(tmp_path), tmp_path)) { ++ wchar_t wtmp_path[MAX_PATH]; ++ if (!GetTempPathW(ARRAYSIZE(wtmp_path), wtmp_path)) { + return WindowsError("GetTempPath", ::GetLastError()); + } ++ std::string tmp_path = toUtf8(std::wstring(wtmp_path)); + std::stringstream ss; + ss << tmp_path << "leveldbtest-" << std::this_thread::get_id(); + *result = ss.str(); +@@ -622,7 +640,8 @@ class WindowsEnv : public Env { + } + + Status NewLogger(const std::string& filename, Logger** result) override { +- std::FILE* fp = std::fopen(filename.c_str(), "w"); ++ auto wFilename = toUtf16(filename); ++ std::FILE* fp = _wfopen(wFilename.c_str(), L"w"); + if (fp == nullptr) { + *result = nullptr; + return WindowsError(filename, ::GetLastError()); +@@ -678,6 +697,31 @@ class WindowsEnv : public Env { + GUARDED_BY(background_work_mutex_); + + Limiter mmap_limiter_; // Thread-safe. ++ ++ // Converts a Windows wide multi-byte UTF-16 string to a UTF-8 string. ++ // See http://utf8everywhere.org/#windows ++ std::string toUtf8(const std::wstring& wstr) { ++ if (wstr.empty()) return std::string(); ++ int size_needed = WideCharToMultiByte( ++ CP_UTF8, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL); ++ std::string strTo(size_needed, 0); ++ WideCharToMultiByte(CP_UTF8, 0, &wstr[0], (int)wstr.size(), &strTo[0], ++ size_needed, NULL, NULL); ++ return strTo; ++ } ++ ++ // Converts a UTF-8 string to a Windows UTF-16 multi-byte wide character ++ // string. ++ // See http://utf8everywhere.org/#windows ++ std::wstring toUtf16(const std::string& str) { ++ if (str.empty()) return std::wstring(); ++ int size_needed = MultiByteToWideChar( ++ CP_UTF8, 0, &str[0], (int)str.size(), NULL, 0); ++ std::wstring strTo(size_needed, 0); ++ MultiByteToWideChar(CP_UTF8, 0, &str[0], (int)str.size(), &strTo[0], ++ size_needed); ++ return strTo; ++ } + }; + + // Return the maximum number of concurrent mmaps. +diff --git a/util/env_windows_test.cc b/util/env_windows_test.cc +index d6822d2..d108ad9 100644 +--- a/util/env_windows_test.cc ++++ b/util/env_windows_test.cc +@@ -55,6 +55,70 @@ TEST_F(EnvWindowsTest, TestOpenOnRead) { + ASSERT_LEVELDB_OK(env_->RemoveFile(test_file)); + } + ++TEST_F(EnvWindowsTest, TestOpenOnRead_Unicode) { ++ // Write some test data to a single file that will be opened |n| times. ++ std::string test_dir; ++ ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); ++ std::string test_file = test_dir + u8"/open_on_run🏃_read.txt"; ++ ++ std::wstring_convert> converter; ++ std::wstring wideUtf8Path = converter.from_bytes(test_file); ++ FILE* f = _wfopen(wideUtf8Path.c_str(), L"w"); ++ ASSERT_TRUE(f != nullptr); ++ const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; ++ fputs(kFileData, f); ++ fclose(f); ++ ++ // Open test file some number above the sum of the two limits to force ++ // leveldb::WindowsEnv to switch from mapping the file into memory ++ // to basic file reading. ++ const int kNumFiles = kMMapLimit + 5; ++ leveldb::RandomAccessFile* files[kNumFiles] = {0}; ++ for (int i = 0; i < kNumFiles; i++) { ++ ASSERT_LEVELDB_OK(env_->NewRandomAccessFile(test_file, &files[i])); ++ } ++ char scratch; ++ Slice read_result; ++ for (int i = 0; i < kNumFiles; i++) { ++ ASSERT_LEVELDB_OK(files[i]->Read(i, 1, &read_result, &scratch)); ++ ASSERT_EQ(kFileData[i], read_result[0]); ++ } ++ for (int i = 0; i < kNumFiles; i++) { ++ delete files[i]; ++ } ++ ASSERT_LEVELDB_OK(env_->DeleteFile(test_file)); ++} ++ ++TEST_F(EnvWindowsTest, TestGetChildrenEmpty) { ++ // Create some dummy files. ++ std::string test_dir; ++ ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); ++ ++ std::vector result; ++ ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result)); ++ ASSERT_EQ(2, result.size()); // "." and ".." are always returned. ++} ++ ++TEST_F(EnvWindowsTest, TestGetChildren_ChildFiles) { ++ // Create some dummy files. ++ std::string test_dir; ++ ASSERT_LEVELDB_OK(env_->GetTestDirectory(&test_dir)); ++ ++ int childFilesCount = 10; ++ for (int i = 0; i < childFilesCount; i++) { ++ std::string test_file = test_dir + u8"/run🏃_and_jump🦘_" + std::to_string(i) + ".txt"; ++ std::wstring_convert> converter; ++ std::wstring wTest_file = converter.from_bytes(test_file); ++ FILE* f = _wfopen(wTest_file.c_str(), L"w"); ++ ASSERT_TRUE(f != nullptr); ++ fclose(f); ++ } ++ ++ std::vector result; ++ ASSERT_LEVELDB_OK(env_->GetChildren(test_dir, &result)); ++ ASSERT_EQ(childFilesCount + 2, result.size()); // "." and ".." are returned. ++} ++ + } // namespace leveldb + + int main(int argc, char** argv) {