diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 2f225988ca711..ed3d3b52e63fd 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -79,8 +79,8 @@ jobs: run: | set -e -x BINARY_SIZE_THRESHOLD_ARGS="" - echo "Binary size threshold in bytes: 1585152" - BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes 1585152" + echo "Binary size threshold in bytes: 1589248" + BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes 1589248" # Ensure ANDROID_NDK_HOME is available and get its real path if [ -z "$ANDROID_NDK_HOME" ]; then diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h index f45f6c088d2a5..8e0f6669a9dbc 100644 --- a/onnxruntime/core/platform/env.h +++ b/onnxruntime/core/platform/env.h @@ -108,6 +108,35 @@ std::ostream& operator<<(std::ostream& os, gsl::span); /// errno and the error message string if errno indicates an error. std::pair GetErrnoInfo(); +/** + * An owned open file supporting concurrent positional reads. + * + * Reads and length queries refer to the same file even if its pathname is replaced. + * This is not a snapshot: callers must not modify the file in place while reading it. + * Keep the object alive until all callers have finished. Its destruction closes the file. + */ +class RandomAccessFile { + public: + virtual ~RandomAccessFile() = default; + + // Query the open file, leaving length unchanged on failure. + virtual common::Status GetLength(size_t& length) const = 0; + + /** + * Fill buffer starting at offset without changing a shared file position. + * Concurrent calls must use disjoint buffers. Returns only after all I/O has completed. + * Negative offsets, unrepresentable ranges, and unexpected EOF are errors. + * An empty buffer succeeds for any nonnegative offset. On failure, buffer may be partially written. + */ + virtual common::Status Read(FileOffsetType offset, gsl::span buffer) const = 0; + + protected: + RandomAccessFile() = default; + + private: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(RandomAccessFile); +}; + /// \brief An interface used by the onnxruntime implementation to /// access operating system functionality like the filesystem etc. /// @@ -286,6 +315,17 @@ class Env { // Returns empty string if there is no such environment variable available virtual std::string GetEnvironmentVar(const std::string& var_name) const = 0; + /** + * Open a regular file for positional reads. Leaves file unchanged on failure. + * Retain the returned object across every read that must use the same file identity, + * for example throughout loading a tensor or all tensors from one external-data file. + * Custom environments can override this to supply their own file implementation. + */ + virtual common::Status OpenRandomAccessFile(const ORTCHAR_T* /*file_path*/, + std::unique_ptr& /*file*/) const { + return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "This environment does not support random-access files."); + } + protected: Env(); diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc index b2a25282ae5da..43b2c4b9a73ae 100644 --- a/onnxruntime/core/platform/posix/env.cc +++ b/onnxruntime/core/platform/posix/env.cc @@ -103,6 +103,72 @@ long int TempFailureRetry(TFunc retriable_operation, TFuncArgs&&... args) { return result; } +common::Status ReportSystemError(const char* operation_name, const std::string& path) { + auto [err_no, err_msg] = GetErrnoInfo(); + std::ostringstream oss; + oss << operation_name << " file \"" << path << "\" failed: " << err_msg; + return common::Status(common::SYSTEM, err_no, oss.str()); +} + +common::Status GetFileLength(int fd, size_t& file_size) { + if (fd < 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid fd was supplied: ", fd); + } + + struct stat buf; + if (TempFailureRetry(fstat, fd, &buf) < 0) { + return ReportSystemError("fstat", ""); + } + if (buf.st_size < 0) { + return ORT_MAKE_STATUS(SYSTEM, FAIL, "Received negative size from stat call"); + } + if (static_cast(buf.st_size) > std::numeric_limits::max()) { + return ORT_MAKE_STATUS(SYSTEM, FAIL, "File is too large."); + } + + file_size = static_cast(buf.st_size); + return common::Status::OK(); +} + +class PosixRandomAccessFile final : public RandomAccessFile { + public: + PosixRandomAccessFile(ScopedFileDescriptor descriptor, std::string path) + : descriptor_(std::move(descriptor)), path_(std::move(path)) {} + + common::Status GetLength(size_t& length) const override { + return GetFileLength(descriptor_.Get(), length); + } + + common::Status Read(FileOffsetType offset, gsl::span buffer) const override { + if (offset < 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "RandomAccessFile::Read: offset must be nonnegative."); + } + if (static_cast(buffer.size()) > + static_cast(std::numeric_limits::max() - offset)) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "RandomAccessFile::Read: file range is not representable."); + } + + size_t total_bytes_read = 0; + while (total_bytes_read < buffer.size()) { + constexpr size_t kMaxBytesToRead = 1 << 30; + const auto bytes_to_read = std::min(buffer.size() - total_bytes_read, kMaxBytesToRead); + const auto bytes_read = TempFailureRetry(pread, descriptor_.Get(), buffer.data() + total_bytes_read, + bytes_to_read, offset + static_cast(total_bytes_read)); + if (bytes_read < 0) { + return ReportSystemError("pread", path_); + } + ORT_RETURN_IF(bytes_read == 0, "RandomAccessFile::Read: unexpected end of file: ", path_); + total_bytes_read += static_cast(bytes_read); + } + return common::Status::OK(); + } + + private: + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PosixRandomAccessFile); + ScopedFileDescriptor descriptor_; + const std::string path_; +}; + // nftw() callback to remove a file int nftw_remove( const char* fpath, const struct stat* /*sb*/, @@ -371,26 +437,30 @@ class PosixEnv : public Env { } common::Status GetFileLength(int fd, /*out*/ size_t& file_size) const override { - using namespace common; - if (fd < 0) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid fd was supplied: ", fd); - } + return onnxruntime::GetFileLength(fd, file_size); + } - struct stat buf; - int rc = fstat(fd, &buf); - if (rc < 0) { - return ReportSystemError("fstat", ""); + common::Status OpenRandomAccessFile(const ORTCHAR_T* file_path, + std::unique_ptr& file) const override { + if (file_path == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "file_path == nullptr"); } - - if (buf.st_size < 0) { - return ORT_MAKE_STATUS(SYSTEM, FAIL, "Received negative size from stat call"); + // Nonblocking open lets us reject FIFOs without waiting for a writer. + int flags = O_RDONLY | O_NONBLOCK; +#ifdef O_CLOEXEC + flags |= O_CLOEXEC; +#endif + // Android's fortified open is overloaded; resolve the call inside a lambda. + ScopedFileDescriptor descriptor{static_cast(TempFailureRetry([&] { return open(file_path, flags); }))}; + if (!descriptor.IsValid()) { + return ReportSystemError("open", file_path); } - - if (static_cast(buf.st_size) > std::numeric_limits::max()) { - return ORT_MAKE_STATUS(SYSTEM, FAIL, "File is too large."); + struct stat info; + if (TempFailureRetry(fstat, descriptor.Get(), &info) < 0) { + return ReportSystemError("fstat", file_path); } - - file_size = static_cast(buf.st_size); + ORT_RETURN_IF_NOT(S_ISREG(info.st_mode), "Random-access reads require a regular file: ", file_path); + file = std::make_unique(std::move(descriptor), file_path); return Status::OK(); } @@ -486,13 +556,6 @@ class PosixEnv : public Env { return Status::OK(); } - static common::Status ReportSystemError(const char* operation_name, const std::string& path) { - auto [err_no, err_msg] = GetErrnoInfo(); - std::ostringstream oss; - oss << operation_name << " file \"" << path << "\" failed: " << err_msg; - return common::Status(common::SYSTEM, err_no, oss.str()); - } - bool FolderExists(const std::string& path) const override { struct stat sb; if (stat(path.c_str(), &sb)) { diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc index 07d5dfc9c0b22..33f8b3e20994d 100644 --- a/onnxruntime/core/platform/windows/env.cc +++ b/onnxruntime/core/platform/windows/env.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include #include +#include #include #include #include @@ -357,6 +358,122 @@ common::Status WindowsEnv::GetFileLength(int fd, /*out*/ size_t& file_size) cons return Status::OK(); } +namespace { + +class WindowsRandomAccessFile final : public RandomAccessFile { + public: + explicit WindowsRandomAccessFile(wil::unique_hfile file_handle) : file_handle_(std::move(file_handle)) {} + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(WindowsRandomAccessFile); + + Status GetLength(size_t& length) const override { + LARGE_INTEGER file_size{}; + if (!GetFileSizeEx(file_handle_.get(), &file_size)) { + return FileError("GetFileSizeEx", GetLastError()); + } + if (file_size.QuadPart < 0 || + static_cast(file_size.QuadPart) > std::numeric_limits::max()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "RandomAccessFile: invalid or unrepresentable file length"); + } + length = static_cast(file_size.QuadPart); + return Status::OK(); + } + + Status Read(FileOffsetType offset, gsl::span buffer) const override { + if (offset < 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "RandomAccessFile: offset < 0"); + } + if (buffer.size() > static_cast(std::numeric_limits::max() - offset)) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "RandomAccessFile: offset + length overflows"); + } + if (buffer.empty()) { + return Status::OK(); + } + + // Each caller owns its event and OVERLAPPED; neither the file cursor nor another caller's event is used. + wil::unique_handle event{CreateEventExW(nullptr, nullptr, CREATE_EVENT_MANUAL_RESET, EVENT_ALL_ACCESS)}; + if (!event) { + return FileError("CreateEventExW", GetLastError()); + } + + size_t total_bytes_read = 0; + while (total_bytes_read < buffer.size()) { + OVERLAPPED overlapped{}; + const auto current_offset = static_cast(offset) + total_bytes_read; + overlapped.Offset = static_cast(current_offset & 0xFFFFFFFF); + overlapped.OffsetHigh = static_cast(current_offset >> 32); + overlapped.hEvent = event.get(); + constexpr size_t kMaxBytesToRead = 1 << 30; + const DWORD bytes_to_read = + static_cast(std::min(buffer.size() - total_bytes_read, kMaxBytesToRead)); + if (!ReadFile(file_handle_.get(), buffer.data() + total_bytes_read, bytes_to_read, nullptr, &overlapped)) { + const auto error_code = GetLastError(); + if (error_code != ERROR_IO_PENDING) { + return FileError("ReadFile", error_code); + } + } + + DWORD bytes_read = 0; + if (!GetOverlappedResult(file_handle_.get(), &overlapped, &bytes_read, TRUE)) { + const auto error_code = GetLastError(); + // A failed wait must not let outstanding I/O outlive the buffer, OVERLAPPED, or event. + if (!HasOverlappedIoCompleted(&overlapped)) { + (void)CancelIoEx(file_handle_.get(), &overlapped); + do { + (void)GetOverlappedResult(file_handle_.get(), &overlapped, &bytes_read, TRUE); + } while (!HasOverlappedIoCompleted(&overlapped)); + } + return FileError("GetOverlappedResult", error_code); + } + if (bytes_read == 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "RandomAccessFile: unexpected end of file"); + } + total_bytes_read += bytes_read; + } + return Status::OK(); + } + + private: + static Status FileError(const char* operation, DWORD error_code) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "RandomAccessFile: ", operation, " failed, errcode = ", + error_code, " - ", std::system_category().message(error_code)); + } + + wil::unique_hfile file_handle_; +}; + +} // namespace + +Status WindowsEnv::OpenRandomAccessFile(_In_z_ const ORTCHAR_T* file_path, + std::unique_ptr& file) const { + if (file_path == nullptr) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "OpenRandomAccessFile: file_path == nullptr"); + } + CREATEFILE2_EXTENDED_PARAMETERS parameters{}; + parameters.dwSize = sizeof(parameters); + parameters.dwFileFlags = FILE_FLAG_OVERLAPPED; + wil::unique_hfile file_handle{ + CreateFile2(file_path, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING, ¶meters)}; + if (file_handle.get() == INVALID_HANDLE_VALUE) { + const auto error_code = GetLastError(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", ToUTF8String(Basename(file_path)), + " fail, errcode = ", error_code, " - ", std::system_category().message(error_code)); + } + if (GetFileType(file_handle.get()) != FILE_TYPE_DISK) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "OpenRandomAccessFile: expected a disk file"); + } + BY_HANDLE_FILE_INFORMATION information{}; + if (!GetFileInformationByHandle(file_handle.get(), &information)) { + const auto error_code = GetLastError(); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "GetFileInformationByHandle failed, errcode = ", + error_code, " - ", std::system_category().message(error_code)); + } + if ((information.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "OpenRandomAccessFile: expected a regular file"); + } + file = std::make_unique(std::move(file_handle)); + return Status::OK(); +} + Status WindowsEnv::ReadFileIntoBuffer(_In_z_ const ORTCHAR_T* const file_path, const FileOffsetType offset, const size_t length, const gsl::span buffer) const { ORT_RETURN_IF_NOT(file_path, "file_path == nullptr"); diff --git a/onnxruntime/core/platform/windows/env.h b/onnxruntime/core/platform/windows/env.h index df8a3e10d512a..ba5f8f97ff260 100644 --- a/onnxruntime/core/platform/windows/env.h +++ b/onnxruntime/core/platform/windows/env.h @@ -61,6 +61,8 @@ class WindowsEnv : public Env { PIDType GetSelfPid() const override; Status GetFileLength(_In_z_ const ORTCHAR_T* file_path, size_t& length) const override; common::Status GetFileLength(int fd, /*out*/ size_t& file_size) const override; + Status OpenRandomAccessFile(_In_z_ const ORTCHAR_T* file_path, + std::unique_ptr& file) const override; Status ReadFileIntoBuffer(_In_z_ const ORTCHAR_T* const file_path, const FileOffsetType offset, const size_t length, const gsl::span buffer) const override; Status MapFileIntoMemory(_In_z_ const ORTCHAR_T* file_path, diff --git a/onnxruntime/test/platform/env_test.cc b/onnxruntime/test/platform/env_test.cc index be7b0ce397c48..f49385e9359fa 100644 --- a/onnxruntime/test/platform/env_test.cc +++ b/onnxruntime/test/platform/env_test.cc @@ -5,11 +5,28 @@ #include #include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include #include "gtest/gtest.h" #include "core/common/path_string.h" +#include "core/common/inlined_containers.h" +#include "core/common/safeint.h" #include "test/util/include/asserts.h" +#include "test/util/include/file_util.h" namespace onnxruntime { namespace test { @@ -55,5 +72,275 @@ TEST(PlatformEnvTest, GetErrnoInfo) { #endif } +namespace { + +void WriteRandomAccessTestFile(const std::string& contents, PathString& path, ScopedFileDeleter& deleter) { + path = ORT_TSTR("random_access_file_XXXXXX"); + FILE* file = nullptr; + ASSERT_NO_FATAL_FAILURE(CreateTestFile(file, path)); + deleter = ScopedFileDeleter(path); + std::unique_ptr owner(file, fclose); + ASSERT_EQ(contents.size(), fwrite(contents.data(), 1, contents.size(), file)); + ASSERT_EQ(0, fclose(owner.release())); +} + +class RandomAccessFileTest : public testing::Test { + protected: + void SetUp() override { + contents_.resize(64 * 1024); + for (size_t i = 0; i < contents_.size(); ++i) { + contents_[i] = static_cast((i * 31 + i / 257) & 0xff); + } + ASSERT_NO_FATAL_FAILURE(WriteRandomAccessTestFile(contents_, path_, deleter_)); + ASSERT_STATUS_OK(Env::Default().OpenRandomAccessFile(path_.c_str(), file_)); + ASSERT_NE(file_, nullptr); + } + + std::string contents_; + PathString path_; + ScopedFileDeleter deleter_; + std::unique_ptr file_; +}; + +TEST_F(RandomAccessFileTest, ReadsRangesAndLengthFromOneOpenFile) { + size_t length = 0; + ASSERT_STATUS_OK(file_->GetLength(length)); + EXPECT_EQ(length, contents_.size()); + size_t legacy_length = 0; + ASSERT_STATUS_OK(Env::Default().GetFileLength(path_.c_str(), legacy_length)); + EXPECT_EQ(length, legacy_length); + std::string output(193, '\0'); + ASSERT_STATUS_OK(file_->Read(271, gsl::span(output))); + EXPECT_EQ(output, contents_.substr(271, output.size())); + ASSERT_STATUS_OK(file_->Read(7, gsl::span(output))); + EXPECT_EQ(output, contents_.substr(7, output.size())); + + std::string legacy_output(output.size(), '\0'); + ASSERT_STATUS_OK(Env::Default().ReadFileIntoBuffer(path_.c_str(), 7, legacy_output.size(), + gsl::span(legacy_output))); + EXPECT_EQ(output, legacy_output); +} + +#ifndef __wasm__ +TEST_F(RandomAccessFileTest, ConcurrentReadsDoNotShareAFilePosition) { + constexpr size_t kReaderCount = 4; + std::array statuses; + std::array matched; + matched.fill(true); + { + InlinedVector readers; + readers.reserve(kReaderCount); + auto join_readers = gsl::finally([&] { + for (auto& reader : readers) { + reader.join(); + } + }); + for (size_t reader = 0; reader < kReaderCount; ++reader) { + readers.emplace_back([&, reader] { + std::string output(4096, '\0'); + for (size_t iteration = 0; iteration < 100; ++iteration) { + const auto offset = (reader * 1009 + iteration * 3277) % (contents_.size() - output.size()); + statuses[reader] = file_->Read(static_cast(offset), gsl::span(output)); + if (!statuses[reader].IsOK()) { + return; + } + if (output != contents_.substr(offset, output.size())) { + matched[reader] = false; + return; + } + } + }); + } + } + for (size_t reader = 0; reader < kReaderCount; ++reader) { + ASSERT_STATUS_OK(statuses[reader]); + EXPECT_TRUE(matched[reader]) << "Reader " << reader; + } +} +#endif + +TEST_F(RandomAccessFileTest, RejectsInvalidRangesAndUnexpectedEof) { + std::array output{}; + EXPECT_EQ(file_->Read(-1, output).Code(), common::INVALID_ARGUMENT); + constexpr auto kMaxOffset = std::numeric_limits::max(); + EXPECT_EQ(file_->Read(kMaxOffset, output).Code(), common::INVALID_ARGUMENT); + ASSERT_STATUS_OK(file_->Read(kMaxOffset, {})); + ASSERT_STATUS_OK(file_->Read(static_cast(contents_.size()), {})); + EXPECT_FALSE(file_->Read(static_cast(contents_.size()), output).IsOK()); + // This request can read a prefix, but must fail rather than accept a short read. + EXPECT_FALSE(file_->Read(static_cast(contents_.size() - 2), output).IsOK()); + ASSERT_STATUS_OK(file_->Read(0, output)); + EXPECT_EQ(std::string(output.data(), output.size()), contents_.substr(0, output.size())); +} + +TEST_F(RandomAccessFileTest, EmptyFileHasZeroLength) { + PathString empty_path; + ScopedFileDeleter empty_deleter; + ASSERT_NO_FATAL_FAILURE(WriteRandomAccessTestFile({}, empty_path, empty_deleter)); + ASSERT_STATUS_OK(Env::Default().OpenRandomAccessFile(empty_path.c_str(), file_)); + size_t length = 123; + ASSERT_STATUS_OK(file_->GetLength(length)); + EXPECT_EQ(length, 0U); + ASSERT_STATUS_OK(file_->Read(0, {})); + char byte; + EXPECT_FALSE(file_->Read(0, gsl::span(&byte, 1)).IsOK()); +} + +TEST_F(RandomAccessFileTest, FailedOpenDoesNotReplaceAnExistingFile) { + const auto* original = file_.get(); + const auto missing_path = path_ + ORT_TSTR(".missing"); + ASSERT_FALSE(Env::Default().FileExists(missing_path)); + EXPECT_FALSE(Env::Default().OpenRandomAccessFile(missing_path.c_str(), file_).IsOK()); + EXPECT_EQ(file_.get(), original); + EXPECT_EQ(Env::Default().OpenRandomAccessFile(nullptr, file_).Code(), common::INVALID_ARGUMENT); + EXPECT_EQ(file_.get(), original); + EXPECT_FALSE(Env::Default().OpenRandomAccessFile(ORT_TSTR("."), file_).IsOK()); + EXPECT_EQ(file_.get(), original); + char byte; + ASSERT_STATUS_OK(file_->Read(1, gsl::span(&byte, 1))); + EXPECT_EQ(byte, contents_[1]); +} + +TEST_F(RandomAccessFileTest, DefaultImplementationReportsUnsupportedWithoutReplacingFile) { + const auto* original = file_.get(); + EXPECT_EQ(Env::Default().Env::OpenRandomAccessFile(path_.c_str(), file_).Code(), common::NOT_IMPLEMENTED); + EXPECT_EQ(file_.get(), original); +} + +TEST_F(RandomAccessFileTest, PathReplacementDoesNotChangeTheOpenFile) { + const std::string replacement_contents = "replacement file"; + PathString replacement_path; + ScopedFileDeleter replacement_deleter; + ASSERT_NO_FATAL_FAILURE(WriteRandomAccessTestFile(replacement_contents, replacement_path, replacement_deleter)); +#ifdef _WIN32 + // Ordinary Windows rename cannot replace an open destination, even with delete sharing. + const HANDLE replacement_handle = + CreateFile2(replacement_path.c_str(), DELETE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + OPEN_EXISTING, nullptr); + ASSERT_NE(replacement_handle, INVALID_HANDLE_VALUE) << GetLastError(); + auto close_replacement = gsl::finally([&] { CloseHandle(replacement_handle); }); + + const auto target_path = std::filesystem::absolute(path_).native(); + const size_t name_bytes = SafeInt(target_path.size()) * sizeof(wchar_t); + const size_t rename_info_bytes = SafeInt(sizeof(FILE_RENAME_INFO)) + name_bytes; + const auto rename_info_size = gsl::narrow(rename_info_bytes); + auto rename_buffer = std::make_unique(rename_info_size); + auto* rename_info = reinterpret_cast(rename_buffer.get()); + rename_info->Flags = FILE_RENAME_FLAG_REPLACE_IF_EXISTS | FILE_RENAME_FLAG_POSIX_SEMANTICS; + rename_info->RootDirectory = nullptr; + rename_info->FileNameLength = gsl::narrow(name_bytes); + std::memcpy(rename_info->FileName, target_path.c_str(), name_bytes); + // Some SDK headers omit FileRenameInfoEx. Its documented FILE_INFO_BY_HANDLE_CLASS value is 22. + constexpr auto kFileRenameInfoEx = static_cast(22); + const BOOL renamed = + SetFileInformationByHandle(replacement_handle, kFileRenameInfoEx, rename_info, rename_info_size); + const DWORD rename_error = GetLastError(); + ASSERT_NE(renamed, FALSE) << rename_error; +#else + std::error_code error; + std::filesystem::rename(replacement_path, path_, error); + ASSERT_FALSE(error) << error.message(); +#endif + + size_t length = 0; + ASSERT_STATUS_OK(file_->GetLength(length)); + EXPECT_EQ(length, contents_.size()); + std::string original_output(contents_.size(), '\0'); + ASSERT_STATUS_OK(file_->Read(0, gsl::span(original_output))); + EXPECT_EQ(original_output, contents_); + + std::unique_ptr replacement; + ASSERT_STATUS_OK(Env::Default().OpenRandomAccessFile(path_.c_str(), replacement)); + ASSERT_STATUS_OK(replacement->GetLength(length)); + EXPECT_EQ(length, replacement_contents.size()); + std::string replacement_output(length, '\0'); + ASSERT_STATUS_OK(replacement->Read(0, gsl::span(replacement_output))); + EXPECT_EQ(replacement_output, replacement_contents); +} + +TEST_F(RandomAccessFileTest, HandlesInPlaceTruncationAccordingToPlatformSharingRules) { +#ifdef _WIN32 + // Windows denies write sharing while the file is open; destruction must release that restriction. + { + std::ofstream writer(path_, std::ios::binary | std::ios::trunc); + EXPECT_FALSE(writer.is_open()); + } + file_.reset(); + std::ofstream writer(path_, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(writer.is_open()); +#else + std::error_code error; + std::filesystem::resize_file(path_, 3, error); + ASSERT_FALSE(error) << error.message(); + size_t length = 0; + ASSERT_STATUS_OK(file_->GetLength(length)); + EXPECT_EQ(length, 3U); + std::array output{}; + EXPECT_FALSE(file_->Read(0, output).IsOK()); + ASSERT_STATUS_OK(file_->Read(0, gsl::span(output.data(), 3))); + EXPECT_EQ(std::string(output.data(), 3), contents_.substr(0, 3)); +#endif +} + +#if !defined(_WIN32) && !defined(__wasm__) +TEST_F(RandomAccessFileTest, RejectsFifosWithoutWaitingForAWriter) { + PathString fifo_path; + ScopedFileDeleter fifo_deleter; + ASSERT_NO_FATAL_FAILURE(WriteRandomAccessTestFile({}, fifo_path, fifo_deleter)); + ASSERT_EQ(std::remove(fifo_path.c_str()), 0); + const int create_result = mkfifo(fifo_path.c_str(), 0600); + const int create_error = errno; +#ifdef __ANDROID__ + if (create_result != 0 && (create_error == EACCES || create_error == EPERM)) { + GTEST_SKIP() << "Android SELinux policy denies FIFO creation: " << std::strerror(create_error); + } +#endif + ASSERT_EQ(create_result, 0) << std::strerror(create_error); + std::unique_ptr fifo; + EXPECT_FALSE(Env::Default().OpenRandomAccessFile(fifo_path.c_str(), fifo).IsOK()); + EXPECT_EQ(fifo, nullptr); +} +#endif + +#ifndef __wasm__ +TEST_F(RandomAccessFileTest, ReadsSparseFileBeyondFourGiB) { + if (sizeof(FileOffsetType) < 8 || sizeof(size_t) < 8) { + GTEST_SKIP() << "Requires 64-bit file offsets and sizes."; + } + constexpr int64_t kOffset = (int64_t{1} << 32) + 123; + file_.reset(); +#ifdef _WIN32 + { + const HANDLE sparse_handle = CreateFile2(path_.c_str(), GENERIC_WRITE, 0, OPEN_EXISTING, nullptr); + ASSERT_NE(sparse_handle, INVALID_HANDLE_VALUE) << GetLastError(); + auto close_sparse = gsl::finally([&] { CloseHandle(sparse_handle); }); + DWORD bytes_returned = 0; + const BOOL marked_sparse = + DeviceIoControl(sparse_handle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &bytes_returned, nullptr); + const DWORD sparse_error = GetLastError(); + ASSERT_NE(marked_sparse, FALSE) << sparse_error; + } +#endif + { + std::fstream writer(path_, std::ios::binary | std::ios::in | std::ios::out); + ASSERT_TRUE(writer.is_open()); + writer.seekp(kOffset); + writer.put('Z'); + writer.close(); + ASSERT_FALSE(writer.fail()); + } + ASSERT_STATUS_OK(Env::Default().OpenRandomAccessFile(path_.c_str(), file_)); + size_t length = 0; + ASSERT_STATUS_OK(file_->GetLength(length)); + EXPECT_EQ(length, static_cast(kOffset + 1)); + std::array output{}; + ASSERT_STATUS_OK(file_->Read(static_cast(kOffset - 1), output)); + EXPECT_EQ(output[0], '\0'); + EXPECT_EQ(output[1], 'Z'); +} +#endif + +} // namespace + } // namespace test } // namespace onnxruntime