diff --git a/Source/CLI/Global.cpp b/Source/CLI/Global.cpp index c53cea70..3de5b443 100644 --- a/Source/CLI/Global.cpp +++ b/Source/CLI/Global.cpp @@ -242,6 +242,40 @@ int global::SetHash(bool Value) return 0; } +//--------------------------------------------------------------------------- +int global::SetFileOpenMethod(const char* Value) +{ + if (strcmp(Value, "mmap") == 0) + { + FileOpenMethod = filemap::method::mmap; + return 0; + } + if (strcmp(Value, "fstream") == 0) + { + FileOpenMethod = filemap::method::fstream; + return 0; + } + if (strcmp(Value, "fopen") == 0) + { + FileOpenMethod = filemap::method::fopen; + return 0; + } + if (strcmp(Value, "open") == 0) + { + FileOpenMethod = filemap::method::open; + return 0; + } + #if defined(_WIN32) || defined(_WINDOWS) + if (strcmp(Value, "createfile") == 0) + { + FileOpenMethod = filemap::method::createfile; + return 0; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + cerr << "Error: unknown io value '" << Value << "'." << endl; + return 1; +} + //--------------------------------------------------------------------------- int global::SetAll(bool Value) { @@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc) IgnoreLicenseKey = !License.IsSupported_License(); SubLicenseId = 0; SubLicenseDur = 1; + FileOpenMethod = (filemap::method)-1; ShowLicenseKey = false; StoreLicenseKey = false; DisplayCommand = false; @@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc) if (auto Value = SetAcceptFiles()) return Value; } + else if (strcmp(argv[i], "--io") == 0) + { + if (i + 1 == argc) + return Error_Missing(argv[i]); + int Value = SetFileOpenMethod(argv[++i]); + if (Value) + return Value; + } else if (!strcmp(argv[i], "-framerate")) { if (OptionsForOtherFiles) diff --git a/Source/CLI/Global.h b/Source/CLI/Global.h index e06e1f18..76eabd53 100644 --- a/Source/CLI/Global.h +++ b/Source/CLI/Global.h @@ -38,6 +38,7 @@ class global string LicenseKey; uint64_t SubLicenseId; uint64_t SubLicenseDur; + filemap::method FileOpenMethod; bool IgnoreLicenseKey; bool ShowLicenseKey; bool StoreLicenseKey; @@ -100,6 +101,7 @@ class global int SetFrameMd5An(bool Value); int SetFrameMd5FileName(const char* FileName); int SetHash(bool Value); + int SetFileOpenMethod(const char* Value); int SetAll(bool Value); private: diff --git a/Source/CLI/Main.cpp b/Source/CLI/Main.cpp index 53efb45e..9e2fb7cd 100644 --- a/Source/CLI/Main.cpp +++ b/Source/CLI/Main.cpp @@ -89,19 +89,19 @@ struct parse_info bool IsContainer = false; bool Problem = false; - bool ParseFile_Input(input_base& Input, bool OverrideCheckPadding = false); + bool ParseFile_Input(input_base& Input, bool OverrideCheckPadding = false, const string* FileName = nullptr); bool ParseFile_Input(input_base_uncompressed& SingleFile, input& Input, size_t Files_Pos); }; //--------------------------------------------------------------------------- -bool parse_info::ParseFile_Input(input_base& SingleFile, bool OverrideCheckPadding) +bool parse_info::ParseFile_Input(input_base& SingleFile, bool OverrideCheckPadding, const string* FileName) { // Init SingleFile.Actions = Global.Actions; if (OverrideCheckPadding) SingleFile.Actions.set(Action_CheckPadding); SingleFile.Hashes = &Global.Hashes; - SingleFile.FileName = &RAWcooked.OutputFileName; + SingleFile.FileName = (!RAWcooked.OutputFileName.empty() || !FileName) ? &RAWcooked.OutputFileName : FileName; SingleFile.InputInfo = &InputInfo; // Parse @@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos) } //--------------------------------------------------------------------------- -int ParseFile_Compressed(parse_info& ParseInfo) +int ParseFile_Compressed(parse_info& ParseInfo, const string* FileName) { // Init string OutputDirectoryName; @@ -522,7 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo) matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors); M->Quiet = Global.Quiet; M->NoOutputCheck = NoOutputCheck; - if (ParseInfo.ParseFile_Input(*M)) + M->OpenStyle = Global.FileOpenMethod; + if (ParseInfo.ParseFile_Input(*M, false, FileName)) { ReturnValue = 1; } @@ -591,7 +592,7 @@ int ParseFile(size_t Files_Pos) return 1; // Compressed content - if (int Value = ParseFile_Compressed(ParseInfo)) + if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name)) return Value; if (ParseInfo.IsDetected) return 0; @@ -755,6 +756,7 @@ int main(int argc, const char* argv[]) if (!Value) { // Configure for a 2nd pass + auto OutputFileName = Global.OutputFileName; ParseInfo.Name = NULL; Global.OutputFileName = Global.Inputs[0]; if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes @@ -772,7 +774,7 @@ int main(int argc, const char* argv[]) // Parse (check mode) Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]); Global.Actions.set(Action_Decode, false); // Override config - Value = ParseFile_Compressed(ParseInfo); + Value = ParseFile_Compressed(ParseInfo, &OutputFileName); if (!Value && !ParseInfo.IsDetected) { cout << '\n' << "Error: " << Global.OutputFileName << endl; diff --git a/Source/Lib/Compressed/Matroska/Matroska.cpp b/Source/Lib/Compressed/Matroska/Matroska.cpp index 550397f4..fd6e7e5c 100644 --- a/Source/Lib/Compressed/Matroska/Matroska.cpp +++ b/Source/Lib/Compressed/Matroska/Matroska.cpp @@ -359,13 +359,16 @@ void matroska::ParseBuffer() // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value { - FileMap->Remap(); + FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024); Buffer = *FileMap; + if (OpenStyle == filemap::method::mmap) + { if (ReversibilityData) ReversibilityData->SetBaseData(Buffer.Data()); for (const auto& TrackInfo_Current : TrackInfo) if (TrackInfo_Current && TrackInfo_Current->ReversibilityData) TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data()); + } Buffer_Offset_LowerLimit = Buffer_Offset; } @@ -376,13 +379,16 @@ void matroska::ParseBuffer() Buffer_Offset = Cluster_Offset; Cluster_Level = (size_t)-1; - FileMap->Remap(); + FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024); Buffer = *FileMap; + if (OpenStyle == filemap::method::mmap) + { if (ReversibilityData) ReversibilityData->SetBaseData(Buffer.Data()); for (const auto& TrackInfo_Current : TrackInfo) if (TrackInfo_Current && TrackInfo_Current->ReversibilityData) TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data()); + } Buffer_Offset_LowerLimit = Buffer_Offset; } } @@ -799,6 +805,11 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV //--------------------------------------------------------------------------- void matroska::Segment_Cluster() { + IsList = true; + + if (FileMap2) + return; + if (RAWcooked_LibraryName.empty()) { memcpy(Cluster_Levels, Levels, sizeof(Levels)); @@ -808,8 +819,6 @@ void matroska::Segment_Cluster() return; } - IsList = true; - // Check if Hashes check is useful if (Hashes_FromRAWcooked) { @@ -850,6 +859,14 @@ void matroska::Segment_Cluster() Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string()); if (ReversibilityData && !FrameWriter_Template->Compound) InitOutput_Find(); + + FileMap2 = FileMap; + if (OpenStyle != filemap::method::mmap && this->FileName) + { + FileMap = new filemap; + FileMap->Open_ReadMode(*this->FileName, OpenStyle, 0, 256 * 1024 * 1024); + Buffer = *FileMap; + } } //--------------------------------------------------------------------------- diff --git a/Source/Lib/Utils/FileIO/FileIO.cpp b/Source/Lib/Utils/FileIO/FileIO.cpp index 0c543a94..bcbc73bc 100644 --- a/Source/Lib/Utils/FileIO/FileIO.cpp +++ b/Source/Lib/Utils/FileIO/FileIO.cpp @@ -11,8 +11,11 @@ #include "Lib/Utils/FileIO/FileIO.h" #include #include +#include #if defined(_WIN32) || defined(_WINDOWS) #include "windows.h" + #include + #include #include // File existence #include // Directory creation #define access _access_s @@ -29,10 +32,96 @@ //--------------------------------------------------------------------------- //--------------------------------------------------------------------------- -int filemap::Open_ReadMode(const char* FileName) +struct private_buffered +{ + void* F; + size_t Data_Shift = 0; + size_t MaxSize = 0; +}; + +//--------------------------------------------------------------------------- +int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End) { Close(); + if (NewStyle != method::mmap) + { + Method = NewStyle; + private_buffered* P = new private_buffered; + P->MaxSize = End - Begin; + size_t FileSize; + + switch (Method) + { + default: // case style::fstream: + { + auto F = new ifstream(FileName, ios::binary); + F->seekg(0, F->end); + FileSize = F->tellg(); + F->seekg(Begin, F->beg); + P->F = F; + break; + } + case method::fopen: + { + auto F = fopen(FileName, "rb"); + fseek(F, 0, SEEK_END); + FileSize = ftell(F); + fseek(F, (long)Begin, SEEK_SET); + P->F = F; + break; + } + case method::open: + { + struct stat Fstat; + if (stat(FileName, &Fstat)) + return 1; + FileSize = Fstat.st_size; + #if defined(_WIN32) || defined(_WINDOWS) + auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD); + #else //defined(_WIN32) || defined(_WINDOWS) + auto F = open(FileName, O_RDONLY); + #endif //defined(_WIN32) || defined(_WINDOWS) + if (F == -1) + return 1; + P->F = (void*)F; + break; + } + #if defined(_WIN32) || defined(_WINDOWS) + case method::createfile: + { + DWORD FileSizeHigh; + auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh); + if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value) + && (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode + { + FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow; + } + else + return 1; + if (Begin) + { + LARGE_INTEGER GoTo; + GoTo.QuadPart = Begin; + if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0)) + return 1; + P->Data_Shift = Begin; + } + P->F = NewFile; + break; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + } + + auto Buffer = new uint8_t[P->MaxSize]; + P->Data_Shift -= P->MaxSize; + AssignBase(Buffer - P->Data_Shift, FileSize); + Private2 = (decltype(Private2))P; + + return Remap(Begin, End); + } + size_t NewSize; #if defined(_WIN32) || defined(_WINDOWS) auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); @@ -99,12 +188,58 @@ inline int munmap_const(const void* addr, size_t length) #pragma GCC diagnostic pop #endif #endif -int filemap::Remap() +int filemap::Remap(size_t Begin, size_t End) { // Special case for 0-byte files if (Empty()) return 0; + if (Method != method::mmap) + { + auto P = (private_buffered*)Private2; + auto Buffer = Data() + P->Data_Shift; + auto Buffer_MaxSize = P->MaxSize; + Begin -= P->Data_Shift; + if (!End) + End = Size(); + End -= P->Data_Shift; + auto Buffer_Middle = Buffer + Begin; + auto Buffer_Middle_Size = Buffer_MaxSize - Begin; + memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size); + P->Data_Shift += Begin; + AssignKeepSizeBase(Buffer - P->Data_Shift); + Buffer += Buffer_Middle_Size; + Buffer_MaxSize -= Buffer_Middle_Size; + + switch (Method) + { + default: // case style::fstream: + { + auto F = (ifstream*)P->F; + F->read((char*)Buffer, Buffer_MaxSize); + break; + } + case method::fopen: + { + auto F = (FILE*)P->F; + if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1) + return 1; + break; + } + case method::open: + return 1; + #if defined(_WIN32) || defined(_WINDOWS) + case method::createfile: + { + ReadFile(P->F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0); + break; + } + #endif //defined(_WIN32) || defined(_WINDOWS) + } + + return 0; + } + // Close previous map if (Data()) { diff --git a/Source/Lib/Utils/FileIO/FileIO.h b/Source/Lib/Utils/FileIO/FileIO.h index 45590707..75be259c 100644 --- a/Source/Lib/Utils/FileIO/FileIO.h +++ b/Source/Lib/Utils/FileIO/FileIO.h @@ -23,19 +23,30 @@ class filemap : public buffer_view ~filemap() { Close(); } // Actions - int Open_ReadMode(const char* FileName); - int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); } + enum class method + { + mmap, + fstream, + fopen, + open, + #if defined(_WIN32) || defined(_WINDOWS) + createfile, + #endif //defined(_WIN32) || defined(_WINDOWS) + }; + int Open_ReadMode(const char* FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}); + int Open_ReadMode(const string& FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}) { return Open_ReadMode(FileName.c_str(), NewMethod, Begin, End); } bool IsOpen() { return Private == (decltype(Private))-1 ? false : true; } - int Remap(); + int Remap(size_t Begin = 0, size_t End = 0); int Close(); private: #if defined(_WIN32) || defined(_WINDOWS) void* Private = (void*)-1; - void* Private2 = (void*)-1; #else //defined(_WIN32) || defined(_WINDOWS) int Private = (int)-1; #endif //defined(_WIN32) || defined(_WINDOWS) + void* Private2 = (void*)-1; + method Method = {}; }; class file diff --git a/Source/Lib/Utils/FileIO/Input_Base.cpp b/Source/Lib/Utils/FileIO/Input_Base.cpp index 1bd39e7f..d21e38a5 100644 --- a/Source/Lib/Utils/FileIO/Input_Base.cpp +++ b/Source/Lib/Utils/FileIO/Input_Base.cpp @@ -38,6 +38,7 @@ input_base::~input_base() bool input_base::Parse(filemap* FileMap_Source, const buffer_view& Buffer_Source, size_t FileSize_Source) { FileMap = FileMap_Source; + FileMap2 = nullptr; FileSize = FileSize_Source == (size_t)-1 ? Buffer_Source.Size() : FileSize_Source; Buffer = Buffer_Source; HashComputed = false; diff --git a/Source/Lib/Utils/FileIO/Input_Base.h b/Source/Lib/Utils/FileIO/Input_Base.h index 0b0383e4..d7c79760 100644 --- a/Source/Lib/Utils/FileIO/Input_Base.h +++ b/Source/Lib/Utils/FileIO/Input_Base.h @@ -76,7 +76,8 @@ class input_base // Config bitset Actions; hashes* Hashes = nullptr; - string* FileName = nullptr; + const string* FileName = nullptr; + filemap::method OpenStyle = {}; // Parse bool Parse(const buffer_view& Buffer, size_t FileSize = (size_t)-1) { return Parse(nullptr, Buffer, FileSize); } @@ -95,6 +96,7 @@ class input_base virtual void ParseBuffer() = 0; virtual void BufferOverflow() = 0; filemap* FileMap; + filemap* FileMap2; size_t FileSize; buffer_view Buffer; size_t Buffer_Offset;