[WIP] Add an alternative method for reading files

Using mmap only seems to be very slow with some NAS

Source/CLI/Global.cpp

@@ -242,6 +242,40 @@ int global::SetHash(bool Value)
     return 0;
 }
 
+//---------------------------------------------------------------------------
+int global::SetFileOpenMethod(const char* Value)
+{
+    if (strcmp(Value, "mmap") == 0)
+    {
+        FileOpenMethod = filemap::method::mmap;
+        return 0;
+    }
+    if (strcmp(Value, "fstream") == 0)
+    {
+        FileOpenMethod = filemap::method::fstream;
+        return 0;
+    }
+    if (strcmp(Value, "fopen") == 0)
+    {
+        FileOpenMethod = filemap::method::fopen;
+        return 0;
+    }
+    if (strcmp(Value, "open") == 0)
+    {
+        FileOpenMethod = filemap::method::open;
+        return 0;
+    }
+    #if defined(_WIN32) || defined(_WINDOWS)
+    if (strcmp(Value, "createfile") == 0)
+    {
+        FileOpenMethod = filemap::method::createfile;
+        return 0;
+    }
+    #endif //defined(_WIN32) || defined(_WINDOWS)
+    cerr << "Error: unknown io value '" << Value << "'." << endl;
+    return 1;
+}
+
 //---------------------------------------------------------------------------
 int global::SetAll(bool Value)
 {
@@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
     IgnoreLicenseKey = !License.IsSupported_License();
     SubLicenseId = 0;
     SubLicenseDur = 1;
+    FileOpenMethod = (filemap::method)-1;
     ShowLicenseKey = false;
     StoreLicenseKey = false;
     DisplayCommand = false;
@@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
             if (auto Value = SetAcceptFiles())
                 return Value;
         }
+        else if (strcmp(argv[i], "--io") == 0)
+        {
+            if (i + 1 == argc)
+                return Error_Missing(argv[i]);
+            int Value = SetFileOpenMethod(argv[++i]);
+            if (Value)
+                return Value;
+        }
         else if (!strcmp(argv[i], "-framerate"))
         {
             if (OptionsForOtherFiles)

Source/CLI/Global.h

@@ -38,6 +38,7 @@ class global
     string                      LicenseKey;
     uint64_t                    SubLicenseId;
     uint64_t                    SubLicenseDur;
+    filemap::method             FileOpenMethod;
     bool                        IgnoreLicenseKey;
     bool                        ShowLicenseKey;
     bool                        StoreLicenseKey;
@@ -100,6 +101,7 @@ class global
     int SetFrameMd5An(bool Value);
     int SetFrameMd5FileName(const char* FileName);
     int SetHash(bool Value);
+    int SetFileOpenMethod(const char* Value);
     int SetAll(bool Value);
 
 private:

Source/CLI/Main.cpp

@@ -89,19 +89,19 @@ struct parse_info
     bool   IsContainer = false;
     bool   Problem = false;
 
-    bool ParseFile_Input(input_base& Input, bool OverrideCheckPadding = false);
+    bool ParseFile_Input(input_base& Input, bool OverrideCheckPadding = false, const string* FileName = nullptr);
     bool ParseFile_Input(input_base_uncompressed& SingleFile, input& Input, size_t Files_Pos);
 };
 
 //---------------------------------------------------------------------------
-bool parse_info::ParseFile_Input(input_base& SingleFile, bool OverrideCheckPadding)
+bool parse_info::ParseFile_Input(input_base& SingleFile, bool OverrideCheckPadding, const string* FileName)
 {
     // Init
     SingleFile.Actions = Global.Actions;
     if (OverrideCheckPadding)
         SingleFile.Actions.set(Action_CheckPadding);
     SingleFile.Hashes = &Global.Hashes;
-    SingleFile.FileName = &RAWcooked.OutputFileName;
+    SingleFile.FileName = (!RAWcooked.OutputFileName.empty() || !FileName) ? &RAWcooked.OutputFileName : FileName;
     SingleFile.InputInfo = &InputInfo;
 
     // Parse
@@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
 }
 
 //---------------------------------------------------------------------------
-int ParseFile_Compressed(parse_info& ParseInfo)
+int ParseFile_Compressed(parse_info& ParseInfo, const string* FileName)
 {
     // Init
     string OutputDirectoryName;
@@ -522,7 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
         matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
         M->Quiet = Global.Quiet;
         M->NoOutputCheck = NoOutputCheck;
-        if (ParseInfo.ParseFile_Input(*M))
+        M->OpenStyle = Global.FileOpenMethod;
+        if (ParseInfo.ParseFile_Input(*M, false, FileName))
         {
             ReturnValue = 1;
         }
@@ -591,7 +592,7 @@ int ParseFile(size_t Files_Pos)
         return 1;
 
     // Compressed content
-    if (int Value = ParseFile_Compressed(ParseInfo))
+    if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
         return Value;
     if (ParseInfo.IsDetected)
         return 0;
@@ -755,6 +756,7 @@ int main(int argc, const char* argv[])
         if (!Value)
         {
             // Configure for a 2nd pass
+            auto OutputFileName = Global.OutputFileName;
             ParseInfo.Name = NULL;
             Global.OutputFileName = Global.Inputs[0];
             if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
@@ -772,7 +774,7 @@ int main(int argc, const char* argv[])
             // Parse (check mode)
             Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
             Global.Actions.set(Action_Decode, false); // Override config
-            Value = ParseFile_Compressed(ParseInfo);
+            Value = ParseFile_Compressed(ParseInfo, &OutputFileName);
             if (!Value && !ParseInfo.IsDetected)
             {
                 cout << '\n' << "Error: " << Global.OutputFileName << endl;

Source/Lib/Compressed/Matroska/Matroska.cpp

@@ -359,13 +359,16 @@ void matroska::ParseBuffer()
         // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
         if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value 
         {
-            FileMap->Remap();
+            FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
             Buffer = *FileMap;
+            if (OpenStyle == filemap::method::mmap)
+            {
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
             for (const auto& TrackInfo_Current : TrackInfo)
                 if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
                     TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
+            }
             Buffer_Offset_LowerLimit = Buffer_Offset;
         }
 
@@ -376,13 +379,16 @@ void matroska::ParseBuffer()
             Buffer_Offset = Cluster_Offset;
             Cluster_Level = (size_t)-1;
 
-            FileMap->Remap();
+            FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
             Buffer = *FileMap;
+            if (OpenStyle == filemap::method::mmap)
+            {
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
             for (const auto& TrackInfo_Current : TrackInfo)
                 if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
                     TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
+            }
             Buffer_Offset_LowerLimit = Buffer_Offset;
         }
     }
@@ -799,6 +805,11 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV
 //---------------------------------------------------------------------------
 void matroska::Segment_Cluster()
 {
+    IsList = true;
+
+    if (FileMap2)
+        return;
+
     if (RAWcooked_LibraryName.empty())
     {
         memcpy(Cluster_Levels, Levels, sizeof(Levels));
@@ -808,8 +819,6 @@ void matroska::Segment_Cluster()
         return;
     }
 
-    IsList = true;
-
     // Check if Hashes check is useful
     if (Hashes_FromRAWcooked)
     {
@@ -850,6 +859,14 @@ void matroska::Segment_Cluster()
             Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
     if (ReversibilityData && !FrameWriter_Template->Compound)
         InitOutput_Find();
+
+    FileMap2 = FileMap;
+    if (OpenStyle != filemap::method::mmap && this->FileName)
+    {
+        FileMap = new filemap;
+        FileMap->Open_ReadMode(*this->FileName, OpenStyle, 0, 256 * 1024 * 1024);
+        Buffer = *FileMap;
+    }
 }
 
 //---------------------------------------------------------------------------

Source/Lib/Utils/FileIO/FileIO.cpp

@@ -11,8 +11,11 @@
 #include "Lib/Utils/FileIO/FileIO.h"
 #include <iostream>
 #include <sstream>
+#include <fstream>
 #if defined(_WIN32) || defined(_WINDOWS)
     #include "windows.h"
+    #include <stdio.h>
+    #include <fcntl.h>
     #include <io.h> // File existence
     #include <direct.h> // Directory creation
     #define access _access_s
@@ -29,10 +32,96 @@
 //---------------------------------------------------------------------------
 
 //---------------------------------------------------------------------------
-int filemap::Open_ReadMode(const char* FileName)
+struct private_buffered
+{
+    void*               F;
+    size_t              Data_Shift = 0;
+    size_t              MaxSize = 0;
+};
+
+//---------------------------------------------------------------------------
+int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End)
 {
     Close();
 
+    if (NewStyle != method::mmap)
+    {
+        Method = NewStyle;
+        private_buffered* P = new private_buffered;
+        P->MaxSize = End - Begin;
+        size_t FileSize;
+
+        switch (Method)
+        {
+        default: // case style::fstream:
+        {
+            auto F = new ifstream(FileName, ios::binary);
+            F->seekg(0, F->end);
+            FileSize = F->tellg();
+            F->seekg(Begin, F->beg);
+            P->F = F;
+            break;
+        }
+        case method::fopen:
+        {
+            auto F = fopen(FileName, "rb");
+            fseek(F, 0, SEEK_END);
+            FileSize = ftell(F);
+            fseek(F, (long)Begin, SEEK_SET);
+            P->F = F;
+            break;
+        }
+        case method::open:
+        {
+            struct stat Fstat;
+            if (stat(FileName, &Fstat))
+                return 1;
+            FileSize = Fstat.st_size;
+            #if defined(_WIN32) || defined(_WINDOWS)
+            auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD);
+            #else //defined(_WIN32) || defined(_WINDOWS)
+            auto F = open(FileName, O_BINARY | O_RDONLY);
+            #endif //defined(_WIN32) || defined(_WINDOWS)
+            if (F == -1)
+                return 1;
+            P->F = (void*)F;
+            break;
+        }
+        #if defined(_WIN32) || defined(_WINDOWS)
+        case method::createfile:
+        {
+            DWORD FileSizeHigh;
+            auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
+            auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh);
+            if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value)
+                && (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode
+            {
+                FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow;
+            }
+            else
+                return 1;
+            if (Begin)
+            {
+                LARGE_INTEGER GoTo;
+                GoTo.QuadPart = Begin;
+                if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0))
+                    return 1;
+                P->Data_Shift = Begin;
+            }
+            P->F = NewFile;
+            break;
+        }
+        #endif //defined(_WIN32) || defined(_WINDOWS)
+        }
+
+        auto Buffer = new uint8_t[P->MaxSize];
+        P->Data_Shift -= P->MaxSize;
+        AssignBase(Buffer - P->Data_Shift, FileSize);
+        Private2 = (decltype(Private2))P;
+
+        return Remap(Begin, End);
+    }
+
     size_t NewSize;
 #if defined(_WIN32) || defined(_WINDOWS)
     auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
@@ -99,12 +188,58 @@ inline int munmap_const(const void* addr, size_t length)
 #pragma GCC diagnostic pop
 #endif
 #endif
-int filemap::Remap()
+int filemap::Remap(size_t Begin, size_t End)
 {
     // Special case for 0-byte files
     if (Empty())
         return 0;
 
+    if (Method != method::mmap)
+    {
+        auto P = (private_buffered*)Private2;
+        auto Buffer = Data() + P->Data_Shift;
+        auto Buffer_MaxSize = P->MaxSize;
+        Begin -= P->Data_Shift;
+        if (!End)
+            End = Size();
+        End -= P->Data_Shift;
+        auto Buffer_Middle = Buffer + Begin;
+        auto Buffer_Middle_Size = Buffer_MaxSize - Begin;
+        memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size);
+        P->Data_Shift += Begin;
+        AssignKeepSizeBase(Buffer - P->Data_Shift);
+        Buffer += Buffer_Middle_Size;
+        Buffer_MaxSize -= Buffer_Middle_Size;
+
+        switch (Method)
+        {
+        default: // case style::fstream:
+        {
+            auto F = (ifstream*)P->F;
+            F->read((char*)Buffer, Buffer_MaxSize);
+            break;
+        }
+        case method::fopen:
+        {
+            auto F = (FILE*)P->F;
+            if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1)
+                return 1;
+            break;
+        }
+        case method::open:
+        {
+            #if defined(_WIN32) || defined(_WINDOWS)
+            ReadFile(P->F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0);
+            #else //defined(_WIN32) || defined(_WINDOWS)
+            return 1;
+            #endif //defined(_WIN32) || defined(_WINDOWS)
+            break;
+        }
+        }
+
+        return 0;
+    }
+
     // Close previous map
     if (Data())
     {