[WIP] Add an alternative method for reading files

Using mmap only seems to be very slow with some NAS

Source/CLI/Global.cpp

@@ -242,6 +242,40 @@ int global::SetHash(bool Value)
     return 0;
 }
 
+//---------------------------------------------------------------------------
+int global::SetFileOpenMethod(const char* Value)
+{
+    if (strcmp(Value, "mmap") == 0)
+    {
+        FileOpenMethod = filemap::method::mmap;
+        return 0;
+    }
+    if (strcmp(Value, "fstream") == 0)
+    {
+        FileOpenMethod = filemap::method::fstream;
+        return 0;
+    }
+    if (strcmp(Value, "fopen") == 0)
+    {
+        FileOpenMethod = filemap::method::fopen;
+        return 0;
+    }
+    if (strcmp(Value, "open") == 0)
+    {
+        FileOpenMethod = filemap::method::open;
+        return 0;
+    }
+    #if defined(_WIN32) || defined(_WINDOWS)
+    if (strcmp(Value, "createfile") == 0)
+    {
+        FileOpenMethod = filemap::method::createfile;
+        return 0;
+    }
+    #endif //defined(_WIN32) || defined(_WINDOWS)
+    cerr << "Error: unknown io value '" << Value << "'." << endl;
+    return 1;
+}
+
 //---------------------------------------------------------------------------
 int global::SetAll(bool Value)
 {
@@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
     IgnoreLicenseKey = !License.IsSupported_License();
     SubLicenseId = 0;
     SubLicenseDur = 1;
+    FileOpenMethod = (filemap::method)-1;
     ShowLicenseKey = false;
     StoreLicenseKey = false;
     DisplayCommand = false;
@@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
             if (auto Value = SetAcceptFiles())
                 return Value;
         }
+        else if (strcmp(argv[i], "--io") == 0)
+        {
+            if (i + 1 == argc)
+                return Error_Missing(argv[i]);
+            int Value = SetFileOpenMethod(argv[++i]);
+            if (Value)
+                return Value;
+        }
         else if (!strcmp(argv[i], "-framerate"))
         {
             if (OptionsForOtherFiles)
@@ -831,6 +874,11 @@ int global::ManageCommandLine(const char* argv[], int argc)
     }
     if (License.ShowLicense(ShowLicenseKey, SubLicenseId, SubLicenseDur))
         return 1;
+    if (FileOpenMethod == (filemap::method)-1)
+    {
+        //cerr << "\nThis is a test version, please use another version if you don't know which option to test\n" << endl;
+        //return 1;
+    }
     if (Inputs.empty() && (ShowLicenseKey || SubLicenseId))
         return 0;
 

Source/CLI/Global.h

@@ -38,6 +38,7 @@ class global
     string                      LicenseKey;
     uint64_t                    SubLicenseId;
     uint64_t                    SubLicenseDur;
+    filemap::method             FileOpenMethod;
     bool                        IgnoreLicenseKey;
     bool                        ShowLicenseKey;
     bool                        StoreLicenseKey;
@@ -100,6 +101,7 @@ class global
     int SetFrameMd5An(bool Value);
     int SetFrameMd5FileName(const char* FileName);
     int SetHash(bool Value);
+    int SetFileOpenMethod(const char* Value);
     int SetAll(bool Value);
 
 private:

Source/CLI/Main.cpp

@@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
 }
 
 //---------------------------------------------------------------------------
-int ParseFile_Compressed(parse_info& ParseInfo)
+int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName)
 {
     // Init
     string OutputDirectoryName;
@@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
         matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
         M->Quiet = Global.Quiet;
         M->NoOutputCheck = NoOutputCheck;
+        M->OpenName = FileOpenName;
+        M->OpenStyle = Global.FileOpenMethod;
         if (ParseInfo.ParseFile_Input(*M))
         {
             ReturnValue = 1;
@@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos)
         return 1;
 
     // Compressed content
-    if (int Value = ParseFile_Compressed(ParseInfo))
+    if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
         return Value;
     if (ParseInfo.IsDetected)
         return 0;
@@ -755,6 +757,7 @@ int main(int argc, const char* argv[])
         if (!Value)
         {
             // Configure for a 2nd pass
+            auto FileOpenName = Global.OutputFileName;
             ParseInfo.Name = NULL;
             Global.OutputFileName = Global.Inputs[0];
             if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
@@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
             // Parse (check mode)
             Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
             Global.Actions.set(Action_Decode, false); // Override config
-            Value = ParseFile_Compressed(ParseInfo);
+            Value = ParseFile_Compressed(ParseInfo, &FileOpenName);
             if (!Value && !ParseInfo.IsDetected)
             {
                 cout << '\n' << "Error: " << Global.OutputFileName << endl;

Source/Lib/Compressed/Matroska/Matroska.cpp

@@ -359,13 +359,22 @@ void matroska::ParseBuffer()
         // Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
         if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value 
         {
-            FileMap->Remap();
+            if (FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024))
+            {
+                //delete FileMap;
+                FileMap = FileMap2;
+                OpenStyle = filemap::method::mmap;
+                FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
+            }
             Buffer = *FileMap;
+            if (OpenStyle == filemap::method::mmap)
+            {
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
             for (const auto& TrackInfo_Current : TrackInfo)
                 if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
                     TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
+            }
             Buffer_Offset_LowerLimit = Buffer_Offset;
         }
 
@@ -376,13 +385,16 @@ void matroska::ParseBuffer()
             Buffer_Offset = Cluster_Offset;
             Cluster_Level = (size_t)-1;
 
-            FileMap->Remap();
+            FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
             Buffer = *FileMap;
+            if (OpenStyle == filemap::method::mmap)
+            {
             if (ReversibilityData)
                 ReversibilityData->SetBaseData(Buffer.Data());
             for (const auto& TrackInfo_Current : TrackInfo)
                 if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
                     TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
+            }
             Buffer_Offset_LowerLimit = Buffer_Offset;
         }
     }
@@ -428,6 +440,7 @@ void matroska::Segment()
 {
     SetDetected();
     IsList = true;
+    Actions[Action_Hash] = false;
 
     // In case of partial check
     if (!Actions[Action_Decode] && !Actions[Action_Info] && Actions[Action_QuickCheckAfterEncode]) // Quick check after encoding
@@ -799,6 +812,11 @@ void matroska::Segment_Attachments_AttachedFile_FileData_RawCookedTrack_LibraryV
 //---------------------------------------------------------------------------
 void matroska::Segment_Cluster()
 {
+    IsList = true;
+
+    if (FileMap2)
+        return;
+
     if (RAWcooked_LibraryName.empty())
     {
         memcpy(Cluster_Levels, Levels, sizeof(Levels));
@@ -808,8 +826,6 @@ void matroska::Segment_Cluster()
         return;
     }
 
-    IsList = true;
-
     // Check if Hashes check is useful
     if (Hashes_FromRAWcooked)
     {
@@ -850,6 +866,19 @@ void matroska::Segment_Cluster()
             Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
     if (ReversibilityData && !FrameWriter_Template->Compound)
         InitOutput_Find();
+
+    FileMap2 = FileMap;
+    if (OpenStyle != filemap::method::mmap && OpenName)
+    {
+        FileMap = new filemap;
+        if (FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024))
+        {
+            //delete FileMap;
+            FileMap = FileMap2;
+            OpenStyle = filemap::method::mmap;
+        }
+        Buffer = *FileMap;
+    }
 }
 
 //---------------------------------------------------------------------------