Skip to content

Commit

Permalink
[WIP] Add an alternative method for reading files
Browse files Browse the repository at this point in the history 
Using mmap only seems to be very slow with some NAS
  • Loading branch information
@JeromeMartinez
JeromeMartinez committed Oct 18, 2024
1 parent fce3a36 commit 868842f
Show file tree
Hide file tree
Showing 8 changed files with 247 additions and 12 deletions.
49 changes: 49 additions & 0 deletions Source/CLI/Global.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,40 @@ int global::SetHash(bool Value)
return 0;
}

//---------------------------------------------------------------------------
int global::SetFileOpenMethod(const char* Value)
{
if (strcmp(Value, "mmap") == 0)
{
FileOpenMethod = filemap::method::mmap;
return 0;
}
if (strcmp(Value, "fstream") == 0)
{
FileOpenMethod = filemap::method::fstream;
return 0;
}
if (strcmp(Value, "fopen") == 0)
{
FileOpenMethod = filemap::method::fopen;
return 0;
}
if (strcmp(Value, "open") == 0)
{
FileOpenMethod = filemap::method::open;
return 0;
}
#if defined(_WIN32) || defined(_WINDOWS)
if (strcmp(Value, "createfile") == 0)
{
FileOpenMethod = filemap::method::createfile;
return 0;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
cerr << "Error: unknown io value '" << Value << "'." << endl;
return 1;
}

//---------------------------------------------------------------------------
int global::SetAll(bool Value)
{
Expand Down Expand Up @@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
IgnoreLicenseKey = !License.IsSupported_License();
SubLicenseId = 0;
SubLicenseDur = 1;
FileOpenMethod = (filemap::method)-1;
ShowLicenseKey = false;
StoreLicenseKey = false;
DisplayCommand = false;
Expand Down Expand Up @@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
if (auto Value = SetAcceptFiles())
return Value;
}
else if (strcmp(argv[i], "--io") == 0)
{
if (i + 1 == argc)
return Error_Missing(argv[i]);
int Value = SetFileOpenMethod(argv[++i]);
if (Value)
return Value;
}
else if (!strcmp(argv[i], "-framerate"))
{
if (OptionsForOtherFiles)
Expand Down Expand Up @@ -831,6 +874,12 @@ int global::ManageCommandLine(const char* argv[], int argc)
}
if (License.ShowLicense(ShowLicenseKey, SubLicenseId, SubLicenseDur))
return 1;
if (FileOpenMethod == (filemap::method)-1)
{
FileOpenMethod = filemap::method::mmap;
//cerr << "\nThis is a test version, please use another version if you don't know which option to test\n" << endl;
//return 1;
}
if (Inputs.empty() && (ShowLicenseKey || SubLicenseId))
return 0;

Expand Down
2 changes: 2 additions & 0 deletions Source/CLI/Global.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class global
string LicenseKey;
uint64_t SubLicenseId;
uint64_t SubLicenseDur;
filemap::method FileOpenMethod;
bool IgnoreLicenseKey;
bool ShowLicenseKey;
bool StoreLicenseKey;
Expand Down Expand Up @@ -100,6 +101,7 @@ class global
int SetFrameMd5An(bool Value);
int SetFrameMd5FileName(const char* FileName);
int SetHash(bool Value);
int SetFileOpenMethod(const char* Value);
int SetAll(bool Value);

private:
Expand Down
9 changes: 6 additions & 3 deletions Source/CLI/Main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
}

//---------------------------------------------------------------------------
int ParseFile_Compressed(parse_info& ParseInfo)
int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName)
{
// Init
string OutputDirectoryName;
Expand Down Expand Up @@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
M->Quiet = Global.Quiet;
M->NoOutputCheck = NoOutputCheck;
M->OpenName = FileOpenName;
M->OpenStyle = Global.FileOpenMethod;
if (ParseInfo.ParseFile_Input(*M))
{
ReturnValue = 1;
Expand Down Expand Up @@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos)
return 1;

// Compressed content
if (int Value = ParseFile_Compressed(ParseInfo))
if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
return Value;
if (ParseInfo.IsDetected)
return 0;
Expand Down Expand Up @@ -755,6 +757,7 @@ int main(int argc, const char* argv[])
if (!Value)
{
// Configure for a 2nd pass
auto FileOpenName = Global.OutputFileName;
ParseInfo.Name = NULL;
Global.OutputFileName = Global.Inputs[0];
if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
Expand All @@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
// Parse (check mode)
Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
Global.Actions.set(Action_Decode, false); // Override config
Value = ParseFile_Compressed(ParseInfo);
Value = ParseFile_Compressed(ParseInfo, &FileOpenName);
if (!Value && !ParseInfo.IsDetected)
{
cout << '\n' << "Error: " << Global.OutputFileName << endl;
Expand Down
20 changes: 18 additions & 2 deletions Source/Lib/Compressed/Matroska/Matroska.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,13 +359,16 @@ void matroska::ParseBuffer()
// Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value
{
FileMap->Remap();
FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
Buffer = *FileMap;
if (OpenStyle == filemap::method::mmap)
{
if (ReversibilityData)
ReversibilityData->SetBaseData(Buffer.Data());
for (const auto& TrackInfo_Current : TrackInfo)
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
}
Buffer_Offset_LowerLimit = Buffer_Offset;
}

Expand All @@ -376,7 +379,7 @@ void matroska::ParseBuffer()
Buffer_Offset = Cluster_Offset;
Cluster_Level = (size_t)-1;

FileMap->Remap();
FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
Buffer = *FileMap;
if (ReversibilityData)
ReversibilityData->SetBaseData(Buffer.Data());
Expand Down Expand Up @@ -850,6 +853,19 @@ void matroska::Segment_Cluster()
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
if (ReversibilityData && !FrameWriter_Template->Compound)
InitOutput_Find();

Actions[Action_Hash] = false;

if (!FileMap2)
{
FileMap2 = FileMap;
if (OpenStyle != filemap::method::mmap && OpenName)
{
FileMap = new filemap;
FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024);
Buffer = *FileMap;
}
}
}

//---------------------------------------------------------------------------
Expand Down
154 changes: 152 additions & 2 deletions Source/Lib/Utils/FileIO/FileIO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@
#include "Lib/Utils/FileIO/FileIO.h"
#include <iostream>
#include <sstream>
#include <fstream>
#if defined(_WIN32) || defined(_WINDOWS)
#include "windows.h"
#include <stdio.h>
#include <fcntl.h>
#include <io.h> // File existence
#include <direct.h> // Directory creation
#define access _access_s
Expand All @@ -29,10 +32,106 @@
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
int filemap::Open_ReadMode(const char* FileName)
struct private_buffered
{
union f
{
FILE* File;
ifstream* Ifstream;
int Int;
#if defined(_WIN32) || defined(_WINDOWS)
HANDLE Handle;
#endif
};
f F;
size_t Data_Shift = 0;
size_t MaxSize = 0;
};

//---------------------------------------------------------------------------
int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End)
{
Close();

if (NewStyle != method::mmap)
{
Method = NewStyle;
private_buffered* P = new private_buffered;
P->MaxSize = End - Begin;
size_t FileSize;

switch (Method)
{
default: // case style::fstream:
{
auto F = new ifstream(FileName, ios::binary);
F->seekg(0, F->end);
FileSize = F->tellg();
F->seekg(Begin, F->beg);
P->F.Ifstream = F;
break;
}
case method::fopen:
{
struct stat Fstat;
if (stat(FileName, &Fstat))
return 1;
FileSize = Fstat.st_size;
auto F = fopen(FileName, "rb");
P->F.File = F;
break;
}
case method::open:
{
struct stat Fstat;
if (stat(FileName, &Fstat))
return 1;
FileSize = Fstat.st_size;
#if defined(_WIN32) || defined(_WINDOWS)
auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD);
#else //defined(_WIN32) || defined(_WINDOWS)
auto F = open(FileName, O_RDONLY);
#endif //defined(_WIN32) || defined(_WINDOWS)
if (F == -1)
return 1;
P->F.Int = F;
break;
}
#if defined(_WIN32) || defined(_WINDOWS)
case method::createfile:
{
DWORD FileSizeHigh;
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh);
if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value)
&& (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode
{
FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow;
}
else
return 1;
if (Begin)
{
LARGE_INTEGER GoTo;
GoTo.QuadPart = Begin;
if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0))
return 1;
P->Data_Shift = Begin;
}
P->F.Handle = NewFile;
break;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
}

auto Buffer = new uint8_t[P->MaxSize];
P->Data_Shift -= P->MaxSize;
AssignBase(Buffer - P->Data_Shift, FileSize);
Private2 = (decltype(Private2))P;

return Remap(Begin, End);
}

size_t NewSize;
#if defined(_WIN32) || defined(_WINDOWS)
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
Expand Down Expand Up @@ -99,12 +198,63 @@ inline int munmap_const(const void* addr, size_t length)
#pragma GCC diagnostic pop
#endif
#endif
int filemap::Remap()
int filemap::Remap(size_t Begin, size_t End)
{
// Special case for 0-byte files
if (Empty())
return 0;

if (Method != method::mmap)
{
auto P = (private_buffered*)Private2;
auto Buffer = Data() + P->Data_Shift;
auto Buffer_MaxSize = P->MaxSize;
Begin -= P->Data_Shift;
if (!End)
End = Size();
End -= P->Data_Shift;
auto Buffer_Middle = Buffer + Begin;
auto Buffer_Middle_Size = Buffer_MaxSize - Begin;
memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size);
P->Data_Shift += Begin;
AssignKeepSizeBase(Buffer - P->Data_Shift);
Buffer += Buffer_Middle_Size;
Buffer_MaxSize -= Buffer_Middle_Size;

switch (Method)
{
default: // case style::fstream:
{
auto F = P->F.Ifstream;
F->read((char*)Buffer, Buffer_MaxSize);
break;
}
case method::fopen:
{
auto F = P->F.File;
if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1)
return 1;
break;
}
case method::open:
{
auto F = P->F.Int;
read(F, (void*)Buffer, Buffer_MaxSize);
break;
}
#if defined(_WIN32) || defined(_WINDOWS)
case method::createfile:
{
auto F = P->F.Handle;
ReadFile(F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0);
break;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
}

return 0;
}

// Close previous map
if (Data())
{
Expand Down
Loading

0 comments on commit 868842f

Please sign in to comment.