Skip to content

Commit

Permalink
Add alternative methods for reading files
Browse files Browse the repository at this point in the history 
Using mmap only seems to be very slow with some NAS
  • Loading branch information
@JeromeMartinez
JeromeMartinez committed Oct 18, 2024
1 parent fce3a36 commit 6530ec3
Show file tree
Hide file tree
Showing 8 changed files with 241 additions and 12 deletions.
43 changes: 43 additions & 0 deletions Source/CLI/Global.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,40 @@ int global::SetHash(bool Value)
return 0;
}

//---------------------------------------------------------------------------
int global::SetFileOpenMethod(const char* Value)
{
if (strcmp(Value, "mmap") == 0)
{
FileOpenMethod = filemap::method::mmap;
return 0;
}
if (strcmp(Value, "fstream") == 0)
{
FileOpenMethod = filemap::method::fstream;
return 0;
}
if (strcmp(Value, "fopen") == 0)
{
FileOpenMethod = filemap::method::fopen;
return 0;
}
if (strcmp(Value, "open") == 0)
{
FileOpenMethod = filemap::method::open;
return 0;
}
#if defined(_WIN32) || defined(_WINDOWS)
if (strcmp(Value, "createfile") == 0)
{
FileOpenMethod = filemap::method::createfile;
return 0;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
cerr << "Error: unknown io value '" << Value << "'." << endl;
return 1;
}

//---------------------------------------------------------------------------
int global::SetAll(bool Value)
{
Expand Down Expand Up @@ -432,6 +466,7 @@ int global::ManageCommandLine(const char* argv[], int argc)
IgnoreLicenseKey = !License.IsSupported_License();
SubLicenseId = 0;
SubLicenseDur = 1;
FileOpenMethod = filemap::method::mmap;
ShowLicenseKey = false;
StoreLicenseKey = false;
DisplayCommand = false;
Expand Down Expand Up @@ -748,6 +783,14 @@ int global::ManageCommandLine(const char* argv[], int argc)
if (auto Value = SetAcceptFiles())
return Value;
}
else if (strcmp(argv[i], "--io") == 0)
{
if (i + 1 == argc)
return Error_Missing(argv[i]);
int Value = SetFileOpenMethod(argv[++i]);
if (Value)
return Value;
}
else if (!strcmp(argv[i], "-framerate"))
{
if (OptionsForOtherFiles)
Expand Down
2 changes: 2 additions & 0 deletions Source/CLI/Global.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class global
string LicenseKey;
uint64_t SubLicenseId;
uint64_t SubLicenseDur;
filemap::method FileOpenMethod;
bool IgnoreLicenseKey;
bool ShowLicenseKey;
bool StoreLicenseKey;
Expand Down Expand Up @@ -100,6 +101,7 @@ class global
int SetFrameMd5An(bool Value);
int SetFrameMd5FileName(const char* FileName);
int SetHash(bool Value);
int SetFileOpenMethod(const char* Value);
int SetAll(bool Value);

private:
Expand Down
9 changes: 6 additions & 3 deletions Source/CLI/Main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ int ParseFile_Uncompressed(parse_info& ParseInfo, size_t Files_Pos)
}

//---------------------------------------------------------------------------
int ParseFile_Compressed(parse_info& ParseInfo)
int ParseFile_Compressed(parse_info& ParseInfo, const string* FileOpenName)
{
// Init
string OutputDirectoryName;
Expand Down Expand Up @@ -522,6 +522,8 @@ int ParseFile_Compressed(parse_info& ParseInfo)
matroska* M = new matroska(OutputDirectoryName, &Global.Mode, Ask_Callback, Thread_Pool, &Global.Errors);
M->Quiet = Global.Quiet;
M->NoOutputCheck = NoOutputCheck;
M->OpenName = FileOpenName;
M->OpenStyle = Global.FileOpenMethod;
if (ParseInfo.ParseFile_Input(*M))
{
ReturnValue = 1;
Expand Down Expand Up @@ -591,7 +593,7 @@ int ParseFile(size_t Files_Pos)
return 1;

// Compressed content
if (int Value = ParseFile_Compressed(ParseInfo))
if (int Value = ParseFile_Compressed(ParseInfo, ParseInfo.Name))
return Value;
if (ParseInfo.IsDetected)
return 0;
Expand Down Expand Up @@ -755,6 +757,7 @@ int main(int argc, const char* argv[])
if (!Value)
{
// Configure for a 2nd pass
auto FileOpenName = Global.OutputFileName;
ParseInfo.Name = NULL;
Global.OutputFileName = Global.Inputs[0];
if (!Global.Actions[Action_Hash]) // If hashes are present in the file, output is checked by using hashes
Expand All @@ -772,7 +775,7 @@ int main(int argc, const char* argv[])
// Parse (check mode)
Global.Actions.set(Action_QuickCheckAfterEncode, !Global.Actions[Action_Check]);
Global.Actions.set(Action_Decode, false); // Override config
Value = ParseFile_Compressed(ParseInfo);
Value = ParseFile_Compressed(ParseInfo, &FileOpenName);
if (!Value && !ParseInfo.IsDetected)
{
cout << '\n' << "Error: " << Global.OutputFileName << endl;
Expand Down
20 changes: 18 additions & 2 deletions Source/Lib/Compressed/Matroska/Matroska.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -359,13 +359,16 @@ void matroska::ParseBuffer()
// Check if we can indicate the system that we'll not need anymore memory below this value, without indicating it too much
if (Buffer_Offset > Buffer_Offset_LowerLimit + 1024 * 1024 && Buffer_Offset < Buffer.Size()) // TODO: when multi-threaded frame decoding is implemented, we need to check that all thread don't need anymore memory below this value
{
FileMap->Remap();
FileMap->Remap(Buffer_Offset, Buffer_Offset + 256 * 1024 * 1024);
Buffer = *FileMap;
if (OpenStyle == filemap::method::mmap)
{
if (ReversibilityData)
ReversibilityData->SetBaseData(Buffer.Data());
for (const auto& TrackInfo_Current : TrackInfo)
if (TrackInfo_Current && TrackInfo_Current->ReversibilityData)
TrackInfo_Current->ReversibilityData->SetBaseData(Buffer.Data());
}
Buffer_Offset_LowerLimit = Buffer_Offset;
}

Expand All @@ -376,7 +379,7 @@ void matroska::ParseBuffer()
Buffer_Offset = Cluster_Offset;
Cluster_Level = (size_t)-1;

FileMap->Remap();
FileMap->Remap(Buffer_Offset, 256 * 1024 * 1024);
Buffer = *FileMap;
if (ReversibilityData)
ReversibilityData->SetBaseData(Buffer.Data());
Expand Down Expand Up @@ -850,6 +853,19 @@ void matroska::Segment_Cluster()
Errors->Error(IO_FileChecker, error::type::Undecodable, (error::generic::code)filechecker_issue::undecodable::Format_Undetected, string());
if (ReversibilityData && !FrameWriter_Template->Compound)
InitOutput_Find();

Actions[Action_Hash] = false;

if (!FileMap2)
{
FileMap2 = FileMap;
if (OpenStyle != filemap::method::mmap && OpenName)
{
FileMap = new filemap;
FileMap->Open_ReadMode(*OpenName, OpenStyle, 0, 256 * 1024 * 1024);
Buffer = *FileMap;
}
}
}

//---------------------------------------------------------------------------
Expand Down
154 changes: 152 additions & 2 deletions Source/Lib/Utils/FileIO/FileIO.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@
#include "Lib/Utils/FileIO/FileIO.h"
#include <iostream>
#include <sstream>
#include <fstream>
#if defined(_WIN32) || defined(_WINDOWS)
#include "windows.h"
#include <stdio.h>
#include <fcntl.h>
#include <io.h> // File existence
#include <direct.h> // Directory creation
#define access _access_s
Expand All @@ -29,10 +32,106 @@
//---------------------------------------------------------------------------

//---------------------------------------------------------------------------
int filemap::Open_ReadMode(const char* FileName)
struct private_buffered
{
union f
{
FILE* File;
ifstream* Ifstream;
int Int;
#if defined(_WIN32) || defined(_WINDOWS)
HANDLE Handle;
#endif
};
f F;
size_t Data_Shift = 0;
size_t MaxSize = 0;
};

//---------------------------------------------------------------------------
int filemap::Open_ReadMode(const char* FileName, method NewStyle, size_t Begin, size_t End)
{
Close();

if (NewStyle != method::mmap)
{
Method = NewStyle;
private_buffered* P = new private_buffered;
P->MaxSize = End - Begin;
size_t FileSize;

switch (Method)
{
default: // case style::fstream:
{
auto F = new ifstream(FileName, ios::binary);
F->seekg(0, F->end);
FileSize = F->tellg();
F->seekg(Begin, F->beg);
P->F.Ifstream = F;
break;
}
case method::fopen:
{
struct stat Fstat;
if (stat(FileName, &Fstat))
return 1;
FileSize = Fstat.st_size;
auto F = fopen(FileName, "rb");
P->F.File = F;
break;
}
case method::open:
{
struct stat Fstat;
if (stat(FileName, &Fstat))
return 1;
FileSize = Fstat.st_size;
#if defined(_WIN32) || defined(_WINDOWS)
auto F = _open(FileName, _O_BINARY | _O_RDONLY | _O_SEQUENTIAL, _S_IREAD);
#else //defined(_WIN32) || defined(_WINDOWS)
auto F = open(FileName, O_RDONLY);
#endif //defined(_WIN32) || defined(_WINDOWS)
if (F == -1)
return 1;
P->F.Int = F;
break;
}
#if defined(_WIN32) || defined(_WINDOWS)
case method::createfile:
{
DWORD FileSizeHigh;
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
auto FileSizeLow = GetFileSize(NewFile, &FileSizeHigh);
if ((FileSizeLow != INVALID_FILE_SIZE || GetLastError() == NO_ERROR) // If no error (special case with 32-bit max value)
&& (!FileSizeHigh || sizeof(size_t) >= 8)) // Mapping 4+ GiB files is not supported in 32-bit mode
{
FileSize = ((size_t)FileSizeHigh) << 32 | FileSizeLow;
}
else
return 1;
if (Begin)
{
LARGE_INTEGER GoTo;
GoTo.QuadPart = Begin;
if (!SetFilePointerEx(NewFile, GoTo, nullptr, 0))
return 1;
P->Data_Shift = Begin;
}
P->F.Handle = NewFile;
break;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
}

auto Buffer = new uint8_t[P->MaxSize];
P->Data_Shift -= P->MaxSize;
AssignBase(Buffer - P->Data_Shift, FileSize);
Private2 = (decltype(Private2))P;

return Remap(Begin, End);
}

size_t NewSize;
#if defined(_WIN32) || defined(_WINDOWS)
auto NewFile = CreateFileA(FileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0);
Expand Down Expand Up @@ -99,12 +198,63 @@ inline int munmap_const(const void* addr, size_t length)
#pragma GCC diagnostic pop
#endif
#endif
int filemap::Remap()
int filemap::Remap(size_t Begin, size_t End)
{
// Special case for 0-byte files
if (Empty())
return 0;

if (Method != method::mmap)
{
auto P = (private_buffered*)Private2;
auto Buffer = Data() + P->Data_Shift;
auto Buffer_MaxSize = P->MaxSize;
Begin -= P->Data_Shift;
if (!End)
End = Size();
End -= P->Data_Shift;
auto Buffer_Middle = Buffer + Begin;
auto Buffer_Middle_Size = Buffer_MaxSize - Begin;
memmove((void*)Buffer, (void*)Buffer_Middle, Buffer_Middle_Size);
P->Data_Shift += Begin;
AssignKeepSizeBase(Buffer - P->Data_Shift);
Buffer += Buffer_Middle_Size;
Buffer_MaxSize -= Buffer_Middle_Size;

switch (Method)
{
default: // case style::fstream:
{
auto F = P->F.Ifstream;
F->read((char*)Buffer, Buffer_MaxSize);
break;
}
case method::fopen:
{
auto F = P->F.File;
if (fread((char*)Buffer, Buffer_MaxSize, 1, F) != 1)
return 1;
break;
}
case method::open:
{
auto F = P->F.Int;
read(F, (void*)Buffer, Buffer_MaxSize);
break;
}
#if defined(_WIN32) || defined(_WINDOWS)
case method::createfile:
{
auto F = P->F.Handle;
ReadFile(F, (LPVOID)Buffer, (DWORD)Buffer_MaxSize, nullptr, 0);
break;
}
#endif //defined(_WIN32) || defined(_WINDOWS)
}

return 0;
}

// Close previous map
if (Data())
{
Expand Down
19 changes: 15 additions & 4 deletions Source/Lib/Utils/FileIO/FileIO.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,30 @@ class filemap : public buffer_view
~filemap() { Close(); }

// Actions
int Open_ReadMode(const char* FileName);
int Open_ReadMode(const string& FileName) { return Open_ReadMode(FileName.c_str()); }
enum class method
{
mmap,
fstream,
fopen,
open,
#if defined(_WIN32) || defined(_WINDOWS)
createfile,
#endif //defined(_WIN32) || defined(_WINDOWS)
};
int Open_ReadMode(const char* FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {});
int Open_ReadMode(const string& FileName, method NewMethod = {}, size_t Begin = {}, size_t End = {}) { return Open_ReadMode(FileName.c_str(), NewMethod, Begin, End); }
bool IsOpen() { return Private == (decltype(Private))-1 ? false : true; }
int Remap();
int Remap(size_t Begin = 0, size_t End = 0);
int Close();

private:
#if defined(_WIN32) || defined(_WINDOWS)
void* Private = (void*)-1;
void* Private2 = (void*)-1;
#else //defined(_WIN32) || defined(_WINDOWS)
int Private = (int)-1;
#endif //defined(_WIN32) || defined(_WINDOWS)
void* Private2 = (void*)-1;
method Method = {};
};

class file
Expand Down
1 change: 1 addition & 0 deletions Source/Lib/Utils/FileIO/Input_Base.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ input_base::~input_base()
bool input_base::Parse(filemap* FileMap_Source, const buffer_view& Buffer_Source, size_t FileSize_Source)
{
FileMap = FileMap_Source;
FileMap2 = nullptr;
FileSize = FileSize_Source == (size_t)-1 ? Buffer_Source.Size() : FileSize_Source;
Buffer = Buffer_Source;
HashComputed = false;
Expand Down
Loading

0 comments on commit 6530ec3

Please sign in to comment.