Skip to content

Commit a47d436

Browse files
awawa-devawawa-dev
committed
Add support for XZ compression for all LUTs file
1 parent 97b138d commit a47d436

File tree

4 files changed

+184
-12
lines changed

4 files changed

+184
-12
lines changed

Diff for: ‎include/utils-xz/utils-xz.h

+3
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,7 @@
44
#define _XZ_SHARED_API
55
#endif
66

7+
#include <image/MemoryBuffer.h>
8+
79
_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName);
10+
_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize);

Diff for: ‎include/utils/LutLoader.h

+4
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#ifndef PCH_ENABLED
44
#include <QString>
55
#include <QList>
6+
#include <QFile>
67
#endif
78

89
#include <utils/PixelFormat.h>
@@ -17,4 +18,7 @@ class LutLoader {
1718
MemoryBuffer<uint8_t> _lut;
1819

1920
void loadLutFile(Logger* _log, PixelFormat color, const QList<QString>& files);
21+
private:
22+
bool decompressLut(Logger* _log, QFile& file, int index);
23+
void hasher(int index, Logger* _log);
2024
};

Diff for: ‎sources/utils-xz/utils-xz.cpp

+97
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include <fstream>
33
#include <vector>
44
#include <lzma.h>
5+
#include <cstring>
56
#include <utils-xz/utils-xz.h>
67

78
_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName)
@@ -83,3 +84,99 @@ _XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t
8384

8485
return error;
8586
}
87+
88+
_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize)
89+
{
90+
size_t outSize = 16842808/2;
91+
std::vector<uint8_t> outBuffer;
92+
const char* error = nullptr;
93+
int currentPos = 0;
94+
int writePos = 0;
95+
96+
try
97+
{
98+
outBuffer.resize(outSize);
99+
}
100+
catch (...)
101+
{
102+
error = "Could not allocate buffer";
103+
}
104+
105+
if (error == nullptr)
106+
{
107+
const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
108+
lzma_stream strm = LZMA_STREAM_INIT;
109+
strm.next_in = downloadedData;
110+
strm.avail_in = downloadedDataSize;
111+
lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags);
112+
if (lzmaRet == LZMA_OK)
113+
{
114+
do {
115+
strm.next_out = outBuffer.data();
116+
strm.avail_out = outSize;
117+
lzmaRet = lzma_code(&strm, LZMA_FINISH);
118+
if (lzmaRet == LZMA_MEMLIMIT_ERROR)
119+
{
120+
outSize = lzma_memusage(&strm);
121+
try
122+
{
123+
outBuffer.resize(outSize);
124+
}
125+
catch (...)
126+
{
127+
error = "Could not increase buffer size";
128+
break;
129+
}
130+
lzma_memlimit_set(&strm, outSize);
131+
strm.avail_out = 0;
132+
}
133+
else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END)
134+
{
135+
error = "LZMA decoder returned error";
136+
break;
137+
}
138+
else
139+
{
140+
int toWrite = outSize - strm.avail_out;
141+
int endPos = currentPos + toWrite;
142+
int chunkSize = -1, chunkPos = 0;
143+
144+
if (currentPos <= destSeek && destSeek + destSize >= endPos)
145+
{
146+
chunkPos = destSeek - currentPos;
147+
chunkSize = toWrite - chunkPos;
148+
}
149+
else if (currentPos <= destSeek + destSize && destSeek + destSize < endPos)
150+
{
151+
chunkSize = (destSeek + destSize) - currentPos;
152+
}
153+
else if (currentPos >= destSeek && destSeek + destSize >= endPos)
154+
{
155+
chunkSize = toWrite;
156+
}
157+
158+
if (chunkSize > 0)
159+
{
160+
memcpy(dest + writePos, outBuffer.data() + chunkPos, chunkSize);
161+
writePos += chunkSize;
162+
}
163+
164+
currentPos = endPos;
165+
}
166+
} while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END && (writePos < destSize));
167+
}
168+
else
169+
{
170+
error = "Could not initialize LZMA decoder";
171+
}
172+
173+
lzma_end(&strm);
174+
}
175+
176+
if (writePos != destSize)
177+
{
178+
error = "Incorrect final LUT size";
179+
}
180+
181+
return error;
182+
}

Diff for: ‎sources/utils/LutLoader.cpp

+80-12
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,18 @@
2424
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
2525
* SOFTWARE.
2626
*/
27-
27+
#include <HyperhdrConfig.h>
2828
#include <utils/LutLoader.h>
2929
#include <image/ColorRgb.h>
3030
#include <utils/FrameDecoder.h>
31+
#include <utils/InternalClock.h>
3132
#include <QFile>
3233

34+
#ifdef ENABLE_XZ
35+
#include <utils-xz/utils-xz.h>
36+
#endif
37+
38+
3339
namespace {
3440
const int LUT_FILE_SIZE = 256 * 256 * 256 *3;
3541
const int LUT_MEMORY_ALIGN = 64;
@@ -52,17 +58,26 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
5258
for (QString fileName3d : files)
5359
{
5460
QFile file(fileName3d);
61+
bool compressed = false;
5562

56-
if (file.open(QIODevice::ReadOnly))
63+
if (file.open(QIODevice::ReadOnly)
64+
#ifdef ENABLE_XZ
65+
|| [&]() {file.setFileName(fileName3d + ".xz"); compressed = true; return file.open(QIODevice::ReadOnly); } ()
66+
#endif
67+
)
5768
{
5869
int length;
59-
if (_log) Debug(_log, "LUT file found: %s", QSTRING_CSTR(fileName3d));
70+
71+
if (_log)
72+
{
73+
Debug(_log, "LUT file found: %s (%s)", QSTRING_CSTR(file.fileName()), (compressed) ? "compressed" : "uncompressed");
74+
}
6075

6176
length = file.size();
6277

63-
if ((length == LUT_FILE_SIZE * 3) || (length == LUT_FILE_SIZE && !is_yuv))
78+
if ((length == LUT_FILE_SIZE * 3) || (length == LUT_FILE_SIZE && !is_yuv) || compressed)
6479
{
65-
qint64 index = 0;
80+
int index = 0;
6681

6782
if (is_yuv && _hdrToneMappingEnabled)
6883
{
@@ -77,21 +92,35 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
7792
else
7893
{
7994
if (_log) Debug(_log, "Index 0 for HDR RGB");
80-
}
81-
82-
file.seek(index);
95+
}
8396

8497
_lut.resize(LUT_FILE_SIZE + LUT_MEMORY_ALIGN);
8598

86-
if (file.read((char*)_lut.data(), LUT_FILE_SIZE) != LUT_FILE_SIZE)
99+
if (!compressed)
87100
{
88-
if (_log) Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
101+
file.seek(index);
102+
103+
if (file.read((char*)_lut.data(), LUT_FILE_SIZE) != LUT_FILE_SIZE)
104+
{
105+
if (_log) Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
106+
}
107+
else
108+
{
109+
_lutBufferInit = true;
110+
if (_log) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
111+
}
89112
}
90113
else
91114
{
92-
_lutBufferInit = true;
93-
if (_log) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
115+
_lutBufferInit = decompressLut(_log, file, index);
116+
if (_log)
117+
{
118+
if (_lutBufferInit) Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
119+
else Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
120+
}
94121
}
122+
123+
// hasher(index / LUT_FILE_SIZE, _log);
95124
}
96125
else
97126
{
@@ -111,3 +140,42 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList<QString
111140
if (_log) Error(_log, "Could not find any required LUT file");
112141
}
113142
}
143+
144+
bool LutLoader::decompressLut(Logger* _log, QFile& file, int index)
145+
{
146+
auto now = InternalClock::nowPrecise();
147+
const char* retVal = "HyperHDR was built without a support for XZ decoder";
148+
QByteArray compressedFile = file.readAll();
149+
#ifdef ENABLE_XZ
150+
retVal = DecompressXZ(compressedFile.size(), reinterpret_cast<uint8_t*>(compressedFile.data()), _lut.data(), index, LUT_FILE_SIZE);
151+
#endif
152+
153+
if (retVal != nullptr && _log)
154+
{
155+
Error(_log, "Error while decompressing LUT: %s", retVal);
156+
}
157+
158+
if (_log) Info(_log, "Decompression took %f seconds", (InternalClock::nowPrecise() - now) / 1000.0);
159+
160+
return retVal == nullptr;
161+
}
162+
163+
void LutLoader::hasher(int index, Logger* _log)
164+
{
165+
if (_log)
166+
{
167+
auto start = _lut.data();
168+
auto end = start + _lut.size();
169+
uint8_t position = 0;
170+
uint16_t fletcher1 = 0;
171+
uint16_t fletcher2 = 0;
172+
uint16_t fletcherExt = 0;
173+
while (start < end)
174+
{
175+
fletcher1 = (fletcher1 + (uint16_t)(*(start))) % 255;
176+
fletcher2 = (fletcher2 + fletcher1) % 255;
177+
fletcherExt = (fletcherExt + (*(start++) ^ (position++))) % 255;
178+
}
179+
Info(_log, "CRC for %i segment: 0x%.2X 0x%.2X 0x%.2X", index, fletcher1, fletcher2, fletcherExt);
180+
}
181+
}

0 commit comments

Comments
 (0)