diff --git a/.gitmodules b/.gitmodules index 712f09c06..d30b0c5d6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,9 +13,6 @@ path = external/qmqtt url = https://github.com/emqx/qmqtt.git ignore = dirty -[submodule "external/xz"] - path = external/xz - url = https://github.com/tukaani-project/xz.git [submodule "external/mdns"] path = external/mdns url = https://github.com/mjansson/mdns.git @@ -34,3 +31,6 @@ [submodule "external/linalg"] path = external/linalg url = https://github.com/sgorsten/linalg.git +[submodule "external/zstd"] + path = external/zstd + url = https://github.com/facebook/zstd.git diff --git a/3RD_PARTY_LICENSES b/3RD_PARTY_LICENSES index 7ad185f6f..66eba44e7 100644 --- a/3RD_PARTY_LICENSES +++ b/3RD_PARTY_LICENSES @@ -3009,3 +3009,38 @@ INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHA FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +============================== + zstd +============================== + +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/CMakeLists.txt b/CMakeLists.txt index c4304e058..7990093a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ SET ( DEFAULT_BONJOUR ON ) SET ( DEFAULT_MQTT ON ) SET ( DEFAULT_STATIC_QT_PLUGINS OFF ) SET ( DEFAULT_PRECOMPILED_HEADERS ON ) -SET ( DEFAULT_XZ ON ) +SET ( DEFAULT_ZSTD ON ) SET ( DEFAULT_POWER_MANAGEMENT ON ) SET ( DEFAULT_SYSTRAY ON ) SET ( DEFAULT_SHARED_LIBS ON ) @@ -382,26 +382,6 @@ if(DEFAULT_CEC) endif() endif() -# 7-zip as support -if (WIN32) - # 7zip - cmake_policy(SET CMP0053 NEW) - set(MYENV "PROGRAMFILES(X86)") - - find_program(SEVENZIP_BIN - NAMES 7z 7za - HINTS "${MINGWDIR}" "${MINGWLIBS}/bin" "$MYENV/7-zip" "$ENV{ProgramFiles}/7-zip" "$ENV{ProgramW6432}/7-zip" - PATH_SUFFIXES bin - DOC "7zip executable" - ) - - if (SEVENZIP_BIN) - message( STATUS "7-zip found: ${SEVENZIP_BIN}") - else() - message( FATAL_ERROR "Please install 7-zip") - endif (SEVENZIP_BIN) -endif() - if (CMAKE_CROSSCOMPILING) set ( DEFAULT_USE_SYSTEM_FLATBUFFERS_LIBS OFF ) SET ( DEFAULT_SHARED_LIBS OFF ) @@ -504,8 +484,8 @@ colorMe("ENABLE_PROTOBUF = " ${ENABLE_PROTOBUF}) option(ENABLE_SYSTRAY "Enable SYSTRAY" ${DEFAULT_SYSTRAY}) colorMe("ENABLE_SYSTRAY = " ${ENABLE_SYSTRAY}) -option(ENABLE_XZ "Enable XZ support" ${DEFAULT_XZ}) -colorMe("ENABLE_XZ = " ${ENABLE_XZ}) +option(ENABLE_ZSTD "Enable ZSTD support" ${DEFAULT_ZSTD}) +colorMe("ENABLE_ZSTD = " ${ENABLE_ZSTD}) message( STATUS "\n${CyanColor}BUILD FEATURES${ColorReset}") @@ -547,8 +527,8 @@ SET ( FLATBUFFERS_INSTALL_LIB_DIR ${CMAKE_BINARY_DIR}/flatbuf ) find_package(GitVersion) -if(ENABLE_XZ) - find_package(LibLZMA) +if(ENABLE_ZSTD) + find_package(zstd) endif() # configure a header file to pass some of the CMake settings @@ -694,13 +674,6 @@ if (WIN32) endif() endif() -# Add the source/lib directories -add_subdirectory(external) -add_subdirectory(sources) - -# Add resources directory -add_subdirectory(resources) - # remove generated files on make cleaan too LIST( APPEND GENERATED_QRC ${CMAKE_BINARY_DIR}/WebConfig.qrc @@ -712,12 +685,19 @@ set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${GENERATED_QRC configure_file( "${OUR_CMAKE_MODULES}/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) +# Add the source/lib directories +add_subdirectory(external) +add_subdirectory(sources) + +# Add resources directory +add_subdirectory(resources) + # enable make package - no code after this line ! include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/packages.cmake) # external targets -if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET liblzma AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers) - set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall liblzma sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets) +if (WIN32 AND TARGET stb AND TARGET flatbuffers AND TARGET protobuf-nanopb AND TARGET lunasvg AND TARGET flatc AND TARGET qmqtt AND TARGET libzstd_static AND TARGET libzstd_shared AND TARGET zstd AND TARGET clean-all AND TARGET sqlite3 AND TARGET precompiled_hyperhdr_headers) + set_target_properties(stb qmqtt flatbuffers protobuf-nanopb lunasvg flatc resources uninstall libzstd_static libzstd_shared zstd clean-all sqlite3 precompiled_hyperhdr_headers PROPERTIES FOLDER ExternalLibsTargets) else() set_target_properties(resources uninstall PROPERTIES FOLDER ExternalLibsTargets) endif() diff --git a/HyperhdrConfig.h.in b/HyperhdrConfig.h.in index fcc8b2565..1cfa4f2e5 100644 --- a/HyperhdrConfig.h.in +++ b/HyperhdrConfig.h.in @@ -66,8 +66,8 @@ // Define to enable system qmqtt #cmakedefine USE_SYSTEM_MQTT_LIBS -// Define to enable protobuf -#cmakedefine ENABLE_XZ +// Define to enable zstd +#cmakedefine ENABLE_ZSTD // Define to enable protobuf #cmakedefine ENABLE_PROTOBUF diff --git a/cmake/cmake_modules/Findzstd.cmake b/cmake/cmake_modules/Findzstd.cmake new file mode 100644 index 000000000..44547365d --- /dev/null +++ b/cmake/cmake_modules/Findzstd.cmake @@ -0,0 +1,29 @@ +# - Find zstd +# Find the zstd compression library and includes +# +# ZSTD_INCLUDE_DIRS - where to find zstd.h, etc. +# ZSTD_LIBRARIES - List of libraries when using zstd. +# ZSTD_FOUND - True if zstd found. + +find_path(ZSTD_INCLUDE_DIRS + NAMES zstd.h + HINTS ${zstd_ROOT_DIR}/include) + +find_library(ZSTD_LIBRARIES + NAMES zstd + HINTS ${zstd_ROOT_DIR}/lib) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS) + +mark_as_advanced( + ZSTD_LIBRARIES + ZSTD_INCLUDE_DIRS) + +if(ZSTD_FOUND AND NOT (TARGET zstd::zstd)) + add_library (zstd::zstd UNKNOWN IMPORTED) + set_target_properties(zstd::zstd + PROPERTIES + IMPORTED_LOCATION ${ZSTD_LIBRARIES} + INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIRS}) +endif() \ No newline at end of file diff --git a/cmake/installers.cmake b/cmake/installers.cmake index 90b0b2dae..378fda4dc 100644 --- a/cmake/installers.cmake +++ b/cmake/installers.cmake @@ -20,9 +20,9 @@ macro(DeployApple TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy utils-xz - if (USE_SHARED_LIBS AND TARGET utils-xz) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy utils-zstd + if (USE_SHARED_LIBS AND TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/hyperhdr.app/Contents/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() # Copy utils-image @@ -243,9 +243,9 @@ macro(DeployUnix TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy UTILS-XZ lib - if (TARGET utils-image) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy UTILS-ZSTD lib + if (TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/share/hyperhdr/lib" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() @@ -354,7 +354,7 @@ macro(DeployUnix TARGET) "librt" "libstdc++" "libudev" - "libz" + "libz.so" "libxrender1" "libxi6" "libxext6" @@ -593,9 +593,9 @@ macro(DeployWindows TARGET) install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() - # Copy utils-xz - if (USE_SHARED_LIBS AND TARGET utils-xz) - install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") + # Copy utils-zstd + if (USE_SHARED_LIBS AND TARGET utils-zstd) + install(CODE [[ file(INSTALL FILES $ DESTINATION "${CMAKE_INSTALL_PREFIX}/bin" TYPE SHARED_LIBRARY) ]] COMPONENT "HyperHDR") endif() # Copy utils-image diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 23fce071d..b4d23ad06 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -237,17 +237,22 @@ if ( ENABLE_MQTT ) endif() #============================================================================= -# XZ +# ZSTD #============================================================================= -if ( ENABLE_XZ ) - if (NOT LIBLZMA_FOUND) - message( STATUS "System library xz could not be found. Using embedded xz library.") - option(BUILD_TESTING "" OFF) - add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/xz) - set_target_properties(liblzma PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $) - set_property(TARGET liblzma PROPERTY POSITION_INDEPENDENT_CODE ON) - add_library(LibLZMA::LibLZMA ALIAS liblzma) +if ( ENABLE_ZSTD ) + if (NOT zstd_FOUND) + message( STATUS "System library zstd could not be found. Using embedded zstd library.") + if(WIN32) + set(CMAKE_RC_FLAGS "${CMAKE_RC_FLAGS} -I${CMAKE_SOURCE_DIR}/external/zstd/lib") + endif() + set(ZSTD_BUILD_TESTS OFF) + set(ZSTD_LEGACY_SUPPORT OFF) + set(ZSTD_BUILD_PROGRAMS OFF) + add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/zstd/build/cmake) + set_target_properties(libzstd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES $) + set_property(TARGET libzstd PROPERTY POSITION_INDEPENDENT_CODE ON) + add_library(zstd::zstd ALIAS libzstd) endif() ENDIF() diff --git a/external/xz b/external/xz deleted file mode 160000 index 238b4e545..000000000 --- a/external/xz +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 238b4e5458b4bd2cadefb768b8ea7c6b70a191ac diff --git a/external/zstd b/external/zstd new file mode 160000 index 000000000..794ea1b0a --- /dev/null +++ b/external/zstd @@ -0,0 +1 @@ +Subproject commit 794ea1b0afca0f020f4e57b6732332231fb23c70 diff --git a/include/utils-xz/utils-xz.h b/include/utils-xz/utils-xz.h deleted file mode 100644 index ca1f1657d..000000000 --- a/include/utils-xz/utils-xz.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -#ifndef _XZ_SHARED_API - #define _XZ_SHARED_API -#endif - -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName); diff --git a/include/utils-zstd/utils-zstd.h b/include/utils-zstd/utils-zstd.h new file mode 100644 index 000000000..c23c31a9b --- /dev/null +++ b/include/utils-zstd/utils-zstd.h @@ -0,0 +1,10 @@ +#pragma once + +#ifndef _ZSTD_SHARED_API + #define _ZSTD_SHARED_API +#endif + +#include + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName); +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize); diff --git a/include/utils/LutLoader.h b/include/utils/LutLoader.h index c79f254bc..93bd91dc3 100644 --- a/include/utils/LutLoader.h +++ b/include/utils/LutLoader.h @@ -3,6 +3,7 @@ #ifndef PCH_ENABLED #include #include + #include #endif #include @@ -17,4 +18,7 @@ class LutLoader { MemoryBuffer _lut; void loadLutFile(Logger* _log, PixelFormat color, const QList& files); + private: + bool decompressLut(Logger* _log, QFile& file, int index); + void hasher(int index, Logger* _log); }; diff --git a/sources/CMakeLists.txt b/sources/CMakeLists.txt index 3ca0cf597..157722a67 100644 --- a/sources/CMakeLists.txt +++ b/sources/CMakeLists.txt @@ -33,8 +33,8 @@ add_subdirectory(utils) add_subdirectory(utils-image) add_subdirectory(webserver) -if(ENABLE_XZ) - add_subdirectory(utils-xz) +if(ENABLE_ZSTD) + add_subdirectory(utils-zstd) endif() diff --git a/sources/api/BaseAPI.cpp b/sources/api/BaseAPI.cpp index 11fd1e5d0..62d582c34 100644 --- a/sources/api/BaseAPI.cpp +++ b/sources/api/BaseAPI.cpp @@ -31,8 +31,8 @@ #include #include -#ifdef ENABLE_XZ - #include +#ifdef ENABLE_ZSTD + #include #endif #ifdef _WIN32 @@ -410,21 +410,21 @@ bool BaseAPI::saveSettings(const QJsonObject& data) QString BaseAPI::installLut(QNetworkReply* reply, QString fileName, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time) { -#ifdef ENABLE_XZ +#ifdef ENABLE_ZSTD QString error = nullptr; if (reply->error() == QNetworkReply::NetworkError::NoError) { QByteArray downloadedData = reply->readAll(); - error = DecompressXZ(downloadedData.size(), reinterpret_cast(downloadedData.data()), QSTRING_CSTR(fileName)); + error = DecompressZSTD(downloadedData.size(), reinterpret_cast(downloadedData.data()), QSTRING_CSTR(fileName)); } else error = "Could not download LUT file"; return error; #else - return "XZ support was disabled in the build configuration"; + return "ZSTD support was disabled in the build configuration"; #endif } diff --git a/sources/api/HyperAPI.cpp b/sources/api/HyperAPI.cpp index b27dd3a69..222a0272f 100644 --- a/sources/api/HyperAPI.cpp +++ b/sources/api/HyperAPI.cpp @@ -568,7 +568,7 @@ void HyperAPI::lutDownloaded(QNetworkReply* reply, int hardware_brightness, int void HyperAPI::handleLutInstallCommand(const QJsonObject& message, const QString& command, int tan) { - const QString& address = QString("%1/lut_lin_tables.3d.xz").arg(message["subcommand"].toString().trimmed()); + const QString& address = QString("%1/lut_lin_tables.3d.zst").arg(message["subcommand"].toString().trimmed()); int hardware_brightness = message["hardware_brightness"].toInt(0); int hardware_contrast = message["hardware_contrast"].toInt(0); int hardware_saturation = message["hardware_saturation"].toInt(0); diff --git a/sources/hyperhdr/CMakeLists.txt b/sources/hyperhdr/CMakeLists.txt index 500c562a5..2e6c2f2eb 100644 --- a/sources/hyperhdr/CMakeLists.txt +++ b/sources/hyperhdr/CMakeLists.txt @@ -54,12 +54,8 @@ else() PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) endif() -if(ENABLE_XZ) - if (LIBLZMA_FOUND) - target_link_libraries(hyperhdr LibLZMA::LibLZMA utils-xz) - else() - target_link_libraries(hyperhdr liblzma utils-xz) - endif() +if(ENABLE_ZSTD) + target_link_libraries(hyperhdr zstd::zstd utils-zstd) endif() if (USE_STATIC_QT_PLUGINS) diff --git a/sources/utils-xz/CMakeLists.txt b/sources/utils-xz/CMakeLists.txt deleted file mode 100644 index 6c2ac048b..000000000 --- a/sources/utils-xz/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -# Define the current source locations - -SET(CURRENT_HEADER_DIR ${CMAKE_SOURCE_DIR}/include/utils-xz) -SET(CURRENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/sources/utils-xz) - -FILE ( GLOB_RECURSE utils_xz_SOURCES "${CURRENT_HEADER_DIR}/*.h" "${CURRENT_SOURCE_DIR}/*.cpp" ) - -if (USE_SHARED_LIBS) - add_library(utils-xz SHARED ${utils_xz_SOURCES}) - if(WIN32) - target_compile_definitions(utils-xz - INTERFACE - "_XZ_SHARED_API=__declspec(dllimport)" - PRIVATE - "_XZ_SHARED_API=__declspec(dllexport)" - ) - else() - target_compile_definitions(utils-xz - INTERFACE - "_XZ_SHARED_API=__attribute__((visibility(\"default\")))" - ) - endif() - install( - TARGETS utils-xz - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ) -else() - add_library(utils-xz STATIC ${utils_xz_SOURCES}) -endif() - -target_link_libraries(utils-xz PRIVATE LibLZMA::LibLZMA) -target_include_directories(utils-xz PRIVATE LibLZMA::LibLZMA) - diff --git a/sources/utils-xz/utils-xz.cpp b/sources/utils-xz/utils-xz.cpp deleted file mode 100644 index bd43dd48f..000000000 --- a/sources/utils-xz/utils-xz.cpp +++ /dev/null @@ -1,85 +0,0 @@ -#include -#include -#include -#include -#include - -_XZ_SHARED_API const char* DecompressXZ(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName) -{ - size_t outSize = 67174456; - std::vector outBuffer; - const char* error = nullptr; - - std::ofstream file; - file.open(fileName, std::ios::out | std::ios::trunc | std::ios::binary); - - if (!file.is_open()) - { - return "Could not open file for writing"; - } - - try - { - outBuffer.resize(outSize); - } - catch(...) - { - error = "Could not allocate buffer"; - } - - if (error == nullptr) - { - const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED; - lzma_stream strm = LZMA_STREAM_INIT; - strm.next_in = downloadedData; - strm.avail_in = downloadedDataSize; - lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags); - if (lzmaRet == LZMA_OK) - { - do { - strm.next_out = outBuffer.data(); - strm.avail_out = outSize; - lzmaRet = lzma_code(&strm, LZMA_FINISH); - if (lzmaRet == LZMA_MEMLIMIT_ERROR) - { - outSize = lzma_memusage(&strm); - try - { - outBuffer.resize(outSize); - } - catch (...) - { - error = "Could not increase buffer size"; - break; - } - lzma_memlimit_set(&strm, outSize); - strm.avail_out = 0; - } - else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END) - { - error = "LZMA decoder returned error"; - break; - } - else - { - std::streamsize toWrite = static_cast(outSize - strm.avail_out); - file.write(reinterpret_cast(outBuffer.data()), toWrite); - } - } while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END); - file.flush(); - } - else - { - error = "Could not initialize LZMA decoder"; - } - - lzma_end(&strm); - } - - file.close(); - - if (error != nullptr) - std::remove(fileName); - - return error; -} diff --git a/sources/utils-zstd/CMakeLists.txt b/sources/utils-zstd/CMakeLists.txt new file mode 100644 index 000000000..c7fa8b096 --- /dev/null +++ b/sources/utils-zstd/CMakeLists.txt @@ -0,0 +1,33 @@ +# Define the current source locations + +SET(CURRENT_HEADER_DIR ${CMAKE_SOURCE_DIR}/include/utils-zstd) +SET(CURRENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/sources/utils-zstd) + +FILE ( GLOB_RECURSE utils_zstd_SOURCES "${CURRENT_HEADER_DIR}/*.h" "${CURRENT_SOURCE_DIR}/*.cpp" ) + +if (USE_SHARED_LIBS) + add_library(utils-zstd SHARED ${utils_zstd_SOURCES}) + if(WIN32) + target_compile_definitions(utils-zstd + INTERFACE + "_ZSTD_SHARED_API=__declspec(dllimport)" + PRIVATE + "_ZSTD_SHARED_API=__declspec(dllexport)" + ) + else() + target_compile_definitions(utils-zstd + INTERFACE + "_ZSTD_SHARED_API=__attribute__((visibility(\"default\")))" + ) + endif() + install( + TARGETS utils-zstd + LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" + ) +else() + add_library(utils-zstd STATIC ${utils_zstd_SOURCES}) +endif() + +target_link_libraries(utils-zstd PRIVATE zstd::zstd) +target_include_directories(utils-zstd PRIVATE zstd::zstd) \ No newline at end of file diff --git a/sources/utils-zstd/utils-zstd.cpp b/sources/utils-zstd/utils-zstd.cpp new file mode 100644 index 000000000..b5c0068d8 --- /dev/null +++ b/sources/utils-zstd/utils-zstd.cpp @@ -0,0 +1,103 @@ +#include +#include +#include +#include +#include +#include +#include + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, const char* fileName) +{ + const size_t SINGLE_LUT_SIZE = (static_cast(256 * 256 * 256) * 3); + + size_t outSize = SINGLE_LUT_SIZE / 2; + std::vector outBuffer; + const char* error = nullptr; + + std::ofstream file; + file.open(fileName, std::ios::out | std::ios::trunc | std::ios::binary); + + if (!file.is_open()) + { + return "Could not open file for writing"; + } + + try + { + outBuffer.resize(outSize); + } + catch(...) + { + error = "Could not allocate buffer"; + } + + + if (error == nullptr) + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + { + error = "ZSTD_createDCtx() failed!"; + } + else + { + int total = SINGLE_LUT_SIZE * 3; + ZSTD_inBuffer input = { downloadedData, downloadedDataSize, 0 }; + while (input.pos < input.size) + { + ZSTD_outBuffer outputSeek = { outBuffer.data(), outBuffer.size(), 0 }; + size_t const ret = ZSTD_decompressStream(dctx, &outputSeek, &input); + if (ZSTD_isError(ret) || outputSeek.pos != outBuffer.size() || total < 0) + { + error = "Error during decompression"; + break; + } + else + { + file.write(reinterpret_cast(outBuffer.data()), outBuffer.size()); + } + total -= outBuffer.size(); + } + ZSTD_freeDCtx(dctx); + } + } + + file.close(); + + if (error != nullptr) + std::remove(fileName); + + return error; +} + +_ZSTD_SHARED_API const char* DecompressZSTD(size_t downloadedDataSize, const uint8_t* downloadedData, uint8_t* dest, int destSeek, int destSize) +{ + const char* error = nullptr; + + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + if (dctx == NULL) + { + error = "ZSTD_createDCtx() failed!"; + } + else + { + int totalOutput = 0; + ZSTD_inBuffer input = { downloadedData, downloadedDataSize, 0 }; + while (totalOutput <= destSeek && input.pos < input.size) + { + ZSTD_outBuffer outputSeek = { dest, static_cast(destSize), static_cast(0) }; + size_t const ret = ZSTD_decompressStream(dctx, &outputSeek, &input); + if (ZSTD_isError(ret) || outputSeek.pos != destSize) + { + error = "Error during decompression"; + break; + } + totalOutput += outputSeek.pos; + } + ZSTD_freeDCtx(dctx); + } + + return error; +} diff --git a/sources/utils/LutLoader.cpp b/sources/utils/LutLoader.cpp index 53018a8a9..55037f2b2 100644 --- a/sources/utils/LutLoader.cpp +++ b/sources/utils/LutLoader.cpp @@ -24,12 +24,18 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - +#include #include #include #include +#include #include +#ifdef ENABLE_ZSTD + #include +#endif + + namespace { const int LUT_FILE_SIZE = 256 * 256 * 256 *3; const int LUT_MEMORY_ALIGN = 64; @@ -52,17 +58,26 @@ void LutLoader::loadLutFile(Logger* _log, PixelFormat color, const QList(compressedFile.data()), _lut.data(), index, LUT_FILE_SIZE); + #endif + + if (retVal != nullptr && _log) + { + Error(_log, "Error while decompressing LUT: %s", retVal); + } + + if (_log) Info(_log, "Decompression took %f seconds", (InternalClock::nowPrecise() - now) / 1000.0); + + return retVal == nullptr; +} + +void LutLoader::hasher(int index, Logger* _log) +{ + if (_log) + { + auto start = _lut.data(); + auto end = start + _lut.size(); + uint8_t position = 0; + uint16_t fletcher1 = 0; + uint16_t fletcher2 = 0; + uint16_t fletcherExt = 0; + while (start < end) + { + fletcher1 = (fletcher1 + (uint16_t)(*(start))) % 255; + fletcher2 = (fletcher2 + fletcher1) % 255; + fletcherExt = (fletcherExt + (*(start++) ^ (position++))) % 255; + } + Info(_log, "CRC for %i segment: 0x%.2X 0x%.2X 0x%.2X", index, fletcher1, fletcher2, fletcherExt); + } +}