Add automatic LUT downloader & installer (#568)

* Add LUT downloader

* Switch to xz Github mirror

* Use cleanpath to join paths

* Pass params

* Add LUT's fast CRC

* Save and apply recommended: brightness, contrast, saturation

.gitmodules

@@ -16,3 +16,6 @@
 	path = dependencies/external/qmqtt
 	url = https://github.com/emqx/qmqtt.git
 	ignore = dirty
+[submodule "dependencies/external/xz"]
+	path = dependencies/external/xz
+	url = https://github.com/tukaani-project/xz.git

CMakeLists.txt

@@ -38,6 +38,7 @@ SET ( DEFAULT_BONJOUR                     ON  )
 SET ( DEFAULT_MQTT                        ON  )
 SET ( DEFAULT_STATIC_QT_PLUGINS           OFF )
 SET ( DEFAULT_PRECOMPILED_HEADERS         ON  )
+SET ( DEFAULT_XZ                          ON  )
 
 # Configure CCache if available
 find_program(CCACHE_FOUND ccache)
@@ -384,6 +385,9 @@ colorMe("ENABLE_PROTOBUF = " ${ENABLE_PROTOBUF})
 
 message( STATUS "\n${CyanColor}BUILD FEATURES${ColorReset}")
 
+option(ENABLE_XZ "Enable XZ support" ${DEFAULT_XZ})
+colorMe("ENABLE_XZ = " ${ENABLE_XZ})
+
 option(USE_STATIC_QT_PLUGINS "Enable static QT plugins" ${DEFAULT_STATIC_QT_PLUGINS})
 colorMe("USE_STATIC_QT_PLUGINS = " ${USE_STATIC_QT_PLUGINS})
 
@@ -580,8 +584,8 @@ add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_D
 include (${CMAKE_CURRENT_SOURCE_DIR}/cmake/packages.cmake)
 
 # external targets
-if (WIN32 AND TARGET precompiled_hyperhdr_headers AND TARGET apidoc AND TARGET flatbuffers AND TARGET flatc AND TARGET mbedcrypto AND TARGET qmqtt)
-	set_target_properties(precompiled_hyperhdr_headers qmqtt apidoc flatbuffers flatc flathash lib mbedcrypto mbedtls mbedx509 resources uninstall PROPERTIES FOLDER ExternalLibsTargets)	
+if (WIN32 AND TARGET precompiled_hyperhdr_headers AND TARGET apidoc AND TARGET flatbuffers AND TARGET flatc AND TARGET mbedcrypto AND TARGET qmqtt AND TARGET liblzma)
+	set_target_properties(precompiled_hyperhdr_headers qmqtt apidoc flatbuffers flatc flathash lib mbedcrypto mbedtls mbedx509 resources uninstall liblzma PROPERTIES FOLDER ExternalLibsTargets)	
 else()
 	set_target_properties(resources uninstall PROPERTIES FOLDER ExternalLibsTargets)
 endif()

HyperhdrConfig.h.in

@@ -54,6 +54,9 @@
 // Define to enable MQTT network client
 #cmakedefine ENABLE_MQTT
 
+// Define to enable protobuf
+#cmakedefine ENABLE_XZ
+
 // Define to enable protobuf
 #cmakedefine ENABLE_PROTOBUF
 

dependencies/CMakeLists.txt

@@ -153,3 +153,8 @@ if ( ENABLE_MQTT )
 
 	add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/qmqtt)	
 ENDIF()
+
+if ( ENABLE_XZ )
+	option(BUILD_TESTING "" OFF)
+	add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/external/xz)	
+ENDIF()

include/api/API.h

@@ -4,6 +4,7 @@
 #include <base/HyperHdrIManager.h>
 #include <base/AuthManager.h>
 
+class QNetworkReply;
 class QTimer;
 class JsonCB;
 
@@ -72,6 +73,8 @@ class API : public QObject
 	///
 	void init();
 
+	QString installLut(QNetworkReply* reply, QString fileName, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time);
+
 	///
 	/// @brief Set a single color
 	/// @param[in] priority The priority of the written color

include/api/JsonAPI.h

@@ -11,6 +11,7 @@
 #include <QJsonObject>
 #include <QString>
 #include <QSemaphore>
+#include <QNetworkReply>
 
 class QTimer;
 class JsonCB;
@@ -96,6 +97,8 @@ private slots:
 
 	void handleLedColorsTimer();
 
+	void lutDownloaded(QNetworkReply* reply, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time);
+
 signals:
 	///
 	/// Signal emits with the reply message provided with handleMessage()
@@ -301,6 +304,8 @@ private slots:
 
 	void handleBenchmarkCommand(const QJsonObject& message, const QString& command, int tan);
 
+	void handleLutInstallCommand(const QJsonObject& message, const QString& command, int tan);
+
 	void handleSmoothingCommand(const QJsonObject& message, const QString& command, int tan);
 
 	void handleTunnel(const QJsonObject& message, const QString& command, int tan);

include/api/JsonCB.h

@@ -56,6 +56,8 @@ class JsonCB : public QObject
 	///
 	void setSubscriptionsTo(HyperHdrInstance* hyperhdr);
 
+	void handleLutInstallUpdate(const QJsonObject& data);
+
 signals:
 	///
 	/// @brief Emits whenever a new json mesage callback is ready to send

include/base/AllHeaders_pch.h

@@ -22,6 +22,7 @@
 #include <QMultiMap>
 #include <QMutex>
 #include <QMutexLocker>
+#include <QNetworkReply>
 #include <QPair>
 #include <QPainter>
 #include <QRectF>

include/base/HyperHdrInstance.h

@@ -116,6 +116,8 @@ public slots:
 
 	void saveCalibration(QString saveData);
 
+	void saveGrabberParams(int hardware_brightness, int hardware_contrast, int hardware_saturation);
+
 	///
 	/// Updates the priority muxer with the current time and (re)writes the led color with applied
 	/// transforms.

sources/api/API.cpp

@@ -18,6 +18,8 @@
 #include <QBuffer>
 #include <QByteArray>
 #include <QTimer>
+#include <QNetworkReply>
+#include <QFile>
 #include <base/GrabberWrapper.h>
 #include <base/SystemWrapper.h>
 #include <utils/jsonschema/QJsonSchemaChecker.h>
@@ -35,6 +37,11 @@
 // api includes
 #include <api/JsonCB.h>
 
+#ifdef ENABLE_XZ
+	// decoder
+	#include <lzma.h>
+#endif
+
 using namespace hyperhdr;
 
 API::API(Logger* log, bool localConnection, QObject* parent)
@@ -569,3 +576,90 @@ void API::logout()
 void API::stopDataConnectionss()
 {
 }
+
+QString API::installLut(QNetworkReply *reply, QString fileName, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time)
+{
+#ifdef ENABLE_XZ
+	QString error = nullptr;
+
+	if (reply->error() == QNetworkReply::NetworkError::NoError)
+	{
+		QByteArray downloadedData = reply->readAll();		
+
+		QFile file(fileName);
+		if (file.open(QIODevice::WriteOnly | QIODevice::Truncate))
+		{
+			size_t outSize = 67174456;
+			uint8_t* outBuf = reinterpret_cast<uint8_t*>(malloc(outSize));
+
+			if (outBuf == nullptr)
+			{
+				error = "Could not allocate buffer";
+			}
+			else
+			{
+				const uint32_t flags = LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED;
+				lzma_stream strm = LZMA_STREAM_INIT;
+				strm.next_in = reinterpret_cast<uint8_t*>(downloadedData.data());
+				strm.avail_in = downloadedData.size();
+				lzma_ret lzmaRet = lzma_stream_decoder(&strm, outSize, flags);
+				if (lzmaRet == LZMA_OK)
+				{
+					do {
+						strm.next_out = outBuf;
+						strm.avail_out = outSize;
+						lzmaRet = lzma_code(&strm, LZMA_FINISH);
+						if (lzmaRet == LZMA_MEMLIMIT_ERROR)
+						{
+							outSize = lzma_memusage(&strm);
+							free(outBuf);
+							outBuf = reinterpret_cast<uint8_t*>(malloc(outSize));
+							if (outBuf == nullptr)
+							{
+								error = QString("Could not increase buffer size");
+								break;
+							}
+							lzma_memlimit_set(&strm, outSize);
+							strm.avail_out = 0;
+						}
+						else if (lzmaRet != LZMA_OK && lzmaRet != LZMA_STREAM_END)
+						{
+							// error
+							error = QString("LZMA decoder return error: %1").arg(lzmaRet);
+							break;
+						}
+						else
+						{
+							size_t toWrite = outSize - strm.avail_out;
+							file.write(QByteArray((char*)outBuf, toWrite));
+						}
+					} while (strm.avail_out == 0 && lzmaRet != LZMA_STREAM_END);
+					file.flush();
+				}
+				else
+				{
+					error = "Could not initialize LZMA decoder";
+				}
+
+				if (time != 0)
+					file.setFileTime(QDateTime::fromMSecsSinceEpoch(time), QFileDevice::FileModificationTime);
+
+				file.close();
+				if (error != nullptr)
+					file.remove();
+
+				lzma_end(&strm);
+				free(outBuf);
+			}
+		}
+		else
+			error = QString("Could not open %1 for writing").arg(fileName);
+	}
+	else
+		error = "Could not download LUT file";
+
+	return error;
+#else
+	return "XZ support was disabled in the build configuration";
+#endif
+}

sources/api/CMakeLists.txt

@@ -20,6 +20,11 @@ target_link_libraries(hyperhdr-api
 	Qt${Qt_VERSION}::Network
 )
 
+if(ENABLE_XZ)
+	target_link_libraries(hyperhdr-api liblzma)
+	target_include_directories(hyperhdr-api PRIVATE "${CMAKE_SOURCE_DIR}/dependencies/external/xz/src/liblzma/api")
+endif()
+
 if(USE_PRECOMPILED_HEADERS AND COMMAND target_precompile_headers)
     target_precompile_headers(hyperhdr-api REUSE_FROM precompiled_hyperhdr_headers)
 endif()

sources/api/JSONRPC_schema/schema-lut-install.json

@@ -0,0 +1,35 @@
+{
+	"type":"object",
+	"required":true,
+	"properties":{
+		"command": {
+			"type" : "string",
+			"required" : true,
+			"enum" : ["lut-install"]
+		},
+		"subcommand": {
+			"type" : "string",
+			"required" : true
+		},
+		"hardware_brightness": {
+			"type" : "integer",
+			"required" : true
+		},
+		"hardware_contrast": {
+			"type" : "integer",
+			"required" : true
+		},
+		"hardware_saturation": {
+			"type" : "integer",
+			"required" : true
+		},
+		"now": {
+			"type" : "integer",
+			"required" : true
+		},
+		"tan" : {
+			"type" : "integer"
+		}
+	},
+	"additionalProperties": false
+}

sources/api/JSONRPC_schema/schema.json

@@ -5,7 +5,7 @@
 		"command": {
 			"type" : "string",
 			"required" : true,
-			"enum": [ "color", "tunnel", "smoothing", "benchmark", "image", "effect", "create-effect", "delete-effect", "serverinfo", "clear", "clearall", "adjustment", "sourceselect", "config", "componentstate", "ledcolors", "load-db", "save-db", "logging", "performance-counters", "lut-calibration", "signal-calibration", "processing", "sysinfo", "videomodehdr", "video-crop", "videomode", "authorize", "instance", "leddevice", "transform", "correction", "temperature", "help", "video-controls" ]
+			"enum": [ "color", "tunnel", "smoothing", "benchmark", "lut-install", "image", "effect", "create-effect", "delete-effect", "serverinfo", "clear", "clearall", "adjustment", "sourceselect", "config", "componentstate", "ledcolors", "load-db", "save-db", "logging", "performance-counters", "lut-calibration", "signal-calibration", "processing", "sysinfo", "videomodehdr", "video-crop", "videomode", "authorize", "instance", "leddevice", "transform", "correction", "temperature", "help", "video-controls" ]
 		}
 	}
 }

sources/api/JSONRPC_schemas.qrc

@@ -17,6 +17,7 @@
         <file alias="schema-componentstate">JSONRPC_schema/schema-componentstate.json</file>
         <file alias="schema-ledcolors">JSONRPC_schema/schema-ledcolors.json</file>
         <file alias="schema-lut-calibration">JSONRPC_schema/schema-lut-calibration.json</file>
+        <file alias="schema-lut-install">JSONRPC_schema/schema-lut-install.json</file>
         <file alias="schema-signal-calibration">JSONRPC_schema/schema-signal-calibration.json</file>
         <file alias="schema-logging">JSONRPC_schema/schema-logging.json</file>
         <file alias="schema-save-db">JSONRPC_schema/schema-save-db.json</file>

sources/api/JsonAPI.cpp

@@ -16,6 +16,7 @@
 #include <QHostAddress>
 #include <QHostInfo>
 #include <QMultiMap>
+#include <QDir>
 
 #include <leddevice/LedDeviceWrapper.h>
 #include <leddevice/LedDevice.h>
@@ -220,6 +221,8 @@ void JsonAPI::handleMessage(const QString& messageString, const QString& httpAut
 			handleVideoControlsCommand(message, command, tan);
 		else if (command == "benchmark")
 			handleBenchmarkCommand(message, command, tan);
+		else if (command == "lut-install")
+			handleLutInstallCommand(message, command, tan);
 		else if (command == "smoothing")
 			handleSmoothingCommand(message, command, tan);
 		else if (command == "transform" || command == "correction" || command == "temperature")
@@ -582,6 +585,17 @@ void JsonAPI::handleServerInfoCommand(const QJsonObject& message, const QString&
 		QJsonObject grabbers;
 	#if defined(ENABLE_V4L2) || defined(ENABLE_MF) || defined(ENABLE_AVF)	
 		grabbers = GrabberWrapper::getInstance()->getJsonInfo();
+		QString lutPath = QDir::cleanPath(_instanceManager->getRootPath() + QDir::separator() + "lut_lin_tables.3d");
+		grabbers["lut_for_hdr_path"] = lutPath;
+		if (QFile(lutPath).exists())
+		{
+			grabbers["lut_for_hdr_exists"] = 1;
+			grabbers["lut_for_hdr_modified_date"] = QFileInfo(lutPath).lastModified().toMSecsSinceEpoch();
+		}
+		else
+		{
+			grabbers["lut_for_hdr_exists"] = 0;
+		}
 	#endif
 		info["grabbers"] = grabbers;
 
@@ -839,6 +853,57 @@ void JsonAPI::handleBenchmarkCommand(const QJsonObject& message, const QString&
 	sendSuccessReply(command, tan);
 }
 
+void JsonAPI::lutDownloaded(QNetworkReply* reply, int hardware_brightness, int hardware_contrast, int hardware_saturation, qint64 time)
+{
+	QString fileName = QDir::cleanPath(_instanceManager->getRootPath() + QDir::separator() + "lut_lin_tables.3d");
+	QString error = installLut(reply, fileName, hardware_brightness, hardware_contrast, hardware_saturation, time);
+
+	if (error == nullptr)
+	{
+		Info(_log, "Reloading LUT...");
+		API::setVideoModeHdr(0);
+		API::setVideoModeHdr(1);
+		QTimer::singleShot(0, _hyperhdr, [=]() { _hyperhdr->saveGrabberParams(hardware_brightness, hardware_contrast, hardware_saturation); });
+		Info(_log, "New LUT has been installed as: %s (from: %s)", QSTRING_CSTR(fileName), QSTRING_CSTR(reply->url().toString()));
+	}
+	else
+	{
+		Error(_log, "Error occured while installing new LUT: %s", QSTRING_CSTR(error));
+	}
+
+	reply->manager()->deleteLater();
+	reply->deleteLater();
+
+	QJsonObject report;
+	report["status"] = (error == nullptr)? 1:0;
+	report["error"] = error;
+	_jsonCB->handleLutInstallUpdate(report);
+}
+
+void JsonAPI::handleLutInstallCommand(const QJsonObject& message, const QString& command, int tan)
+{
+	const QString& address = QString("%1/lut_lin_tables.3d.xz").arg(message["subcommand"].toString().trimmed());
+	int hardware_brightness = message["hardware_brightness"].toInt(0);
+	int hardware_contrast = message["hardware_contrast"].toInt(0);
+	int hardware_saturation = message["hardware_saturation"].toInt(0);
+	qint64 time = message["now"].toInt(0);
+
+	Debug(_log, "Request to install LUT from: %s (params => [%i, %i, %i])", QSTRING_CSTR(address),
+										hardware_brightness, hardware_contrast, hardware_saturation);
+	if (_adminAuthorized)
+	{
+		QNetworkAccessManager *mgr = new QNetworkAccessManager(this);
+		connect(mgr, &QNetworkAccessManager::finished, this, [=](QNetworkReply* reply) {
+			lutDownloaded(reply, hardware_brightness, hardware_contrast, hardware_saturation, time);
+		});
+		QNetworkRequest request(address);
+		mgr->get(request);
+		sendSuccessReply(command, tan);
+	}
+	else
+		sendErrorReply("No Authorization", command, tan);
+}
+
 void JsonAPI::handleSmoothingCommand(const QJsonObject& message, const QString& command, int tan)
 {
 	const QString& subc = message["subcommand"].toString().trimmed().toLower();