Add NV12 image format support for flatbuffers (#920)

Author: awawa-dev

Diff for: include/base/Grabber.h

@@ -17,6 +17,7 @@
 #include <utils/Components.h>
 #include <image/MemoryBuffer.h>
 #include <utils/FrameDecoder.h>
+#include <utils/LutLoader.h>
 #include <base/DetectionManual.h>
 #include <base/DetectionAutomatic.h>
 #include <performance-counters/PerformanceCounters.h>
@@ -32,11 +33,9 @@
 	#include <CoreGraphics/CoreGraphics.h>
 #endif
 
-#define LUT_FILE_SIZE 50331648
-
 #define UNSUPPORTED_DECODER "UNSUPPORTED YUV DECODER"
 
-class Grabber : public DetectionAutomatic, public DetectionManual
+class Grabber : public DetectionAutomatic, public DetectionManual, protected LutLoader
 {
 	Q_OBJECT
 
@@ -174,7 +173,6 @@ public slots:
 	void SignalBenchmarkUpdate(int status, QString message);
 
 protected:
-	void loadLutFile(PixelFormat color, const QList<QString>& files);
 
 	int getTargetSystemFrameDimension(int& targetSizeX, int& targetSizeY);
 
@@ -234,9 +232,6 @@ public slots:
 
 	bool _enabled;
 
-	// enable/disable HDR tone mapping
-	uint8_t _hdrToneMappingEnabled;
-
 	/// logger instance
 	Logger* _log;
 
@@ -266,8 +261,6 @@ public slots:
 	int			_actualWidth, _actualHeight, _actualFPS;
 	QString		_actualDeviceName;
 	uint		_targetMonitorNits;
-	MemoryBuffer<uint8_t>	_lut;
-	bool		_lutBufferInit;
 
 	int			_lineLength;
 	int			_frameByteSize;

Diff for: include/flatbuffers/parser/FlatBuffersParser.h

@@ -32,6 +32,30 @@
 namespace FlatBuffersParser
 {
 	enum FLATBUFFERS_PACKAGE_TYPE { COLOR = 1, IMAGE, CLEAR, PRIORITY, ERROR };
+	enum FLATBUFFERS_IMAGE_FORMAT { RGB = 1, NV12 };
+
+	struct FlatbuffersColor
+	{
+		uint8_t red;
+		uint8_t green;
+		uint8_t blue;
+	};
+
+	struct FlatbuffersTransientImage
+	{
+		FLATBUFFERS_IMAGE_FORMAT format;
+
+		struct
+		{
+			uint8_t* data;
+			int	size;
+			int	stride;
+		} firstPlane, secondPlane;
+		
+		int width;
+		int height;
+		int size;
+	};
 
 
 	void* createFlatbuffersBuilder();
@@ -43,8 +67,8 @@ namespace FlatBuffersParser
 	void encodeColorIntoFlatbuffers(void* builder, int red, int green, int blue, int priority, int duration, uint8_t** buffer, size_t* bufferSize);
 	bool verifyFlatbuffersReplyBuffer(const uint8_t* messageData, size_t messageSize, bool* _sent, bool* _registered, int* _priority);
 	int decodeIncomingFlatbuffersFrame(void* builder, const uint8_t* messageData, size_t messageSize,
-		uint8_t* red, uint8_t* green, uint8_t* blue,
 		int* priority, std::string* clientDescription, int* duration,
-		uint8_t** imageData, int* imageWidth, int* imageHeight, size_t* imageSize,
+		FlatbuffersTransientImage& image,
+		FlatbuffersColor& color,
 		uint8_t** buffer, size_t* bufferSize);
 }

Diff for: include/flatbuffers/parser/hyperhdr_request.fbs

@@ -12,7 +12,16 @@ table RawImage {
   height:int = -1;
 }
 
-union ImageType {RawImage}
+table NV12Image {
+  data_y:[ubyte];
+  data_uv:[ubyte];
+  width:int;
+  height:int;
+  stride_y:int = 0;
+  stride_uv:int = 0;
+}
+
+union ImageType {RawImage, NV12Image}
 
 table Image {
   data:ImageType (required);

Diff for: include/flatbuffers/server/FlatBuffersServer.h

@@ -9,17 +9,23 @@
 #include <utils/settings.h>
 #include <image/Image.h>
 #include <utils/Components.h>
+#include <utils/LutLoader.h>
 
 class BonjourServiceRegister;
 class QTcpServer;
 class QLocalServer;
 class FlatBuffersServerConnection;
 class NetOrigin;
 
+namespace FlatBuffersParser
+{
+	struct FlatbuffersTransientImage;
+};
+
 #define HYPERHDR_DOMAIN_SERVER QStringLiteral("hyperhdr-domain")
 #define BASEAPI_FLATBUFFER_USER_LUT_FILE QStringLiteral("BASEAPI_user_lut_file")
 
-class FlatBuffersServer : public QObject
+class FlatBuffersServer : public QObject, protected LutLoader
 {
 	Q_OBJECT
 public:
@@ -35,7 +41,7 @@ public slots:
 	void initServer();
 	int getHdrToneMappingEnabled();
 	void handlerImportFromProto(int priority, int duration, const Image<ColorRgb>& image, QString clientDescription);
-	void handlerImageReceived(int priority, const Image<ColorRgb>& image, int timeout_ms, hyperhdr::Components origin, QString clientDescription);
+	void handlerImageReceived(int priority, FlatBuffersParser::FlatbuffersTransientImage* flatImage, int timeout_ms, hyperhdr::Components origin, QString clientDescription);
 	void signalRequestSourceHandler(hyperhdr::Components component, int instanceIndex, bool listen);
 
 private slots:
@@ -60,11 +66,8 @@ private slots:
 	BonjourServiceRegister* _serviceRegister = nullptr;
 	QVector<FlatBuffersServerConnection*> _openConnections;
 
-	int			_hdrToneMappingMode;
-	int			_realHdrToneMappingMode;	
-	bool		_lutBufferInit;
 	QString		_configurationPath;
 	QString		_userLutFile;
-
-	MemoryBuffer<uint8_t> _lut;
+	PixelFormat	_currentLutPixelFormat;
+	int			_flatbufferToneMappingMode;
 };

Diff for: include/flatbuffers/server/FlatBuffersServerConnection.h

@@ -37,6 +37,10 @@
 class QTcpSocket;
 class QLocalSocket;
 class QTimer;
+namespace FlatBuffersParser
+{
+	struct FlatbuffersTransientImage;
+};
 
 class FlatBuffersServerConnection : public QObject
 {
@@ -49,7 +53,7 @@ class FlatBuffersServerConnection : public QObject
 
 signals:
 	void SignalClearGlobalInput(int priority, bool forceClearAll);
-	void SignalImageReceived(int priority, const Image<ColorRgb>& image, int timeout_ms, hyperhdr::Components origin, QString clientDescription);
+	void SignalDirectImageReceivedInTempBuffer(int priority, FlatBuffersParser::FlatbuffersTransientImage* image, int timeout_ms, hyperhdr::Components origin, QString clientDescription);
 	void SignalSetGlobalColor(int priority, const std::vector<ColorRgb>& ledColor, int timeout_ms, hyperhdr::Components origin, QString clientDescription);
 	void SignalClientDisconnected(FlatBuffersServerConnection* client);
 

Diff for: include/image/ColorRgb.h

@@ -91,7 +91,7 @@ struct ColorRgb
 	operator std::string()
 	{
 		std::stringstream ss;
-		ss << "(" << red << ", " << green << ", " << blue << ")";
+		ss << "(" << std::to_string(red) << ", " << std::to_string(green) << ", " << std::to_string(blue) << ")";
 		return ss.str();
 	}
 

Diff for: include/utils/FrameDecoder.h

@@ -13,7 +13,7 @@ class FrameDecoder
 public:
 	static void processImage(
 		int _cropLeft, int _cropRight, int _cropTop, int _cropBottom,
-		const uint8_t* data, int width, int height, int lineLength,
+		const uint8_t* data, const uint8_t* dataUV, int width, int height, int lineLength,
 		const PixelFormat pixelFormat, const uint8_t* lutBuffer, Image<ColorRgb>& outputImage);
 
 	static void processQImage(

Diff for: include/utils/LutLoader.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#ifndef PCH_ENABLED
+	#include <QString>
+	#include <QList>
+#endif
+
+#include <utils/PixelFormat.h>
+#include <image/MemoryBuffer.h>
+#include <utils/Logger.h>
+
+class LutLoader {
+	public:
+		int		_hdrToneMappingEnabled = 0;
+		bool	_lutBufferInit = false;
+
+		MemoryBuffer<uint8_t>	_lut;		
+
+		void loadLutFile(Logger* _log, PixelFormat color, const QList<QString>& files);
+};

Diff for: sources/base/Grabber.cpp

@@ -49,7 +49,6 @@ Grabber::Grabber(const QString& configurationPath, const QString& grabberName)
 	, _cropTop(0)
 	, _cropBottom(0)
 	, _enabled(true)
-	, _hdrToneMappingEnabled(0)
 	, _log(Logger::getInstance(grabberName.toUpper()))
 	, _currentFrame(0)
 	, _deviceName()
@@ -70,7 +69,6 @@ Grabber::Grabber(const QString& configurationPath, const QString& grabberName)
 	, _actualFPS(0)
 	, _actualDeviceName("")
 	, _targetMonitorNits(200)
-	, _lutBufferInit(false)
 	, _lineLength(-1)
 	, _frameByteSize(-1)
 	, _signalDetectionEnabled(false)
@@ -410,100 +408,6 @@ int Grabber::getHdrToneMappingEnabled()
 	return _hdrToneMappingEnabled;
 }
 
-void Grabber::loadLutFile(PixelFormat color, const QList<QString>& files)
-{
-	bool is_yuv = (color == PixelFormat::YUYV);
-
-	_lutBufferInit = false;
-
-	if (color != PixelFormat::NO_CHANGE && color != PixelFormat::RGB24 && color != PixelFormat::YUYV)
-	{
-		Error(_log, "Unsupported mode for loading LUT table: %s", QSTRING_CSTR(pixelFormatToString(color)));
-		return;
-	}
-
-	if (color == PixelFormat::NO_CHANGE)
-	{
-		_lut.resize(LUT_FILE_SIZE + 4);
-
-		if (_lut.data() != nullptr)
-		{
-			for (int y = 0; y < 256; y++)
-				for (int u = 0; u < 256; u++)
-					for (int v = 0; v < 256; v++)
-					{
-						uint32_t ind_lutd = LUT_INDEX(y, u, v);
-						ColorRgb::yuv2rgb(y, u, v,
-							_lut.data()[ind_lutd],
-							_lut.data()[ind_lutd + 1],
-							_lut.data()[ind_lutd + 2]);
-					}
-			_lutBufferInit = true;
-		}
-
-		Error(_log, "You have forgotten to put lut_lin_tables.3d file in the HyperHDR configuration folder. Internal LUT table for YUV conversion has been created instead.");
-		return;
-	}
-
-	if (_hdrToneMappingEnabled || is_yuv)
-	{
-		for (QString fileName3d : files)
-		{
-			QFile file(fileName3d);
-
-			if (file.open(QIODevice::ReadOnly))
-			{
-				size_t length;
-				Debug(_log, "LUT file found: %s", QSTRING_CSTR(fileName3d));
-
-				length = file.size();
-
-				if (length == LUT_FILE_SIZE * 3)
-				{
-					qint64 index = 0;
-
-					if (is_yuv && _hdrToneMappingEnabled)
-					{
-						Debug(_log, "Index 1 for HDR YUV");
-						index = LUT_FILE_SIZE;
-					}
-					else if (is_yuv)
-					{
-						Debug(_log, "Index 2 for YUV");
-						index = LUT_FILE_SIZE * 2;
-					}
-					else
-						Debug(_log, "Index 0 for HDR RGB");
-
-					file.seek(index);
-
-					_lut.resize(LUT_FILE_SIZE + 64);
-
-					if (file.read((char*)_lut.data(), LUT_FILE_SIZE) != LUT_FILE_SIZE)
-					{
-						Error(_log, "Error reading LUT file %s", QSTRING_CSTR(fileName3d));
-					}
-					else
-					{
-						_lutBufferInit = true;
-						Info(_log, "Found and loaded LUT: '%s'", QSTRING_CSTR(fileName3d));
-					}
-				}
-				else
-					Error(_log, "LUT file has invalid length: %i %s. Please generate new one LUT table using the generator page.", length, QSTRING_CSTR(fileName3d));
-
-				file.close();
-
-				return;
-			}
-			else
-				Warning(_log, "LUT file is not found here: %s", QSTRING_CSTR(fileName3d));
-		}
-
-		Error(_log, "Could not find any required LUT file");
-	}
-}
-
 QMap<Grabber::VideoControls, int> Grabber::getVideoDeviceControls(const QString& devicePath)
 {
 	QMap<Grabber::VideoControls, int> retVal;

Diff for: sources/flatbuffers/parser/FlatBuffersParser.cpp

@@ -138,10 +138,10 @@ static void sendErrorReply(flatbuffers::FlatBufferBuilder* _builder, const char*
 	_builder->Finish(reply);
 }
 
-int FlatBuffersParser::decodeIncomingFlatbuffersFrame(void* builder, const uint8_t* messageData, size_t messageSize,
-									uint8_t* red, uint8_t* green, uint8_t* blue,
+int FlatBuffersParser::decodeIncomingFlatbuffersFrame(void* builder, const uint8_t* messageData, size_t messageSize,									
 									int* priority, std::string* clientDescription, int* duration,
-									uint8_t** imageData, int* imageWidth, int* imageHeight, size_t* imageSize,
+									FlatbuffersTransientImage& image,
+									FlatbuffersColor& color,
 									uint8_t** buffer, size_t* bufferSize)
 {
 	int retType = FLATBUFFERS_PACKAGE_TYPE::ERROR;
@@ -158,9 +158,9 @@ int FlatBuffersParser::decodeIncomingFlatbuffersFrame(void* builder, const uint8
 			auto colorReq = static_cast<const hyperhdrnet::Color*>(reqPtr);
 			const int32_t rgbData = colorReq->data();
 
-			*red = uint8_t((rgbData >> 16) & 0xff);
-			*green = uint8_t((rgbData >> 8) & 0xff);
-			*blue = uint8_t(rgbData & 0xff);
+			color.red = uint8_t((rgbData >> 16) & 0xff);
+			color.green = uint8_t((rgbData >> 8) & 0xff);
+			color.blue = uint8_t(rgbData & 0xff);
 			*duration = colorReq->duration();
 
 			retType = FLATBUFFERS_PACKAGE_TYPE::COLOR;
@@ -177,10 +177,33 @@ int FlatBuffersParser::decodeIncomingFlatbuffersFrame(void* builder, const uint8
 				const auto* img = static_cast<const hyperhdrnet::RawImage*>(rawFlatImage);
 				const auto& imgD = img->data();
 
-				*imageData = const_cast<uint8_t*>(imgD->data());
-				*imageSize = imgD->size();
-				*imageWidth = img->width();
-				*imageHeight = img->height();
+				image.format = FLATBUFFERS_IMAGE_FORMAT::RGB;
+				image.firstPlane.data = const_cast<uint8_t*>(imgD->data());
+				image.firstPlane.size = static_cast<int>(imgD->size());
+				image.size = imgD->size();
+				image.width = img->width();
+				image.height = img->height();
+
+				retType = FLATBUFFERS_PACKAGE_TYPE::IMAGE;
+				auto reply = hyperhdrnet::CreateReplyDirect(*_builder, nullptr, -1, *priority);
+				_builder->Finish(reply);
+			}
+			else if ((rawFlatImage = flatImage->data_as_NV12Image()) != nullptr)
+			{
+				const auto* img = static_cast<const hyperhdrnet::NV12Image*>(rawFlatImage);
+				const auto& imgD = img->data_y();
+				const auto& imgUvD = img->data_uv();
+
+				image.format = FLATBUFFERS_IMAGE_FORMAT::NV12;
+				image.firstPlane.data = const_cast<uint8_t*>(imgD->data());
+				image.firstPlane.size = static_cast<int>(imgD->size());
+				image.firstPlane.stride = img->stride_y();
+				image.secondPlane.data = const_cast<uint8_t*>(imgUvD->data());
+				image.secondPlane.size = static_cast<int>(imgUvD->size());
+				image.secondPlane.stride = img->stride_uv();
+				image.size = imgD->size() + imgUvD->size();
+				image.width = img->width();
+				image.height = img->height();
 
 				retType = FLATBUFFERS_PACKAGE_TYPE::IMAGE;
 				auto reply = hyperhdrnet::CreateReplyDirect(*_builder, nullptr, -1, *priority);