Merge pull request #1181 from alicevision/DenoiseNLMeansOpenCV

Add NLMeans denoiser in ImageProcessing
alicevision · Dec 13, 2022 · b2abc45 · b2abc45
2 parents c095170 + 7e029a7
commit b2abc45
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 24 deletions.
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -474,7 +474,7 @@ set(ALICEVISION_HAVE_OCVSIFT 0)
 
 if(ALICEVISION_BUILD_SFM)
   if(NOT ALICEVISION_USE_OPENCV STREQUAL "OFF")
-    find_package(OpenCV COMPONENTS core imgproc video imgcodecs videoio features2d)
+    find_package(OpenCV COMPONENTS core imgproc video imgcodecs videoio features2d photo)
 
     if(OpenCV_FOUND)
       # We do not set the minimal version directly in find_package

diff --git a/src/aliceVision/image/convertionOpenCV.hpp b/src/aliceVision/image/convertionOpenCV.hpp
@@ -29,7 +29,6 @@ namespace image
  * @param[in] color - value to set
  * @param[in] factor - optional scale factor
  * @param[in] delta - optional delta added to the scaled values
- * @return the resulting openCV image
  */
 template <typename VecType, typename ValueType>
 inline void setValueCvMatBGR(cv::Mat& mat, int i, int j, const image::RGBAfColor& color, float factor = 1.f,
@@ -42,11 +41,11 @@ inline void setValueCvMatBGR(cv::Mat& mat, int i, int j, const image::RGBAfColor
 
 
 /**
- * @brief Converts an aliceVision image to an openCv image (cv::Mat) in BGR
+ * @brief Converts an aliceVision image to an OpenCV image (cv::Mat) in BGR
  * Ignores the alpha channel of the source image
- * @param[in] img - Input RGBA aliceVision image
+ * @param[in] img - input RGBA aliceVision image
  * @param[in] cvtype - OpenCV mat type (supported values: CV_32FC3, CV_8UC3)
- * @return the resulting openCV image
+ * @return the resulting OpenCV image
  */
 inline cv::Mat imageRGBAToCvMatBGR(const image::Image<image::RGBAfColor>& img, int cvtype = CV_32FC3)
 {
@@ -73,25 +72,51 @@ inline cv::Mat imageRGBAToCvMatBGR(const image::Image<image::RGBAfColor>& img, i
 
 
 /**
- * @brief Converts an openCv image (cv::Mat) in BGR to an aliceVision image
+ * @brief Implements the conversion of an OpenCV image (cv::Mat) in BGR to an aliceVision image
  * Keeps the alpha channel of the output image unchanged
- * @param[in] img - Input openCV image (cv::Mat)
- * @param[out] img - output RGBA aliceVision image
- * @return the resulting regex
+ * @tparam VecType - OpenCV vector type to interpret the img values with
+ * @param[in] img - input OpenCV image (supported OpenCV image types: CV_32FC3, CV_8UC3)
+ * @param[inout] imageOut - output RGBA aliceVision image
+ * @param[in] factor - optional scale factor
  */
-inline void cvMatBGRToImageRGBA(const cv::Mat& img, image::Image<image::RGBAfColor>& imageOut)
+template <typename VecType>
+inline void cvMatBGRToImageRGBAImpl(const cv::Mat& img, image::Image<image::RGBAfColor>& imageOut, float factor = 1.f)
 {
     for(int row = 0; row < imageOut.Height(); row++)
     {
-        const cv::Vec3f* rowPtr = img.ptr<cv::Vec3f>(row);
+        const VecType* rowPtr = img.ptr<VecType>(row);
         for(int col = 0; col < imageOut.Width(); col++)
         {
-            const cv::Vec3f& matPixel = rowPtr[col];
-            imageOut(row, col) = image::RGBAfColor(matPixel[2], matPixel[1], matPixel[0], imageOut(row, col).a());
+            const VecType& matPixel = rowPtr[col];
+            imageOut(row, col) = image::RGBAfColor(matPixel[2] * factor, matPixel[1] * factor,
+                                                   matPixel[0] * factor, imageOut(row, col).a());
         }
     }
 }
 
+
+/**
+ * @brief Converts an OpenCV image (cv::Mat) in BGR to an aliceVision image
+ * Keeps the alpha channel of the output image unchanged
+ * @param[in] img - input OpenCV image (supported OpenCV image types: CV_32FC3, CV_8UC3)
+ * @param[inout] imageOut - output RGBA aliceVision image
+ * @return the resulting aliceVision image
+ */
+inline void cvMatBGRToImageRGBA(const cv::Mat& img, image::Image<image::RGBAfColor>& imageOut)
+{
+    switch(img.type())
+    {
+        case CV_32FC3:
+            cvMatBGRToImageRGBAImpl<cv::Vec3f>(img, imageOut);
+            break;
+        case CV_8UC3:
+            cvMatBGRToImageRGBAImpl<cv::Vec3b>(img, imageOut, 1.0f / 255.0f);
+            break;
+        default:
+            std::runtime_error("Cannot handle OpenCV matrix type '" + std::to_string(img.type()) + "'.");
+    }
+}
+
 } // namespace image
 } // namespace aliceVision
 

diff --git a/src/software/utils/main_imageProcessing.cpp b/src/software/utils/main_imageProcessing.cpp
@@ -1,4 +1,3 @@
-
 #include <aliceVision/image/all.hpp>
 #include <aliceVision/system/cmdline.hpp>
 #include <aliceVision/system/Logger.hpp>
@@ -18,6 +17,7 @@
 
 #if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV)
 #include <opencv2/imgproc.hpp>
+#include <opencv2/photo.hpp>
 #endif
 
 #include <OpenImageIO/imageio.h>
@@ -213,6 +213,39 @@ inline std::istream& operator>>(std::istream& in, EImageFormat& e)
     return in;
 }
 
+struct NLMeansFilterParams
+{
+    bool enabled;
+    float filterStrength;
+    float filterStrengthColor;
+    int templateWindowSize;
+    int searchWindowSize;
+};
+
+std::istream& operator>>(std::istream& in, NLMeansFilterParams& nlmParams)
+{
+    std::string token;
+    in >> token;
+    std::vector<std::string> splitParams;
+    boost::split(splitParams, token, boost::algorithm::is_any_of(":"));
+    if(splitParams.size() != 5)
+        throw std::invalid_argument("Failed to parse NLMeansFilterParams from: " + token);
+    nlmParams.enabled = boost::to_lower_copy(splitParams[0]) == "true";
+    nlmParams.filterStrength = boost::lexical_cast<float>(splitParams[1]);
+    nlmParams.filterStrengthColor = boost::lexical_cast<float>(splitParams[2]);
+    nlmParams.templateWindowSize = boost::lexical_cast<int>(splitParams[3]);
+    nlmParams.searchWindowSize = boost::lexical_cast<int>(splitParams[4]);
+
+    return in;
+}
+
+inline std::ostream& operator<<(std::ostream& os, const NLMeansFilterParams& nlmParams)
+{
+    os << nlmParams.enabled << ":" << nlmParams.filterStrength << ":" << nlmParams.filterStrengthColor << ":"
+       << nlmParams.templateWindowSize << ":" << nlmParams.searchWindowSize;
+    return os;
+}
+
 std::string getColorProfileDatabaseFolder()
 {
     const char* value = std::getenv("ALICEVISION_COLOR_PROFILE_DB");
@@ -262,6 +295,14 @@ struct ProcessingParams
         true // mono
     };
 
+    NLMeansFilterParams nlmFilter = 
+    {
+        false, // enable
+        5.0f,  // filterStrength
+        10.0f, // filterStrengthColor
+        7,     // templateWindowSize
+        21     // searchWindowSize
+    };
 };
 
 
@@ -408,6 +449,26 @@ void processImage(image::Image<image::RGBAfColor>& image, const ProcessingParams
         oiio::ImageBuf inBuf(oiio::ImageSpec(image.Width(), image.Height(), nchannels, oiio::TypeDesc::FLOAT), image.data());
         oiio::ImageBufAlgo::noise(inBuf, ENoiseMethod_enumToString(pParams.noise.method), pParams.noise.A, pParams.noise.B, pParams.noise.mono);
     }
+
+    if(pParams.nlmFilter.enabled)
+    {
+#if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV)
+        // Create temporary OpenCV Mat (keep only 3 channels) to handle Eigen data of our image
+        cv::Mat openCVMatIn = image::imageRGBAToCvMatBGR(image, CV_8UC3);
+        cv::Mat openCVMatOut(image.Width(), image.Height(), CV_8UC3);
+
+        cv::fastNlMeansDenoisingColored(openCVMatIn, openCVMatOut, pParams.nlmFilter.filterStrength,
+                                        pParams.nlmFilter.filterStrengthColor, pParams.nlmFilter.templateWindowSize,
+                                        pParams.nlmFilter.searchWindowSize);
+
+        // Copy filtered data from OpenCV Mat(3 channels) to our image (keep the alpha channel unfiltered)
+        image::cvMatBGRToImageRGBA(openCVMatOut, image);
+
+#else
+        throw std::invalid_argument(
+            "Unsupported mode! If you intended to use a non-local means filter, please add OpenCV support.");
+#endif
+    }
 }
 
 void saveImage(image::Image<image::RGBAfColor>& image, const std::string& inputPath, const std::string& outputPath, std::map<std::string, std::string> inputMetadata,
@@ -582,14 +643,22 @@ int aliceVision_main(int argc, char * argv[])
             " * TileGridSize: Sets Size Of Grid For Histogram Equalization. Input Image Will Be Divided Into Equally Sized Rectangular Tiles.")
 
         ("noiseFilter", po::value<NoiseFilterParams>(&pParams.noise)->default_value(pParams.noise),
-                                 "Noise Filter parameters:\n"
-                                 " * Enabled: Add Noise.\n"
-                                 " * method: There are several noise types to choose from:\n"
-                                 "    - uniform: adds noise values uninformly distributed on range [A,B).\n"
-                                 "    - gaussian: adds Gaussian (normal distribution) noise values with mean value A and standard deviation B.\n"
-                                 "    - salt: changes to value A a portion of pixels given by B.\n"
-                                 " * A, B: parameters that have a different interpretation depending on the method chosen.\n"
-                                 " * mono: If is true, a single noise value will be applied to all channels otherwise a separate noise value will be computed for each channel.")
+            "Noise Filter parameters:\n"
+            " * Enabled: Add Noise.\n"
+            " * method: There are several noise types to choose from:\n"
+            "    - uniform: adds noise values uninformly distributed on range [A,B).\n"
+            "    - gaussian: adds Gaussian (normal distribution) noise values with mean value A and standard deviation B.\n"
+            "    - salt: changes to value A a portion of pixels given by B.\n"
+            " * A, B: parameters that have a different interpretation depending on the method chosen.\n"
+            " * mono: If is true, a single noise value will be applied to all channels otherwise a separate noise value will be computed for each channel.")
+
+        ("nlmFilter", po::value<NLMeansFilterParams>(&pParams.nlmFilter)->default_value(pParams.nlmFilter),
+            "Non local means Filter parameters:\n"
+            " * Enabled: Use non local means Filter.\n"
+            " * H: Parameter regulating filter strength. Bigger H value perfectly removes noise but also removes image details, smaller H value preserves details but also preserves some noise.\n"
+            " * HColor: Parameter regulating filter strength for color images only. Normally same as Filtering Parameter H. Not necessary for grayscale images\n "
+            " * templateWindowSize: Size in pixels of the template patch that is used to compute weights. Should be odd. \n"
+            " * searchWindowSize:Size in pixels of the window that is used to compute weighted average for given pixel. Should be odd. Affect performance linearly: greater searchWindowsSize - greater denoising time.")
 
         ("workingColorSpace", po::value<image::EImageColorSpace>(&workingColorSpace)->default_value(workingColorSpace),
          ("Working color space: " + image::EImageColorSpace_informations()).c_str())
@@ -635,9 +704,10 @@ int aliceVision_main(int argc, char * argv[])
     }
 
 #if !ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV)
-    if(pParams.bilateralFilter.enabled || pParams.claheFilter.enabled)
+    if(pParams.bilateralFilter.enabled || pParams.claheFilter.enabled || pParams.nlmFilter.enabled)
     {
-        ALICEVISION_LOG_ERROR("Invalid option: BilateralFilter and claheFilter can't be used without openCV !");
+        ALICEVISION_LOG_ERROR(
+            "Invalid option: BilateralFilter, claheFilter and nlmFilter can't be used without openCV !");
         return EXIT_FAILURE;
     }
 #endif