Skip to content

Commit

Permalink
Merge pull request #1469 from alicevision/dev/hdrMergingNoiseReduction
Browse files Browse the repository at this point in the history 
Noise reduction in HDR merging
  • Loading branch information
@mugulmd
mugulmd authored Jul 5, 2023
2 parents 8fff765 + a95dd3c commit 3e9f63a
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 67 deletions.
6 changes: 4 additions & 2 deletions src/aliceVision/hdr/brackets.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,9 @@ bool estimateBracketsFromSfmData(std::vector<std::vector<std::shared_ptr<sfmData
return true;
}

void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetViews, const std::vector<std::vector<std::shared_ptr<sfmData::View>>> & groups, int offsetRefBracketIndex, const std::string& lumaStatFilepath, const double meanTargetedLuma)
int selectTargetViews(std::vector<std::shared_ptr<sfmData::View>>& out_targetViews,
std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex,
const std::string& lumaStatFilepath, const double meanTargetedLuma)
{
// If targetIndexesFilename cannot be opened or is not valid an error is thrown
// For odd number, there is no ambiguity on the middle image.
Expand Down Expand Up @@ -219,7 +221,7 @@ void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetV

out_targetViews.push_back(group[targetIndex]);
}
return;
return targetIndex;
}

}
Expand Down
5 changes: 4 additions & 1 deletion src/aliceVision/hdr/brackets.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,11 @@ bool estimateBracketsFromSfmData(std::vector<std::vector<std::shared_ptr<sfmData
* @param[in] targetIndexesFilename: in case offsetRefBracketIndex is out of range the number of views in a group target indexes can be read from a text file
* if the file cannot be read or does not contain the expected number of values (same as view group number) and
* if offsetRefBracketIndex is out of range the number of views then a clamped values of offsetRefBracketIndex is considered
* @return Index of the targetView
*/
void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetViews, const std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename = "", const double meanTargetedLuma = 0.4);
int selectTargetViews(std::vector<std::shared_ptr<sfmData::View>>& out_targetViews,
std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex,
const std::string& targetIndexesFilename = "", const double meanTargetedLuma = 0.4);

}
}
161 changes: 107 additions & 54 deletions src/aliceVision/hdr/hdrMerge.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ void hdrMerge::process(const std::vector< image::Image<image::RGBfColor> > &imag
const rgbCurve &weight,
const rgbCurve &response,
image::Image<image::RGBfColor> &radiance,
float targetCameraExposure)
image::Image<image::RGBfColor> &lowLight,
image::Image<image::RGBfColor> &highLight,
image::Image<image::RGBfColor> &noMidLight,
MergingParams &mergingParams)
{
//checks
assert(!response.isEmpty());
Expand All @@ -72,73 +75,123 @@ void hdrMerge::process(const std::vector< image::Image<image::RGBfColor> > &imag
rgbCurve weightLongestExposure = weight;
weightLongestExposure.freezeFirstPartValues();

const std::vector<double> v_minValue = {response(mergingParams.minSignificantValue, 0),
response(mergingParams.minSignificantValue, 1),
response(mergingParams.minSignificantValue, 2)};
const std::vector<double> v_maxValue = {response(mergingParams.maxSignificantValue, 0),
response(mergingParams.maxSignificantValue, 1),
response(mergingParams.maxSignificantValue, 2)};

highLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));
lowLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));
noMidLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));

#pragma omp parallel for
for(int y = 0; y < height; ++y)
{
for(int x = 0; x < width; ++x)
{
//for each pixels
image::RGBfColor &radianceColor = radiance(y, x);
//for each pixels
image::RGBfColor& radianceColor = radiance(y, x);
image::RGBfColor& highLightColor = highLight(y, x);
image::RGBfColor& lowLightColor = lowLight(y, x);
image::RGBfColor& noMidLightColor = noMidLight(y, x);

std::vector<std::vector<double>> vv_coeff;
std::vector<std::vector<double>> vv_value;
std::vector<std::vector<double>> vv_normalizedValue;

// Compute merging range
std::vector<int> v_firstIndex;
std::vector<int> v_lastIndex;
for(std::size_t channel = 0; channel < 3; ++channel)
{
int firstIndex = mergingParams.refImageIndex;
while(firstIndex > 0 && (response(images[firstIndex](y, x)(channel), channel) > v_minValue[channel] ||
firstIndex == images.size() - 1))
{
firstIndex--;
}
v_firstIndex.push_back(firstIndex);

for(std::size_t channel = 0; channel < 3; ++channel)
{
double wsum = 0.0;
double wdiv = 0.0;
int lastIndex = v_firstIndex[channel] + 1;
while(lastIndex < images.size() - 1 && response(images[lastIndex](y, x)(channel), channel) < v_maxValue[channel])
{
lastIndex++;
}
v_lastIndex.push_back(lastIndex);
}

// Merge shortest exposure
// Compute merging coeffs and values to be merged
for(std::size_t channel = 0; channel < 3; ++channel)
{
int exposureIndex = 0;

// for each image
const double value = images[exposureIndex](y, x)(channel);
const double time = times[exposureIndex];
//
// weightShortestExposure: _______
// _______/
// 0 1
double w = std::max(0.001f, weightShortestExposure(value, channel));
std::vector<double> v_coeff;
std::vector<double> v_normalizedValue;
std::vector<double> v_value;

const double r = response(value, channel);
for(std::size_t e = 0; e < images.size(); ++e)
{
const double value = images[e](y, x)(channel);
const double resp = response(value, channel);
const double normalizedValue = resp / times[e];
double coeff = std::max(0.001f, e == 0 ? weightShortestExposure(value, channel) :
(e == images.size() - 1 ? weightLongestExposure(value, channel) :
weight(value, channel)));

v_value.push_back(value);
v_normalizedValue.push_back(normalizedValue);
v_coeff.push_back(coeff);
}

wsum += w * r / time;
wdiv += w;
vv_coeff.push_back(v_coeff);
vv_normalizedValue.push_back(v_normalizedValue);
vv_value.push_back(v_value);
}
// Merge intermediate exposures
for(std::size_t i = 1; i < images.size() - 1; ++i)

// Compute light masks if required (monitoring and debug purposes)
if(mergingParams.computeLightMasks)
{
// for each image
const double value = images[i](y, x)(channel);
const double time = times[i];
//
// weight: ____
// _______/ \________
// 0 1
double w = std::max(0.001f, weight(value, channel));

const double r = response(value, channel);
wsum += w * r / time;
wdiv += w;
for(std::size_t channel = 0; channel < 3; ++channel)
{
int idxMaxValue = 0;
int idxMinValue = 0;
double maxValue = 0.0;
double minValue = 10000.0;
bool jump = true;
for(std::size_t e = 0; e < images.size(); ++e)
{
if(vv_value[channel][e] > maxValue)
{
maxValue = vv_value[channel][e];
idxMaxValue = e;
}
if(vv_value[channel][e] < minValue)
{
minValue = vv_value[channel][e];
idxMinValue = e;
}
jump = jump && ((vv_value[channel][e] < mergingParams.minSignificantValue && e < images.size() - 1) ||
(vv_value[channel][e] > mergingParams.maxSignificantValue && e > 0));
}
highLightColor(channel) = minValue > mergingParams.maxSignificantValue ? 1.0 : 0.0;
lowLightColor(channel) = maxValue < mergingParams.minSignificantValue ? 1.0 : 0.0;
noMidLightColor(channel) = jump ? 1.0 : 0.0;
}
}
// Merge longest exposure
{
int exposureIndex = images.size() - 1;

// for each image
const double value = images[exposureIndex](y, x)(channel);
const double time = times[exposureIndex];
//
// weightLongestExposure: ____________
// \_______
// 0 1
double w = std::max(0.001f, weightLongestExposure(value, channel));

const double r = response(value, channel);

wsum += w * r / time;
wdiv += w;
// Compute the final result and adjust the exposure to the reference one.
for(std::size_t channel = 0; channel < 3; ++channel)
{
double v = 0.0;
double sumCoeff = 0.0;
for(std::size_t i = v_firstIndex[channel]; i <= v_lastIndex[channel]; ++i)
{
v += vv_coeff[channel][i] * vv_normalizedValue[channel][i];
sumCoeff += vv_coeff[channel][i];
}
radianceColor(channel) = mergingParams.targetCameraExposure *
(sumCoeff != 0.0 ? v / sumCoeff : vv_normalizedValue[channel][mergingParams.refImageIndex]);
}
radianceColor(channel) = wsum / std::max(0.001, wdiv) * targetCameraExposure;
}
}
}
}
Expand Down Expand Up @@ -166,8 +219,8 @@ void hdrMerge::postProcessHighlight(const std::vector< image::Image<image::RGBfC

// get images width, height
const std::size_t width = inputImage.Width();
const std::size_t height = inputImage.Height();

const std::size_t height = inputImage.Height();
image::Image<float> isPixelClamped(width, height);

#pragma omp parallel for
Expand Down
19 changes: 13 additions & 6 deletions src/aliceVision/hdr/hdrMerge.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@

namespace aliceVision {
namespace hdr {

struct MergingParams
{
double minSignificantValue = 0.05;
double maxSignificantValue = 0.995;
float targetCameraExposure;
int refImageIndex;
bool computeLightMasks = false;
};

class hdrMerge {
public:
Expand All @@ -24,12 +33,10 @@ class hdrMerge {
* @param targetCameraExposure
* @param response
*/
void process(const std::vector< image::Image<image::RGBfColor> > &images,
const std::vector<double> &times,
const rgbCurve &weight,
const rgbCurve &response,
image::Image<image::RGBfColor> &radiance,
float targetCameraExposure);
void process(const std::vector<image::Image<image::RGBfColor>>& images, const std::vector<double>& times,
const rgbCurve& weight, const rgbCurve& response, image::Image<image::RGBfColor>& radiance,
image::Image<image::RGBfColor>& lowLight, image::Image<image::RGBfColor>& highLight, image::Image<image::RGBfColor>& noMidLight,
MergingParams& mergingParams);

void postProcessHighlight(const std::vector< image::Image<image::RGBfColor> > &images,
const std::vector<double> &times,
Expand Down
61 changes: 57 additions & 4 deletions src/software/pipeline/main_LdrToHdrMerge.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ using namespace aliceVision;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const std::string& rootname="")
std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const std::string& rootname = "")
{
// Output image file path
std::stringstream sstream;
if (rootname == "")
if(rootname == "")
{
sstream << "hdr_" << std::setfill('0') << std::setw(4) << g << ".exr";
}
Expand All @@ -52,6 +52,21 @@ std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const
return hdrImagePath;
}

std::string getHdrMaskPath(const std::string& outputPath, std::size_t g, const std::string& maskname, const std::string& rootname = "")
{
// Output image file path
std::stringstream sstream;
if(rootname == "")
{
sstream << "hdrMask_" << maskname << "_" << std::setfill('0') << std::setw(4) << g << ".exr";
}
else
{
sstream << rootname << "_" << maskname << ".exr";
}
const std::string hdrImagePath = (fs::path(outputPath) / sstream.str()).string();
return hdrImagePath;
}

int aliceVision_main(int argc, char** argv)
{
Expand All @@ -64,6 +79,9 @@ int aliceVision_main(int argc, char** argv)
int channelQuantizationPower = 10;
int offsetRefBracketIndex = 1000; // By default, use the automatic selection
double meanTargetedLumaForMerging = 0.4;
double minSignificantValue = 0.05;
double maxSignificantValue = 0.995;
bool computeLightMasks = false;
image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB;

hdr::EFunctionType fusionWeightFunction = hdr::EFunctionType::GAUSSIAN;
Expand Down Expand Up @@ -103,6 +121,12 @@ int aliceVision_main(int argc, char** argv)
"Zero to use the center bracket. +N to use a more exposed bracket or -N to use a less exposed backet.")
("meanTargetedLumaForMerging", po::value<double>(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging),
"Mean expected luminance after merging step when input LDR images are decoded in sRGB color space. Must be in the range [0, 1].")
("minSignificantValue", po::value<double>(&minSignificantValue)->default_value(minSignificantValue),
"Minimum channel input value to be considered in advanced pixelwise merging. Used in advanced pixelwise merging.")
("maxSignificantValue", po::value<double>(&maxSignificantValue)->default_value(maxSignificantValue),
"Maximum channel input value to be considered in advanced pixelwise merging. Used in advanced pixelwise merging.")
("computeLightMasks", po::value<bool>(&computeLightMasks)->default_value(computeLightMasks),
"Compute masks of dark and high lights and missing mid lights info.")
("highlightTargetLux", po::value<float>(&highlightTargetLux)->default_value(highlightTargetLux),
"Highlights maximum luminance.")
("highlightCorrectionFactor", po::value<float>(&highlightCorrectionFactor)->default_value(highlightCorrectionFactor),
Expand Down Expand Up @@ -195,6 +219,7 @@ int aliceVision_main(int argc, char** argv)
}
}
std::vector<std::shared_ptr<sfmData::View>> targetViews;
int estimatedTargetIndex;

if (!byPass)
{
Expand All @@ -215,7 +240,7 @@ int aliceVision_main(int argc, char** argv)
{
meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2);
}
hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging);
estimatedTargetIndex = hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging);

if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid)
{
Expand Down Expand Up @@ -337,12 +362,23 @@ int aliceVision_main(int argc, char** argv)

// Merge HDR images
image::Image<image::RGBfColor> HDRimage;
image::Image<image::RGBfColor> lowLightMask;
image::Image<image::RGBfColor> highLightMask;
image::Image<image::RGBfColor> noMidLightMask;
if(images.size() > 1)
{
hdr::hdrMerge merge;
sfmData::ExposureSetting targetCameraSetting = targetView->getCameraExposureSetting();
ALICEVISION_LOG_INFO("[" << g - rangeStart << "/" << rangeSize << "] Merge " << group.size() << " LDR images " << g << "/" << groupedViews.size());
merge.process(images, exposures, fusionWeight, response, HDRimage, targetCameraSetting.getExposure());

hdr::MergingParams mergingParams;
mergingParams.targetCameraExposure = targetCameraSetting.getExposure();
mergingParams.refImageIndex = estimatedTargetIndex;
mergingParams.minSignificantValue = minSignificantValue;
mergingParams.maxSignificantValue = maxSignificantValue;
mergingParams.computeLightMasks = computeLightMasks;

merge.process(images, exposures, fusionWeight, response, HDRimage, lowLightMask, highLightMask, noMidLightMask, mergingParams);//, targetCameraSetting.getExposure(), estimatedTargetIndex);
if(highlightCorrectionFactor > 0.0f)
{
merge.postProcessHighlight(images, exposures, fusionWeight, response, HDRimage, targetCameraSetting.getExposure(), highlightCorrectionFactor, highlightTargetLux);
Expand Down Expand Up @@ -381,6 +417,23 @@ int aliceVision_main(int argc, char** argv)
writeOptions.storageDataType(storageDataType);

image::writeImage(hdrImagePath, HDRimage, writeOptions, targetMetadata);

if(computeLightMasks)
{
const std::string hdrMaskLowLightPath =
getHdrMaskPath(outputPath, g, "lowLight", keepSourceImageName ? p.stem().string() : "");
const std::string hdrMaskHighLightPath =
getHdrMaskPath(outputPath, g, "highLight", keepSourceImageName ? p.stem().string() : "");
const std::string hdrMaskNoMidLightPath =
getHdrMaskPath(outputPath, g, "noMidLight", keepSourceImageName ? p.stem().string() : "");

image::ImageWriteOptions maskWriteOptions;
maskWriteOptions.exrCompressionMethod(image::EImageExrCompression::None);

image::writeImage(hdrMaskLowLightPath, lowLightMask, maskWriteOptions);
image::writeImage(hdrMaskHighLightPath, highLightMask, maskWriteOptions);
image::writeImage(hdrMaskNoMidLightPath, noMidLightMask, maskWriteOptions);
}
}

return EXIT_SUCCESS;
Expand Down

0 comments on commit 3e9f63a

Please sign in to comment.