Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Noise reduction in HDR merging #1469

Merged
merged 6 commits into from
Jul 5, 2023
Merged

Noise reduction in HDR merging #1469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3c910a3
[HDR merge] Add a test on mean luminance of set of images with same …
demoulinv Jun 12, 2023
86cfafb
[HDR merging] For each pixel, select exposures to be merged.
demoulinv Jun 14, 2023
73fc65f
[HDR merging] Add option and parameters dedicated to advanced HDR mer…
demoulinv Jun 19, 2023
fbe3674
[HDR] Merging algorithm simplification.
demoulinv Jun 27, 2023
796a992
[HDR] merging algorithm: bugfix on min and max values.
demoulinv Jun 28, 2023
a95dd3c
[HDR] merging algorithm: remove noise param and code cleaning
demoulinv Jun 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/aliceVision/hdr/brackets.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,9 @@ bool estimateBracketsFromSfmData(std::vector<std::vector<std::shared_ptr<sfmData
return true;
}

void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetViews, const std::vector<std::vector<std::shared_ptr<sfmData::View>>> & groups, int offsetRefBracketIndex, const std::string& lumaStatFilepath, const double meanTargetedLuma)
int selectTargetViews(std::vector<std::shared_ptr<sfmData::View>>& out_targetViews,
std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex,
const std::string& lumaStatFilepath, const double meanTargetedLuma)
{
// If targetIndexesFilename cannot be opened or is not valid an error is thrown
// For odd number, there is no ambiguity on the middle image.
Expand Down Expand Up @@ -219,7 +221,7 @@ void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetV

out_targetViews.push_back(group[targetIndex]);
}
return;
return targetIndex;
}

}
Expand Down
5 changes: 4 additions & 1 deletion src/aliceVision/hdr/brackets.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,11 @@ bool estimateBracketsFromSfmData(std::vector<std::vector<std::shared_ptr<sfmData
* @param[in] targetIndexesFilename: in case offsetRefBracketIndex is out of range the number of views in a group target indexes can be read from a text file
* if the file cannot be read or does not contain the expected number of values (same as view group number) and
* if offsetRefBracketIndex is out of range the number of views then a clamped values of offsetRefBracketIndex is considered
* @return Index of the targetView
*/
void selectTargetViews(std::vector<std::shared_ptr<sfmData::View>> & out_targetViews, const std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename = "", const double meanTargetedLuma = 0.4);
int selectTargetViews(std::vector<std::shared_ptr<sfmData::View>>& out_targetViews,
std::vector<std::vector<std::shared_ptr<sfmData::View>>>& groups, int offsetRefBracketIndex,
const std::string& targetIndexesFilename = "", const double meanTargetedLuma = 0.4);

}
}
161 changes: 107 additions & 54 deletions src/aliceVision/hdr/hdrMerge.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@ void hdrMerge::process(const std::vector< image::Image<image::RGBfColor> > &imag
const rgbCurve &weight,
const rgbCurve &response,
image::Image<image::RGBfColor> &radiance,
float targetCameraExposure)
image::Image<image::RGBfColor> &lowLight,
image::Image<image::RGBfColor> &highLight,
image::Image<image::RGBfColor> &noMidLight,
MergingParams &mergingParams)
{
//checks
assert(!response.isEmpty());
Expand All @@ -72,73 +75,123 @@ void hdrMerge::process(const std::vector< image::Image<image::RGBfColor> > &imag
rgbCurve weightLongestExposure = weight;
weightLongestExposure.freezeFirstPartValues();

const std::vector<double> v_minValue = {response(mergingParams.minSignificantValue, 0),
response(mergingParams.minSignificantValue, 1),
response(mergingParams.minSignificantValue, 2)};
const std::vector<double> v_maxValue = {response(mergingParams.maxSignificantValue, 0),
response(mergingParams.maxSignificantValue, 1),
response(mergingParams.maxSignificantValue, 2)};

highLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));
lowLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));
noMidLight.resize(width, height, true, image::RGBfColor(0.f, 0.f, 0.f));

#pragma omp parallel for
for(int y = 0; y < height; ++y)
{
for(int x = 0; x < width; ++x)
{
//for each pixels
image::RGBfColor &radianceColor = radiance(y, x);
//for each pixels
image::RGBfColor& radianceColor = radiance(y, x);
image::RGBfColor& highLightColor = highLight(y, x);
image::RGBfColor& lowLightColor = lowLight(y, x);
image::RGBfColor& noMidLightColor = noMidLight(y, x);

std::vector<std::vector<double>> vv_coeff;
std::vector<std::vector<double>> vv_value;
std::vector<std::vector<double>> vv_normalizedValue;

// Compute merging range
std::vector<int> v_firstIndex;
std::vector<int> v_lastIndex;
for(std::size_t channel = 0; channel < 3; ++channel)
{
int firstIndex = mergingParams.refImageIndex;
while(firstIndex > 0 && (response(images[firstIndex](y, x)(channel), channel) > v_minValue[channel] ||
firstIndex == images.size() - 1))
{
firstIndex--;
}
v_firstIndex.push_back(firstIndex);

for(std::size_t channel = 0; channel < 3; ++channel)
{
double wsum = 0.0;
double wdiv = 0.0;
int lastIndex = v_firstIndex[channel] + 1;
while(lastIndex < images.size() - 1 && response(images[lastIndex](y, x)(channel), channel) < v_maxValue[channel])
{
lastIndex++;
}
v_lastIndex.push_back(lastIndex);
}

// Merge shortest exposure
// Compute merging coeffs and values to be merged
for(std::size_t channel = 0; channel < 3; ++channel)
{
int exposureIndex = 0;

// for each image
const double value = images[exposureIndex](y, x)(channel);
const double time = times[exposureIndex];
//
// weightShortestExposure: _______
// _______/
// 0 1
double w = std::max(0.001f, weightShortestExposure(value, channel));
std::vector<double> v_coeff;
std::vector<double> v_normalizedValue;
std::vector<double> v_value;

const double r = response(value, channel);
for(std::size_t e = 0; e < images.size(); ++e)
{
const double value = images[e](y, x)(channel);
const double resp = response(value, channel);
const double normalizedValue = resp / times[e];
double coeff = std::max(0.001f, e == 0 ? weightShortestExposure(value, channel) :
(e == images.size() - 1 ? weightLongestExposure(value, channel) :
weight(value, channel)));

v_value.push_back(value);
v_normalizedValue.push_back(normalizedValue);
v_coeff.push_back(coeff);
}

wsum += w * r / time;
wdiv += w;
vv_coeff.push_back(v_coeff);
vv_normalizedValue.push_back(v_normalizedValue);
vv_value.push_back(v_value);
}
// Merge intermediate exposures
for(std::size_t i = 1; i < images.size() - 1; ++i)

// Compute light masks if required (monitoring and debug purposes)
if(mergingParams.computeLightMasks)
{
// for each image
const double value = images[i](y, x)(channel);
const double time = times[i];
//
// weight: ____
// _______/ \________
// 0 1
double w = std::max(0.001f, weight(value, channel));

const double r = response(value, channel);
wsum += w * r / time;
wdiv += w;
for(std::size_t channel = 0; channel < 3; ++channel)
{
int idxMaxValue = 0;
int idxMinValue = 0;
double maxValue = 0.0;
double minValue = 10000.0;
bool jump = true;
for(std::size_t e = 0; e < images.size(); ++e)
{
if(vv_value[channel][e] > maxValue)
{
maxValue = vv_value[channel][e];
idxMaxValue = e;
}
if(vv_value[channel][e] < minValue)
{
minValue = vv_value[channel][e];
idxMinValue = e;
}
jump = jump && ((vv_value[channel][e] < mergingParams.minSignificantValue && e < images.size() - 1) ||
(vv_value[channel][e] > mergingParams.maxSignificantValue && e > 0));
}
highLightColor(channel) = minValue > mergingParams.maxSignificantValue ? 1.0 : 0.0;
lowLightColor(channel) = maxValue < mergingParams.minSignificantValue ? 1.0 : 0.0;
noMidLightColor(channel) = jump ? 1.0 : 0.0;
}
}
// Merge longest exposure
{
int exposureIndex = images.size() - 1;

// for each image
const double value = images[exposureIndex](y, x)(channel);
const double time = times[exposureIndex];
//
// weightLongestExposure: ____________
// \_______
// 0 1
double w = std::max(0.001f, weightLongestExposure(value, channel));

const double r = response(value, channel);

wsum += w * r / time;
wdiv += w;
// Compute the final result and adjust the exposure to the reference one.
for(std::size_t channel = 0; channel < 3; ++channel)
{
double v = 0.0;
double sumCoeff = 0.0;
for(std::size_t i = v_firstIndex[channel]; i <= v_lastIndex[channel]; ++i)
{
v += vv_coeff[channel][i] * vv_normalizedValue[channel][i];
sumCoeff += vv_coeff[channel][i];
}
radianceColor(channel) = mergingParams.targetCameraExposure *
(sumCoeff != 0.0 ? v / sumCoeff : vv_normalizedValue[channel][mergingParams.refImageIndex]);
}
radianceColor(channel) = wsum / std::max(0.001, wdiv) * targetCameraExposure;
}
}
}
}
Expand Down Expand Up @@ -166,8 +219,8 @@ void hdrMerge::postProcessHighlight(const std::vector< image::Image<image::RGBfC

// get images width, height
const std::size_t width = inputImage.Width();
const std::size_t height = inputImage.Height();

const std::size_t height = inputImage.Height();
image::Image<float> isPixelClamped(width, height);

#pragma omp parallel for
Expand Down
19 changes: 13 additions & 6 deletions src/aliceVision/hdr/hdrMerge.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@

namespace aliceVision {
namespace hdr {

struct MergingParams
{
double minSignificantValue = 0.05;
double maxSignificantValue = 0.995;
float targetCameraExposure;
int refImageIndex;
bool computeLightMasks = false;
};

class hdrMerge {
public:
Expand All @@ -24,12 +33,10 @@ class hdrMerge {
* @param targetCameraExposure
* @param response
*/
void process(const std::vector< image::Image<image::RGBfColor> > &images,
const std::vector<double> &times,
const rgbCurve &weight,
const rgbCurve &response,
image::Image<image::RGBfColor> &radiance,
float targetCameraExposure);
void process(const std::vector<image::Image<image::RGBfColor>>& images, const std::vector<double>& times,
const rgbCurve& weight, const rgbCurve& response, image::Image<image::RGBfColor>& radiance,
image::Image<image::RGBfColor>& lowLight, image::Image<image::RGBfColor>& highLight, image::Image<image::RGBfColor>& noMidLight,
MergingParams& mergingParams);

void postProcessHighlight(const std::vector< image::Image<image::RGBfColor> > &images,
const std::vector<double> &times,
Expand Down
61 changes: 57 additions & 4 deletions src/software/pipeline/main_LdrToHdrMerge.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ using namespace aliceVision;
namespace po = boost::program_options;
namespace fs = boost::filesystem;

std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const std::string& rootname="")
std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const std::string& rootname = "")
{
// Output image file path
std::stringstream sstream;
if (rootname == "")
if(rootname == "")
{
sstream << "hdr_" << std::setfill('0') << std::setw(4) << g << ".exr";
}
Expand All @@ -52,6 +52,21 @@ std::string getHdrImagePath(const std::string& outputPath, std::size_t g, const
return hdrImagePath;
}

std::string getHdrMaskPath(const std::string& outputPath, std::size_t g, const std::string& maskname, const std::string& rootname = "")
{
// Output image file path
std::stringstream sstream;
if(rootname == "")
{
sstream << "hdrMask_" << maskname << "_" << std::setfill('0') << std::setw(4) << g << ".exr";
}
else
{
sstream << rootname << "_" << maskname << ".exr";
}
const std::string hdrImagePath = (fs::path(outputPath) / sstream.str()).string();
return hdrImagePath;
}

int aliceVision_main(int argc, char** argv)
{
Expand All @@ -64,6 +79,9 @@ int aliceVision_main(int argc, char** argv)
int channelQuantizationPower = 10;
int offsetRefBracketIndex = 1000; // By default, use the automatic selection
double meanTargetedLumaForMerging = 0.4;
double minSignificantValue = 0.05;
double maxSignificantValue = 0.995;
bool computeLightMasks = false;
image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB;

hdr::EFunctionType fusionWeightFunction = hdr::EFunctionType::GAUSSIAN;
Expand Down Expand Up @@ -103,6 +121,12 @@ int aliceVision_main(int argc, char** argv)
"Zero to use the center bracket. +N to use a more exposed bracket or -N to use a less exposed backet.")
("meanTargetedLumaForMerging", po::value<double>(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging),
"Mean expected luminance after merging step when input LDR images are decoded in sRGB color space. Must be in the range [0, 1].")
("minSignificantValue", po::value<double>(&minSignificantValue)->default_value(minSignificantValue),
"Minimum channel input value to be considered in advanced pixelwise merging. Used in advanced pixelwise merging.")
("maxSignificantValue", po::value<double>(&maxSignificantValue)->default_value(maxSignificantValue),
"Maximum channel input value to be considered in advanced pixelwise merging. Used in advanced pixelwise merging.")
("computeLightMasks", po::value<bool>(&computeLightMasks)->default_value(computeLightMasks),
"Compute masks of dark and high lights and missing mid lights info.")
("highlightTargetLux", po::value<float>(&highlightTargetLux)->default_value(highlightTargetLux),
"Highlights maximum luminance.")
("highlightCorrectionFactor", po::value<float>(&highlightCorrectionFactor)->default_value(highlightCorrectionFactor),
Expand Down Expand Up @@ -195,6 +219,7 @@ int aliceVision_main(int argc, char** argv)
}
}
std::vector<std::shared_ptr<sfmData::View>> targetViews;
int estimatedTargetIndex;

if (!byPass)
{
Expand All @@ -215,7 +240,7 @@ int aliceVision_main(int argc, char** argv)
{
meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2);
}
hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging);
estimatedTargetIndex = hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging);

if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid)
{
Expand Down Expand Up @@ -337,12 +362,23 @@ int aliceVision_main(int argc, char** argv)

// Merge HDR images
image::Image<image::RGBfColor> HDRimage;
image::Image<image::RGBfColor> lowLightMask;
image::Image<image::RGBfColor> highLightMask;
image::Image<image::RGBfColor> noMidLightMask;
if(images.size() > 1)
{
hdr::hdrMerge merge;
sfmData::ExposureSetting targetCameraSetting = targetView->getCameraExposureSetting();
ALICEVISION_LOG_INFO("[" << g - rangeStart << "/" << rangeSize << "] Merge " << group.size() << " LDR images " << g << "/" << groupedViews.size());
merge.process(images, exposures, fusionWeight, response, HDRimage, targetCameraSetting.getExposure());

hdr::MergingParams mergingParams;
mergingParams.targetCameraExposure = targetCameraSetting.getExposure();
mergingParams.refImageIndex = estimatedTargetIndex;
mergingParams.minSignificantValue = minSignificantValue;
mergingParams.maxSignificantValue = maxSignificantValue;
mergingParams.computeLightMasks = computeLightMasks;

merge.process(images, exposures, fusionWeight, response, HDRimage, lowLightMask, highLightMask, noMidLightMask, mergingParams);//, targetCameraSetting.getExposure(), estimatedTargetIndex);
if(highlightCorrectionFactor > 0.0f)
{
merge.postProcessHighlight(images, exposures, fusionWeight, response, HDRimage, targetCameraSetting.getExposure(), highlightCorrectionFactor, highlightTargetLux);
Expand Down Expand Up @@ -381,6 +417,23 @@ int aliceVision_main(int argc, char** argv)
writeOptions.storageDataType(storageDataType);

image::writeImage(hdrImagePath, HDRimage, writeOptions, targetMetadata);

if(computeLightMasks)
{
const std::string hdrMaskLowLightPath =
getHdrMaskPath(outputPath, g, "lowLight", keepSourceImageName ? p.stem().string() : "");
const std::string hdrMaskHighLightPath =
getHdrMaskPath(outputPath, g, "highLight", keepSourceImageName ? p.stem().string() : "");
const std::string hdrMaskNoMidLightPath =
getHdrMaskPath(outputPath, g, "noMidLight", keepSourceImageName ? p.stem().string() : "");

image::ImageWriteOptions maskWriteOptions;
maskWriteOptions.exrCompressionMethod(image::EImageExrCompression::None);

image::writeImage(hdrMaskLowLightPath, lowLightMask, maskWriteOptions);
image::writeImage(hdrMaskHighLightPath, highLightMask, maskWriteOptions);
image::writeImage(hdrMaskNoMidLightPath, noMidLightMask, maskWriteOptions);
}
}

return EXIT_SUCCESS;
Expand Down