From 0a29bc139fdd65dc8d83bee3233545af0d373550 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Fri, 9 Dec 2022 07:34:55 +0100 Subject: [PATCH 01/18] [LdrToHdrCalibration]&[LdrToHdrMerge] Use samples in calibration node to determine the reference exposure for each grouped views. Write results in a text file in the same directory as the response file. In the merge node, read the file containing the reference indexes and use them to compute the HDR image. --- .../pipeline/main_LdrToHdrCalibration.cpp | 111 +++++++++++++++--- src/software/pipeline/main_LdrToHdrMerge.cpp | 24 +++- 2 files changed, 115 insertions(+), 20 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index c4e62d4333..eb7e157f51 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -33,6 +33,7 @@ #include #include +#include // These constants define the current software version. @@ -218,6 +219,8 @@ int aliceVision_main(int argc, char** argv) std::vector> calibrationSamples; hdr::rgbCurve calibrationWeight(channelQuantization); + std::vector> groupedExposures; + std::vector refIndexes; if(calibrationMethod == ECalibrationMethod::LINEAR) { @@ -234,9 +237,30 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); return EXIT_FAILURE; } + + // Build camera exposure table + for (int i = 0; i < groupedViews.size(); ++i) + { + const std::vector>& group = groupedViews[i]; + std::vector exposuresSetting; + + for (int j = 0; j < group.size(); ++j) + { + const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(/*group[0]->getMetadataISO(), group[0]->getMetadataFNumber()*/); + exposuresSetting.push_back(exp); + } + if (!sfmData::hasComparableExposures(exposuresSetting)) + { + ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + } + groupedExposures.push_back(getExposures(exposuresSetting)); + } + size_t group_pos = 0; hdr::Sampling sampling; + double meanTargetedLuma = 0.4; // should be a parameter + ALICEVISION_LOG_INFO("Analyzing samples for each group"); for(auto & group : groupedViews) { @@ -260,6 +284,59 @@ int aliceVision_main(int argc, char** argv) sampling.analyzeSource(samples, channelQuantization, group_pos); + std::map> meanLum; + std::map> rangeLum; + for (int i = 0; i < groupedExposures[group_pos].size(); i++) + { + meanLum[(int)(groupedExposures[group_pos][i] * 1000)].first = 0; + meanLum[(int)(groupedExposures[group_pos][i] * 1000)].second = 0.0; + rangeLum[(int)(groupedExposures[group_pos][i] * 1000)].first = 1000.0; + rangeLum[(int)(groupedExposures[group_pos][i] * 1000)].second = 0.0; + } + for (int i = 0; i < samples.size(); i++) + { + for (int j = 0; j < samples[i].descriptions.size(); j++) + { + double lum = 0.2126 * samples[i].descriptions[j].mean[0] + 0.7152 * samples[i].descriptions[j].mean[1] + 0.0722 * samples[i].descriptions[j].mean[2]; + meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].second += lum; + meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].first++; + if (lum < rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first) + { + rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first = lum; + } + if (lum > rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second) + { + rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second = lum; + } + } + } + + double minDiffWithLumaTarget = 1000.0; + int minDiffWithLumaTargetIdx = 0; + + int k = 0; + for (const auto& n : meanLum) + { + double lumaMean = n.second.second / (double)n.second.first; + + std::cout << '[' << n.first << "] = " << lumaMean << std::endl; + + if (fabs(lumaMean - meanTargetedLuma) < minDiffWithLumaTarget) + { + minDiffWithLumaTarget = fabs(lumaMean - meanTargetedLuma); + minDiffWithLumaTargetIdx = k; + } + ++k; + } + std::cout << std::endl; + for (const auto& n : rangeLum) + { + std::cout << '[' << n.first << " , " << n.second << "]" << std::endl; + } + std::cout << std::endl; + + refIndexes.push_back(minDiffWithLumaTargetIdx); + ++group_pos; } @@ -321,25 +398,6 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_INFO("Start calibration"); hdr::rgbCurve response(channelQuantization); - // Build camera exposure table - std::vector> groupedExposures; - for(int i = 0; i < groupedViews.size(); ++i) - { - const std::vector>& group = groupedViews[i]; - std::vector exposuresSetting; - - for(int j = 0; j < group.size(); ++j) - { - const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(/*group[0]->getMetadataISO(), group[0]->getMetadataFNumber()*/); - exposuresSetting.push_back(exp); - } - if(!sfmData::hasComparableExposures(exposuresSetting)) - { - ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); - } - groupedExposures.push_back(getExposures(exposuresSetting)); - } - switch(calibrationMethod) { case ECalibrationMethod::LINEAR: @@ -382,5 +440,20 @@ int aliceVision_main(int argc, char** argv) response.write(outputResponsePath); response.writeHtml(htmlOutput, "response"); + const std::string expRefIndexesFilename = (fs::path(outputResponsePath).parent_path() / (std::string("exposureRefIndexes") + std::string(".txt"))).string(); + + std::ofstream file(expRefIndexesFilename); + if (!file) + { + throw std::logic_error("Can't create file for exposure reference Indexes"); + } + std::string text = ""; + for (std::size_t index = 0; index < refIndexes.size(); ++index) + { + text += std::to_string(refIndexes[index]) + "\n"; + } + file << text; + file.close(); + return EXIT_SUCCESS; } diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 879c9c3bf5..e18a2714aa 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -178,7 +178,29 @@ int aliceVision_main(int argc, char** argv) } } std::vector> targetViews; - hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex); + //hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex); + + const std::string targetIndexFilename = (fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndexes") + std::string(".txt"))).string(); + + std::ifstream file(targetIndexFilename); + std::vector targetIndexes; + if (!file) + { + throw std::logic_error("Can't open target indexes file"); + } + //create fileData + while (file) + { + std::string line; + if (!getline(file, line)) break; + targetIndexes.push_back(atoi(line.c_str())); + } + file.close(); + + for (int i = 0; i < groupedViews.size(); ++i) + { + targetViews.push_back(groupedViews[i][targetIndexes[i]]); + } // Define range to compute if(rangeStart != -1) From 1dde0a30abc265fac99b2382819239125b0cf9b0 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Wed, 14 Dec 2022 13:38:53 +0100 Subject: [PATCH 02/18] LdrToHdrCalibration: Computation of targeted view index for merging code refactoring. LdrToHdrMerging: Use offsetRefBracketIndex parameter if it is in a suitable range regarding the number of views in a group. If not, use the indexes computed in the calibration node. --- src/aliceVision/hdr/brackets.cpp | 62 ++++++++-- src/aliceVision/hdr/brackets.hpp | 5 +- .../pipeline/main_LdrToHdrCalibration.cpp | 116 +++++++++--------- src/software/pipeline/main_LdrToHdrMerge.cpp | 23 +--- 4 files changed, 120 insertions(+), 86 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 3b9e49d703..cef586e05f 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -1,5 +1,7 @@ #include "brackets.hpp" +#include + #include #include @@ -101,15 +103,55 @@ bool estimateBracketsFromSfmData(std::vector> & out_targetViews, const std::vector>> & groups, int offsetRefBracketIndex) +void selectTargetViews(std::vector> & out_targetViews, const std::vector>> & groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename) { - for(auto & group : groups) + // If targetIndexesFilename cannot be opened or is not valid target views are derived from the middle exposed views + // For odd number, there is no ambiguity on the middle image. + // For even number, we arbitrarily choose the more exposed view (as we usually have more under-exposed images than over-exposed). + const int viewNumberPerGroup = groups[0].size(); + const int middleIndex = viewNumberPerGroup / 2; + const int targetIndex = middleIndex + offsetRefBracketIndex; + + if ((targetIndex >= 0) && (targetIndex < viewNumberPerGroup)) { - // Target views are the middle exposed views - // For add number, there is no ambiguity on the middle image. - // For even number, we arbitrarily choose the more exposed view (as we usually have more under-exposed images than over-exposed). - const int middleIndex = int(group.size()) / 2; - const int targetIndex = clamp(middleIndex + offsetRefBracketIndex, 0, int(group.size()) - 1); + ALICEVISION_LOG_INFO("Use offsetRefBracketIndex parameter"); + for(auto & group : groups) + { + out_targetViews.push_back(group[targetIndex]); + } + } + else // try to use indexes in the file + { + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing target indexes"); + std::vector targetIndexes; + std::ifstream file(targetIndexesFilename); + if (!file) + { + ALICEVISION_LOG_WARNING("Unable to open the file " << targetIndexesFilename << " with the selection of exposures. This file would be needed to select the optimal exposure for the creation of the HDR images. Use clamped offsetRefBracketIndex parameter."); + for (int i = 0; i < groups.size(); ++i) + { + targetIndexes.push_back(clamp(targetIndex, 0, int(groups[i].size()) - 1)); + } + } + else + { + while (file) + { + std::string line; + if (!getline(file, line)) break; + targetIndexes.push_back(atoi(line.c_str())); + } + file.close(); + + if (targetIndexes.size() != groups.size()) + { + ALICEVISION_LOG_WARNING("Non consistent number of reference indexes in file " << targetIndexesFilename << ", use clamped offsetRefBracketIndex parameter"); + for (int i = 0; i < groups.size(); ++i) + { + targetIndexes.push_back(clamp(targetIndex, 0, int(groups[i].size()) - 1)); + } + } + } //Set the ldr ancestors id for (auto v : group) @@ -117,8 +159,12 @@ void selectTargetViews(std::vector> & out_targetV group[targetIndex]->addAncestor(v->getViewId()); } - out_targetViews.push_back(group[targetIndex]); + for (int i = 0; i < groups.size(); ++i) + { + out_targetViews.push_back(groups[i][targetIndexes[i]]); + } } + } } } diff --git a/src/aliceVision/hdr/brackets.hpp b/src/aliceVision/hdr/brackets.hpp index 490868bc99..d56a0973d6 100644 --- a/src/aliceVision/hdr/brackets.hpp +++ b/src/aliceVision/hdr/brackets.hpp @@ -25,8 +25,11 @@ bool estimateBracketsFromSfmData(std::vector> & out_targetViews, const std::vector>>& groups, int offsetRefBracketIndex); +void selectTargetViews(std::vector> & out_targetViews, const std::vector>>& groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename = ""); } } diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index eb7e157f51..14c57741da 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -111,6 +111,60 @@ inline std::istream& operator>>(std::istream& in, ECalibrationMethod& calibratio return in; } +int computeMergingRefIndex(const std::vector& groupedExposure, const std::vector& samples, const double meanTargetedLuma) +{ + std::map> meanLum; + std::map> rangeLum; + for (int i = 0; i < groupedExposure.size(); i++) + { + meanLum[(int)(groupedExposure[i] * 1000)].first = 0; + meanLum[(int)(groupedExposure[i] * 1000)].second = 0.0; + rangeLum[(int)(groupedExposure[i] * 1000)].first = 1000.0; + rangeLum[(int)(groupedExposure[i] * 1000)].second = 0.0; + } + for (int i = 0; i < samples.size(); i++) + { + for (int j = 0; j < samples[i].descriptions.size(); j++) + { + double lum = image::Rgb2GrayLinear(samples[i].descriptions[j].mean[0], samples[i].descriptions[j].mean[1], samples[i].descriptions[j].mean[2]); + meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].second += lum; + meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].first++; + if (lum < rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first) + { + rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first = lum; + } + if (lum > rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second) + { + rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second = lum; + } + } + } + + double minDiffWithLumaTarget = 1000.0; + int minDiffWithLumaTargetIdx = 0; + + int k = 0; + for (const auto& n : meanLum) + { + double lumaMean = n.second.second / (double)n.second.first; + + ALICEVISION_LOG_DEBUG('[' << n.first << "] = " << lumaMean); + + if (fabs(lumaMean - meanTargetedLuma) < minDiffWithLumaTarget) + { + minDiffWithLumaTarget = fabs(lumaMean - meanTargetedLuma); + minDiffWithLumaTargetIdx = k; + } + ++k; + } + for (const auto& n : rangeLum) + { + ALICEVISION_LOG_DEBUG('[' << n.first << " , " << n.second << "]"); + } + + return minDiffWithLumaTargetIdx; +} + int aliceVision_main(int argc, char** argv) { std::string sfmInputDataFilename; @@ -121,6 +175,7 @@ int aliceVision_main(int argc, char** argv) int nbBrackets = 0; int channelQuantizationPower = 10; size_t maxTotalPoints = 1000000; + double meanTargetedLumaForMerging = 0.4; // Command line parameters @@ -145,8 +200,10 @@ int aliceVision_main(int argc, char** argv) ("channelQuantizationPower", po::value(&channelQuantizationPower)->default_value(channelQuantizationPower), "Quantization level like 8 bits or 10 bits.") ("maxTotalPoints", po::value(&maxTotalPoints)->default_value(maxTotalPoints), - "Max number of points used from the sampling. This ensures that the number of pixels values extracted by the sampling " - "can be managed by the calibration step (in term of computation time and memory usage).") + "Max number of points used from the sampling. This ensures that the number of pixels values extracted by the sampling " + "can be managed by the calibration step (in term of computation time and memory usage).") + ("meanTargetedLumaForMerging", po::value(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging), + "Mean expected luminance after merging step. Must be in the range [0, 1].") ; CmdLine cmdline("This program recovers the Camera Response Function (CRF) from samples extracted from LDR images with multi-bracketing.\n" @@ -259,8 +316,6 @@ int aliceVision_main(int argc, char** argv) size_t group_pos = 0; hdr::Sampling sampling; - double meanTargetedLuma = 0.4; // should be a parameter - ALICEVISION_LOG_INFO("Analyzing samples for each group"); for(auto & group : groupedViews) { @@ -284,58 +339,7 @@ int aliceVision_main(int argc, char** argv) sampling.analyzeSource(samples, channelQuantization, group_pos); - std::map> meanLum; - std::map> rangeLum; - for (int i = 0; i < groupedExposures[group_pos].size(); i++) - { - meanLum[(int)(groupedExposures[group_pos][i] * 1000)].first = 0; - meanLum[(int)(groupedExposures[group_pos][i] * 1000)].second = 0.0; - rangeLum[(int)(groupedExposures[group_pos][i] * 1000)].first = 1000.0; - rangeLum[(int)(groupedExposures[group_pos][i] * 1000)].second = 0.0; - } - for (int i = 0; i < samples.size(); i++) - { - for (int j = 0; j < samples[i].descriptions.size(); j++) - { - double lum = 0.2126 * samples[i].descriptions[j].mean[0] + 0.7152 * samples[i].descriptions[j].mean[1] + 0.0722 * samples[i].descriptions[j].mean[2]; - meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].second += lum; - meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].first++; - if (lum < rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first) - { - rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first = lum; - } - if (lum > rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second) - { - rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second = lum; - } - } - } - - double minDiffWithLumaTarget = 1000.0; - int minDiffWithLumaTargetIdx = 0; - - int k = 0; - for (const auto& n : meanLum) - { - double lumaMean = n.second.second / (double)n.second.first; - - std::cout << '[' << n.first << "] = " << lumaMean << std::endl; - - if (fabs(lumaMean - meanTargetedLuma) < minDiffWithLumaTarget) - { - minDiffWithLumaTarget = fabs(lumaMean - meanTargetedLuma); - minDiffWithLumaTargetIdx = k; - } - ++k; - } - std::cout << std::endl; - for (const auto& n : rangeLum) - { - std::cout << '[' << n.first << " , " << n.second << "]" << std::endl; - } - std::cout << std::endl; - - refIndexes.push_back(minDiffWithLumaTargetIdx); + refIndexes.push_back(computeMergingRefIndex(groupedExposures[group_pos], samples, meanTargetedLumaForMerging)); ++group_pos; } diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index e18a2714aa..55502e6877 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -178,29 +178,10 @@ int aliceVision_main(int argc, char** argv) } } std::vector> targetViews; - //hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex); - const std::string targetIndexFilename = (fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndexes") + std::string(".txt"))).string(); + const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndexes.txt"))); - std::ifstream file(targetIndexFilename); - std::vector targetIndexes; - if (!file) - { - throw std::logic_error("Can't open target indexes file"); - } - //create fileData - while (file) - { - std::string line; - if (!getline(file, line)) break; - targetIndexes.push_back(atoi(line.c_str())); - } - file.close(); - - for (int i = 0; i < groupedViews.size(); ++i) - { - targetViews.push_back(groupedViews[i][targetIndexes[i]]); - } + hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, targetIndexFilepath.string()); // Define range to compute if(rangeStart != -1) From 1f865fb53788051c79db6d57e423ba4b2704cac6 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Fri, 16 Dec 2022 17:32:42 +0100 Subject: [PATCH 03/18] [HDRCalib & merging] compute and use a unique reference index for hdr image merging --- src/aliceVision/hdr/brackets.cpp | 41 +++++++------------ .../pipeline/main_LdrToHdrCalibration.cpp | 20 +++++---- src/software/pipeline/main_LdrToHdrMerge.cpp | 23 +++++++++-- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index cef586e05f..425d354ed3 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -110,48 +110,36 @@ void selectTargetViews(std::vector> & out_targetV // For even number, we arbitrarily choose the more exposed view (as we usually have more under-exposed images than over-exposed). const int viewNumberPerGroup = groups[0].size(); const int middleIndex = viewNumberPerGroup / 2; - const int targetIndex = middleIndex + offsetRefBracketIndex; + int targetIndex = middleIndex + offsetRefBracketIndex; + + out_targetViews.clear(); if ((targetIndex >= 0) && (targetIndex < viewNumberPerGroup)) { ALICEVISION_LOG_INFO("Use offsetRefBracketIndex parameter"); - for(auto & group : groups) - { - out_targetViews.push_back(group[targetIndex]); - } } - else // try to use indexes in the file + else // try to use the index in the file { - ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing target indexes"); - std::vector targetIndexes; + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing the target index"); std::ifstream file(targetIndexesFilename); if (!file) { - ALICEVISION_LOG_WARNING("Unable to open the file " << targetIndexesFilename << " with the selection of exposures. This file would be needed to select the optimal exposure for the creation of the HDR images. Use clamped offsetRefBracketIndex parameter."); - for (int i = 0; i < groups.size(); ++i) - { - targetIndexes.push_back(clamp(targetIndex, 0, int(groups[i].size()) - 1)); - } + return; } else { - while (file) + std::string line; + if (getline(file, line)) { - std::string line; - if (!getline(file, line)) break; - targetIndexes.push_back(atoi(line.c_str())); + targetIndex = atoi(line.c_str()); + file.close(); } - file.close(); - - if (targetIndexes.size() != groups.size()) + else { - ALICEVISION_LOG_WARNING("Non consistent number of reference indexes in file " << targetIndexesFilename << ", use clamped offsetRefBracketIndex parameter"); - for (int i = 0; i < groups.size(); ++i) - { - targetIndexes.push_back(clamp(targetIndex, 0, int(groups[i].size()) - 1)); - } + return; } } + } //Set the ldr ancestors id for (auto v : group) @@ -164,7 +152,8 @@ void selectTargetViews(std::vector> & out_targetV out_targetViews.push_back(groups[i][targetIndexes[i]]); } } - + return; } + } } diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 14c57741da..ba54c4bbd4 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -444,20 +444,24 @@ int aliceVision_main(int argc, char** argv) response.write(outputResponsePath); response.writeHtml(htmlOutput, "response"); - const std::string expRefIndexesFilename = (fs::path(outputResponsePath).parent_path() / (std::string("exposureRefIndexes") + std::string(".txt"))).string(); + double meanIndex = 0.0; + for (int i = 0; i < refIndexes.size(); i++) + { + meanIndex += static_cast(refIndexes[i]); + } + const int refIndex = static_cast(std::round(meanIndex /= refIndexes.size())); - std::ofstream file(expRefIndexesFilename); + const std::string expRefIndexFilename = (fs::path(outputResponsePath).parent_path() / (std::string("exposureRefIndex") + std::string(".txt"))).string(); + std::ofstream file(expRefIndexFilename); if (!file) { - throw std::logic_error("Can't create file for exposure reference Indexes"); + ALICEVISION_LOG_WARNING("Unable to create file " << expRefIndexFilename << " for storing the exposure reference Index"); } - std::string text = ""; - for (std::size_t index = 0; index < refIndexes.size(); ++index) + else { - text += std::to_string(refIndexes[index]) + "\n"; + file << std::to_string(refIndex) + "\n"; + file.close(); } - file << text; - file.close(); return EXIT_SUCCESS; } diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 55502e6877..10a0cf0ae9 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -55,7 +55,7 @@ int aliceVision_main(int argc, char** argv) bool byPass = false; int channelQuantizationPower = 10; image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; - int offsetRefBracketIndex = 0; + int offsetRefBracketIndex = 1000; hdr::EFunctionType fusionWeightFunction = hdr::EFunctionType::GAUSSIAN; float highlightCorrectionFactor = 0.0f; @@ -154,6 +154,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + std::size_t usedNbBrackets; { std::set sizeOfGroups; for(auto& group : groupedViews) @@ -162,7 +163,7 @@ int aliceVision_main(int argc, char** argv) } if(sizeOfGroups.size() == 1) { - std::size_t usedNbBrackets = *sizeOfGroups.begin(); + usedNbBrackets = *sizeOfGroups.begin(); if(usedNbBrackets == 1) { ALICEVISION_LOG_INFO("No multi-bracketing."); @@ -179,10 +180,26 @@ int aliceVision_main(int argc, char** argv) } std::vector> targetViews; - const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndexes.txt"))); + const int middleIndex = usedNbBrackets / 2; + const int targetIndex = middleIndex + offsetRefBracketIndex; + const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); + + const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndex.txt"))); + + if (!fs::is_regular_file(targetIndexFilepath) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("Unable to open the file " << targetIndexFilepath.string() << " with the selection of exposures. This file is needed to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, targetIndexFilepath.string()); + if (targetViews.empty() && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("File " << targetIndexFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } + // Define range to compute if(rangeStart != -1) { From e64730dc0ad8e2ca524bf73060d8e6f318f2e40d Mon Sep 17 00:00:00 2001 From: demoulinv Date: Mon, 19 Dec 2022 11:56:07 +0100 Subject: [PATCH 04/18] [hdr] merging: take bypass mode into account calibration: compute a ref index for final exposure in linear mode --- .../pipeline/main_LdrToHdrCalibration.cpp | 53 ++++++++++--------- src/software/pipeline/main_LdrToHdrMerge.cpp | 31 ++++++----- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index ba54c4bbd4..fe252fc2c5 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -148,7 +148,7 @@ int computeMergingRefIndex(const std::vector& groupedExposure, const std { double lumaMean = n.second.second / (double)n.second.first; - ALICEVISION_LOG_DEBUG('[' << n.first << "] = " << lumaMean); + ALICEVISION_LOG_DEBUG('[' << n.first << "] : mean = " << lumaMean << " ; min = " << rangeLum[k].first << " ; max = " << rangeLum[k].second); if (fabs(lumaMean - meanTargetedLuma) < minDiffWithLumaTarget) { @@ -157,10 +157,6 @@ int computeMergingRefIndex(const std::vector& groupedExposure, const std } ++k; } - for (const auto& n : rangeLum) - { - ALICEVISION_LOG_DEBUG('[' << n.first << " , " << n.second << "]"); - } return minDiffWithLumaTargetIdx; } @@ -287,14 +283,22 @@ int aliceVision_main(int argc, char** argv) ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); } } - else + + if(samplesFolder.empty()) { - if(samplesFolder.empty()) + if (calibrationMethod != ECalibrationMethod::LINEAR) { ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); return EXIT_FAILURE; } - + else + { + ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); + ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); + } + } + else + { // Build camera exposure table for (int i = 0; i < groupedViews.size(); ++i) { @@ -303,7 +307,7 @@ int aliceVision_main(int argc, char** argv) for (int j = 0; j < group.size(); ++j) { - const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(/*group[0]->getMetadataISO(), group[0]->getMetadataFNumber()*/); + const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); exposuresSetting.push_back(exp); } if (!sfmData::hasComparableExposures(exposuresSetting)) @@ -317,7 +321,7 @@ int aliceVision_main(int argc, char** argv) hdr::Sampling sampling; ALICEVISION_LOG_INFO("Analyzing samples for each group"); - for(auto & group : groupedViews) + for (auto& group : groupedViews) { // Read from file const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); @@ -329,7 +333,7 @@ int aliceVision_main(int argc, char** argv) } std::size_t size; - fileSamples.read((char *)&size, sizeof(size)); + fileSamples.read((char*)&size, sizeof(size)); std::vector samples(size); for (std::size_t i = 0; i < size; ++i) @@ -351,7 +355,7 @@ int aliceVision_main(int argc, char** argv) group_pos = 0; std::size_t total = 0; - for(auto & group : groupedViews) + for (auto& group : groupedViews) { // Read from file const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); @@ -363,7 +367,7 @@ int aliceVision_main(int argc, char** argv) } std::size_t size = 0; - fileSamples.read((char *)&size, sizeof(size)); + fileSamples.read((char*)&size, sizeof(size)); std::vector samples(size); for (int i = 0; i < size; ++i) @@ -381,19 +385,19 @@ int aliceVision_main(int argc, char** argv) // Define calibration weighting curve from name boost::algorithm::to_lower(calibrationWeightFunction); - if(calibrationWeightFunction == "default") + if (calibrationWeightFunction == "default") { - switch(calibrationMethod) + switch (calibrationMethod) { - case ECalibrationMethod::DEBEVEC: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); - break; - case ECalibrationMethod::LINEAR: - case ECalibrationMethod::GROSSBERG: - case ECalibrationMethod::LAGUERRE: - default: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); - break; + case ECalibrationMethod::DEBEVEC: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); + break; + case ECalibrationMethod::LINEAR: + case ECalibrationMethod::GROSSBERG: + case ECalibrationMethod::LAGUERRE: + default: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); + break; } } calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); @@ -456,6 +460,7 @@ int aliceVision_main(int argc, char** argv) if (!file) { ALICEVISION_LOG_WARNING("Unable to create file " << expRefIndexFilename << " for storing the exposure reference Index"); + ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); } else { diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 10a0cf0ae9..815160b15e 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -180,24 +180,27 @@ int aliceVision_main(int argc, char** argv) } std::vector> targetViews; - const int middleIndex = usedNbBrackets / 2; - const int targetIndex = middleIndex + offsetRefBracketIndex; - const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); + if (!byPass) + { + const int middleIndex = usedNbBrackets / 2; + const int targetIndex = middleIndex + offsetRefBracketIndex; + const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); - const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndex.txt"))); + const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndex.txt"))); - if (!fs::is_regular_file(targetIndexFilepath) && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("Unable to open the file " << targetIndexFilepath.string() << " with the selection of exposures. This file is needed to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; - } + if (!fs::is_regular_file(targetIndexFilepath) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("Unable to open the file " << targetIndexFilepath.string() << " with the selection of exposures. This file is needed to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } - hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, targetIndexFilepath.string()); + hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, targetIndexFilepath.string()); - if (targetViews.empty() && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("File " << targetIndexFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; + if (targetViews.empty() && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("File " << targetIndexFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } } // Define range to compute From 131c6a1c5a7d367c778d5f9f443443e0dcf6cc58 Mon Sep 17 00:00:00 2001 From: demoulinv <99878110+demoulinv@users.noreply.github.com> Date: Fri, 23 Dec 2022 18:05:14 +0100 Subject: [PATCH 05/18] Update src/aliceVision/hdr/brackets.cpp code simplification Co-authored-by: Fabien Castan --- src/aliceVision/hdr/brackets.cpp | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 425d354ed3..292793635e 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -123,22 +123,11 @@ void selectTargetViews(std::vector> & out_targetV ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing the target index"); std::ifstream file(targetIndexesFilename); if (!file) - { - return; - } - else - { - std::string line; - if (getline(file, line)) - { - targetIndex = atoi(line.c_str()); - file.close(); - } - else - { - return; - } - } + ALICEVISION_THROW_ERROR("Failed to open file: " << targetIndexesFilename); + std::string line; + if (!std::getline(file, line)) + ALICEVISION_THROW_ERROR("Failed to read from file: " << targetIndexesFilename); + targetIndex = atoi(line.c_str()); } //Set the ldr ancestors id From 1cdacbd97e5c262a496c1860e29c82eeefa2fe3a Mon Sep 17 00:00:00 2001 From: demoulinv <99878110+demoulinv@users.noreply.github.com> Date: Fri, 23 Dec 2022 18:09:15 +0100 Subject: [PATCH 06/18] Update main_LdrToHdrCalibration.cpp Remove useless ofstream closing --- src/software/pipeline/main_LdrToHdrCalibration.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index fe252fc2c5..898fe210ba 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -465,7 +465,6 @@ int aliceVision_main(int argc, char** argv) else { file << std::to_string(refIndex) + "\n"; - file.close(); } return EXIT_SUCCESS; From d8856344a683da3d1759a3b1f6cb9865f92560e2 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Thu, 5 Jan 2023 18:37:22 +0100 Subject: [PATCH 07/18] [HDR Calibration and Merging] Compute reference index for merging within the merging node after reading luminance statistics provided by the calibration node through a text file. --- src/aliceVision/hdr/brackets.cpp | 58 ++++++++++-- src/aliceVision/hdr/brackets.hpp | 2 +- .../pipeline/main_LdrToHdrCalibration.cpp | 89 ++++++++----------- src/software/pipeline/main_LdrToHdrMerge.cpp | 19 ++-- 4 files changed, 101 insertions(+), 67 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 292793635e..dc48d6d016 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -103,7 +103,7 @@ bool estimateBracketsFromSfmData(std::vector> & out_targetViews, const std::vector>> & groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename) +void selectTargetViews(std::vector> & out_targetViews, const std::vector>> & groups, int offsetRefBracketIndex, const std::string& lumaStatFilepath, const double meanTargetedLuma) { // If targetIndexesFilename cannot be opened or is not valid target views are derived from the middle exposed views // For odd number, there is no ambiguity on the middle image. @@ -118,16 +118,58 @@ void selectTargetViews(std::vector> & out_targetV { ALICEVISION_LOG_INFO("Use offsetRefBracketIndex parameter"); } - else // try to use the index in the file + else // try to use the luminance statistics of the LDR images stored in the file { - ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing the target index"); - std::ifstream file(targetIndexesFilename); + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter out of range, read file containing luminance statistics to compute an estimation"); + std::ifstream file(lumaStatFilepath); if (!file) - ALICEVISION_THROW_ERROR("Failed to open file: " << targetIndexesFilename); + ALICEVISION_THROW_ERROR("Failed to open file: " << lumaStatFilepath); + std::vector lines; std::string line; - if (!std::getline(file, line)) - ALICEVISION_THROW_ERROR("Failed to read from file: " << targetIndexesFilename); - targetIndex = atoi(line.c_str()); + while (std::getline(file, line)) + { + lines.push_back(line); + } + if ((lines.size() < 2) || (atoi(lines[0].c_str()) != groups.size()) || (atoi(lines[1].c_str()) < groups[0].size())) + { + ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); + } + int nbGroup = atoi(lines[0].c_str()); + int nbExp = atoi(lines[1].c_str()); + + std::vector v_lumaMeanMean; + + for (int i = 0; i < nbExp; ++i) + { + double lumaMeanMean = 0.0; + for (int j = 0; j < nbGroup; ++j) + { + std::istringstream iss(lines[2 + j * nbExp + i]); + int exposure, nbItem; + double lumaMean, lumaMin, lumaMax; + if (!(iss >> exposure >> nbItem >> lumaMean >> lumaMin >> lumaMax)) + { + ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); + } + lumaMeanMean += lumaMean; + } + v_lumaMeanMean.push_back(lumaMeanMean / nbGroup); + } + + double minDiffWithLumaTarget = 1000.0; + targetIndex = 0; + + for (int k = 0; k < v_lumaMeanMean.size(); ++k) + { + const double diffWithLumaTarget = fabs(v_lumaMeanMean[k] - meanTargetedLuma); + if (diffWithLumaTarget < minDiffWithLumaTarget) + { + minDiffWithLumaTarget = diffWithLumaTarget; + targetIndex = k; + } + } + + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex); } //Set the ldr ancestors id diff --git a/src/aliceVision/hdr/brackets.hpp b/src/aliceVision/hdr/brackets.hpp index d56a0973d6..af9cfa750a 100644 --- a/src/aliceVision/hdr/brackets.hpp +++ b/src/aliceVision/hdr/brackets.hpp @@ -29,7 +29,7 @@ bool estimateBracketsFromSfmData(std::vector> & out_targetViews, const std::vector>>& groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename = ""); +void selectTargetViews(std::vector> & out_targetViews, const std::vector>>& groups, int offsetRefBracketIndex, const std::string& targetIndexesFilename = "", const double meanTargetedLuma = 0.4); } } diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 898fe210ba..ffd44a139b 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -111,54 +111,40 @@ inline std::istream& operator>>(std::istream& in, ECalibrationMethod& calibratio return in; } -int computeMergingRefIndex(const std::vector& groupedExposure, const std::vector& samples, const double meanTargetedLuma) +struct luminanceInfo +{ + double meanLum; + double minLum; + double maxLum; + int itemNb; +}; + +void computeLuminanceStat(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) { - std::map> meanLum; - std::map> rangeLum; for (int i = 0; i < groupedExposure.size(); i++) { - meanLum[(int)(groupedExposure[i] * 1000)].first = 0; - meanLum[(int)(groupedExposure[i] * 1000)].second = 0.0; - rangeLum[(int)(groupedExposure[i] * 1000)].first = 1000.0; - rangeLum[(int)(groupedExposure[i] * 1000)].second = 0.0; + luminanceInfos[(int)(groupedExposure[i] * 1000)].itemNb = 0; + luminanceInfos[(int)(groupedExposure[i] * 1000)].meanLum = 0.0; + luminanceInfos[(int)(groupedExposure[i] * 1000)].minLum = 1000.0; + luminanceInfos[(int)(groupedExposure[i] * 1000)].maxLum = 0.0; } for (int i = 0; i < samples.size(); i++) { for (int j = 0; j < samples[i].descriptions.size(); j++) { double lum = image::Rgb2GrayLinear(samples[i].descriptions[j].mean[0], samples[i].descriptions[j].mean[1], samples[i].descriptions[j].mean[2]); - meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].second += lum; - meanLum[(int)(samples[i].descriptions[j].exposure * 1000)].first++; - if (lum < rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first) + luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].meanLum += lum; + luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].itemNb++; + if (lum < luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].minLum) { - rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].first = lum; + luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].minLum = lum; } - if (lum > rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second) + if (lum > luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].maxLum) { - rangeLum[(int)(samples[i].descriptions[j].exposure * 1000)].second = lum; + luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].maxLum = lum; } } } - - double minDiffWithLumaTarget = 1000.0; - int minDiffWithLumaTargetIdx = 0; - - int k = 0; - for (const auto& n : meanLum) - { - double lumaMean = n.second.second / (double)n.second.first; - - ALICEVISION_LOG_DEBUG('[' << n.first << "] : mean = " << lumaMean << " ; min = " << rangeLum[k].first << " ; max = " << rangeLum[k].second); - - if (fabs(lumaMean - meanTargetedLuma) < minDiffWithLumaTarget) - { - minDiffWithLumaTarget = fabs(lumaMean - meanTargetedLuma); - minDiffWithLumaTargetIdx = k; - } - ++k; - } - - return minDiffWithLumaTargetIdx; } int aliceVision_main(int argc, char** argv) @@ -171,7 +157,6 @@ int aliceVision_main(int argc, char** argv) int nbBrackets = 0; int channelQuantizationPower = 10; size_t maxTotalPoints = 1000000; - double meanTargetedLumaForMerging = 0.4; // Command line parameters @@ -198,8 +183,6 @@ int aliceVision_main(int argc, char** argv) ("maxTotalPoints", po::value(&maxTotalPoints)->default_value(maxTotalPoints), "Max number of points used from the sampling. This ensures that the number of pixels values extracted by the sampling " "can be managed by the calibration step (in term of computation time and memory usage).") - ("meanTargetedLumaForMerging", po::value(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging), - "Mean expected luminance after merging step. Must be in the range [0, 1].") ; CmdLine cmdline("This program recovers the Camera Response Function (CRF) from samples extracted from LDR images with multi-bracketing.\n" @@ -273,7 +256,7 @@ int aliceVision_main(int argc, char** argv) std::vector> calibrationSamples; hdr::rgbCurve calibrationWeight(channelQuantization); std::vector> groupedExposures; - std::vector refIndexes; + std::vector> v_luminanceInfos; if(calibrationMethod == ECalibrationMethod::LINEAR) { @@ -343,7 +326,9 @@ int aliceVision_main(int argc, char** argv) sampling.analyzeSource(samples, channelQuantization, group_pos); - refIndexes.push_back(computeMergingRefIndex(groupedExposures[group_pos], samples, meanTargetedLumaForMerging)); + std::map luminanceInfos; + computeLuminanceStat(groupedExposures[group_pos], samples, luminanceInfos); + v_luminanceInfos.push_back(luminanceInfos); ++group_pos; } @@ -448,23 +433,27 @@ int aliceVision_main(int argc, char** argv) response.write(outputResponsePath); response.writeHtml(htmlOutput, "response"); - double meanIndex = 0.0; - for (int i = 0; i < refIndexes.size(); i++) - { - meanIndex += static_cast(refIndexes[i]); - } - const int refIndex = static_cast(std::round(meanIndex /= refIndexes.size())); - - const std::string expRefIndexFilename = (fs::path(outputResponsePath).parent_path() / (std::string("exposureRefIndex") + std::string(".txt"))).string(); - std::ofstream file(expRefIndexFilename); + const std::string lumastatFilename = (fs::path(outputResponsePath).parent_path() / (std::string("luminanceStatistics") + std::string(".txt"))).string(); + std::ofstream file(lumastatFilename); if (!file) { - ALICEVISION_LOG_WARNING("Unable to create file " << expRefIndexFilename << " for storing the exposure reference Index"); - ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); + ALICEVISION_LOG_ERROR("Unable to create file " << lumastatFilename << " for storing luminance statistics"); + return EXIT_FAILURE; } else { - file << std::to_string(refIndex) + "\n"; + file << v_luminanceInfos.size() << std::endl; + file << v_luminanceInfos[0].size() << std::endl; + + for (int i = 0; i < v_luminanceInfos.size(); ++i) + { + for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) + { + file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; + file << (it->second).minLum << " " << (it->second).maxLum << std::endl; + } + } + } return EXIT_SUCCESS; diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 815160b15e..35b2a7fa13 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -54,8 +54,9 @@ int aliceVision_main(int argc, char** argv) int nbBrackets = 3; bool byPass = false; int channelQuantizationPower = 10; + int offsetRefBracketIndex = 1000; // By default, use the automatic selection + double meanTargetedLumaForMerging = 0.4; image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; - int offsetRefBracketIndex = 1000; hdr::EFunctionType fusionWeightFunction = hdr::EFunctionType::GAUSSIAN; float highlightCorrectionFactor = 0.0f; @@ -90,6 +91,8 @@ int aliceVision_main(int argc, char** argv) "Weight function used to fuse all LDR images together (gaussian, triangle, plateau).") ("offsetRefBracketIndex", po::value(&offsetRefBracketIndex)->default_value(offsetRefBracketIndex), "Zero to use the center bracket. +N to use a more exposed bracket or -N to use a less exposed backet.") + ("meanTargetedLumaForMerging", po::value(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging), + "Mean expected luminance after merging step. Must be in the range [0, 1].") ("highlightTargetLux", po::value(&highlightTargetLux)->default_value(highlightTargetLux), "Highlights maximum luminance.") ("highlightCorrectionFactor", po::value(&highlightCorrectionFactor)->default_value(highlightCorrectionFactor), @@ -98,9 +101,9 @@ int aliceVision_main(int argc, char** argv) ("storageDataType", po::value(&storageDataType)->default_value(storageDataType), ("Storage data type: " + image::EStorageDataType_informations()).c_str()) ("rangeStart", po::value(&rangeStart)->default_value(rangeStart), - "Range image index start.") + "Range image index start.") ("rangeSize", po::value(&rangeSize)->default_value(rangeSize), - "Range size."); + "Range size."); CmdLine cmdline("This program merges LDR images into HDR images.\n" "AliceVision LdrToHdrMerge"); @@ -186,19 +189,19 @@ int aliceVision_main(int argc, char** argv) const int targetIndex = middleIndex + offsetRefBracketIndex; const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); - const fs::path targetIndexFilepath(fs::path(inputResponsePath).parent_path() / (std::string("exposureRefIndex.txt"))); + const fs::path lumaStatFilepath(fs::path(inputResponsePath).parent_path() / (std::string("luminanceStatistics.txt"))); - if (!fs::is_regular_file(targetIndexFilepath) && !isOffsetRefBracketIndexValid) + if (!fs::is_regular_file(lumaStatFilepath) && !isOffsetRefBracketIndexValid) { - ALICEVISION_LOG_ERROR("Unable to open the file " << targetIndexFilepath.string() << " with the selection of exposures. This file is needed to select the optimal exposure for the creation of HDR images."); + ALICEVISION_LOG_ERROR("Unable to open the file " << lumaStatFilepath.string() << " with luminance statistics. This file is needed to select the optimal exposure for the creation of HDR images."); return EXIT_FAILURE; } - hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, targetIndexFilepath.string()); + hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); if (targetViews.empty() && !isOffsetRefBracketIndexValid) { - ALICEVISION_LOG_ERROR("File " << targetIndexFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); + ALICEVISION_LOG_ERROR("File " << lumaStatFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); return EXIT_FAILURE; } } From 89960d67b198c9d2c0382fe9eb94b01fbfa10bd0 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Mon, 16 Jan 2023 14:54:03 +0100 Subject: [PATCH 08/18] [HDR Calib & Merge] Code simplification and bypass management update. --- .../pipeline/main_LdrToHdrCalibration.cpp | 21 ++++++------- src/software/pipeline/main_LdrToHdrMerge.cpp | 31 +++++++++---------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index ffd44a139b..6c479273cf 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -433,27 +433,24 @@ int aliceVision_main(int argc, char** argv) response.write(outputResponsePath); response.writeHtml(htmlOutput, "response"); - const std::string lumastatFilename = (fs::path(outputResponsePath).parent_path() / (std::string("luminanceStatistics") + std::string(".txt"))).string(); + const std::string lumastatFilename = (fs::path(outputResponsePath).parent_path() / "luminanceStatistics.txt").string(); std::ofstream file(lumastatFilename); if (!file) { ALICEVISION_LOG_ERROR("Unable to create file " << lumastatFilename << " for storing luminance statistics"); return EXIT_FAILURE; } - else - { - file << v_luminanceInfos.size() << std::endl; - file << v_luminanceInfos[0].size() << std::endl; - for (int i = 0; i < v_luminanceInfos.size(); ++i) + file << v_luminanceInfos.size() << std::endl; + file << v_luminanceInfos[0].size() << std::endl; + + for (int i = 0; i < v_luminanceInfos.size(); ++i) + { + for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) { - for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) - { - file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; - file << (it->second).minLum << " " << (it->second).maxLum << std::endl; - } + file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; + file << (it->second).minLum << " " << (it->second).maxLum << std::endl; } - } return EXIT_SUCCESS; diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 35b2a7fa13..9d0d5d3929 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -183,27 +183,24 @@ int aliceVision_main(int argc, char** argv) } std::vector> targetViews; - if (!byPass) - { - const int middleIndex = usedNbBrackets / 2; - const int targetIndex = middleIndex + offsetRefBracketIndex; - const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); + const int middleIndex = usedNbBrackets / 2; + const int targetIndex = middleIndex + offsetRefBracketIndex; + const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); - const fs::path lumaStatFilepath(fs::path(inputResponsePath).parent_path() / (std::string("luminanceStatistics.txt"))); + const fs::path lumaStatFilepath(fs::path(inputResponsePath).parent_path() / (std::string("luminanceStatistics.txt"))); - if (!fs::is_regular_file(lumaStatFilepath) && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("Unable to open the file " << lumaStatFilepath.string() << " with luminance statistics. This file is needed to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; - } + if (!fs::is_regular_file(lumaStatFilepath) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("Unable to open the file " << lumaStatFilepath.string() << " with luminance statistics. This file is needed to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } - hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); + hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); - if (targetViews.empty() && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("File " << lumaStatFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; - } + if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("File " << lumaStatFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; } // Define range to compute From 7418e8cc35043c787bd599ccbf33d5cddb467f09 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Wed, 18 Jan 2023 11:33:44 +0100 Subject: [PATCH 09/18] [HDR Calib & Merge] Manage bypass mode by computing luminance statistics from images at the calibration stage. Adjust the targeted luminance level for the output HDR image in case of non sRGB working space. --- .../pipeline/main_LdrToHdrCalibration.cpp | 307 +++++++++++------- src/software/pipeline/main_LdrToHdrMerge.cpp | 4 + 2 files changed, 200 insertions(+), 111 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 6c479273cf..1b4cd9cab3 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -119,7 +119,7 @@ struct luminanceInfo int itemNb; }; -void computeLuminanceStat(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) +void computeLuminanceStatFromSamples(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) { for (int i = 0; i < groupedExposure.size(); i++) { @@ -147,6 +147,49 @@ void computeLuminanceStat(const std::vector& groupedExposure, const std: } } +void computeLuminanceInfoFromImage(image::Image& image, luminanceInfo& lumaInfo) +{ + // Luminance statistics are calculated from a subsampled square, centered and rotated by 45°. + // 2 vertices of this square are the centers of the longest sides of the image. + // Such a shape is suitable for both fisheye and classic images. + + double meanLuminance = 0.0; + double maxLuminance = 0.0; + double minLuminance = 1000.0; + int sampleNb = 0; + + const int imgH = image.Height(); + const int imgW = image.Width(); + + const int a1 = (imgH <= imgW) ? imgW / 2 : imgH / 2; + const int a2 = (imgH <= imgW) ? imgW / 2 : imgW - (imgH / 2); + const int a3 = (imgH <= imgW) ? imgH - (imgW / 2) : imgH / 2; + const int a4 = (imgH <= imgW) ? (imgW / 2) + imgH : imgW + (imgH / 2); + + const int rmin = (imgH <= imgW) ? 0 : (imgH - imgW) / 2; + const int rmax = (imgH <= imgW) ? imgH : (imgH + imgW) / 2; + + for (int r = rmin; r < rmax; r = r + 16) + { + const int cmin = (r < imgH / 2) ? a1 - r : r - a3; + const int cmax = (r < imgH / 2) ? a2 + r : a4 - r; + + for (int c = cmin; c < cmax; c = c + 16) + { + double luma = image::Rgb2GrayLinear(image(r, c)[0], image(r, c)[1], image(r, c)[2]); + meanLuminance += luma; + minLuminance = (luma < minLuminance) ? luma : minLuminance; + maxLuminance = (luma > maxLuminance) ? luma : maxLuminance; + sampleNb++; + } + } + + lumaInfo.itemNb = sampleNb; + lumaInfo.minLum = minLuminance; + lumaInfo.maxLum = maxLuminance; + lumaInfo.meanLum = meanLuminance; +} + int aliceVision_main(int argc, char** argv) { std::string sfmInputDataFilename; @@ -156,7 +199,9 @@ int aliceVision_main(int argc, char** argv) std::string calibrationWeightFunction = "default"; int nbBrackets = 0; int channelQuantizationPower = 10; + image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; size_t maxTotalPoints = 1000000; + bool byPass = false; // Command line parameters @@ -175,11 +220,15 @@ int aliceVision_main(int argc, char** argv) "Name of method used for camera calibration: linear, debevec, grossberg, laguerre.") ("calibrationWeight,w", po::value(&calibrationWeightFunction)->default_value(calibrationWeightFunction), "Weight function used to calibrate camera response (default depends on the calibration method, gaussian, " - "triangle, plateau).") + "triangle, plateau).") ("nbBrackets,b", po::value(&nbBrackets)->default_value(nbBrackets), "bracket count per HDR image (0 means automatic).") + ("byPass", po::value(&byPass)->default_value(byPass), + "bypass HDR creation and use a single bracket as input for next steps") ("channelQuantizationPower", po::value(&channelQuantizationPower)->default_value(channelQuantizationPower), "Quantization level like 8 bits or 10 bits.") + ("workingColorSpace", po::value(&workingColorSpace)->default_value(workingColorSpace), + ("Working color space: " + image::EImageColorSpace_informations()).c_str()) ("maxTotalPoints", po::value(&maxTotalPoints)->default_value(maxTotalPoints), "Max number of points used from the sampling. This ensures that the number of pixels values extracted by the sampling " "can be managed by the calibration step (in term of computation time and memory usage).") @@ -253,146 +302,181 @@ int aliceVision_main(int argc, char** argv) } } - std::vector> calibrationSamples; - hdr::rgbCurve calibrationWeight(channelQuantization); - std::vector> groupedExposures; std::vector> v_luminanceInfos; - if(calibrationMethod == ECalibrationMethod::LINEAR) - { - ALICEVISION_LOG_INFO("No calibration needed in Linear."); - if(!samplesFolder.empty()) - { - ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); - } - } - - if(samplesFolder.empty()) + if (byPass) { - if (calibrationMethod != ECalibrationMethod::LINEAR) - { - ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); - return EXIT_FAILURE; - } - else + // Compute luminance statistics from raw images + + std::vector> meanLuminances(groupedViews.size(), std::vector(groupedViews[0].size(), 0.0)); + v_luminanceInfos.resize(groupedViews.size()); + + #pragma omp parallel for + for (int g = 0; g < groupedViews.size(); ++g) { - ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); - ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); + std::vector> group = groupedViews[g]; + std::map luminanceInfos; + + for (std::size_t i = 0; i < group.size(); ++i) + { + image::Image image; + + const std::string filepath = group[i]->getImagePath(); + + image::ImageReadOptions options; + options.workingColorSpace = workingColorSpace; + options.rawColorInterpretation = image::ERawColorInterpretation_stringToEnum(group[i]->getRawColorInterpretation()); + options.colorProfileFileName = group[i]->getColorProfileFileName(); + image::readImage(filepath, image, options); + + computeLuminanceInfoFromImage(image, luminanceInfos[i]); + } + v_luminanceInfos[g] = luminanceInfos; } } else { - // Build camera exposure table - for (int i = 0; i < groupedViews.size(); ++i) - { - const std::vector>& group = groupedViews[i]; - std::vector exposuresSetting; - for (int j = 0; j < group.size(); ++j) - { - const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); - exposuresSetting.push_back(exp); - } - if (!sfmData::hasComparableExposures(exposuresSetting)) + std::vector> calibrationSamples; + hdr::rgbCurve calibrationWeight(channelQuantization); + std::vector> groupedExposures; + //std::vector> v_luminanceInfos; + + if (calibrationMethod == ECalibrationMethod::LINEAR) + { + ALICEVISION_LOG_INFO("No calibration needed in Linear."); + if (!samplesFolder.empty()) { - ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); } - groupedExposures.push_back(getExposures(exposuresSetting)); } - size_t group_pos = 0; - hdr::Sampling sampling; - - ALICEVISION_LOG_INFO("Analyzing samples for each group"); - for (auto& group : groupedViews) + if (samplesFolder.empty()) { - // Read from file - const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); - std::ifstream fileSamples(samplesFilepath, std::ios::binary); - if (!fileSamples.is_open()) + if (calibrationMethod != ECalibrationMethod::LINEAR) { - ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); return EXIT_FAILURE; } - - std::size_t size; - fileSamples.read((char*)&size, sizeof(size)); - - std::vector samples(size); - for (std::size_t i = 0; i < size; ++i) + else { - fileSamples >> samples[i]; + ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); + ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); } - - sampling.analyzeSource(samples, channelQuantization, group_pos); - - std::map luminanceInfos; - computeLuminanceStat(groupedExposures[group_pos], samples, luminanceInfos); - v_luminanceInfos.push_back(luminanceInfos); - - ++group_pos; } - - // We need to trim samples list - sampling.filter(maxTotalPoints); - - ALICEVISION_LOG_INFO("Extracting samples for each group"); - group_pos = 0; - - std::size_t total = 0; - for (auto& group : groupedViews) + else { - // Read from file - const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); - std::ifstream fileSamples(samplesFilepath, std::ios::binary); - if (!fileSamples.is_open()) + // Build camera exposure table + for (int i = 0; i < groupedViews.size(); ++i) { - ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); - return EXIT_FAILURE; + const std::vector>& group = groupedViews[i]; + std::vector exposuresSetting; + + for (int j = 0; j < group.size(); ++j) + { + const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); + exposuresSetting.push_back(exp); + } + if (!sfmData::hasComparableExposures(exposuresSetting)) + { + ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + } + groupedExposures.push_back(getExposures(exposuresSetting)); } - std::size_t size = 0; - fileSamples.read((char*)&size, sizeof(size)); + size_t group_pos = 0; + hdr::Sampling sampling; - std::vector samples(size); - for (int i = 0; i < size; ++i) + ALICEVISION_LOG_INFO("Analyzing samples for each group"); + for (auto& group : groupedViews) { - fileSamples >> samples[i]; + // Read from file + const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); + std::ifstream fileSamples(samplesFilepath, std::ios::binary); + if (!fileSamples.is_open()) + { + ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + return EXIT_FAILURE; + } + + std::size_t size; + fileSamples.read((char*)&size, sizeof(size)); + + std::vector samples(size); + for (std::size_t i = 0; i < size; ++i) + { + fileSamples >> samples[i]; + } + + sampling.analyzeSource(samples, channelQuantization, group_pos); + + std::map luminanceInfos; + computeLuminanceStatFromSamples(groupedExposures[group_pos], samples, luminanceInfos); + v_luminanceInfos.push_back(luminanceInfos); + + ++group_pos; } - std::vector out_samples; - sampling.extractUsefulSamples(out_samples, samples, group_pos); + // We need to trim samples list + sampling.filter(maxTotalPoints); - calibrationSamples.push_back(out_samples); + ALICEVISION_LOG_INFO("Extracting samples for each group"); + group_pos = 0; - ++group_pos; - } + std::size_t total = 0; + for (auto& group : groupedViews) + { + // Read from file + const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); + std::ifstream fileSamples(samplesFilepath, std::ios::binary); + if (!fileSamples.is_open()) + { + ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + return EXIT_FAILURE; + } + + std::size_t size = 0; + fileSamples.read((char*)&size, sizeof(size)); + + std::vector samples(size); + for (int i = 0; i < size; ++i) + { + fileSamples >> samples[i]; + } + + std::vector out_samples; + sampling.extractUsefulSamples(out_samples, samples, group_pos); + + calibrationSamples.push_back(out_samples); + + ++group_pos; + } - // Define calibration weighting curve from name - boost::algorithm::to_lower(calibrationWeightFunction); - if (calibrationWeightFunction == "default") - { - switch (calibrationMethod) + // Define calibration weighting curve from name + boost::algorithm::to_lower(calibrationWeightFunction); + if (calibrationWeightFunction == "default") { - case ECalibrationMethod::DEBEVEC: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); - break; - case ECalibrationMethod::LINEAR: - case ECalibrationMethod::GROSSBERG: - case ECalibrationMethod::LAGUERRE: - default: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); - break; + switch (calibrationMethod) + { + case ECalibrationMethod::DEBEVEC: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); + break; + case ECalibrationMethod::LINEAR: + case ECalibrationMethod::GROSSBERG: + case ECalibrationMethod::LAGUERRE: + default: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); + break; + } } + calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); } - calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); - } - ALICEVISION_LOG_INFO("Start calibration"); - hdr::rgbCurve response(channelQuantization); + ALICEVISION_LOG_INFO("Start calibration"); + hdr::rgbCurve response(channelQuantization); - switch(calibrationMethod) - { + switch (calibrationMethod) + { case ECalibrationMethod::LINEAR: { // set the response function to linear @@ -425,13 +509,14 @@ int aliceVision_main(int argc, char** argv) calibration.process(calibrationSamples, groupedExposures, channelQuantization, false, response); break; } - } + } - const std::string methodName = ECalibrationMethod_enumToString(calibrationMethod); - const std::string htmlOutput = (fs::path(outputResponsePath).parent_path() / (std::string("response_") + methodName + std::string(".html"))).string(); + const std::string methodName = ECalibrationMethod_enumToString(calibrationMethod); + const std::string htmlOutput = (fs::path(outputResponsePath).parent_path() / (std::string("response_") + methodName + std::string(".html"))).string(); - response.write(outputResponsePath); - response.writeHtml(htmlOutput, "response"); + response.write(outputResponsePath); + response.writeHtml(htmlOutput, "response"); + } const std::string lumastatFilename = (fs::path(outputResponsePath).parent_path() / "luminanceStatistics.txt").string(); std::ofstream file(lumastatFilename); diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 9d0d5d3929..362cd13647 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -195,6 +195,10 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + if (workingColorSpace != image::EImageColorSpace::SRGB) + { + meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2); + } hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid) From 85b7c210b8e45346fd42aa41586c9545dfff4d9b Mon Sep 17 00:00:00 2001 From: demoulinv Date: Wed, 18 Jan 2023 20:47:25 +0100 Subject: [PATCH 10/18] [HDR Calib and Merge] Code adjustment and cleaning. --- .../pipeline/main_LdrToHdrCalibration.cpp | 25 +++++++++++-------- src/software/pipeline/main_LdrToHdrMerge.cpp | 3 ++- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 1b4cd9cab3..fb2f571fea 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -166,15 +166,19 @@ void computeLuminanceInfoFromImage(image::Image& image, lumina const int a3 = (imgH <= imgW) ? imgH - (imgW / 2) : imgH / 2; const int a4 = (imgH <= imgW) ? (imgW / 2) + imgH : imgW + (imgH / 2); + // All rows must be considered if image orientation is landscape + // Only imgW rows centered on imgH/2 must be considered if image orientation is portrait const int rmin = (imgH <= imgW) ? 0 : (imgH - imgW) / 2; const int rmax = (imgH <= imgW) ? imgH : (imgH + imgW) / 2; - for (int r = rmin; r < rmax; r = r + 16) + const int sampling = 16; + + for (int r = rmin; r < rmax; r = r + sampling) { const int cmin = (r < imgH / 2) ? a1 - r : r - a3; const int cmax = (r < imgH / 2) ? a2 + r : a4 - r; - for (int c = cmin; c < cmax; c = c + 16) + for (int c = cmin; c < cmax; c = c + sampling) { double luma = image::Rgb2GrayLinear(image(r, c)[0], image(r, c)[1], image(r, c)[2]); meanLuminance += luma; @@ -336,11 +340,9 @@ int aliceVision_main(int argc, char** argv) } else { - std::vector> calibrationSamples; hdr::rgbCurve calibrationWeight(channelQuantization); std::vector> groupedExposures; - //std::vector> v_luminanceInfos; if (calibrationMethod == ECalibrationMethod::LINEAR) { @@ -527,14 +529,17 @@ int aliceVision_main(int argc, char** argv) } file << v_luminanceInfos.size() << std::endl; - file << v_luminanceInfos[0].size() << std::endl; - - for (int i = 0; i < v_luminanceInfos.size(); ++i) + if (!v_luminanceInfos.empty()) { - for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) + file << v_luminanceInfos[0].size() << std::endl; + + for (int i = 0; i < v_luminanceInfos.size(); ++i) { - file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; - file << (it->second).minLum << " " << (it->second).maxLum << std::endl; + for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) + { + file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; + file << (it->second).minLum << " " << (it->second).maxLum << std::endl; + } } } diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 362cd13647..61fe39903f 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -92,7 +92,7 @@ int aliceVision_main(int argc, char** argv) ("offsetRefBracketIndex", po::value(&offsetRefBracketIndex)->default_value(offsetRefBracketIndex), "Zero to use the center bracket. +N to use a more exposed bracket or -N to use a less exposed backet.") ("meanTargetedLumaForMerging", po::value(&meanTargetedLumaForMerging)->default_value(meanTargetedLumaForMerging), - "Mean expected luminance after merging step. Must be in the range [0, 1].") + "Mean expected luminance after merging step when input LDR images are decoded in sRGB color space. Must be in the range [0, 1].") ("highlightTargetLux", po::value(&highlightTargetLux)->default_value(highlightTargetLux), "Highlights maximum luminance.") ("highlightCorrectionFactor", po::value(&highlightCorrectionFactor)->default_value(highlightCorrectionFactor), @@ -195,6 +195,7 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + // Adjust the targeted luminance level by removing the corresponding gamma if the working color space is not sRGB. if (workingColorSpace != image::EImageColorSpace::SRGB) { meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2); From 53f5d562ad8d5e16b45cbeaae83c413f9dc3dee6 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Fri, 20 Jan 2023 14:39:32 +0100 Subject: [PATCH 11/18] [HDR Smpling and Calib] Add bypass and calibrationMethod as parameters of sampling.cpp. Move calibration method definition in brackets.hpp. Redispatch luminance statistics computation between sampling and calib. Create a simplified sampling mode. Update luminance statistics file format. --- src/aliceVision/hdr/brackets.cpp | 7 +- src/aliceVision/hdr/brackets.hpp | 65 ++++ src/aliceVision/hdr/sampling.cpp | 295 +++++++++++------- src/aliceVision/hdr/sampling.hpp | 2 +- .../pipeline/main_LdrToHdrCalibration.cpp | 212 ++++--------- .../pipeline/main_LdrToHdrSampling.cpp | 12 +- 6 files changed, 314 insertions(+), 279 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index dc48d6d016..1bb59aea25 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -130,7 +130,7 @@ void selectTargetViews(std::vector> & out_targetV { lines.push_back(line); } - if ((lines.size() < 2) || (atoi(lines[0].c_str()) != groups.size()) || (atoi(lines[1].c_str()) < groups[0].size())) + if ((lines.size() < 3) || (atoi(lines[0].c_str()) != groups.size()) || (atoi(lines[1].c_str()) < groups[0].size())) { ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); } @@ -144,10 +144,11 @@ void selectTargetViews(std::vector> & out_targetV double lumaMeanMean = 0.0; for (int j = 0; j < nbGroup; ++j) { - std::istringstream iss(lines[2 + j * nbExp + i]); + std::istringstream iss(lines[3 + j * nbExp + i]); + aliceVision::IndexT srcId; int exposure, nbItem; double lumaMean, lumaMin, lumaMax; - if (!(iss >> exposure >> nbItem >> lumaMean >> lumaMin >> lumaMax)) + if (!(iss >> srcId >> exposure >> nbItem >> lumaMean >> lumaMin >> lumaMax)) { ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); } diff --git a/src/aliceVision/hdr/brackets.hpp b/src/aliceVision/hdr/brackets.hpp index af9cfa750a..756a0ac92a 100644 --- a/src/aliceVision/hdr/brackets.hpp +++ b/src/aliceVision/hdr/brackets.hpp @@ -11,6 +11,71 @@ namespace aliceVision { namespace hdr { +enum class ECalibrationMethod +{ + LINEAR, + DEBEVEC, + GROSSBERG, + LAGUERRE, +}; + +/** + * @brief convert an enum ECalibrationMethod to its corresponding string + * @param ECalibrationMethod + * @return String + */ +inline std::string ECalibrationMethod_enumToString(const ECalibrationMethod calibrationMethod) +{ + switch (calibrationMethod) + { + case ECalibrationMethod::LINEAR: + return "linear"; + case ECalibrationMethod::DEBEVEC: + return "debevec"; + case ECalibrationMethod::GROSSBERG: + return "grossberg"; + case ECalibrationMethod::LAGUERRE: + return "laguerre"; + } + throw std::out_of_range("Invalid method name enum"); +} + +/** + * @brief convert a string calibration method name to its corresponding enum ECalibrationMethod + * @param ECalibrationMethod + * @return String + */ +inline ECalibrationMethod ECalibrationMethod_stringToEnum(const std::string& calibrationMethodName) +{ + std::string methodName = calibrationMethodName; + std::transform(methodName.begin(), methodName.end(), methodName.begin(), ::tolower); + + if (methodName == "linear") + return ECalibrationMethod::LINEAR; + if (methodName == "debevec") + return ECalibrationMethod::DEBEVEC; + if (methodName == "grossberg") + return ECalibrationMethod::GROSSBERG; + if (methodName == "laguerre") + return ECalibrationMethod::LAGUERRE; + + throw std::out_of_range("Invalid method name : '" + calibrationMethodName + "'"); +} + +inline std::ostream& operator<<(std::ostream& os, ECalibrationMethod calibrationMethodName) +{ + os << ECalibrationMethod_enumToString(calibrationMethodName); + return os; +} + +inline std::istream& operator>>(std::istream& in, ECalibrationMethod& calibrationMethod) +{ + std::string token; + in >> token; + calibrationMethod = ECalibrationMethod_stringToEnum(token); + return in; +} + /** * @brief Estimate brackets information from sfm data * @param[out] groups: estimated groups diff --git a/src/aliceVision/hdr/sampling.cpp b/src/aliceVision/hdr/sampling.cpp index 4aafeaf797..e0b77e6cf6 100644 --- a/src/aliceVision/hdr/sampling.cpp +++ b/src/aliceVision/hdr/sampling.cpp @@ -146,7 +146,7 @@ void square(image::Image & dest, const Eigen::Matrix& out_samples, const std::vector& imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Sampling::Params params) +bool Sampling::extractSamplesFromImages(std::vector& out_samples, const std::vector& imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Sampling::Params params, const bool simplified) { const int radiusp1 = params.radius + 1; const int diameter = (params.radius * 2) + 1; @@ -184,46 +184,96 @@ bool Sampling::extractSamplesFromImages(std::vector& out_samples, c throw std::runtime_error(ss.str()); } - #pragma omp parallel for - for (int idx = 0; idx < vec_blocks.size(); ++idx) + if (simplified) { - int cx = vec_blocks[idx].first; - int cy = vec_blocks[idx].second; + // Luminance statistics are calculated from a subsampled square, centered and rotated by 45°. + // 2 vertices of this square are the centers of the longest sides of the image. + // Such a shape is suitable for both fisheye and classic images. - int blockWidth = ((img.Width() - cx) > params.blockSize) ? params.blockSize : img.Width() - cx; - int blockHeight = ((img.Height() - cy) > params.blockSize) ? params.blockSize : img.Height() - cy; + const int H = imageHeight; + const int W = imageWidth; + const int hH = imageHeight / 2; + const int hW = imageWidth / 2; - auto blockInput = img.block(cy, cx, blockHeight, blockWidth); - auto blockOutput = samples.block(cy, cx, blockHeight, blockWidth); + const int a1 = (H <= W) ? hW : hH; + const int a2 = (H <= W) ? hW : W - hH; + const int a3 = (H <= W) ? H - hW : hH; + const int a4 = (H <= W) ? hW + H : W + hH; - // Stats for deviation - Image> imgIntegral, imgIntegralSquare; - Image imgSquare; + // All rows must be considered if image orientation is landscape (H < W) + // Only imgW rows centered on imgH/2 must be considered if image orientation is portrait (H > W) + const int rmin = (H <= W) ? 0 : (H - W) / 2; + const int rmax = (H <= W) ? H : (H + W) / 2; - square(imgSquare, blockInput); - integral(imgIntegral, blockInput); - integral(imgIntegralSquare, imgSquare); + const int sampling = 16; - for(int y = radiusp1; y < imgIntegral.Height() - params.radius; ++y) + #pragma omp parallel for + for (int r = rmin; r < rmax; r = r + sampling) { - for(int x = radiusp1; x < imgIntegral.Width() - params.radius; ++x) + const int cmin = (r < hH) ? a1 - r : r - a3; + const int cmax = (r < hH) ? a2 + r : a4 - r; + + for (int c = cmin; c < cmax; c = c + sampling) { - image::Rgb S1 = imgIntegral(y + params.radius, x + params.radius) + imgIntegral(y - radiusp1, x - radiusp1) - imgIntegral(y + params.radius, x - radiusp1) - imgIntegral(y - radiusp1, x + params.radius); - image::Rgb S2 = imgIntegralSquare(y + params.radius, x + params.radius) + imgIntegralSquare(y - radiusp1, x - radiusp1) - imgIntegralSquare(y + params.radius, x - radiusp1) - imgIntegralSquare(y - radiusp1, x + params.radius); - PixelDescription pd; - + pd.exposure = exposure; - pd.mean.r() = blockInput(y,x).r(); - pd.mean.g() = blockInput(y,x).g(); - pd.mean.b() = blockInput(y,x).b(); - pd.variance.r() = (S2.r() - (S1.r()*S1.r()) / area) / area; - pd.variance.g() = (S2.g() - (S1.g()*S1.g()) / area) / area; - pd.variance.b() = (S2.b() - (S1.b()*S1.b()) / area) / area; - - blockOutput(y, x).x = cx + x; - blockOutput(y, x).y = cy + y; - blockOutput(y, x).descriptions.push_back(pd); + pd.mean.r() = img(r, c).r(); + pd.mean.g() = img(r, c).g(); + pd.mean.b() = img(r, c).b(); + pd.variance.r() = 0.0; + pd.variance.g() = 0.0; + pd.variance.b() = 0.0; + + samples(r, c).x = c; + samples(r, c).y = r; + samples(r, c).descriptions.push_back(pd); + } + } + } + else + { + #pragma omp parallel for + for (int idx = 0; idx < vec_blocks.size(); ++idx) + { + int cx = vec_blocks[idx].first; + int cy = vec_blocks[idx].second; + + int blockWidth = ((img.Width() - cx) > params.blockSize) ? params.blockSize : img.Width() - cx; + int blockHeight = ((img.Height() - cy) > params.blockSize) ? params.blockSize : img.Height() - cy; + + auto blockInput = img.block(cy, cx, blockHeight, blockWidth); + auto blockOutput = samples.block(cy, cx, blockHeight, blockWidth); + + // Stats for deviation + Image> imgIntegral, imgIntegralSquare; + Image imgSquare; + + square(imgSquare, blockInput); + integral(imgIntegral, blockInput); + integral(imgIntegralSquare, imgSquare); + + for (int y = radiusp1; y < imgIntegral.Height() - params.radius; ++y) + { + for (int x = radiusp1; x < imgIntegral.Width() - params.radius; ++x) + { + image::Rgb S1 = imgIntegral(y + params.radius, x + params.radius) + imgIntegral(y - radiusp1, x - radiusp1) - imgIntegral(y + params.radius, x - radiusp1) - imgIntegral(y - radiusp1, x + params.radius); + image::Rgb S2 = imgIntegralSquare(y + params.radius, x + params.radius) + imgIntegralSquare(y - radiusp1, x - radiusp1) - imgIntegralSquare(y + params.radius, x - radiusp1) - imgIntegralSquare(y - radiusp1, x + params.radius); + + PixelDescription pd; + + pd.exposure = exposure; + pd.mean.r() = blockInput(y, x).r(); + pd.mean.g() = blockInput(y, x).g(); + pd.mean.b() = blockInput(y, x).b(); + pd.variance.r() = (S2.r() - (S1.r() * S1.r()) / area) / area; + pd.variance.g() = (S2.g() - (S1.g() * S1.g()) / area) / area; + pd.variance.b() = (S2.b() - (S1.b() * S1.b()) / area) / area; + + blockOutput(y, x).x = cx + x; + blockOutput(y, x).y = cy + y; + blockOutput(y, x).descriptions.push_back(pd); + } } } } @@ -235,124 +285,127 @@ bool Sampling::extractSamplesFromImages(std::vector& out_samples, c return false; } - // Create samples image - #pragma omp parallel for - for (int y = params.radius; y < samples.Height() - params.radius; ++y) + if (!simplified) { - for (int x = params.radius; x < samples.Width() - params.radius; ++x) + // Create samples image + #pragma omp parallel for + for (int y = params.radius; y < samples.Height() - params.radius; ++y) { - ImageSample & sample = samples(y, x); - if (sample.descriptions.size() < 2) - { - continue; - } - - int last_ok = 0; - - // Make sure we don't have a patch with high variance on any bracket. - // If the variance is too high somewhere, ignore the whole coordinate samples - bool valid = true; - const float maxVariance = 0.05f; - for (int k = 0; k < sample.descriptions.size(); ++k) + for (int x = params.radius; x < samples.Width() - params.radius; ++x) { - if (sample.descriptions[k].variance.r() > maxVariance || - sample.descriptions[k].variance.g() > maxVariance || - sample.descriptions[k].variance.b() > maxVariance) + ImageSample& sample = samples(y, x); + if (sample.descriptions.size() < 2) { - valid = false; - break; + continue; } - } - if (!valid) - { - sample.descriptions.clear(); - continue; - } + int last_ok = 0; - // Makes sure the curve is monotonic - int firstvalid = -1; - int lastvalid = 0; - for (std::size_t k = 1; k < sample.descriptions.size(); ++k) - { - bool valid = false; - - // Threshold on the max values, to avoid using fully saturated pixels - // TODO: on RAW images, values can be higher. May need to be computed dynamically? - const float maxValue = 0.99f; - if(sample.descriptions[k].mean.r() > maxValue || - sample.descriptions[k].mean.g() > maxValue || - sample.descriptions[k].mean.b() > maxValue) + // Make sure we don't have a patch with high variance on any bracket. + // If the variance is too high somewhere, ignore the whole coordinate samples + bool valid = true; + const float maxVariance = 0.05f; + for (int k = 0; k < sample.descriptions.size(); ++k) { - continue; + if (sample.descriptions[k].variance.r() > maxVariance || + sample.descriptions[k].variance.g() > maxVariance || + sample.descriptions[k].variance.b() > maxVariance) + { + valid = false; + break; + } } - // Ensures that at least one channel is strictly increasing with increasing exposure - // TODO: check "exposure" params, we may have the same exposure multiple times - const float minIncreaseRatio = 1.004f; - if(sample.descriptions[k].mean.r() > minIncreaseRatio * sample.descriptions[k - 1].mean.r() || - sample.descriptions[k].mean.g() > minIncreaseRatio * sample.descriptions[k - 1].mean.g() || - sample.descriptions[k].mean.b() > minIncreaseRatio * sample.descriptions[k - 1].mean.b()) + if (!valid) { - valid = true; + sample.descriptions.clear(); + continue; } - // Ensures that the values of each channel are increasing with increasing exposure - if (sample.descriptions[k].mean.r() < sample.descriptions[k - 1].mean.r() || - sample.descriptions[k].mean.g() < sample.descriptions[k - 1].mean.g() || - sample.descriptions[k].mean.b() < sample.descriptions[k - 1].mean.b()) + // Makes sure the curve is monotonic + int firstvalid = -1; + int lastvalid = 0; + for (std::size_t k = 1; k < sample.descriptions.size(); ++k) { - valid = false; - } + bool valid = false; + + // Threshold on the max values, to avoid using fully saturated pixels + // TODO: on RAW images, values can be higher. May need to be computed dynamically? + const float maxValue = 0.99f; + if (sample.descriptions[k].mean.r() > maxValue || + sample.descriptions[k].mean.g() > maxValue || + sample.descriptions[k].mean.b() > maxValue) + { + continue; + } - // If we have enough information to analyze the chrominance - const float minGlobalValue = 0.1f; - if(sample.descriptions[k - 1].mean.norm() > minGlobalValue) - { - // Check that both colors are similars - const float n1 = sample.descriptions[k - 1].mean.norm(); - const float n2 = sample.descriptions[k].mean.norm(); - const float dot = sample.descriptions[k - 1].mean.dot(sample.descriptions[k].mean); - const float cosa = dot / (n1*n2); - - const float maxCosa = 0.95f; // ~ 18deg - if(cosa < maxCosa) + // Ensures that at least one channel is strictly increasing with increasing exposure + // TODO: check "exposure" params, we may have the same exposure multiple times + const float minIncreaseRatio = 1.004f; + if (sample.descriptions[k].mean.r() > minIncreaseRatio * sample.descriptions[k - 1].mean.r() || + sample.descriptions[k].mean.g() > minIncreaseRatio * sample.descriptions[k - 1].mean.g() || + sample.descriptions[k].mean.b() > minIncreaseRatio * sample.descriptions[k - 1].mean.b()) + { + valid = true; + } + + // Ensures that the values of each channel are increasing with increasing exposure + if (sample.descriptions[k].mean.r() < sample.descriptions[k - 1].mean.r() || + sample.descriptions[k].mean.g() < sample.descriptions[k - 1].mean.g() || + sample.descriptions[k].mean.b() < sample.descriptions[k - 1].mean.b()) { valid = false; } - } - if (valid) - { - if (firstvalid < 0) + // If we have enough information to analyze the chrominance + const float minGlobalValue = 0.1f; + if (sample.descriptions[k - 1].mean.norm() > minGlobalValue) { - firstvalid = int(k) - 1; + // Check that both colors are similars + const float n1 = sample.descriptions[k - 1].mean.norm(); + const float n2 = sample.descriptions[k].mean.norm(); + const float dot = sample.descriptions[k - 1].mean.dot(sample.descriptions[k].mean); + const float cosa = dot / (n1 * n2); + + const float maxCosa = 0.95f; // ~ 18deg + if (cosa < maxCosa) + { + valid = false; + } } - lastvalid = int(k); - } - else - { - if (lastvalid != 0) + + if (valid) { - break; + if (firstvalid < 0) + { + firstvalid = int(k) - 1; + } + lastvalid = int(k); + } + else + { + if (lastvalid != 0) + { + break; + } } } - } - if (lastvalid == 0 || firstvalid < 0) - { - sample.descriptions.clear(); - continue; - } + if (lastvalid == 0 || firstvalid < 0) + { + sample.descriptions.clear(); + continue; + } - if (firstvalid > 0 || lastvalid < int(sample.descriptions.size()) - 1) - { - std::vector replace; - for (int pos = firstvalid; pos <= lastvalid; ++pos) + if (firstvalid > 0 || lastvalid < int(sample.descriptions.size()) - 1) { - replace.push_back(sample.descriptions[pos]); + std::vector replace; + for (int pos = firstvalid; pos <= lastvalid; ++pos) + { + replace.push_back(sample.descriptions[pos]); + } + sample.descriptions = replace; } - sample.descriptions = replace; } } } diff --git a/src/aliceVision/hdr/sampling.hpp b/src/aliceVision/hdr/sampling.hpp index 4992a5c8c9..601639dc50 100644 --- a/src/aliceVision/hdr/sampling.hpp +++ b/src/aliceVision/hdr/sampling.hpp @@ -66,7 +66,7 @@ class Sampling void filter(size_t maxTotalPoints); void extractUsefulSamples(std::vector & out_samples, const std::vector & samples, int imageIndex) const; - static bool extractSamplesFromImages(std::vector& out_samples, const std::vector & imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Params params); + static bool extractSamplesFromImages(std::vector& out_samples, const std::vector & imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Params params, const bool simplified = false); private: MapSampleRefList _positions; diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index fb2f571fea..9d408b9883 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -42,77 +42,14 @@ #define ALICEVISION_SOFTWARE_VERSION_MINOR 1 using namespace aliceVision; +using namespace aliceVision::hdr; namespace po = boost::program_options; namespace fs = boost::filesystem; -enum class ECalibrationMethod -{ - LINEAR, - DEBEVEC, - GROSSBERG, - LAGUERRE, -}; - -/** - * @brief convert an enum ECalibrationMethod to its corresponding string - * @param ECalibrationMethod - * @return String - */ -inline std::string ECalibrationMethod_enumToString(const ECalibrationMethod calibrationMethod) -{ - switch(calibrationMethod) - { - case ECalibrationMethod::LINEAR: - return "linear"; - case ECalibrationMethod::DEBEVEC: - return "debevec"; - case ECalibrationMethod::GROSSBERG: - return "grossberg"; - case ECalibrationMethod::LAGUERRE: - return "laguerre"; - } - throw std::out_of_range("Invalid method name enum"); -} - -/** - * @brief convert a string calibration method name to its corresponding enum ECalibrationMethod - * @param ECalibrationMethod - * @return String - */ -inline ECalibrationMethod ECalibrationMethod_stringToEnum(const std::string& calibrationMethodName) -{ - std::string methodName = calibrationMethodName; - std::transform(methodName.begin(), methodName.end(), methodName.begin(), ::tolower); - - if(methodName == "linear") - return ECalibrationMethod::LINEAR; - if(methodName == "debevec") - return ECalibrationMethod::DEBEVEC; - if(methodName == "grossberg") - return ECalibrationMethod::GROSSBERG; - if(methodName == "laguerre") - return ECalibrationMethod::LAGUERRE; - - throw std::out_of_range("Invalid method name : '" + calibrationMethodName + "'"); -} - -inline std::ostream& operator<<(std::ostream& os, ECalibrationMethod calibrationMethodName) -{ - os << ECalibrationMethod_enumToString(calibrationMethodName); - return os; -} - -inline std::istream& operator>>(std::istream& in, ECalibrationMethod& calibrationMethod) -{ - std::string token; - in >> token; - calibrationMethod = ECalibrationMethod_stringToEnum(token); - return in; -} - struct luminanceInfo { + aliceVision::IndexT srcId; double meanLum; double minLum; double maxLum; @@ -199,7 +136,7 @@ int aliceVision_main(int argc, char** argv) std::string sfmInputDataFilename; std::string samplesFolder; std::string outputResponsePath; - ECalibrationMethod calibrationMethod = ECalibrationMethod::LINEAR; + ECalibrationMethod calibrationMethod = ECalibrationMethod::DEBEVEC; std::string calibrationWeightFunction = "default"; int nbBrackets = 0; int channelQuantizationPower = 10; @@ -307,118 +244,84 @@ int aliceVision_main(int argc, char** argv) } std::vector> v_luminanceInfos; + std::vector> calibrationSamples; + hdr::rgbCurve calibrationWeight(channelQuantization); + std::vector> groupedExposures; - if (byPass) + if (samplesFolder.empty()) { - // Compute luminance statistics from raw images - - std::vector> meanLuminances(groupedViews.size(), std::vector(groupedViews[0].size(), 0.0)); - v_luminanceInfos.resize(groupedViews.size()); - - #pragma omp parallel for - for (int g = 0; g < groupedViews.size(); ++g) - { - std::vector> group = groupedViews[g]; - std::map luminanceInfos; - - for (std::size_t i = 0; i < group.size(); ++i) - { - image::Image image; - - const std::string filepath = group[i]->getImagePath(); - - image::ImageReadOptions options; - options.workingColorSpace = workingColorSpace; - options.rawColorInterpretation = image::ERawColorInterpretation_stringToEnum(group[i]->getRawColorInterpretation()); - options.colorProfileFileName = group[i]->getColorProfileFileName(); - image::readImage(filepath, image, options); - - computeLuminanceInfoFromImage(image, luminanceInfos[i]); - } - v_luminanceInfos[g] = luminanceInfos; - } + ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF) and/or estimate the hdr output exposure level."); + return EXIT_FAILURE; } else { - std::vector> calibrationSamples; - hdr::rgbCurve calibrationWeight(channelQuantization); - std::vector> groupedExposures; - - if (calibrationMethod == ECalibrationMethod::LINEAR) + // Build camera exposure table + for (int i = 0; i < groupedViews.size(); ++i) { - ALICEVISION_LOG_INFO("No calibration needed in Linear."); - if (!samplesFolder.empty()) + const std::vector>& group = groupedViews[i]; + std::vector exposuresSetting; + + for (int j = 0; j < group.size(); ++j) { - ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); + const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); + exposuresSetting.push_back(exp); } + if (!sfmData::hasComparableExposures(exposuresSetting)) + { + ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + } + groupedExposures.push_back(getExposures(exposuresSetting)); } - if (samplesFolder.empty()) + size_t group_pos = 0; + hdr::Sampling sampling; + + ALICEVISION_LOG_INFO("Analyzing samples for each group"); + for (auto& group : groupedViews) { - if (calibrationMethod != ECalibrationMethod::LINEAR) + // Read from file + const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); + std::ifstream fileSamples(samplesFilepath, std::ios::binary); + if (!fileSamples.is_open()) { - ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); + ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); return EXIT_FAILURE; } - else + + std::size_t size; + fileSamples.read((char*)&size, sizeof(size)); + + std::vector samples(size); + for (std::size_t i = 0; i < size; ++i) { - ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); - ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); + fileSamples >> samples[i]; } - } - else - { - // Build camera exposure table - for (int i = 0; i < groupedViews.size(); ++i) - { - const std::vector>& group = groupedViews[i]; - std::vector exposuresSetting; - for (int j = 0; j < group.size(); ++j) - { - const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); - exposuresSetting.push_back(exp); - } - if (!sfmData::hasComparableExposures(exposuresSetting)) - { - ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); - } - groupedExposures.push_back(getExposures(exposuresSetting)); - } + sampling.analyzeSource(samples, channelQuantization, group_pos); - size_t group_pos = 0; - hdr::Sampling sampling; + std::map luminanceInfos; + computeLuminanceStatFromSamples(groupedExposures[group_pos], samples, luminanceInfos); - ALICEVISION_LOG_INFO("Analyzing samples for each group"); - for (auto& group : groupedViews) + int v = 0; + for (auto it = luminanceInfos.begin(); it != luminanceInfos.end(); it++) { - // Read from file - const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); - std::ifstream fileSamples(samplesFilepath, std::ios::binary); - if (!fileSamples.is_open()) + if (v < group.size()) { - ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); - return EXIT_FAILURE; + (it->second).srcId = group[v]->getViewId(); + v++; } - - std::size_t size; - fileSamples.read((char*)&size, sizeof(size)); - - std::vector samples(size); - for (std::size_t i = 0; i < size; ++i) + else { - fileSamples >> samples[i]; + (it->second).srcId = 0; } - - sampling.analyzeSource(samples, channelQuantization, group_pos); - - std::map luminanceInfos; - computeLuminanceStatFromSamples(groupedExposures[group_pos], samples, luminanceInfos); - v_luminanceInfos.push_back(luminanceInfos); - - ++group_pos; } + v_luminanceInfos.push_back(luminanceInfos); + + ++group_pos; + } + if (!byPass) + { // We need to trim samples list sampling.filter(maxTotalPoints); @@ -473,7 +376,10 @@ int aliceVision_main(int argc, char** argv) } calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); } + } + if (!byPass) + { ALICEVISION_LOG_INFO("Start calibration"); hdr::rgbCurve response(channelQuantization); @@ -532,11 +438,13 @@ int aliceVision_main(int argc, char** argv) if (!v_luminanceInfos.empty()) { file << v_luminanceInfos[0].size() << std::endl; + file << "# viewId ; exposure ; sampleNumber ; meanLuminance ; minLuminance ; maxLuminance" << std::endl; for (int i = 0; i < v_luminanceInfos.size(); ++i) { for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) { + file << (it->second).srcId << " "; file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; file << (it->second).minLum << " " << (it->second).maxLum << std::endl; } diff --git a/src/software/pipeline/main_LdrToHdrSampling.cpp b/src/software/pipeline/main_LdrToHdrSampling.cpp index e412d64a91..9be8dc5063 100644 --- a/src/software/pipeline/main_LdrToHdrSampling.cpp +++ b/src/software/pipeline/main_LdrToHdrSampling.cpp @@ -44,6 +44,7 @@ #define ALICEVISION_SOFTWARE_VERSION_MINOR 1 using namespace aliceVision; +using namespace aliceVision::hdr; namespace po = boost::program_options; namespace fs = boost::filesystem; @@ -54,8 +55,10 @@ int aliceVision_main(int argc, char** argv) std::string outputFolder; int nbBrackets = 0; int channelQuantizationPower = 10; + ECalibrationMethod calibrationMethod = ECalibrationMethod::DEBEVEC; image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; hdr::Sampling::Params params; + bool byPass = false; bool debug = false; int rangeStart = -1; @@ -74,10 +77,14 @@ int aliceVision_main(int argc, char** argv) optionalParams.add_options() ("nbBrackets,b", po::value(&nbBrackets)->default_value(nbBrackets), "bracket count per HDR image (0 means automatic).") + ("byPass", po::value(&byPass)->default_value(byPass), + "bypass HDR creation and use a single bracket as input for next steps") ("channelQuantizationPower", po::value(&channelQuantizationPower)->default_value(channelQuantizationPower), "Quantization level like 8 bits or 10 bits.") ("workingColorSpace", po::value(&workingColorSpace)->default_value(workingColorSpace), ("Working color space: " + image::EImageColorSpace_informations()).c_str()) + ("calibrationMethod,m", po::value(&calibrationMethod)->default_value(calibrationMethod), + "Name of method used for camera calibration: linear, debevec, grossberg, laguerre.") ("blockSize", po::value(¶ms.blockSize)->default_value(params.blockSize), "Size of the image tile to extract a sample.") ("radius", po::value(¶ms.radius)->default_value(params.radius), @@ -105,7 +112,6 @@ int aliceVision_main(int argc, char** argv) HardwareContext hwc = cmdline.getHardwareContext(); omp_set_num_threads(hwc.getMaxThreads()); - const std::size_t channelQuantization = std::pow(2, channelQuantizationPower); // Read sfm data @@ -216,8 +222,10 @@ int aliceVision_main(int argc, char** argv) imgReadOptions.rawColorInterpretation = rawColorInterpretation; imgReadOptions.colorProfileFileName = colorProfileFileName; + const bool simplifiedSampling = byPass || (calibrationMethod == ECalibrationMethod::LINEAR); + std::vector out_samples; - const bool res = hdr::Sampling::extractSamplesFromImages(out_samples, paths, exposures, width, height, channelQuantization, imgReadOptions, params); + const bool res = hdr::Sampling::extractSamplesFromImages(out_samples, paths, exposures, width, height, channelQuantization, imgReadOptions, params, simplifiedSampling); if (!res) { ALICEVISION_LOG_ERROR("Error while extracting samples from group " << groupIdx); From b3e8a8e3b10c33ce40c16ef4740ee2a707ed61ed Mon Sep 17 00:00:00 2001 From: demoulinv Date: Tue, 24 Jan 2023 15:31:30 +0100 Subject: [PATCH 12/18] [HDR merge] make sure the luminance stat curve is monotonic before searching the best exposure. --- src/aliceVision/hdr/brackets.cpp | 61 +++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 1bb59aea25..53b7e2e4bd 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -157,32 +157,67 @@ void selectTargetViews(std::vector> & out_targetV v_lumaMeanMean.push_back(lumaMeanMean / nbGroup); } - double minDiffWithLumaTarget = 1000.0; - targetIndex = 0; - - for (int k = 0; k < v_lumaMeanMean.size(); ++k) + // Makes sure the luminance curve is monotonic + int firstvalid = -1; + int lastvalid = 0; + for (std::size_t k = 1; k < v_lumaMeanMean.size(); ++k) { - const double diffWithLumaTarget = fabs(v_lumaMeanMean[k] - meanTargetedLuma); - if (diffWithLumaTarget < minDiffWithLumaTarget) + bool valid = false; + + if (v_lumaMeanMean[k] > v_lumaMeanMean[k - 1]) + { + valid = true; + } + + if (valid) + { + if (firstvalid < 0) + { + firstvalid = int(k) - 1; + } + lastvalid = int(k); + } + else { - minDiffWithLumaTarget = diffWithLumaTarget; - targetIndex = k; + if (lastvalid != 0) + { + break; + } } } - ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex); + if (lastvalid >= firstvalid && firstvalid >= 0) + { + double minDiffWithLumaTarget = 1000.0; + targetIndex = 0; + + for (int k = firstvalid; k <= lastvalid; ++k) + { + const double diffWithLumaTarget = fabs(v_lumaMeanMean[k] - meanTargetedLuma); + if (diffWithLumaTarget < minDiffWithLumaTarget) + { + minDiffWithLumaTarget = diffWithLumaTarget; + targetIndex = k; + } + } + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex); + } + else + { + targetIndex = middleIndex; + ALICEVISION_LOG_WARNING("Non valid luminance statistics file, offsetRefBracketIndex parameter set to medium exposure " << targetIndex); + } } + for (auto& group : groups) + { //Set the ldr ancestors id for (auto v : group) { group[targetIndex]->addAncestor(v->getViewId()); } - for (int i = 0; i < groups.size(); ++i) - { - out_targetViews.push_back(groups[i][targetIndexes[i]]); - } + out_targetViews.push_back(group[targetIndex]); } return; } From a2e442e16c2f60fbaf99794c7138e53bbfd16012 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Fri, 3 Feb 2023 15:56:30 +0100 Subject: [PATCH 13/18] [HDR merge] Bugfix: single exposure case --- src/software/pipeline/main_LdrToHdrMerge.cpp | 75 ++++++++++++-------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 61fe39903f..b84091ae0c 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -183,45 +183,47 @@ int aliceVision_main(int argc, char** argv) } std::vector> targetViews; - const int middleIndex = usedNbBrackets / 2; - const int targetIndex = middleIndex + offsetRefBracketIndex; - const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); + if (!byPass) + { + const int middleIndex = usedNbBrackets / 2; + const int targetIndex = middleIndex + offsetRefBracketIndex; + const bool isOffsetRefBracketIndexValid = (targetIndex >= 0) && (targetIndex < usedNbBrackets); - const fs::path lumaStatFilepath(fs::path(inputResponsePath).parent_path() / (std::string("luminanceStatistics.txt"))); + const fs::path lumaStatFilepath(fs::path(inputResponsePath).parent_path() / (std::string("luminanceStatistics.txt"))); - if (!fs::is_regular_file(lumaStatFilepath) && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("Unable to open the file " << lumaStatFilepath.string() << " with luminance statistics. This file is needed to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; - } + if (!fs::is_regular_file(lumaStatFilepath) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("Unable to open the file " << lumaStatFilepath.string() << " with luminance statistics. This file is needed to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } - // Adjust the targeted luminance level by removing the corresponding gamma if the working color space is not sRGB. - if (workingColorSpace != image::EImageColorSpace::SRGB) - { - meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2); - } - hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); + // Adjust the targeted luminance level by removing the corresponding gamma if the working color space is not sRGB. + if (workingColorSpace != image::EImageColorSpace::SRGB) + { + meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2); + } + hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); - if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid) - { - ALICEVISION_LOG_ERROR("File " << lumaStatFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); - return EXIT_FAILURE; + if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid) + { + ALICEVISION_LOG_ERROR("File " << lumaStatFilepath.string() << " is not valid. This file is required to select the optimal exposure for the creation of HDR images."); + return EXIT_FAILURE; + } } // Define range to compute if(rangeStart != -1) { - if(rangeStart < 0 || rangeSize < 0 || - rangeStart > groupedViews.size()) - { - ALICEVISION_LOG_ERROR("Range is incorrect"); - return EXIT_FAILURE; - } + if(rangeStart < 0 || rangeSize < 0 || rangeStart > groupedViews.size()) + { + ALICEVISION_LOG_ERROR("Range is incorrect"); + return EXIT_FAILURE; + } - if(rangeStart + rangeSize > groupedViews.size()) - { - rangeSize = groupedViews.size() - rangeStart; - } + if(rangeStart + rangeSize > groupedViews.size()) + { + rangeSize = groupedViews.size() - rangeStart; + } } else { @@ -239,7 +241,20 @@ int aliceVision_main(int argc, char** argv) // Export a new sfmData with HDR images as new Views. for(std::size_t g = 0; g < groupedViews.size(); ++g) { - std::shared_ptr hdrView = std::make_shared(*targetViews[g]); + std::shared_ptr hdrView; + if (groupedViews[g].size() == 1) + { + hdrView = std::make_shared(*groupedViews[g][0]); + } + else if (targetViews.empty()) + { + ALICEVISION_LOG_ERROR("Target view for HDR merging has not been computed"); + return EXIT_FAILURE; + } + else + { + hdrView = std::make_shared(*targetViews[g]); + } if(!byPass) { const std::string hdrImagePath = getHdrImagePath(outputPath, g); From fd0f221da2dfca144516e2f50a94523d86fafc9f Mon Sep 17 00:00:00 2001 From: demoulinv Date: Fri, 3 Feb 2023 16:01:51 +0100 Subject: [PATCH 14/18] [HDR merging] Update trace about automatic offset bracket index. --- src/aliceVision/hdr/brackets.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 53b7e2e4bd..bf6f8506dd 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -200,12 +200,12 @@ void selectTargetViews(std::vector> & out_targetV targetIndex = k; } } - ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex); + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex - middleIndex); } else { targetIndex = middleIndex; - ALICEVISION_LOG_WARNING("Non valid luminance statistics file, offsetRefBracketIndex parameter set to medium exposure " << targetIndex); + ALICEVISION_LOG_WARNING("Non valid luminance statistics file, offsetRefBracketIndex parameter set to 0 (medium exposure)"); } } From 731eeb5687cd1c557d1f34b7a5948b28f9b2f4aa Mon Sep 17 00:00:00 2001 From: demoulinv Date: Mon, 6 Feb 2023 15:55:43 +0100 Subject: [PATCH 15/18] [HDR sampling] Add check about set of exposure consistency. Return error if non consistent group of exposures are detected. --- src/aliceVision/hdr/brackets.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index bf6f8506dd..3288c8d9a2 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -89,6 +89,7 @@ bool estimateBracketsFromSfmData(std::vector> v_exposuresSetting; for(auto & group : groups) { // Sort all images by exposure time @@ -98,6 +99,28 @@ bool estimateBracketsFromSfmData(std::vectorgetCameraExposureSetting().getExposure() < b->getCameraExposureSetting().getExposure()); }); + std::vector exposuresSetting; + for (auto& v : group) + { + exposuresSetting.push_back(v->getCameraExposureSetting()); + } + v_exposuresSetting.push_back(exposuresSetting); + } + + // Check exposure consistency between group + if (v_exposuresSetting.size() > 1) + { + for (int g=1 ; g < v_exposuresSetting.size(); g++) + { + for (int e = 0; e < v_exposuresSetting[g].size(); e++) + { + if (!(v_exposuresSetting[g][e] == v_exposuresSetting[g - 1][e])) + { + ALICEVISION_LOG_ERROR("Non consistant exposures between groups have been detected. Check your dataset metadata."); + return false; + } + } + } } return true; From 2c676a78b94e7c3c19257cb5bd1593b133b9ff29 Mon Sep 17 00:00:00 2001 From: demoulinv Date: Thu, 9 Feb 2023 09:32:27 +0100 Subject: [PATCH 16/18] [HDR fusion] Accept unaligned exposures between poses. Add origin viewID in image samples. --- src/aliceVision/hdr/brackets.cpp | 19 +++-- src/aliceVision/hdr/hdrTestCommon.hpp | 8 +- src/aliceVision/hdr/sampling.cpp | 10 ++- src/aliceVision/hdr/sampling.hpp | 3 +- .../pipeline/main_LdrToHdrCalibration.cpp | 85 ++++++++++--------- .../pipeline/main_LdrToHdrSampling.cpp | 4 +- 6 files changed, 76 insertions(+), 53 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 3288c8d9a2..e923295d81 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -110,14 +110,15 @@ bool estimateBracketsFromSfmData(std::vector 1) { - for (int g=1 ; g < v_exposuresSetting.size(); g++) + for (int g = 1 ; g < v_exposuresSetting.size(); ++g) { - for (int e = 0; e < v_exposuresSetting[g].size(); e++) + for (int e = 0; e < v_exposuresSetting[g].size(); ++e) { if (!(v_exposuresSetting[g][e] == v_exposuresSetting[g - 1][e])) { - ALICEVISION_LOG_ERROR("Non consistant exposures between groups have been detected. Check your dataset metadata."); - return false; + ALICEVISION_LOG_WARNING("Non consistant exposures between poses have been detected. Most likely the dataset has been captured with an automatic exposure mode enabled. Final result can be impacted."); + g = v_exposuresSetting.size(); + break; } } } @@ -153,12 +154,12 @@ void selectTargetViews(std::vector> & out_targetV { lines.push_back(line); } - if ((lines.size() < 3) || (atoi(lines[0].c_str()) != groups.size()) || (atoi(lines[1].c_str()) < groups[0].size())) + if ((lines.size() < 3) || (std::stoi(lines[0].c_str()) != groups.size()) || (std::stoi(lines[1].c_str()) < groups[0].size())) { ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); } - int nbGroup = atoi(lines[0].c_str()); - int nbExp = atoi(lines[1].c_str()); + int nbGroup = std::stoi(lines[0].c_str()); + int nbExp = std::stoi(lines[1].c_str()); std::vector v_lumaMeanMean; @@ -169,8 +170,8 @@ void selectTargetViews(std::vector> & out_targetV { std::istringstream iss(lines[3 + j * nbExp + i]); aliceVision::IndexT srcId; - int exposure, nbItem; - double lumaMean, lumaMin, lumaMax; + int nbItem; + double exposure, lumaMean, lumaMin, lumaMax; if (!(iss >> srcId >> exposure >> nbItem >> lumaMean >> lumaMin >> lumaMax)) { ALICEVISION_THROW_ERROR("File: " << lumaStatFilepath << " is not a valid file"); diff --git a/src/aliceVision/hdr/hdrTestCommon.hpp b/src/aliceVision/hdr/hdrTestCommon.hpp index ebfa9fccb3..43c49e9278 100644 --- a/src/aliceVision/hdr/hdrTestCommon.hpp +++ b/src/aliceVision/hdr/hdrTestCommon.hpp @@ -38,8 +38,14 @@ bool extractSamplesGroups(std::vector>& out_samples, imgReadOptions.rawColorInterpretation = image::ERawColorInterpretation::LibRawWhiteBalancing; imgReadOptions.workingColorSpace = image::EImageColorSpace::LINEAR; + std::vector viewIds; + for (size_t id = 0; id < imagePaths[idGroup].size(); ++id) + { + viewIds.push_back(id); + } + std::vector groupSamples; - if (!Sampling::extractSamplesFromImages(groupSamples, imagePaths[idGroup], + if (!Sampling::extractSamplesFromImages(groupSamples, imagePaths[idGroup], viewIds, times[idGroup], width, height, channelQuantization, imgReadOptions, Sampling::Params{})) diff --git a/src/aliceVision/hdr/sampling.cpp b/src/aliceVision/hdr/sampling.cpp index e0b77e6cf6..db9245aaa5 100644 --- a/src/aliceVision/hdr/sampling.cpp +++ b/src/aliceVision/hdr/sampling.cpp @@ -65,7 +65,8 @@ std::istream & operator>>(std::istream& is, ImageSample & s) std::ostream & operator<<(std::ostream& os, const PixelDescription & p) { - os.write((const char *)&p.exposure, sizeof(p.exposure)); + os.write((const char*)&p.srcId, sizeof(p.srcId)); + os.write((const char*)&p.exposure, sizeof(p.exposure)); os.write((const char *)&p.mean.r(), sizeof(p.mean.r())); os.write((const char *)&p.mean.g(), sizeof(p.mean.g())); os.write((const char *)&p.mean.b(), sizeof(p.mean.b())); @@ -78,7 +79,8 @@ std::ostream & operator<<(std::ostream& os, const PixelDescription & p) std::istream & operator>>(std::istream& is, PixelDescription & p) { - is.read((char *)&p.exposure, sizeof(p.exposure)); + is.read((char*)&p.srcId, sizeof(p.srcId)); + is.read((char*)&p.exposure, sizeof(p.exposure)); is.read((char *)&p.mean.r(), sizeof(p.mean.r())); is.read((char *)&p.mean.g(), sizeof(p.mean.g())); is.read((char *)&p.mean.b(), sizeof(p.mean.b())); @@ -146,7 +148,7 @@ void square(image::Image & dest, const Eigen::Matrix& out_samples, const std::vector& imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Sampling::Params params, const bool simplified) +bool Sampling::extractSamplesFromImages(std::vector& out_samples, const std::vector& imagePaths, const std::vector& viewIds, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Sampling::Params params, const bool simplified) { const int radiusp1 = params.radius + 1; const int diameter = (params.radius * 2) + 1; @@ -217,6 +219,7 @@ bool Sampling::extractSamplesFromImages(std::vector& out_samples, c { PixelDescription pd; + pd.srcId = viewIds[idBracket]; pd.exposure = exposure; pd.mean.r() = img(r, c).r(); pd.mean.g() = img(r, c).g(); @@ -262,6 +265,7 @@ bool Sampling::extractSamplesFromImages(std::vector& out_samples, c PixelDescription pd; + pd.srcId = viewIds[idBracket]; pd.exposure = exposure; pd.mean.r() = blockInput(y, x).r(); pd.mean.g() = blockInput(y, x).g(); diff --git a/src/aliceVision/hdr/sampling.hpp b/src/aliceVision/hdr/sampling.hpp index 601639dc50..c5ca4fb717 100644 --- a/src/aliceVision/hdr/sampling.hpp +++ b/src/aliceVision/hdr/sampling.hpp @@ -24,6 +24,7 @@ struct UniqueDescriptor struct PixelDescription { + aliceVision::IndexT srcId = 0; float exposure; image::Rgb mean; image::Rgb variance; @@ -66,7 +67,7 @@ class Sampling void filter(size_t maxTotalPoints); void extractUsefulSamples(std::vector & out_samples, const std::vector & samples, int imageIndex) const; - static bool extractSamplesFromImages(std::vector& out_samples, const std::vector & imagePaths, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Params params, const bool simplified = false); + static bool extractSamplesFromImages(std::vector& out_samples, const std::vector & imagePaths, const std::vector& viewIds, const std::vector& times, const size_t imageWidth, const size_t imageHeight, const size_t channelQuantization, const image::ImageReadOptions & imgReadOptions, const Params params, const bool simplified = false); private: MapSampleRefList _positions; diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 9d408b9883..fdc2af1997 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -49,36 +49,43 @@ namespace fs = boost::filesystem; struct luminanceInfo { - aliceVision::IndexT srcId; - double meanLum; - double minLum; - double maxLum; - int itemNb; + double exposure = 0.0; + double meanLum = 0.0; + double minLum = 1e6; + double maxLum = 0.0; + int itemNb = 0; + + luminanceInfo() = default; }; -void computeLuminanceStatFromSamples(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) +void computeLuminanceStatFromSamples(const std::vector& samples, std::map& luminanceInfos) { - for (int i = 0; i < groupedExposure.size(); i++) - { - luminanceInfos[(int)(groupedExposure[i] * 1000)].itemNb = 0; - luminanceInfos[(int)(groupedExposure[i] * 1000)].meanLum = 0.0; - luminanceInfos[(int)(groupedExposure[i] * 1000)].minLum = 1000.0; - luminanceInfos[(int)(groupedExposure[i] * 1000)].maxLum = 0.0; - } + luminanceInfo lumaInfo; + luminanceInfos.clear(); + for (int i = 0; i < samples.size(); i++) { for (int j = 0; j < samples[i].descriptions.size(); j++) { - double lum = image::Rgb2GrayLinear(samples[i].descriptions[j].mean[0], samples[i].descriptions[j].mean[1], samples[i].descriptions[j].mean[2]); - luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].meanLum += lum; - luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].itemNb++; - if (lum < luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].minLum) + const IndexT key = samples[i].descriptions[j].srcId; + + const double lum = image::Rgb2GrayLinear(samples[i].descriptions[j].mean[0], samples[i].descriptions[j].mean[1], samples[i].descriptions[j].mean[2]); + + if (luminanceInfos.find(key) == luminanceInfos.end()) + { + luminanceInfos[key] = lumaInfo; + luminanceInfos[key].exposure = samples[i].descriptions[j].exposure; + } + + luminanceInfos[key].meanLum += lum; + luminanceInfos[key].itemNb++; + if (lum < luminanceInfos[key].minLum) { - luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].minLum = lum; + luminanceInfos[key].minLum = lum; } - if (lum > luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].maxLum) + if (lum > luminanceInfos[key].maxLum) { - luminanceInfos[(int)(samples[i].descriptions[j].exposure * 1000)].maxLum = lum; + luminanceInfos[key].maxLum = lum; } } } @@ -300,21 +307,8 @@ int aliceVision_main(int argc, char** argv) sampling.analyzeSource(samples, channelQuantization, group_pos); std::map luminanceInfos; - computeLuminanceStatFromSamples(groupedExposures[group_pos], samples, luminanceInfos); + computeLuminanceStatFromSamples(samples, luminanceInfos); - int v = 0; - for (auto it = luminanceInfos.begin(); it != luminanceInfos.end(); it++) - { - if (v < group.size()) - { - (it->second).srcId = group[v]->getViewId(); - v++; - } - else - { - (it->second).srcId = 0; - } - } v_luminanceInfos.push_back(luminanceInfos); ++group_pos; @@ -442,11 +436,26 @@ int aliceVision_main(int argc, char** argv) for (int i = 0; i < v_luminanceInfos.size(); ++i) { - for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) + while (!v_luminanceInfos[i].empty()) { - file << (it->second).srcId << " "; - file << it->first << " " << (it->second).itemNb << " " << (it->second).meanLum / (it->second).itemNb << " "; - file << (it->second).minLum << " " << (it->second).maxLum << std::endl; + // search min exposure + IndexT srcIdWithMinimalExposure = UndefinedIndexT; + double exposureMin = 1e9; + for (auto it = v_luminanceInfos[i].begin(); it != v_luminanceInfos[i].end(); it++) + { + if ((it->second).exposure < exposureMin) + { + exposureMin = (it->second).exposure; + srcIdWithMinimalExposure = it->first; + } + } + // write in file + file << srcIdWithMinimalExposure << " "; + file << v_luminanceInfos[i][srcIdWithMinimalExposure].exposure << " " << v_luminanceInfos[i][srcIdWithMinimalExposure].itemNb << " "; + file << v_luminanceInfos[i][srcIdWithMinimalExposure].meanLum / v_luminanceInfos[i][srcIdWithMinimalExposure].itemNb << " "; + file << v_luminanceInfos[i][srcIdWithMinimalExposure].minLum << " " << v_luminanceInfos[i][srcIdWithMinimalExposure].maxLum << std::endl; + // erase from map + v_luminanceInfos[i].erase(srcIdWithMinimalExposure); } } } diff --git a/src/software/pipeline/main_LdrToHdrSampling.cpp b/src/software/pipeline/main_LdrToHdrSampling.cpp index 9be8dc5063..1d98092bd5 100644 --- a/src/software/pipeline/main_LdrToHdrSampling.cpp +++ b/src/software/pipeline/main_LdrToHdrSampling.cpp @@ -196,6 +196,7 @@ int aliceVision_main(int argc, char** argv) std::vector paths; std::vector exposuresSetting; + std::vector viewIds; image::ERawColorInterpretation rawColorInterpretation = image::ERawColorInterpretation::LibRawWhiteBalancing; std::string colorProfileFileName = ""; @@ -204,6 +205,7 @@ int aliceVision_main(int argc, char** argv) { paths.push_back(v->getImagePath()); exposuresSetting.push_back(v->getCameraExposureSetting()); + viewIds.push_back(v->getViewId()); const std::string rawColorInterpretation_str = v->getRawColorInterpretation(); rawColorInterpretation = image::ERawColorInterpretation_stringToEnum(rawColorInterpretation_str); @@ -225,7 +227,7 @@ int aliceVision_main(int argc, char** argv) const bool simplifiedSampling = byPass || (calibrationMethod == ECalibrationMethod::LINEAR); std::vector out_samples; - const bool res = hdr::Sampling::extractSamplesFromImages(out_samples, paths, exposures, width, height, channelQuantization, imgReadOptions, params, simplifiedSampling); + const bool res = hdr::Sampling::extractSamplesFromImages(out_samples, paths, viewIds, exposures, width, height, channelQuantization, imgReadOptions, params, simplifiedSampling); if (!res) { ALICEVISION_LOG_ERROR("Error while extracting samples from group " << groupIdx); From 63b7366490b5a15fa8a337ddb80dfabaf01901bd Mon Sep 17 00:00:00 2001 From: demoulinv Date: Thu, 9 Feb 2023 15:57:28 +0100 Subject: [PATCH 17/18] [HDR Merge] Remove monotonicity check of the mean luminance curve. Such a check is done for every poses when the luminance statistic file is built. --- src/aliceVision/hdr/brackets.cpp | 55 ++++++-------------------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index e923295d81..2c31db15c2 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -129,7 +129,7 @@ bool estimateBracketsFromSfmData(std::vector> & out_targetViews, const std::vector>> & groups, int offsetRefBracketIndex, const std::string& lumaStatFilepath, const double meanTargetedLuma) { - // If targetIndexesFilename cannot be opened or is not valid target views are derived from the middle exposed views + // If targetIndexesFilename cannot be opened or is not valid an error is thrown // For odd number, there is no ambiguity on the middle image. // For even number, we arbitrarily choose the more exposed view (as we usually have more under-exposed images than over-exposed). const int viewNumberPerGroup = groups[0].size(); @@ -181,56 +181,19 @@ void selectTargetViews(std::vector> & out_targetV v_lumaMeanMean.push_back(lumaMeanMean / nbGroup); } - // Makes sure the luminance curve is monotonic - int firstvalid = -1; - int lastvalid = 0; - for (std::size_t k = 1; k < v_lumaMeanMean.size(); ++k) - { - bool valid = false; - - if (v_lumaMeanMean[k] > v_lumaMeanMean[k - 1]) - { - valid = true; - } + double minDiffWithLumaTarget = 1000.0; + targetIndex = 0; - if (valid) - { - if (firstvalid < 0) - { - firstvalid = int(k) - 1; - } - lastvalid = int(k); - } - else - { - if (lastvalid != 0) - { - break; - } - } - } - - if (lastvalid >= firstvalid && firstvalid >= 0) + for (int k = 0; k < v_lumaMeanMean.size(); ++k) { - double minDiffWithLumaTarget = 1000.0; - targetIndex = 0; - - for (int k = firstvalid; k <= lastvalid; ++k) + const double diffWithLumaTarget = (v_lumaMeanMean[k] > meanTargetedLuma) ? (v_lumaMeanMean[k] - meanTargetedLuma) : (meanTargetedLuma - v_lumaMeanMean[k]); + if (diffWithLumaTarget < minDiffWithLumaTarget) { - const double diffWithLumaTarget = fabs(v_lumaMeanMean[k] - meanTargetedLuma); - if (diffWithLumaTarget < minDiffWithLumaTarget) - { - minDiffWithLumaTarget = diffWithLumaTarget; - targetIndex = k; - } + minDiffWithLumaTarget = diffWithLumaTarget; + targetIndex = k; } - ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex - middleIndex); - } - else - { - targetIndex = middleIndex; - ALICEVISION_LOG_WARNING("Non valid luminance statistics file, offsetRefBracketIndex parameter set to 0 (medium exposure)"); } + ALICEVISION_LOG_INFO("offsetRefBracketIndex parameter automaticaly set to " << targetIndex - middleIndex); } for (auto& group : groups) From 8fed92a45e8b163eb1f1e5b2413ed659312d40bf Mon Sep 17 00:00:00 2001 From: demoulinv Date: Mon, 13 Feb 2023 19:26:19 +0100 Subject: [PATCH 18/18] [HDR merging] Check luminance curve increase wrt exposure in highlights. --- src/aliceVision/hdr/brackets.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/hdr/brackets.cpp b/src/aliceVision/hdr/brackets.cpp index 2c31db15c2..f3e6df4215 100644 --- a/src/aliceVision/hdr/brackets.cpp +++ b/src/aliceVision/hdr/brackets.cpp @@ -181,10 +181,17 @@ void selectTargetViews(std::vector> & out_targetV v_lumaMeanMean.push_back(lumaMeanMean / nbGroup); } + // adjust last index to avoid non increasing luminance curve due to saturation in highlights + int lastIdx = v_lumaMeanMean.size() - 1; + while ((lastIdx > 1) && ((v_lumaMeanMean[lastIdx] < v_lumaMeanMean[lastIdx - 1]) || (v_lumaMeanMean[lastIdx] < v_lumaMeanMean[lastIdx - 2]))) + { + lastIdx--; + } + double minDiffWithLumaTarget = 1000.0; targetIndex = 0; - for (int k = 0; k < v_lumaMeanMean.size(); ++k) + for (int k = 0; k < lastIdx; ++k) { const double diffWithLumaTarget = (v_lumaMeanMean[k] > meanTargetedLuma) ? (v_lumaMeanMean[k] - meanTargetedLuma) : (meanTargetedLuma - v_lumaMeanMean[k]); if (diffWithLumaTarget < minDiffWithLumaTarget)