From 659717c3639a7d17639b5b20e2538fa15bc6de7b Mon Sep 17 00:00:00 2001 From: demoulinv Date: Wed, 18 Jan 2023 11:33:44 +0100 Subject: [PATCH] [HDR Calib & Merge] Manage bypass mode by computing luminance statistics from images at the calibration stage. Adjust the targeted luminance level for the output HDR image in case of non sRGB working space. --- .../pipeline/main_LdrToHdrCalibration.cpp | 307 +++++++++++------- src/software/pipeline/main_LdrToHdrMerge.cpp | 4 + 2 files changed, 200 insertions(+), 111 deletions(-) diff --git a/src/software/pipeline/main_LdrToHdrCalibration.cpp b/src/software/pipeline/main_LdrToHdrCalibration.cpp index 6c479273cf..1b4cd9cab3 100644 --- a/src/software/pipeline/main_LdrToHdrCalibration.cpp +++ b/src/software/pipeline/main_LdrToHdrCalibration.cpp @@ -119,7 +119,7 @@ struct luminanceInfo int itemNb; }; -void computeLuminanceStat(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) +void computeLuminanceStatFromSamples(const std::vector& groupedExposure, const std::vector& samples, std::map& luminanceInfos) { for (int i = 0; i < groupedExposure.size(); i++) { @@ -147,6 +147,49 @@ void computeLuminanceStat(const std::vector& groupedExposure, const std: } } +void computeLuminanceInfoFromImage(image::Image& image, luminanceInfo& lumaInfo) +{ + // Luminance statistics are calculated from a subsampled square, centered and rotated by 45°. + // 2 vertices of this square are the centers of the longest sides of the image. + // Such a shape is suitable for both fisheye and classic images. + + double meanLuminance = 0.0; + double maxLuminance = 0.0; + double minLuminance = 1000.0; + int sampleNb = 0; + + const int imgH = image.Height(); + const int imgW = image.Width(); + + const int a1 = (imgH <= imgW) ? imgW / 2 : imgH / 2; + const int a2 = (imgH <= imgW) ? imgW / 2 : imgW - (imgH / 2); + const int a3 = (imgH <= imgW) ? imgH - (imgW / 2) : imgH / 2; + const int a4 = (imgH <= imgW) ? (imgW / 2) + imgH : imgW + (imgH / 2); + + const int rmin = (imgH <= imgW) ? 0 : (imgH - imgW) / 2; + const int rmax = (imgH <= imgW) ? imgH : (imgH + imgW) / 2; + + for (int r = rmin; r < rmax; r = r + 16) + { + const int cmin = (r < imgH / 2) ? a1 - r : r - a3; + const int cmax = (r < imgH / 2) ? a2 + r : a4 - r; + + for (int c = cmin; c < cmax; c = c + 16) + { + double luma = image::Rgb2GrayLinear(image(r, c)[0], image(r, c)[1], image(r, c)[2]); + meanLuminance += luma; + minLuminance = (luma < minLuminance) ? luma : minLuminance; + maxLuminance = (luma > maxLuminance) ? luma : maxLuminance; + sampleNb++; + } + } + + lumaInfo.itemNb = sampleNb; + lumaInfo.minLum = minLuminance; + lumaInfo.maxLum = maxLuminance; + lumaInfo.meanLum = meanLuminance; +} + int aliceVision_main(int argc, char** argv) { std::string sfmInputDataFilename; @@ -156,7 +199,9 @@ int aliceVision_main(int argc, char** argv) std::string calibrationWeightFunction = "default"; int nbBrackets = 0; int channelQuantizationPower = 10; + image::EImageColorSpace workingColorSpace = image::EImageColorSpace::SRGB; size_t maxTotalPoints = 1000000; + bool byPass = false; // Command line parameters @@ -175,11 +220,15 @@ int aliceVision_main(int argc, char** argv) "Name of method used for camera calibration: linear, debevec, grossberg, laguerre.") ("calibrationWeight,w", po::value(&calibrationWeightFunction)->default_value(calibrationWeightFunction), "Weight function used to calibrate camera response (default depends on the calibration method, gaussian, " - "triangle, plateau).") + "triangle, plateau).") ("nbBrackets,b", po::value(&nbBrackets)->default_value(nbBrackets), "bracket count per HDR image (0 means automatic).") + ("byPass", po::value(&byPass)->default_value(byPass), + "bypass HDR creation and use a single bracket as input for next steps") ("channelQuantizationPower", po::value(&channelQuantizationPower)->default_value(channelQuantizationPower), "Quantization level like 8 bits or 10 bits.") + ("workingColorSpace", po::value(&workingColorSpace)->default_value(workingColorSpace), + ("Working color space: " + image::EImageColorSpace_informations()).c_str()) ("maxTotalPoints", po::value(&maxTotalPoints)->default_value(maxTotalPoints), "Max number of points used from the sampling. This ensures that the number of pixels values extracted by the sampling " "can be managed by the calibration step (in term of computation time and memory usage).") @@ -253,146 +302,181 @@ int aliceVision_main(int argc, char** argv) } } - std::vector> calibrationSamples; - hdr::rgbCurve calibrationWeight(channelQuantization); - std::vector> groupedExposures; std::vector> v_luminanceInfos; - if(calibrationMethod == ECalibrationMethod::LINEAR) - { - ALICEVISION_LOG_INFO("No calibration needed in Linear."); - if(!samplesFolder.empty()) - { - ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); - } - } - - if(samplesFolder.empty()) + if (byPass) { - if (calibrationMethod != ECalibrationMethod::LINEAR) - { - ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); - return EXIT_FAILURE; - } - else + // Compute luminance statistics from raw images + + std::vector> meanLuminances(groupedViews.size(), std::vector(groupedViews[0].size(), 0.0)); + v_luminanceInfos.resize(groupedViews.size()); + + #pragma omp parallel for + for (int g = 0; g < groupedViews.size(); ++g) { - ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); - ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); + std::vector> group = groupedViews[g]; + std::map luminanceInfos; + + for (std::size_t i = 0; i < group.size(); ++i) + { + image::Image image; + + const std::string filepath = group[i]->getImagePath(); + + image::ImageReadOptions options; + options.workingColorSpace = workingColorSpace; + options.rawColorInterpretation = image::ERawColorInterpretation_stringToEnum(group[i]->getRawColorInterpretation()); + options.colorProfileFileName = group[i]->getColorProfileFileName(); + image::readImage(filepath, image, options); + + computeLuminanceInfoFromImage(image, luminanceInfos[i]); + } + v_luminanceInfos[g] = luminanceInfos; } } else { - // Build camera exposure table - for (int i = 0; i < groupedViews.size(); ++i) - { - const std::vector>& group = groupedViews[i]; - std::vector exposuresSetting; - for (int j = 0; j < group.size(); ++j) - { - const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); - exposuresSetting.push_back(exp); - } - if (!sfmData::hasComparableExposures(exposuresSetting)) + std::vector> calibrationSamples; + hdr::rgbCurve calibrationWeight(channelQuantization); + std::vector> groupedExposures; + //std::vector> v_luminanceInfos; + + if (calibrationMethod == ECalibrationMethod::LINEAR) + { + ALICEVISION_LOG_INFO("No calibration needed in Linear."); + if (!samplesFolder.empty()) { - ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + ALICEVISION_LOG_WARNING("The provided input sampling folder will not be used."); } - groupedExposures.push_back(getExposures(exposuresSetting)); } - size_t group_pos = 0; - hdr::Sampling sampling; - - ALICEVISION_LOG_INFO("Analyzing samples for each group"); - for (auto& group : groupedViews) + if (samplesFolder.empty()) { - // Read from file - const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); - std::ifstream fileSamples(samplesFilepath, std::ios::binary); - if (!fileSamples.is_open()) + if (calibrationMethod != ECalibrationMethod::LINEAR) { - ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + ALICEVISION_LOG_ERROR("A folder with selected samples is required to calibrate the Camera Response Function (CRF)."); return EXIT_FAILURE; } - - std::size_t size; - fileSamples.read((char*)&size, sizeof(size)); - - std::vector samples(size); - for (std::size_t i = 0; i < size; ++i) + else { - fileSamples >> samples[i]; + ALICEVISION_LOG_WARNING("A folder with selected samples is required to estimate the hdr output exposure level."); + ALICEVISION_LOG_WARNING("You will have to set manually the dedicated paramater at merging stage."); } - - sampling.analyzeSource(samples, channelQuantization, group_pos); - - std::map luminanceInfos; - computeLuminanceStat(groupedExposures[group_pos], samples, luminanceInfos); - v_luminanceInfos.push_back(luminanceInfos); - - ++group_pos; } - - // We need to trim samples list - sampling.filter(maxTotalPoints); - - ALICEVISION_LOG_INFO("Extracting samples for each group"); - group_pos = 0; - - std::size_t total = 0; - for (auto& group : groupedViews) + else { - // Read from file - const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); - std::ifstream fileSamples(samplesFilepath, std::ios::binary); - if (!fileSamples.is_open()) + // Build camera exposure table + for (int i = 0; i < groupedViews.size(); ++i) { - ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); - return EXIT_FAILURE; + const std::vector>& group = groupedViews[i]; + std::vector exposuresSetting; + + for (int j = 0; j < group.size(); ++j) + { + const sfmData::ExposureSetting exp = group[j]->getCameraExposureSetting(); + exposuresSetting.push_back(exp); + } + if (!sfmData::hasComparableExposures(exposuresSetting)) + { + ALICEVISION_THROW_ERROR("Camera exposure settings are inconsistent."); + } + groupedExposures.push_back(getExposures(exposuresSetting)); } - std::size_t size = 0; - fileSamples.read((char*)&size, sizeof(size)); + size_t group_pos = 0; + hdr::Sampling sampling; - std::vector samples(size); - for (int i = 0; i < size; ++i) + ALICEVISION_LOG_INFO("Analyzing samples for each group"); + for (auto& group : groupedViews) { - fileSamples >> samples[i]; + // Read from file + const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); + std::ifstream fileSamples(samplesFilepath, std::ios::binary); + if (!fileSamples.is_open()) + { + ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + return EXIT_FAILURE; + } + + std::size_t size; + fileSamples.read((char*)&size, sizeof(size)); + + std::vector samples(size); + for (std::size_t i = 0; i < size; ++i) + { + fileSamples >> samples[i]; + } + + sampling.analyzeSource(samples, channelQuantization, group_pos); + + std::map luminanceInfos; + computeLuminanceStatFromSamples(groupedExposures[group_pos], samples, luminanceInfos); + v_luminanceInfos.push_back(luminanceInfos); + + ++group_pos; } - std::vector out_samples; - sampling.extractUsefulSamples(out_samples, samples, group_pos); + // We need to trim samples list + sampling.filter(maxTotalPoints); - calibrationSamples.push_back(out_samples); + ALICEVISION_LOG_INFO("Extracting samples for each group"); + group_pos = 0; - ++group_pos; - } + std::size_t total = 0; + for (auto& group : groupedViews) + { + // Read from file + const std::string samplesFilepath = (fs::path(samplesFolder) / (std::to_string(group_pos) + "_samples.dat")).string(); + std::ifstream fileSamples(samplesFilepath, std::ios::binary); + if (!fileSamples.is_open()) + { + ALICEVISION_LOG_ERROR("Impossible to read samples from file " << samplesFilepath); + return EXIT_FAILURE; + } + + std::size_t size = 0; + fileSamples.read((char*)&size, sizeof(size)); + + std::vector samples(size); + for (int i = 0; i < size; ++i) + { + fileSamples >> samples[i]; + } + + std::vector out_samples; + sampling.extractUsefulSamples(out_samples, samples, group_pos); + + calibrationSamples.push_back(out_samples); + + ++group_pos; + } - // Define calibration weighting curve from name - boost::algorithm::to_lower(calibrationWeightFunction); - if (calibrationWeightFunction == "default") - { - switch (calibrationMethod) + // Define calibration weighting curve from name + boost::algorithm::to_lower(calibrationWeightFunction); + if (calibrationWeightFunction == "default") { - case ECalibrationMethod::DEBEVEC: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); - break; - case ECalibrationMethod::LINEAR: - case ECalibrationMethod::GROSSBERG: - case ECalibrationMethod::LAGUERRE: - default: - calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); - break; + switch (calibrationMethod) + { + case ECalibrationMethod::DEBEVEC: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::TRIANGLE); + break; + case ECalibrationMethod::LINEAR: + case ECalibrationMethod::GROSSBERG: + case ECalibrationMethod::LAGUERRE: + default: + calibrationWeightFunction = hdr::EFunctionType_enumToString(hdr::EFunctionType::GAUSSIAN); + break; + } } + calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); } - calibrationWeight.setFunction(hdr::EFunctionType_stringToEnum(calibrationWeightFunction)); - } - ALICEVISION_LOG_INFO("Start calibration"); - hdr::rgbCurve response(channelQuantization); + ALICEVISION_LOG_INFO("Start calibration"); + hdr::rgbCurve response(channelQuantization); - switch(calibrationMethod) - { + switch (calibrationMethod) + { case ECalibrationMethod::LINEAR: { // set the response function to linear @@ -425,13 +509,14 @@ int aliceVision_main(int argc, char** argv) calibration.process(calibrationSamples, groupedExposures, channelQuantization, false, response); break; } - } + } - const std::string methodName = ECalibrationMethod_enumToString(calibrationMethod); - const std::string htmlOutput = (fs::path(outputResponsePath).parent_path() / (std::string("response_") + methodName + std::string(".html"))).string(); + const std::string methodName = ECalibrationMethod_enumToString(calibrationMethod); + const std::string htmlOutput = (fs::path(outputResponsePath).parent_path() / (std::string("response_") + methodName + std::string(".html"))).string(); - response.write(outputResponsePath); - response.writeHtml(htmlOutput, "response"); + response.write(outputResponsePath); + response.writeHtml(htmlOutput, "response"); + } const std::string lumastatFilename = (fs::path(outputResponsePath).parent_path() / "luminanceStatistics.txt").string(); std::ofstream file(lumastatFilename); diff --git a/src/software/pipeline/main_LdrToHdrMerge.cpp b/src/software/pipeline/main_LdrToHdrMerge.cpp index 9d0d5d3929..362cd13647 100644 --- a/src/software/pipeline/main_LdrToHdrMerge.cpp +++ b/src/software/pipeline/main_LdrToHdrMerge.cpp @@ -195,6 +195,10 @@ int aliceVision_main(int argc, char** argv) return EXIT_FAILURE; } + if (workingColorSpace != image::EImageColorSpace::SRGB) + { + meanTargetedLumaForMerging = std::pow((meanTargetedLumaForMerging + 0.055) / 1.055, 2.2); + } hdr::selectTargetViews(targetViews, groupedViews, offsetRefBracketIndex, lumaStatFilepath.string(), meanTargetedLumaForMerging); if ((targetViews.empty() || targetViews.size() != groupedViews.size()) && !isOffsetRefBracketIndexValid)