From c7207c741ac548333a937b8cbc7ab0188e7a9b25 Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Fri, 5 Aug 2022 16:25:19 +0300 Subject: [PATCH 1/7] [feature] Extract FeatureExtractor class from main_featureExtraction.cpp This allows the class to be reused by other components. --- src/aliceVision/feature/CMakeLists.txt | 2 + src/aliceVision/feature/FeatureExtractor.cpp | 286 +++++++++++++++++ src/aliceVision/feature/FeatureExtractor.hpp | 67 ++++ .../pipeline/main_featureExtraction.cpp | 302 +----------------- 4 files changed, 357 insertions(+), 300 deletions(-) create mode 100644 src/aliceVision/feature/FeatureExtractor.cpp create mode 100644 src/aliceVision/feature/FeatureExtractor.hpp diff --git a/src/aliceVision/feature/CMakeLists.txt b/src/aliceVision/feature/CMakeLists.txt index 64f41f4841..2e09a4a722 100644 --- a/src/aliceVision/feature/CMakeLists.txt +++ b/src/aliceVision/feature/CMakeLists.txt @@ -14,6 +14,7 @@ set(features_files_headers sift/SIFT.hpp Descriptor.hpp feature.hpp + FeatureExtractor.hpp FeaturesPerView.hpp Hamming.hpp ImageDescriber.hpp @@ -34,6 +35,7 @@ set(features_files_sources akaze/ImageDescriber_AKAZE.cpp sift/SIFT.cpp sift/ImageDescriber_DSPSIFT_vlfeat.cpp + FeatureExtractor.cpp FeaturesPerView.cpp ImageDescriber.cpp imageDescriberCommon.cpp diff --git a/src/aliceVision/feature/FeatureExtractor.cpp b/src/aliceVision/feature/FeatureExtractor.cpp new file mode 100644 index 0000000000..826dd40c0f --- /dev/null +++ b/src/aliceVision/feature/FeatureExtractor.cpp @@ -0,0 +1,286 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2022 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "FeatureExtractor.hpp" +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +namespace aliceVision { +namespace feature { + +struct FeatureExtractor::ViewJob +{ + const sfmData::View& view; + std::size_t memoryConsuption = 0; + std::string outputBasename; + std::vector cpuImageDescriberIndexes; + std::vector gpuImageDescriberIndexes; + + ViewJob(const sfmData::View& view, + const std::string& outputFolder) : + view(view), + outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string()) + {} + + ~ViewJob() = default; + + bool useGPU() const + { + return !gpuImageDescriberIndexes.empty(); + } + + bool useCPU() const + { + return !cpuImageDescriberIndexes.empty(); + } + + std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const + { + return outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".feat"; + } + + std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const + { + return outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".desc"; + } + + void setImageDescribers(const std::vector>& imageDescribers) + { + for (std::size_t i = 0; i < imageDescribers.size(); ++i) + { + const std::shared_ptr& imageDescriber = imageDescribers.at(i); + feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); + + if (fs::exists(getFeaturesPath(imageDescriberType)) && + fs::exists(getDescriptorPath(imageDescriberType))) + { + continue; + } + + memoryConsuption += imageDescriber->getMemoryConsumption(view.getWidth(), view.getHeight()); + + if(imageDescriber->useCuda()) + gpuImageDescriberIndexes.push_back(i); + else + cpuImageDescriberIndexes.push_back(i); + } + } +}; + + +FeatureExtractor::FeatureExtractor(const sfmData::SfMData& sfmData) : + _sfmData(sfmData) +{} + +FeatureExtractor::~FeatureExtractor() = default; + +void FeatureExtractor::process() +{ + // iteration on each view in the range in order + // to prepare viewJob stack + sfmData::Views::const_iterator itViewBegin = _sfmData.getViews().begin(); + sfmData::Views::const_iterator itViewEnd = _sfmData.getViews().end(); + + if(_rangeStart != -1) + { + std::advance(itViewBegin, _rangeStart); + itViewEnd = itViewBegin; + std::advance(itViewEnd, _rangeSize); + } + + std::size_t jobMaxMemoryConsuption = 0; + + for (auto it = itViewBegin; it != itViewEnd; ++it) + { + const sfmData::View& view = *(it->second.get()); + ViewJob viewJob(view, _outputFolder); + + viewJob.setImageDescribers(_imageDescribers); + jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption); + + if (viewJob.useCPU()) + _cpuJobs.push_back(viewJob); + + if (viewJob.useGPU()) + _gpuJobs.push_back(viewJob); + } + + if (!_cpuJobs.empty()) + { + system::MemoryInfo memoryInformation = system::getMemoryInfo(); + + ALICEVISION_LOG_INFO("Job max memory consumption for one image: " + << jobMaxMemoryConsuption / (1024*1024) << " MB"); + ALICEVISION_LOG_INFO("Memory information: " << std::endl << memoryInformation); + + if (jobMaxMemoryConsuption == 0) + throw std::runtime_error("Cannot compute feature extraction job max memory consumption."); + + // How many buffers can fit in 90% of the available RAM? + // This is used to estimate how many jobs can be computed in parallel without SWAP. + const std::size_t memoryImageCapacity = + std::size_t((0.9 * memoryInformation.availableRam) / jobMaxMemoryConsuption); + + std::size_t nbThreads = std::max(std::size_t(1), memoryImageCapacity); + ALICEVISION_LOG_INFO("Max number of threads regarding memory usage: " << nbThreads); + const double oneGB = 1024.0 * 1024.0 * 1024.0; + if (jobMaxMemoryConsuption > memoryInformation.availableRam) + { + ALICEVISION_LOG_WARNING("The amount of RAM available is critical to extract features."); + if (jobMaxMemoryConsuption <= memoryInformation.totalRam) + { + ALICEVISION_LOG_WARNING("But the total amount of RAM is enough to extract features, " + << "so you should close other running applications."); + ALICEVISION_LOG_WARNING(" => " << std::size_t(std::round((double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB))) + << " GB are used by other applications for a total RAM capacity of " + << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) + << " GB."); + } + } + else + { + if (memoryInformation.availableRam < 0.5 * memoryInformation.totalRam) + { + ALICEVISION_LOG_WARNING("More than half of the RAM is used by other applications. It would be more efficient to close them."); + ALICEVISION_LOG_WARNING(" => " + << std::size_t(std::round(double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB)) + << " GB are used by other applications for a total RAM capacity of " + << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) + << " GB."); + } + } + + if(memoryInformation.availableRam == 0) + { + ALICEVISION_LOG_WARNING("Cannot find available system memory, this can be due to OS limitation.\n" + "Use only one thread for CPU feature extraction."); + nbThreads = 1; + } + + // nbThreads should not be higher than user maxThreads param + if(_maxThreads > 0) + nbThreads = std::min(static_cast(_maxThreads), nbThreads); + + // nbThreads should not be higher than the core number + nbThreads = std::min(static_cast(omp_get_num_procs()), nbThreads); + + // nbThreads should not be higher than the number of jobs + nbThreads = std::min(_cpuJobs.size(), nbThreads); + + ALICEVISION_LOG_INFO("# threads for extraction: " << nbThreads); + omp_set_nested(1); + +#pragma omp parallel for num_threads(nbThreads) + for (int i = 0; i < _cpuJobs.size(); ++i) + computeViewJob(_cpuJobs.at(i), false); + } + + if (!_gpuJobs.empty()) + { + for (const auto& job : _gpuJobs) + computeViewJob(job, true); + } +} + +void FeatureExtractor::computeViewJob(const ViewJob& job, bool useGPU) +{ + image::Image imageGrayFloat; + image::Image imageGrayUChar; + image::Image mask; + + image::readImage(job.view.getImagePath(), imageGrayFloat, image::EImageColorSpace::SRGB); + + if (!_masksFolder.empty() && fs::exists(_masksFolder)) + { + const auto masksFolder = fs::path(_masksFolder); + const auto idMaskPath = masksFolder / + fs::path(std::to_string(job.view.getViewId())).replace_extension("png"); + const auto nameMaskPath = masksFolder / + fs::path(job.view.getImagePath()).filename().replace_extension("png"); + + if (fs::exists(idMaskPath)) + { + image::readImage(idMaskPath.string(), mask, image::EImageColorSpace::LINEAR); + } + else if (fs::exists(nameMaskPath)) + { + image::readImage(nameMaskPath.string(), mask, image::EImageColorSpace::LINEAR); + } + } + + const auto imageDescriberIndexes = useGPU ? job.gpuImageDescriberIndexes + : job.cpuImageDescriberIndexes; + + for (const auto & imageDescriberIndex : imageDescriberIndexes) + { + const auto& imageDescriber = _imageDescribers.at(imageDescriberIndex); + const feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); + const std::string imageDescriberTypeName = + feature::EImageDescriberType_enumToString(imageDescriberType); + + // Compute features and descriptors and export them to files + ALICEVISION_LOG_INFO("Extracting " << imageDescriberTypeName << " features from view '" + << job.view.getImagePath() << "' " << (useGPU ? "[gpu]" : "[cpu]")); + + std::unique_ptr regions; + if (imageDescriber->useFloatImage()) + { + // image buffer use float image, use the read buffer + imageDescriber->describe(imageGrayFloat, regions); + } + else + { + // image buffer can't use float image + if (imageGrayUChar.Width() == 0) // the first time, convert the float buffer to uchar + imageGrayUChar = (imageGrayFloat.GetMat() * 255.f).cast(); + imageDescriber->describe(imageGrayUChar, regions); + } + + if (mask.Height() > 0) + { + std::vector selectedIndices; + for (size_t i=0, n=regions->RegionCount(); i != n; ++i) + { + const Vec2 position = regions->GetRegionPosition(i); + const int x = int(position.x()); + const int y = int(position.y()); + + bool masked = false; + if (x < mask.Width() && y < mask.Height()) + { + if (mask(y, x) == 0) + { + masked = true; + } + } + + if (!masked) + { + selectedIndices.push_back({IndexT(i), 0}); + } + } + + std::vector out_associated3dPoint; + std::map out_mapFullToLocal; + regions = regions->createFilteredRegions(selectedIndices, out_associated3dPoint, + out_mapFullToLocal); + } + + imageDescriber->Save(regions.get(), job.getFeaturesPath(imageDescriberType), + job.getDescriptorPath(imageDescriberType)); + ALICEVISION_LOG_INFO(std::left << std::setw(6) << " " << regions->RegionCount() << " " + << imageDescriberTypeName << " features extracted from view '" + << job.view.getImagePath() << "'"); + } +} + +} // namespace feature +} // namespace aliceVision diff --git a/src/aliceVision/feature/FeatureExtractor.hpp b/src/aliceVision/feature/FeatureExtractor.hpp new file mode 100644 index 0000000000..b18d91a86e --- /dev/null +++ b/src/aliceVision/feature/FeatureExtractor.hpp @@ -0,0 +1,67 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2022 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "ImageDescriber.hpp" +#include +#include + +namespace aliceVision { +namespace feature { + +class FeatureExtractor +{ + struct ViewJob; + + public: + + explicit FeatureExtractor(const sfmData::SfMData& sfmData); + ~FeatureExtractor(); + + void setRange(int rangeStart, int rangeSize) + { + _rangeStart = rangeStart; + _rangeSize = rangeSize; + } + + void setMaxThreads(int maxThreads) + { + _maxThreads = maxThreads; + } + + void setMasksFolder(const std::string& folder) + { + _masksFolder = folder; + } + + void setOutputFolder(const std::string& folder) + { + _outputFolder = folder; + } + + void addImageDescriber(std::shared_ptr& imageDescriber) + { + _imageDescribers.push_back(imageDescriber); + } + + void process(); + +private: + + void computeViewJob(const ViewJob& job, bool useGPU); + + const sfmData::SfMData& _sfmData; + std::vector> _imageDescribers; + std::string _masksFolder; + std::string _outputFolder; + int _rangeStart = -1; + int _rangeSize = -1; + int _maxThreads = -1; + std::vector _cpuJobs; + std::vector _gpuJobs; +}; + +} // namespace feature +} // namespace aliceVision diff --git a/src/software/pipeline/main_featureExtraction.cpp b/src/software/pipeline/main_featureExtraction.cpp index c977d3a7c7..7eb6baf00e 100644 --- a/src/software/pipeline/main_featureExtraction.cpp +++ b/src/software/pipeline/main_featureExtraction.cpp @@ -5,7 +5,7 @@ // v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at https://mozilla.org/MPL/2.0/. -#include +#include #include #include #include @@ -15,8 +15,6 @@ #define ALICEVISION_HAVE_GPU_FEATURES #include #endif -#include -#include #include #include #include @@ -27,13 +25,9 @@ #include #include -#include -#include #include #include #include -#include -#include // These constants define the current software version. // They must be updated when the command line is changed. @@ -45,298 +39,6 @@ using namespace aliceVision; namespace po = boost::program_options; namespace fs = boost::filesystem; -class FeatureExtractor -{ - struct ViewJob - { - const sfmData::View& view; - std::size_t memoryConsuption = 0; - std::string outputBasename; - std::vector cpuImageDescriberIndexes; - std::vector gpuImageDescriberIndexes; - - ViewJob(const sfmData::View& view, - const std::string& outputFolder) - : view(view) - , outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string()) - {} - - ~ViewJob() = default; - - bool useGPU() const - { - return !gpuImageDescriberIndexes.empty(); - } - - bool useCPU() const - { - return !cpuImageDescriberIndexes.empty(); - } - - std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const - { - return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".feat"; - } - - std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const - { - return outputBasename + "." + feature::EImageDescriberType_enumToString(imageDescriberType) + ".desc"; - } - - void setImageDescribers(const std::vector>& imageDescribers) - { - for(std::size_t i = 0; i < imageDescribers.size(); ++i) - { - const std::shared_ptr& imageDescriber = imageDescribers.at(i); - feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); - - if(fs::exists(getFeaturesPath(imageDescriberType)) && - fs::exists(getDescriptorPath(imageDescriberType))) - continue; - - memoryConsuption += imageDescriber->getMemoryConsumption(view.getWidth(), view.getHeight()); - - if(imageDescriber->useCuda()) - gpuImageDescriberIndexes.push_back(i); - else - cpuImageDescriberIndexes.push_back(i); - } - } - }; - -public: - - explicit FeatureExtractor(const sfmData::SfMData& sfmData) - : _sfmData(sfmData) - {} - - void setRange(int rangeStart, int rangeSize) - { - _rangeStart = rangeStart; - _rangeSize = rangeSize; - } - - void setMaxThreads(int maxThreads) - { - _maxThreads = maxThreads; - } - - void setMasksFolder(const std::string& folder) - { - _masksFolder = folder; - } - - void setOutputFolder(const std::string& folder) - { - _outputFolder = folder; - } - - void addImageDescriber(std::shared_ptr& imageDescriber) - { - _imageDescribers.push_back(imageDescriber); - } - - void process() - { - // iteration on each view in the range in order - // to prepare viewJob stack - sfmData::Views::const_iterator itViewBegin = _sfmData.getViews().begin(); - sfmData::Views::const_iterator itViewEnd = _sfmData.getViews().end(); - - if(_rangeStart != -1) - { - std::advance(itViewBegin, _rangeStart); - itViewEnd = itViewBegin; - std::advance(itViewEnd, _rangeSize); - } - - std::size_t jobMaxMemoryConsuption = 0; - - for(auto it = itViewBegin; it != itViewEnd; ++it) - { - const sfmData::View& view = *(it->second.get()); - ViewJob viewJob(view, _outputFolder); - - viewJob.setImageDescribers(_imageDescribers); - jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption); - - if(viewJob.useCPU()) - _cpuJobs.push_back(viewJob); - - if(viewJob.useGPU()) - _gpuJobs.push_back(viewJob); - } - - if(!_cpuJobs.empty()) - { - system::MemoryInfo memoryInformation = system::getMemoryInfo(); - - ALICEVISION_LOG_INFO("Job max memory consumption for one image: " << jobMaxMemoryConsuption / (1024*1024) << " MB"); - ALICEVISION_LOG_INFO("Memory information: " << std::endl << memoryInformation); - - if(jobMaxMemoryConsuption == 0) - throw std::runtime_error("Cannot compute feature extraction job max memory consumption."); - - // How many buffers can fit in 90% of the available RAM? - // This is used to estimate how many jobs can be computed in parallel without SWAP. - const std::size_t memoryImageCapacity = std::size_t((0.9 * memoryInformation.availableRam) / jobMaxMemoryConsuption); - std::size_t nbThreads = std::max(std::size_t(1), memoryImageCapacity); - ALICEVISION_LOG_INFO("Max number of threads regarding memory usage: " << nbThreads); - const double oneGB = 1024.0 * 1024.0 * 1024.0; - if(jobMaxMemoryConsuption > memoryInformation.availableRam) - { - ALICEVISION_LOG_WARNING("The amount of RAM available is critical to extract features."); - if(jobMaxMemoryConsuption <= memoryInformation.totalRam) - { - ALICEVISION_LOG_WARNING("But the total amount of RAM is enough to extract features, so you should close other running applications."); - ALICEVISION_LOG_WARNING(" => " << std::size_t(std::round((double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB))) - << " GB are used by other applications for a total RAM capacity of " - << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) << " GB."); - } - } - else - { - if(memoryInformation.availableRam < 0.5 * memoryInformation.totalRam) - { - ALICEVISION_LOG_WARNING("More than half of the RAM is used by other applications. It would be more efficient to close them."); - ALICEVISION_LOG_WARNING(" => " - << std::size_t(std::round(double(memoryInformation.totalRam - memoryInformation.availableRam) / oneGB)) - << " GB are used by other applications for a total RAM capacity of " - << std::size_t(std::round(double(memoryInformation.totalRam) / oneGB)) << " GB."); - } - } - - if(memoryInformation.availableRam == 0) - { - ALICEVISION_LOG_WARNING("Cannot find available system memory, this can be due to OS limitation.\n" - "Use only one thread for CPU feature extraction."); - nbThreads = 1; - } - - // nbThreads should not be higher than user maxThreads param - if(_maxThreads > 0) - nbThreads = std::min(static_cast(_maxThreads), nbThreads); - - // nbThreads should not be higher than the core number - nbThreads = std::min(static_cast(omp_get_num_procs()), nbThreads); - - // nbThreads should not be higher than the number of jobs - nbThreads = std::min(_cpuJobs.size(), nbThreads); - - ALICEVISION_LOG_INFO("# threads for extraction: " << nbThreads); - omp_set_nested(1); - -#pragma omp parallel for num_threads(nbThreads) - for(int i = 0; i < _cpuJobs.size(); ++i) - computeViewJob(_cpuJobs.at(i)); - } - - if(!_gpuJobs.empty()) - { - for(const auto& job : _gpuJobs) - computeViewJob(job, true); - } - } - - ~FeatureExtractor() = default; - -private: - - void computeViewJob(const ViewJob& job, bool useGPU = false) - { - image::Image imageGrayFloat; - image::Image imageGrayUChar; - image::Image mask; - - image::readImage(job.view.getImagePath(), imageGrayFloat, image::EImageColorSpace::SRGB); - - if(!_masksFolder.empty() && fs::exists(_masksFolder)) - { - const auto masksFolder = fs::path(_masksFolder); - const auto idMaskPath = masksFolder / fs::path(std::to_string(job.view.getViewId())).replace_extension("png"); - const auto nameMaskPath = masksFolder / fs::path(job.view.getImagePath()).filename().replace_extension("png"); - - if(fs::exists(idMaskPath)) - { - image::readImage(idMaskPath.string(), mask, image::EImageColorSpace::LINEAR); - } - else if(fs::exists(nameMaskPath)) - { - image::readImage(nameMaskPath.string(), mask, image::EImageColorSpace::LINEAR); - } - } - - const auto imageDescriberIndexes = useGPU ? job.gpuImageDescriberIndexes : job.cpuImageDescriberIndexes; - - for(const auto & imageDescriberIndex : imageDescriberIndexes) - { - const auto& imageDescriber = _imageDescribers.at(imageDescriberIndex); - const feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); - const std::string imageDescriberTypeName = feature::EImageDescriberType_enumToString(imageDescriberType); - - // Compute features and descriptors and export them to files - ALICEVISION_LOG_INFO("Extracting " << imageDescriberTypeName << " features from view '" << job.view.getImagePath() << "' " << (useGPU ? "[gpu]" : "[cpu]")); - - std::unique_ptr regions; - if(imageDescriber->useFloatImage()) - { - // image buffer use float image, use the read buffer - imageDescriber->describe(imageGrayFloat, regions); - } - else - { - // image buffer can't use float image - if(imageGrayUChar.Width() == 0) // the first time, convert the float buffer to uchar - imageGrayUChar = (imageGrayFloat.GetMat() * 255.f).cast(); - imageDescriber->describe(imageGrayUChar, regions); - } - - if(mask.Height() > 0) - { - std::vector selectedIndices; - for(size_t i=0, n=regions->RegionCount(); i != n; ++i) - { - const Vec2 position = regions->GetRegionPosition(i); - const int x = int(position.x()); - const int y = int(position.y()); - - bool masked = false; - if(x < mask.Width() && y < mask.Height()) - { - if(mask(y, x) == 0) - { - masked = true; - } - } - - if(!masked) - { - selectedIndices.push_back({IndexT(i), 0}); - } - } - - std::vector out_associated3dPoint; - std::map out_mapFullToLocal; - regions = regions->createFilteredRegions(selectedIndices, out_associated3dPoint, out_mapFullToLocal); - } - - imageDescriber->Save(regions.get(), job.getFeaturesPath(imageDescriberType), job.getDescriptorPath(imageDescriberType)); - ALICEVISION_LOG_INFO(std::left << std::setw(6) << " " << regions->RegionCount() << " " << imageDescriberTypeName << " features extracted from view '" << job.view.getImagePath() << "'"); - } - } - - const sfmData::SfMData& _sfmData; - std::vector> _imageDescribers; - std::string _masksFolder; - std::string _outputFolder; - int _rangeStart = -1; - int _rangeSize = -1; - int _maxThreads = -1; - std::vector _cpuJobs; - std::vector _gpuJobs; -}; - - /// - Compute view image description (feature & descriptor extraction) /// - Export computed data int aliceVision_main(int argc, char **argv) @@ -463,7 +165,7 @@ int aliceVision_main(int argc, char **argv) } // create feature extractor - FeatureExtractor extractor(sfmData); + feature::FeatureExtractor extractor(sfmData); extractor.setMasksFolder(masksFolder); extractor.setOutputFolder(outputFolder); From 378e55f60503d26fb7cef65f5a172036badb791d Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Fri, 5 Aug 2022 21:49:56 +0300 Subject: [PATCH 2/7] [imageMatching] Extract matching code from main_imageMatching.cpp This allows the code to be reused by other components. --- src/aliceVision/CMakeLists.txt | 1 + src/aliceVision/imageMatching/CMakeLists.txt | 16 + .../imageMatching/ImageMatching.cpp | 458 +++++++++++++++ .../imageMatching/ImageMatching.hpp | 150 +++++ src/software/pipeline/CMakeLists.txt | 1 + src/software/pipeline/main_imageMatching.cpp | 525 +----------------- 6 files changed, 629 insertions(+), 522 deletions(-) create mode 100644 src/aliceVision/imageMatching/CMakeLists.txt create mode 100644 src/aliceVision/imageMatching/ImageMatching.cpp create mode 100644 src/aliceVision/imageMatching/ImageMatching.hpp diff --git a/src/aliceVision/CMakeLists.txt b/src/aliceVision/CMakeLists.txt index 0598a99be9..01eeb3770f 100644 --- a/src/aliceVision/CMakeLists.txt +++ b/src/aliceVision/CMakeLists.txt @@ -17,6 +17,7 @@ if(ALICEVISION_BUILD_SFM) add_subdirectory(geometry) add_subdirectory(graph) add_subdirectory(gpu) + add_subdirectory(imageMatching) add_subdirectory(keyframe) add_subdirectory(linearProgramming) add_subdirectory(localization) diff --git a/src/aliceVision/imageMatching/CMakeLists.txt b/src/aliceVision/imageMatching/CMakeLists.txt new file mode 100644 index 0000000000..b02116e6d5 --- /dev/null +++ b/src/aliceVision/imageMatching/CMakeLists.txt @@ -0,0 +1,16 @@ +# Headers +set(imageMatching_files_headers + ImageMatching.hpp +) + +# Sources +set(imageMatching_files_sources + ImageMatching.cpp +) + +alicevision_add_library(aliceVision_imageMatching + SOURCES ${imageMatching_files_headers} ${imageMatching_files_sources} + PRIVATE_LINKS + aliceVision_image + aliceVision_voctree +) diff --git a/src/aliceVision/imageMatching/ImageMatching.cpp b/src/aliceVision/imageMatching/ImageMatching.cpp new file mode 100644 index 0000000000..ed66ce07f1 --- /dev/null +++ b/src/aliceVision/imageMatching/ImageMatching.cpp @@ -0,0 +1,458 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2019 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#include "ImageMatching.hpp" +#include + +namespace aliceVision { +namespace imageMatching { + +std::ostream& operator<<(std::ostream& os, const PairList& pl) +{ + for (PairList::const_iterator plIter = pl.begin(); plIter != pl.end(); ++plIter) + { + os << plIter->first; + for (ImageID id : plIter->second) + { + os << " " << id; + } + os << "\n"; + } + return os; +} + + +std::string EImageMatchingMethod_enumToString(EImageMatchingMethod m) +{ + switch(m) + { + case EImageMatchingMethod::EXHAUSTIVE: + return "Exhaustive"; + case EImageMatchingMethod::VOCABULARYTREE: + return "VocabularyTree"; + case EImageMatchingMethod::SEQUENTIAL: + return "Sequential"; + case EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE: + return "SequentialAndVocabularyTree"; + case EImageMatchingMethod::FRUSTUM: + return "Frustum"; + case EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE: + return "FrustumOrVocabularyTree"; + } + throw std::out_of_range("Invalid EImageMatchingMethod enum: " + std::to_string(int(m))); +} + +EImageMatchingMethod EImageMatchingMethod_stringToEnum(const std::string& m) +{ + std::string mode = m; + std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); + + if (mode == "exhaustive") + return EImageMatchingMethod::EXHAUSTIVE; + if (mode == "vocabularytree") + return EImageMatchingMethod::VOCABULARYTREE; + if (mode == "sequential") + return EImageMatchingMethod::SEQUENTIAL; + if (mode == "sequentialandvocabularytree") + return EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE; + if (mode == "frustum") + return EImageMatchingMethod::FRUSTUM; + if (mode == "frustumorvocabularytree") + return EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE; + + throw std::out_of_range("Invalid EImageMatchingMethod: " + m); +} + +std::ostream& operator<<(std::ostream& os, EImageMatchingMethod m) +{ + return os << EImageMatchingMethod_enumToString(m); +} + +std::istream& operator>>(std::istream& in, EImageMatchingMethod& m) +{ + std::string token; + in >> token; + m = EImageMatchingMethod_stringToEnum(token); + return in; +} + +std::string EImageMatchingMode_description() +{ + return "The mode to combine image matching between the input SfMData A and B: \n" + "* a/a+a/b : A with A + A with B\n" + "* a/ab : A with A and B\n" + "* a/b : A with B\n" + "* a/a : A with A"; +} + + +std::string EImageMatchingMode_enumToString(EImageMatchingMode modeMultiSfM) +{ + switch(modeMultiSfM) + { + case EImageMatchingMode::A_A_AND_A_B: return "a/a+a/b"; + case EImageMatchingMode::A_AB: return "a/ab"; + case EImageMatchingMode::A_B: return "a/b"; + case EImageMatchingMode::A_A: return "a/a"; + } + throw std::out_of_range("Invalid modeMultiSfM enum"); +} + +EImageMatchingMode EImageMatchingMode_stringToEnum(const std::string& modeMultiSfM) +{ + std::string mode = modeMultiSfM; + std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); //tolower + + if (mode == "a/a+a/b") return EImageMatchingMode::A_A_AND_A_B; + if (mode == "a/ab") return EImageMatchingMode::A_AB; + if (mode == "a/b") return EImageMatchingMode::A_B; + if (mode == "a/a") return EImageMatchingMode::A_A; + + throw std::out_of_range("Invalid modeMultiSfM : " + modeMultiSfM); +} + +void convertAllMatchesToPairList(const PairList &allMatches, std::size_t numMatches, + OrderedPairList &outPairList) +{ + outPairList.clear(); + + if (numMatches == 0) + numMatches = allMatches.size(); // disable image matching limit + + for (const auto& match : allMatches) + { + ImageID currImageId = match.first; + OrderedListOfImageID bestMatches; + + for (const ImageID currMatchId : match.second) + { + // avoid self-matching + if (currMatchId == currImageId) + continue; + + // if the currMatchId ID is lower than the current image ID and + // the current image ID is not already in the list of currMatchId + //BOOST_ASSERT( ( currMatchId < currImageId ) && + // ( outPairList.find( currMatchId ) != outPairList.end() ) ); + if (currMatchId < currImageId) + { + OrderedPairList::const_iterator currMatches = outPairList.find(currMatchId); + if (currMatches != outPairList.end() && + currMatches->second.find(currImageId) == currMatches->second.end()) + { + // then add it to the list + bestMatches.insert(currMatchId); + } + } + else + { + bestMatches.insert(currMatchId); + } + + // stop if numMatches is satisfied + if(bestMatches.size() == numMatches) + break; + } + + // fill the output if we have matches + if (!bestMatches.empty()) + { + outPairList[currImageId] = bestMatches; + } + } +} + +void generateSequentialMatches(const sfmData::SfMData& sfmData, size_t nbMatches, + OrderedPairList& outPairList) +{ + std::vector> sortedImagePaths; + sortedImagePaths.reserve(sfmData.getViews().size()); + for (const auto& vIt: sfmData.getViews()) + { + sortedImagePaths.emplace_back(vIt.second->getImagePath(), vIt.first); + } + std::sort(sortedImagePaths.begin(), sortedImagePaths.end()); + for (size_t i = 0; i < sortedImagePaths.size(); ++i) + { + for (size_t n = i+1, nMax = std::min(i+nbMatches+1, sortedImagePaths.size()); n < nMax; ++n) + { + size_t a = sortedImagePaths[i].second; + size_t b = sortedImagePaths[n].second; + outPairList[std::min(a, b)].insert(std::max(a, b)); + } + } +} + +void generateAllMatchesInOneMap(const std::set& viewIds, OrderedPairList& outPairList) +{ + for (const IndexT imgA : viewIds) + { + OrderedListOfImageID outPerImg; + + for (const IndexT imgB : viewIds) + { + if (imgB > imgA) + outPerImg.insert(imgB); + } + + if (!outPerImg.empty()) + { + OrderedPairList::iterator itFind = outPairList.find(imgA); + + if (itFind == outPairList.end()) + outPairList[imgA] = outPerImg; + else + itFind->second.insert(outPerImg.begin(), outPerImg.end()); + } + } +} + +void generateAllMatchesBetweenTwoMap(const std::set& viewIdsA, + const std::set& viewIdsB, + OrderedPairList& outPairList) +{ + for (const IndexT imgA : viewIdsA) + { + OrderedListOfImageID outPerImg; + + for (const IndexT imgB : viewIdsB) + outPerImg.insert(imgB); + + if (!outPerImg.empty()) + { + OrderedPairList::iterator itFind = outPairList.find(imgA); + + if (itFind == outPairList.end()) + outPairList[imgA] = outPerImg; + else + itFind->second.insert(outPerImg.begin(), outPerImg.end()); + } + } +} + +void generateFromVoctree(PairList& allMatches, + const std::map& descriptorsFiles, + const aliceVision::voctree::Database& db, + const aliceVision::voctree::VocabularyTree& tree, + EImageMatchingMode modeMultiSfM, + std::size_t nbMaxDescriptors, + std::size_t numImageQuery) +{ + ALICEVISION_LOG_INFO("Generate matches in mode: " + + EImageMatchingMode_enumToString(modeMultiSfM)); + + if (numImageQuery == 0) + { + // if 0 retrieve the score for all the documents of the database + numImageQuery = db.size(); + } + + //initialize allMatches + + for (const auto& descriptorPair : descriptorsFiles) + { + if (allMatches.find(descriptorPair.first) == allMatches.end()) + allMatches[descriptorPair.first] = {}; + } + + // query each document +#pragma omp parallel for + for (ptrdiff_t i = 0; i < static_cast(descriptorsFiles.size()); ++i) + { + auto itA = descriptorsFiles.cbegin(); + std::advance(itA, i); + const IndexT viewIdA = itA->first; + const std::string featuresPathA = itA->second; + + aliceVision::voctree::SparseHistogram imageSH; + + if (modeMultiSfM != EImageMatchingMode::A_B) + { + // sparse histogram of A is already computed in the DB + imageSH = db.getSparseHistogramPerImage().at(viewIdA); + } + else // mode AB + { + // compute the sparse histogram of each image A + std::vector descriptors; + // read the descriptors + loadDescsFromBinFile(featuresPathA, descriptors, false, nbMaxDescriptors); + imageSH = tree.quantizeToSparse(descriptors); + } + + std::vector matches; + + db.find(imageSH, numImageQuery, matches); + + ListOfImageID& imgMatches = allMatches.at(viewIdA); + imgMatches.reserve(imgMatches.size() + matches.size()); + + for (const aliceVision::voctree::DocMatch& m : matches) + { + imgMatches.push_back(m.id); + } + } +} + +void conditionVocTree(const std::string& treeName, bool withWeights, + const std::string& weightsName, + const EImageMatchingMode matchingMode, + const std::vector& featuresFolders, + const sfmData::SfMData& sfmDataA, + std::size_t nbMaxDescriptors, + const std::string& sfmDataFilenameA, + const sfmData::SfMData& sfmDataB, + const std::string& sfmDataFilenameB, + bool useMultiSfM, + const std::map& descriptorsFilesA, + std::size_t numImageQuery, + OrderedPairList& selectedPairs) +{ + if (treeName.empty()) + { + throw std::runtime_error("No vocabulary tree argument."); + } + + // load vocabulary tree + ALICEVISION_LOG_INFO("Loading vocabulary tree"); + + auto loadVoctree_start = std::chrono::steady_clock::now(); + aliceVision::voctree::VocabularyTree tree(treeName); + auto loadVoctree_elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - loadVoctree_start); + { + std::stringstream ss; + ss << "tree loaded with:" << std::endl << "\t- " << tree.levels() << " levels" << std::endl; + ss << "\t- " << tree.splits() << " branching factor" << std::endl; + ss << "\tin " << loadVoctree_elapsed.count() << " seconds" << std::endl; + ALICEVISION_LOG_INFO(ss.str()); + } + + // create the databases + ALICEVISION_LOG_INFO("Creating the databases..."); + + // add each object (document) to the database + aliceVision::voctree::Database db(tree.words()); + aliceVision::voctree::Database db2; + + if(withWeights) + { + ALICEVISION_LOG_INFO("Loading weights..."); + db.loadWeights(weightsName); + } + else + { + ALICEVISION_LOG_INFO("No weights specified, skipping..."); + } + + if (matchingMode == EImageMatchingMode::A_A_AND_A_B) + db2 = db; // initialize database2 with database1 initialization + + // read the descriptors and populate the databases + { + std::stringstream ss; + + for (const std::string& featuresFolder : featuresFolders) + ss << "\t- " << featuresFolder << std::endl; + + ALICEVISION_LOG_INFO("Reading descriptors from: " << std::endl << ss.str()); + + std::size_t nbFeaturesLoadedInputA = 0; + std::size_t nbFeaturesLoadedInputB = 0; + std::size_t nbSetDescriptors = 0; + + auto detect_start = std::chrono::steady_clock::now(); + { + if ((matchingMode == EImageMatchingMode::A_A_AND_A_B) || + (matchingMode == EImageMatchingMode::A_AB) || + (matchingMode == EImageMatchingMode::A_A)) + { + nbFeaturesLoadedInputA = voctree::populateDatabase( + sfmDataA, featuresFolders, tree, db, nbMaxDescriptors); + nbSetDescriptors = db.getSparseHistogramPerImage().size(); + + if(nbFeaturesLoadedInputA == 0) + { + throw std::runtime_error("No descriptors loaded in '" + sfmDataFilenameA + "'"); + } + } + + if ((matchingMode == EImageMatchingMode::A_AB) || + (matchingMode == EImageMatchingMode::A_B)) + { + nbFeaturesLoadedInputB = voctree::populateDatabase( + sfmDataB, featuresFolders, tree, db, nbMaxDescriptors); + nbSetDescriptors = db.getSparseHistogramPerImage().size(); + } + + if (matchingMode == EImageMatchingMode::A_A_AND_A_B) + { + nbFeaturesLoadedInputB = voctree::populateDatabase( + sfmDataB, featuresFolders, tree, db2, nbMaxDescriptors); + nbSetDescriptors += db2.getSparseHistogramPerImage().size(); + } + + if (useMultiSfM && (nbFeaturesLoadedInputB == 0)) + { + throw std::runtime_error("No descriptors loaded in '" + sfmDataFilenameB + "'"); + } + } + auto detect_elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - detect_start); + + ALICEVISION_LOG_INFO("Read " << nbSetDescriptors << " sets of descriptors for a total of " + << (nbFeaturesLoadedInputA + nbFeaturesLoadedInputB) << " features"); + ALICEVISION_LOG_INFO("Reading took " << detect_elapsed.count() << " sec."); + } + + if (!withWeights) + { + // compute and save the word weights + ALICEVISION_LOG_INFO("Computing weights..."); + + db.computeTfIdfWeights(); + + if (matchingMode == EImageMatchingMode::A_A_AND_A_B) + db2.computeTfIdfWeights(); + } + + { + PairList allMatches; + + ALICEVISION_LOG_INFO("Query all documents"); + + auto detect_start = std::chrono::steady_clock::now(); + + if (matchingMode == EImageMatchingMode::A_A_AND_A_B) + { + generateFromVoctree(allMatches, descriptorsFilesA, db, tree, EImageMatchingMode::A_A, + nbMaxDescriptors, numImageQuery); + generateFromVoctree(allMatches, descriptorsFilesA, db2, tree, EImageMatchingMode::A_B, + nbMaxDescriptors, numImageQuery); + } + else + { + generateFromVoctree(allMatches, descriptorsFilesA, db, tree, matchingMode, + nbMaxDescriptors, numImageQuery); + } + + auto detect_elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - detect_start); + ALICEVISION_LOG_INFO("Query all documents took " << detect_elapsed.count() << " sec."); + + // process pair list + detect_start = std::chrono::steady_clock::now(); + + ALICEVISION_LOG_INFO("Convert all matches to pairList"); + convertAllMatchesToPairList(allMatches, numImageQuery, selectedPairs); + detect_elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - detect_start); + ALICEVISION_LOG_INFO("Convert all matches to pairList took " << detect_elapsed.count() << " sec."); + } +} + +} // namespace imageMatching +} // namespace aliceVision diff --git a/src/aliceVision/imageMatching/ImageMatching.hpp b/src/aliceVision/imageMatching/ImageMatching.hpp new file mode 100644 index 0000000000..8c04151eb0 --- /dev/null +++ b/src/aliceVision/imageMatching/ImageMatching.hpp @@ -0,0 +1,150 @@ +// This file is part of the AliceVision project. +// Copyright (c) 2019 AliceVision contributors. +// This Source Code Form is subject to the terms of the Mozilla Public License, +// v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace aliceVision { +namespace imageMatching { + +static const int DIMENSION = 128; + +using DescriptorFloat = feature::Descriptor ; +using DescriptorUChar = feature::Descriptor ; + +using ImageID = std::size_t; + +// just a list of doc id +using ListOfImageID = std::vector; + +// An ordered and unique list of doc id +using OrderedListOfImageID = std::set; + +// For each image ID it contains the list of matching imagess +using PairList = std::map; + +// For each image ID it contains the ordered list of matching images +using OrderedPairList = std::map; + + +/** + * @brief Function that prints a PairList + * @param os The stream on which to print + * @param pl The pair list + * @return the stream + */ +std::ostream& operator<<(std::ostream& os, const PairList& pl); + +/** + * @brief Mode to select the type of image matching + */ +enum class EImageMatchingMethod +{ + EXHAUSTIVE = 0, + VOCABULARYTREE = 1, + SEQUENTIAL = 2, + SEQUENTIAL_AND_VOCABULARYTREE = 3, + FRUSTUM = 4, + FRUSTUM_OR_VOCABULARYTREE = 5 +}; + +/** + *@brief convert an enum EImageMatchingMethod to its corresponding string + */ +std::string EImageMatchingMethod_enumToString(EImageMatchingMethod m); + +/** + * @brief convert a string treeMode to its corresponding enum treeMode + * @param String + * @return EImageMatchingMethod + */ +EImageMatchingMethod EImageMatchingMethod_stringToEnum(const std::string& m); + +std::ostream& operator<<(std::ostream& os, EImageMatchingMethod m); +std::istream& operator>>(std::istream& in, EImageMatchingMethod& m); + +/** + * @brief Mode to combine image matching between two SfMDatas + */ +enum class EImageMatchingMode +{ + A_A_AND_A_B, + A_AB, + A_B, + A_A +}; + +/** + * @brief get informations about each EImageMatchingMode + * @return String + */ +std::string EImageMatchingMode_description(); + +/** + * @brief convert an enum EImageMatchingMode to its corresponding string + * @param modeMultiSfM + * @return String + */ +std::string EImageMatchingMode_enumToString(EImageMatchingMode modeMultiSfM); + +/** + * @brief convert a string modeMultiSfM to its corresponding enum modeMultiSfM + * @param String + * @return EImageMatchingMode + */ +EImageMatchingMode EImageMatchingMode_stringToEnum(const std::string& modeMultiSfM); + +/** + * It processes a pairlist containing all the matching images for each image ID and return + * a similar list limited to a numMatches number of matching images and such that + * there is no repetitions: eg if the image 1 matches with 2 in the list of image 2 + * there won't be the image 1 + * + * @param[in] allMatches A pairlist containing all the matching images for each image of the dataset + * @param[in] numMatches The maximum number of matching images to consider for each image (if 0, consider all matches) + * @param[out] matches A processed version of allMatches that consider only the first numMatches without repetitions + */ +void convertAllMatchesToPairList(const PairList &allMatches, std::size_t numMatches, + OrderedPairList &outPairList); + +void generateSequentialMatches(const sfmData::SfMData& sfmData, size_t nbMatches, + OrderedPairList& outPairList); +void generateAllMatchesInOneMap(const std::set& viewIds, OrderedPairList& outPairList); +void generateAllMatchesBetweenTwoMap(const std::set& viewIdsA, + const std::set& viewIdsB, + OrderedPairList& outPairList); + +void generateFromVoctree(PairList& allMatches, + const std::map& descriptorsFiles, + const voctree::Database& db, + const voctree::VocabularyTree& tree, + EImageMatchingMode modeMultiSfM, + std::size_t nbMaxDescriptors, + std::size_t numImageQuery); + +void conditionVocTree(const std::string& treeName, bool withWeights, + const std::string& weightsName, + const EImageMatchingMode matchingMode, + const std::vector& featuresFolders, + const sfmData::SfMData& sfmDataA, + std::size_t nbMaxDescriptors, + const std::string& sfmDataFilenameA, + const sfmData::SfMData& sfmDataB, + const std::string& sfmDataFilenameB, + bool useMultiSfM, + const std::map& descriptorsFilesA, + std::size_t numImageQuery, + OrderedPairList& selectedPairs); + +} // namespace imageMatching +} // namespace aliceVision diff --git a/src/software/pipeline/CMakeLists.txt b/src/software/pipeline/CMakeLists.txt index 2aa211a973..25a1973ff3 100644 --- a/src/software/pipeline/CMakeLists.txt +++ b/src/software/pipeline/CMakeLists.txt @@ -76,6 +76,7 @@ if(ALICEVISION_BUILD_SFM) SOURCE main_imageMatching.cpp FOLDER ${FOLDER_SOFTWARE_PIPELINE} LINKS aliceVision_system + aliceVision_imageMatching aliceVision_sfm aliceVision_sfmData aliceVision_sfmDataIO diff --git a/src/software/pipeline/main_imageMatching.cpp b/src/software/pipeline/main_imageMatching.cpp index e04b78a9b7..7eca3f4927 100644 --- a/src/software/pipeline/main_imageMatching.cpp +++ b/src/software/pipeline/main_imageMatching.cpp @@ -6,10 +6,9 @@ #include #include +#include +#include #include -#include -#include -#include #include #include #include @@ -32,531 +31,13 @@ #define ALICEVISION_SOFTWARE_VERSION_MAJOR 1 #define ALICEVISION_SOFTWARE_VERSION_MINOR 0 -static const int DIMENSION = 128; - using namespace aliceVision; using namespace aliceVision::voctree; +using namespace aliceVision::imageMatching; namespace po = boost::program_options; namespace fs = boost::filesystem; -typedef aliceVision::feature::Descriptor DescriptorFloat; -typedef aliceVision::feature::Descriptor DescriptorUChar; - -typedef std::size_t ImageID; - -using aliceVision::IndexT; - -// just a list of doc id -typedef std::vector ListOfImageID; - -// An ordered and unique list of doc id -typedef std::set OrderedListOfImageID; - -// For each image ID it contains the list of matching imagess -typedef std::map PairList; - -// For each image ID it contains the ordered list of matching images -typedef std::map OrderedPairList; - -/** - * @brief Function that prints a PairList - * @param os The stream on which to print - * @param pl The pair list - * @return the stream - */ -std::ostream& operator<<(std::ostream& os, const PairList & pl) -{ - for(PairList::const_iterator plIter = pl.begin(); plIter != pl.end(); ++plIter) - { - os << plIter->first; - for(ImageID id : plIter->second) - { - os << " " << id; - } - os << "\n"; - } - return os; -} - -/** - * @brief Function that prints a OrderedPairList - * @param os The stream on which to print - * @param pl The pair list - * @return the stream - */ -std::ostream& operator<<(std::ostream& os, const OrderedPairList & pl) -{ - for(OrderedPairList::const_iterator plIter = pl.begin(); plIter != pl.end(); ++plIter) - { - os << plIter->first; - for(ImageID id : plIter->second) - { - os << " " << id; - } - os << "\n"; - } - return os; -} - -/** -* @brief Mode to select the type of image matching -*/ -enum class EImageMatchingMethod -{ - EXHAUSTIVE = 0, - VOCABULARYTREE = 1, - SEQUENTIAL = 2, - SEQUENTIAL_AND_VOCABULARYTREE = 3, - FRUSTUM = 4, - FRUSTUM_OR_VOCABULARYTREE = 5 -}; - -/** -*@brief convert an enum EImageMatchingMethod to its corresponding string -* -*/ -inline std::string EImageMatchingMethod_enumToString(EImageMatchingMethod m) -{ - switch(m) - { - case EImageMatchingMethod::EXHAUSTIVE: - return "Exhaustive"; - case EImageMatchingMethod::VOCABULARYTREE: - return "VocabularyTree"; - case EImageMatchingMethod::SEQUENTIAL: - return "Sequential"; - case EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE: - return "SequentialAndVocabularyTree"; - case EImageMatchingMethod::FRUSTUM: - return "Frustum"; - case EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE: - return "FrustumOrVocabularyTree"; - } - throw std::out_of_range("Invalid EImageMatchingMethod enum: " + std::to_string(int(m))); -} - -/** -* @brief convert a string treeMode to its corresponding enum treeMode -* @param String -* @return EImageMatchingMethod -*/ -inline EImageMatchingMethod EImageMatchingMethod_stringToEnum(const std::string& m) -{ - std::string mode = m; - std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); - - if(mode == "exhaustive") - return EImageMatchingMethod::EXHAUSTIVE; - if(mode == "vocabularytree") - return EImageMatchingMethod::VOCABULARYTREE; - if(mode == "sequential") - return EImageMatchingMethod::SEQUENTIAL; - if(mode == "sequentialandvocabularytree") - return EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE; - if(mode == "frustum") - return EImageMatchingMethod::FRUSTUM; - if(mode == "frustumorvocabularytree") - return EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE; - - throw std::out_of_range("Invalid EImageMatchingMethod: " + m); -} - -inline std::ostream& operator<<(std::ostream& os, EImageMatchingMethod m) -{ - return os << EImageMatchingMethod_enumToString(m); -} - -inline std::istream& operator>>(std::istream& in, EImageMatchingMethod& m) -{ - std::string token; - in >> token; - m = EImageMatchingMethod_stringToEnum(token); - return in; -} - - -/** - * @brief Mode to combine image matching between two SfMDatas - */ -enum class EImageMatchingMode -{ - A_A_AND_A_B, - A_AB, - A_B, - A_A -}; - -/** - * @brief get informations about each EImageMatchingMode - * @return String - */ -std::string EImageMatchingMode_description() -{ - return "The mode to combine image matching between the input SfMData A and B: \n" - "* a/a+a/b : A with A + A with B\n" - "* a/ab : A with A and B\n" - "* a/b : A with B\n" - "* a/a : A with A"; -} - -/** - * @brief convert an enum EImageMatchingMode to its corresponding string - * @param modeMultiSfM - * @return String - */ -std::string EImageMatchingMode_enumToString(EImageMatchingMode modeMultiSfM) -{ - switch(modeMultiSfM) - { - case EImageMatchingMode::A_A_AND_A_B: return "a/a+a/b"; - case EImageMatchingMode::A_AB: return "a/ab"; - case EImageMatchingMode::A_B: return "a/b"; - case EImageMatchingMode::A_A: return "a/a"; - } - throw std::out_of_range("Invalid modeMultiSfM enum"); -} - -/** - * @brief convert a string modeMultiSfM to its corresponding enum modeMultiSfM - * @param String - * @return EImageMatchingMode - */ - EImageMatchingMode EImageMatchingMode_stringToEnum(const std::string& modeMultiSfM) -{ - std::string mode = modeMultiSfM; - std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); //tolower - - if(mode == "a/a+a/b") return EImageMatchingMode::A_A_AND_A_B; - if(mode == "a/ab") return EImageMatchingMode::A_AB; - if(mode == "a/b") return EImageMatchingMode::A_B; - if(mode == "a/a") return EImageMatchingMode::A_A; - - throw std::out_of_range("Invalid modeMultiSfM : " + modeMultiSfM); -} - -/** - * It processes a pairlist containing all the matching images for each image ID and return - * a similar list limited to a numMatches number of matching images and such that - * there is no repetitions: eg if the image 1 matches with 2 in the list of image 2 - * there won't be the image 1 - * - * @param[in] allMatches A pairlist containing all the matching images for each image of the dataset - * @param[in] numMatches The maximum number of matching images to consider for each image (if 0, consider all matches) - * @param[out] matches A processed version of allMatches that consider only the first numMatches without repetitions - */ -void convertAllMatchesToPairList(const PairList &allMatches, std::size_t numMatches, OrderedPairList &outPairList) -{ - outPairList.clear(); - - if(numMatches == 0) - numMatches = allMatches.size(); // disable image matching limit - - for(const auto& match : allMatches) - { - ImageID currImageId = match.first; - OrderedListOfImageID bestMatches; - - for(const ImageID currMatchId : match.second) - { - // avoid self-matching - if(currMatchId == currImageId) - continue; - - // if the currMatchId ID is lower than the current image ID and - // the current image ID is not already in the list of currMatchId - //BOOST_ASSERT( ( currMatchId < currImageId ) && ( outPairList.find( currMatchId ) != outPairList.end() ) ); - if(currMatchId < currImageId) - { - OrderedPairList::const_iterator currMatches = outPairList.find(currMatchId); - if(currMatches != outPairList.end() && - currMatches->second.find(currImageId) == currMatches->second.end()) - { - // then add it to the list - bestMatches.insert(currMatchId); - } - } - else - { - bestMatches.insert(currMatchId); - } - - // stop if numMatches is satisfied - if(bestMatches.size() == numMatches) - break; - } - - // fill the output if we have matches - if(!bestMatches.empty()) - outPairList[currImageId] = bestMatches; - } -} - -void generateSequentialMatches(const sfmData::SfMData& sfmData, size_t nbMatches, OrderedPairList& outPairList) -{ - std::vector> sortedImagePaths; - sortedImagePaths.reserve(sfmData.getViews().size()); - for(const auto& vIt: sfmData.getViews()) - { - sortedImagePaths.emplace_back(vIt.second->getImagePath(), vIt.first); - } - std::sort(sortedImagePaths.begin(), sortedImagePaths.end()); - for(size_t i = 0; i < sortedImagePaths.size(); ++i) - { - for(size_t n = i+1, nMax = std::min(i+nbMatches+1, sortedImagePaths.size()); n < nMax; ++n) - { - size_t a = sortedImagePaths[i].second; - size_t b = sortedImagePaths[n].second; - outPairList[std::min(a, b)].insert(std::max(a, b)); - } - } -} - -void generateAllMatchesInOneMap(const std::set& viewIds, OrderedPairList& outPairList) -{ - for(const IndexT imgA : viewIds) - { - OrderedListOfImageID outPerImg; - - for(const IndexT imgB : viewIds) - { - if(imgB > imgA) - outPerImg.insert(imgB); - } - - if(!outPerImg.empty()) - { - OrderedPairList::iterator itFind = outPairList.find(imgA); - - if(itFind == outPairList.end()) - outPairList[imgA] = outPerImg; - else - itFind->second.insert(outPerImg.begin(), outPerImg.end()); - } - } -} - -void generateAllMatchesBetweenTwoMap(const std::set& viewIdsA, - const std::set& viewIdsB, OrderedPairList& outPairList) -{ - for(const IndexT imgA : viewIdsA) - { - OrderedListOfImageID outPerImg; - - for(const IndexT imgB : viewIdsB) - outPerImg.insert(imgB); - - if(!outPerImg.empty()) - { - OrderedPairList::iterator itFind = outPairList.find(imgA); - - if(itFind == outPairList.end()) - outPairList[imgA] = outPerImg; - else - itFind->second.insert(outPerImg.begin(), outPerImg.end()); - } - } -} - -void generateFromVoctree(PairList& allMatches, - const std::map& descriptorsFiles, - const aliceVision::voctree::Database& db, - const aliceVision::voctree::VocabularyTree& tree, - EImageMatchingMode modeMultiSfM, - std::size_t nbMaxDescriptors, - std::size_t numImageQuery) -{ - ALICEVISION_LOG_INFO("Generate matches in mode: " + EImageMatchingMode_enumToString(modeMultiSfM)); - - if(numImageQuery == 0) - { - // if 0 retrieve the score for all the documents of the database - numImageQuery = db.size(); - } - - //initialize allMatches - - for(const auto& descriptorPair : descriptorsFiles) - { - if(allMatches.find(descriptorPair.first) == allMatches.end()) - allMatches[descriptorPair.first] = {}; - } - - // query each document - #pragma omp parallel for - for(ptrdiff_t i = 0; i < static_cast(descriptorsFiles.size()); ++i) - { - auto itA = descriptorsFiles.cbegin(); - std::advance(itA, i); - const IndexT viewIdA = itA->first; - const std::string featuresPathA = itA->second; - - aliceVision::voctree::SparseHistogram imageSH; - - if(modeMultiSfM != EImageMatchingMode::A_B) - { - // sparse histogram of A is already computed in the DB - imageSH = db.getSparseHistogramPerImage().at(viewIdA); - } - else // mode AB - { - // compute the sparse histogram of each image A - std::vector descriptors; - // read the descriptors - loadDescsFromBinFile(featuresPathA, descriptors, false, nbMaxDescriptors); - imageSH = tree.quantizeToSparse(descriptors); - } - - std::vector matches; - - db.find(imageSH, numImageQuery, matches); - - ListOfImageID& imgMatches = allMatches.at(viewIdA); - imgMatches.reserve(imgMatches.size() + matches.size()); - - for(const aliceVision::voctree::DocMatch& m : matches) - { - imgMatches.push_back(m.id); - } - } -} - -void conditionVocTree(const std::string& treeName, bool withWeights, const std::string& weightsName, const EImageMatchingMode matchingMode, const std::vector& featuresFolders, - const sfmData::SfMData& sfmDataA, std::size_t nbMaxDescriptors, const std::string& sfmDataFilenameA, const sfmData::SfMData& sfmDataB, const std::string& sfmDataFilenameB, - bool useMultiSfM, const std::map& descriptorsFilesA, std::size_t numImageQuery, OrderedPairList& selectedPairs) -{ - if(treeName.empty()) - { - throw std::runtime_error("No vocabulary tree argument."); - } - - // load vocabulary tree - ALICEVISION_LOG_INFO("Loading vocabulary tree"); - - auto loadVoctree_start = std::chrono::steady_clock::now(); - aliceVision::voctree::VocabularyTree tree(treeName); - auto loadVoctree_elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - loadVoctree_start); - { - std::stringstream ss; - ss << "tree loaded with:" << std::endl << "\t- " << tree.levels() << " levels" << std::endl; - ss << "\t- " << tree.splits() << " branching factor" << std::endl; - ss << "\tin " << loadVoctree_elapsed.count() << " seconds" << std::endl; - ALICEVISION_LOG_INFO(ss.str()); - } - - // create the databases - ALICEVISION_LOG_INFO("Creating the databases..."); - - // add each object (document) to the database - aliceVision::voctree::Database db(tree.words()); - aliceVision::voctree::Database db2; - - if(withWeights) - { - ALICEVISION_LOG_INFO("Loading weights..."); - db.loadWeights(weightsName); - } - else - { - ALICEVISION_LOG_INFO("No weights specified, skipping..."); - } - - if(matchingMode == EImageMatchingMode::A_A_AND_A_B) - db2 = db; // initialize database2 with database1 initialization - - // read the descriptors and populate the databases - { - std::stringstream ss; - - for(const std::string& featuresFolder : featuresFolders) - ss << "\t- " << featuresFolder << std::endl; - - ALICEVISION_LOG_INFO("Reading descriptors from: " << std::endl << ss.str()); - - std::size_t nbFeaturesLoadedInputA = 0; - std::size_t nbFeaturesLoadedInputB = 0; - std::size_t nbSetDescriptors = 0; - - auto detect_start = std::chrono::steady_clock::now(); - { - if((matchingMode == EImageMatchingMode::A_A_AND_A_B) || - (matchingMode == EImageMatchingMode::A_AB) || - (matchingMode == EImageMatchingMode::A_A)) - { - nbFeaturesLoadedInputA = voctree::populateDatabase(sfmDataA, featuresFolders, tree, db, nbMaxDescriptors); - nbSetDescriptors = db.getSparseHistogramPerImage().size(); - - if(nbFeaturesLoadedInputA == 0) - { - throw std::runtime_error("No descriptors loaded in '" + sfmDataFilenameA + "'"); - } - } - - if((matchingMode == EImageMatchingMode::A_AB) || - (matchingMode == EImageMatchingMode::A_B)) - { - nbFeaturesLoadedInputB = voctree::populateDatabase(sfmDataB, featuresFolders, tree, db, nbMaxDescriptors); - nbSetDescriptors = db.getSparseHistogramPerImage().size(); - } - - if(matchingMode == EImageMatchingMode::A_A_AND_A_B) - { - nbFeaturesLoadedInputB = voctree::populateDatabase(sfmDataB, featuresFolders, tree, db2, nbMaxDescriptors); - nbSetDescriptors += db2.getSparseHistogramPerImage().size(); - } - - if(useMultiSfM && (nbFeaturesLoadedInputB == 0)) - { - throw std::runtime_error("No descriptors loaded in '" + sfmDataFilenameB + "'"); - } - } - auto detect_elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - detect_start); - - ALICEVISION_LOG_INFO("Read " << nbSetDescriptors << " sets of descriptors for a total of " << (nbFeaturesLoadedInputA + nbFeaturesLoadedInputB) << " features"); - ALICEVISION_LOG_INFO("Reading took " << detect_elapsed.count() << " sec."); - } - - if(!withWeights) - { - // compute and save the word weights - ALICEVISION_LOG_INFO("Computing weights..."); - - db.computeTfIdfWeights(); - - if(matchingMode == EImageMatchingMode::A_A_AND_A_B) - db2.computeTfIdfWeights(); - } - - { - PairList allMatches; - - ALICEVISION_LOG_INFO("Query all documents"); - - auto detect_start = std::chrono::steady_clock::now(); - - if(matchingMode == EImageMatchingMode::A_A_AND_A_B) - { - generateFromVoctree(allMatches, descriptorsFilesA, db, tree, EImageMatchingMode::A_A, nbMaxDescriptors, numImageQuery); - generateFromVoctree(allMatches, descriptorsFilesA, db2, tree, EImageMatchingMode::A_B, nbMaxDescriptors, numImageQuery); - } - else - { - generateFromVoctree(allMatches, descriptorsFilesA, db, tree, matchingMode, nbMaxDescriptors, numImageQuery); - } - - auto detect_elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - detect_start); - ALICEVISION_LOG_INFO("Query all documents took " << detect_elapsed.count() << " sec."); - - // process pair list - detect_start = std::chrono::steady_clock::now(); - - ALICEVISION_LOG_INFO("Convert all matches to pairList"); - convertAllMatchesToPairList(allMatches, numImageQuery, selectedPairs); - detect_elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - detect_start); - ALICEVISION_LOG_INFO("Convert all matches to pairList took " << detect_elapsed.count() << " sec."); - } -} - int aliceVision_main(int argc, char** argv) { // command-line parameters From 4c79c489879c6a08fc72c4556cdbe99dc3084b4c Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Sat, 6 Aug 2022 11:39:10 +0300 Subject: [PATCH 3/7] [imageMatching] Extract more code from main() in main_imageMatching.cpp This allows the code to be reused by other components. --- .../imageMatching/ImageMatching.cpp | 56 +++++++++++++++++++ .../imageMatching/ImageMatching.hpp | 5 ++ src/software/pipeline/main_imageMatching.cpp | 48 +--------------- 3 files changed, 62 insertions(+), 47 deletions(-) diff --git a/src/aliceVision/imageMatching/ImageMatching.cpp b/src/aliceVision/imageMatching/ImageMatching.cpp index ed66ce07f1..8e244b7bd5 100644 --- a/src/aliceVision/imageMatching/ImageMatching.cpp +++ b/src/aliceVision/imageMatching/ImageMatching.cpp @@ -454,5 +454,61 @@ void conditionVocTree(const std::string& treeName, bool withWeights, } } +EImageMatchingMethod selectImageMatchingMethod(EImageMatchingMethod method, + const sfmData::SfMData& sfmDataA, + const sfmData::SfMData& sfmDataB, + std::size_t minNbImages) +{ + if (method == EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE) + { + // Frustum intersection is only implemented for pinhole cameras + bool onlyPinhole = true; + for (auto& cam : sfmDataA.getIntrinsics()) + { + if(!camera::isPinhole(cam.second->getType())) + { + onlyPinhole = false; + break; + } + } + + const std::size_t reconstructedViews = sfmDataA.getValidViews().size(); + if (reconstructedViews == 0) + { + ALICEVISION_LOG_INFO("FRUSTUM_OR_VOCABULARYTREE: Use VOCABULARYTREE matching, as there is no known pose."); + method = EImageMatchingMethod::VOCABULARYTREE; + } + else if (!onlyPinhole) + { + ALICEVISION_LOG_INFO( + "FRUSTUM_OR_VOCABULARYTREE: Use VOCABULARYTREE matching, as the scene contains non-pinhole cameras."); + method = EImageMatchingMethod::VOCABULARYTREE; + } + else if (reconstructedViews == sfmDataA.getViews().size()) + { + ALICEVISION_LOG_INFO("FRUSTUM_OR_VOCABULARYTREE: Use FRUSTUM intersection from known poses."); + method = EImageMatchingMethod::FRUSTUM; + } + else + { + ALICEVISION_LOG_ERROR(reconstructedViews << " reconstructed views for " << sfmDataA.getViews().size() + << " views."); + throw std::runtime_error("FRUSTUM_OR_VOCABULARYTREE: Mixing reconstructed and unreconstructed Views."); + } + } + + // if not enough images to use the VOCABULARYTREE use the EXHAUSTIVE method + if (method == EImageMatchingMethod::VOCABULARYTREE || + method == EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE) + { + if ((sfmDataA.getViews().size() + sfmDataB.getViews().size()) < minNbImages) + { + ALICEVISION_LOG_DEBUG("Use EXHAUSTIVE method instead of VOCABULARYTREE (less images than minNbImages)."); + method = EImageMatchingMethod::EXHAUSTIVE; + } + } + return method; +} + } // namespace imageMatching } // namespace aliceVision diff --git a/src/aliceVision/imageMatching/ImageMatching.hpp b/src/aliceVision/imageMatching/ImageMatching.hpp index 8c04151eb0..80bad5b579 100644 --- a/src/aliceVision/imageMatching/ImageMatching.hpp +++ b/src/aliceVision/imageMatching/ImageMatching.hpp @@ -146,5 +146,10 @@ void conditionVocTree(const std::string& treeName, bool withWeights, std::size_t numImageQuery, OrderedPairList& selectedPairs); +EImageMatchingMethod selectImageMatchingMethod(EImageMatchingMethod method, + const sfmData::SfMData& sfmDataA, + const sfmData::SfMData& sfmDataB, + std::size_t minNbImages); + } // namespace imageMatching } // namespace aliceVision diff --git a/src/software/pipeline/main_imageMatching.cpp b/src/software/pipeline/main_imageMatching.cpp index 7eca3f4927..6fa4b523c9 100644 --- a/src/software/pipeline/main_imageMatching.cpp +++ b/src/software/pipeline/main_imageMatching.cpp @@ -206,53 +206,7 @@ int aliceVision_main(int argc, char** argv) } } - if(method == EImageMatchingMethod::FRUSTUM_OR_VOCABULARYTREE) - { - // Frustum intersection is only implemented for pinhole cameras - bool onlyPinhole = true; - for(auto& cam : sfmDataA.getIntrinsics()) - { - if(!camera::isPinhole(cam.second->getType())) - { - onlyPinhole = false; - break; - } - } - - const std::size_t reconstructedViews = sfmDataA.getValidViews().size(); - if(reconstructedViews == 0) - { - ALICEVISION_LOG_INFO("FRUSTUM_OR_VOCABULARYTREE: Use VOCABULARYTREE matching, as there is no known pose."); - method = EImageMatchingMethod::VOCABULARYTREE; - } - else if(!onlyPinhole) - { - ALICEVISION_LOG_INFO( - "FRUSTUM_OR_VOCABULARYTREE: Use VOCABULARYTREE matching, as the scene contains non-pinhole cameras."); - method = EImageMatchingMethod::VOCABULARYTREE; - } - else if(reconstructedViews == sfmDataA.getViews().size()) - { - ALICEVISION_LOG_INFO("FRUSTUM_OR_VOCABULARYTREE: Use FRUSTUM intersection from known poses."); - method = EImageMatchingMethod::FRUSTUM; - } - else - { - ALICEVISION_LOG_ERROR(reconstructedViews << " reconstructed views for " << sfmDataA.getViews().size() - << " views."); - throw std::runtime_error("FRUSTUM_OR_VOCABULARYTREE: Mixing reconstructed and unreconstructed Views."); - } - } - - // if not enough images to use the VOCABULARYTREE use the EXHAUSTIVE method - if(method == EImageMatchingMethod::VOCABULARYTREE || method == EImageMatchingMethod::SEQUENTIAL_AND_VOCABULARYTREE) - { - if((sfmDataA.getViews().size() + sfmDataB.getViews().size()) < minNbImages) - { - ALICEVISION_LOG_DEBUG("Use EXHAUSTIVE method instead of VOCABULARYTREE (less images than minNbImages)."); - method = EImageMatchingMethod::EXHAUSTIVE; - } - } + method = selectImageMatchingMethod(method, sfmDataA, sfmDataB, minNbImages); std::map descriptorsFilesA, descriptorsFilesB; From 537f9d8e6478cf1aee71a16b0b7af599bc328803 Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Mon, 8 Aug 2022 13:59:47 +0300 Subject: [PATCH 4/7] [featureMatching] Extract filterMatchesByMin2DMotion() --- src/aliceVision/matching/matchesFiltering.cpp | 49 +++++++++++++++++++ src/aliceVision/matching/matchesFiltering.hpp | 5 ++ .../pipeline/main_featureMatching.cpp | 45 +---------------- 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/src/aliceVision/matching/matchesFiltering.cpp b/src/aliceVision/matching/matchesFiltering.cpp index cb1d17aeed..f550002668 100644 --- a/src/aliceVision/matching/matchesFiltering.cpp +++ b/src/aliceVision/matching/matchesFiltering.cpp @@ -139,5 +139,54 @@ void matchesGridFiltering(const aliceVision::feature::Regions& lRegions, outMatches.swap(finalMatches); } +void filterMatchesByMin2DMotion(PairwiseMatches& mapPutativesMatches, + const feature::RegionsPerView& regionPerView, + double minRequired2DMotion) +{ + if (minRequired2DMotion < 0.0f) + return; + + //For each image pair + for (auto& imgPair: mapPutativesMatches) + { + const Pair viewPair = imgPair.first; + IndexT viewI = viewPair.first; + IndexT viewJ = viewPair.second; + + //For each descriptors in this image + for (auto& descType: imgPair.second) + { + const feature::EImageDescriberType type = descType.first; + + const feature::Regions & regions_I = regionPerView.getRegions(viewI, type); + const feature::Regions & regions_J = regionPerView.getRegions(viewJ, type); + + const auto & features_I = regions_I.Features(); + const auto & features_J = regions_J.Features(); + + IndMatches & matches = descType.second; + IndMatches updated_matches; + + for (auto & match : matches) + { + Vec2f pi = features_I[match._i].coords(); + Vec2f pj = features_J[match._j].coords(); + + float scale = std::max(features_I[match._i].scale(), features_J[match._j].scale()); + float coeff = pow(2, scale); + + if ((pi - pj).norm() < (minRequired2DMotion * coeff)) + { + continue; + } + + updated_matches.push_back(match); + } + + matches = updated_matches; + } + } +} + } // namespace sfm } // namespace aliceVision diff --git a/src/aliceVision/matching/matchesFiltering.hpp b/src/aliceVision/matching/matchesFiltering.hpp index 9c9e593fba..2898167bc9 100644 --- a/src/aliceVision/matching/matchesFiltering.hpp +++ b/src/aliceVision/matching/matchesFiltering.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -66,5 +67,9 @@ void matchesGridFiltering(const aliceVision::feature::Regions& lRegions, const aliceVision::Pair& indexImagePair, aliceVision::matching::IndMatches& outMatches, size_t gridSize = 3); +void filterMatchesByMin2DMotion(PairwiseMatches& mapPutativesMatches, + const feature::RegionsPerView& regionPerView, + double minRequired2DMotion); + } // namespace sfm } // namespace aliceVision diff --git a/src/software/pipeline/main_featureMatching.cpp b/src/software/pipeline/main_featureMatching.cpp index 375f513079..ff97d32acd 100644 --- a/src/software/pipeline/main_featureMatching.cpp +++ b/src/software/pipeline/main_featureMatching.cpp @@ -383,50 +383,7 @@ int aliceVision_main(int argc, char **argv) } - if (minRequired2DMotion >= 0.0f) - { - //For each image pair - for (auto& imgPair: mapPutativesMatches) - { - const Pair viewPair = imgPair.first; - IndexT viewI = viewPair.first; - IndexT viewJ = viewPair.second; - - //For each descriptors in this image - for (auto& descType: imgPair.second) - { - const feature::EImageDescriberType type = descType.first; - - const feature::Regions & regions_I = regionPerView.getRegions(viewI, type); - const feature::Regions & regions_J = regionPerView.getRegions(viewJ, type); - - const auto & features_I = regions_I.Features(); - const auto & features_J = regions_J.Features(); - - IndMatches & matches = descType.second; - IndMatches updated_matches; - - for (auto & match : matches) - { - - Vec2f pi = features_I[match._i].coords(); - Vec2f pj = features_J[match._j].coords(); - - float scale = std::max(features_I[match._i].scale(), features_J[match._j].scale()); - float coeff = pow(2, scale); - - if ((pi - pj).norm() < (minRequired2DMotion * coeff)) - { - continue; - } - - updated_matches.push_back(match); - } - - matches = updated_matches; - } - } - } + filterMatchesByMin2DMotion(mapPutativesMatches, regionPerView, minRequired2DMotion); if(mapPutativesMatches.empty()) { From 5caaaf4737b14684368fb12ffd439cd11c16e992 Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Mon, 8 Aug 2022 20:19:23 +0300 Subject: [PATCH 5/7] [featureMatching] Extract matchesGridFilteringForAllPairs() --- src/aliceVision/matching/matchesFiltering.cpp | 54 +++++++++++++++++ src/aliceVision/matching/matchesFiltering.hpp | 7 +++ .../pipeline/main_featureMatching.cpp | 58 +++---------------- 3 files changed, 70 insertions(+), 49 deletions(-) diff --git a/src/aliceVision/matching/matchesFiltering.cpp b/src/aliceVision/matching/matchesFiltering.cpp index f550002668..87d643f5fc 100644 --- a/src/aliceVision/matching/matchesFiltering.cpp +++ b/src/aliceVision/matching/matchesFiltering.cpp @@ -139,6 +139,60 @@ void matchesGridFiltering(const aliceVision::feature::Regions& lRegions, outMatches.swap(finalMatches); } +void matchesGridFilteringForAllPairs(const PairwiseMatches& geometricMatches, + const sfmData::SfMData& sfmData, + const feature::RegionsPerView& regionPerView, + bool useGridSort, + std::size_t numMatchesToKeep, + PairwiseMatches& outPairwiseMatches) +{ + for (const auto& geometricMatch: geometricMatches) + { + //Get the image pair and their matches. + const Pair& indexImagePair = geometricMatch.first; + const MatchesPerDescType& matchesPerDesc = geometricMatch.second; + + for (const auto& match: matchesPerDesc) + { + const feature::EImageDescriberType descType = match.first; + assert(descType != feature::EImageDescriberType::UNINITIALIZED); + const IndMatches& inputMatches = match.second; + + const feature::Regions* rRegions = ®ionPerView.getRegions(indexImagePair.second, descType); + const feature::Regions* lRegions = ®ionPerView.getRegions(indexImagePair.first, descType); + + // get the regions for the current view pair: + if(rRegions && lRegions) + { + // sorting function: + aliceVision::matching::IndMatches outMatches; + sortMatches_byFeaturesScale(inputMatches, *lRegions, *rRegions, outMatches); + + if(useGridSort) + { + // TODO: rename as matchesGridOrdering + matchesGridFiltering(*lRegions, sfmData.getView(indexImagePair.first).getImgSize(), + *rRegions, sfmData.getView(indexImagePair.second).getImgSize(), + indexImagePair, outMatches); + } + + if (numMatchesToKeep > 0) + { + size_t finalSize = std::min(numMatchesToKeep, outMatches.size()); + outMatches.resize(finalSize); + } + + // std::cout << "Left features: " << lRegions->Features().size() << ", right features: " << rRegions->Features().size() << ", num matches: " << inputMatches.size() << ", num filtered matches: " << outMatches.size() << std::endl; + outPairwiseMatches[indexImagePair].insert(std::make_pair(descType, outMatches)); + } + else + { + ALICEVISION_LOG_INFO("You cannot perform the grid filtering with these regions"); + } + } + } +} + void filterMatchesByMin2DMotion(PairwiseMatches& mapPutativesMatches, const feature::RegionsPerView& regionPerView, double minRequired2DMotion) diff --git a/src/aliceVision/matching/matchesFiltering.hpp b/src/aliceVision/matching/matchesFiltering.hpp index 2898167bc9..3d9671f07f 100644 --- a/src/aliceVision/matching/matchesFiltering.hpp +++ b/src/aliceVision/matching/matchesFiltering.hpp @@ -67,6 +67,13 @@ void matchesGridFiltering(const aliceVision::feature::Regions& lRegions, const aliceVision::Pair& indexImagePair, aliceVision::matching::IndMatches& outMatches, size_t gridSize = 3); +void matchesGridFilteringForAllPairs(const PairwiseMatches& geometricMatches, + const sfmData::SfMData& sfmData, + const feature::RegionsPerView& regionPerView, + bool useGridSort, std::size_t numMatchesToKeep, + PairwiseMatches& outPairwiseMatches); + + void filterMatchesByMin2DMotion(PairwiseMatches& mapPutativesMatches, const feature::RegionsPerView& regionPerView, double minRequired2DMotion); diff --git a/src/software/pipeline/main_featureMatching.cpp b/src/software/pipeline/main_featureMatching.cpp index ff97d32acd..b68013d5ff 100644 --- a/src/software/pipeline/main_featureMatching.cpp +++ b/src/software/pipeline/main_featureMatching.cpp @@ -558,57 +558,17 @@ int aliceVision_main(int argc, char **argv) ALICEVISION_LOG_INFO("Grid filtering"); PairwiseMatches finalMatches; - - { - for(const auto& geometricMatch: geometricMatches) - { - //Get the image pair and their matches. - const Pair& indexImagePair = geometricMatch.first; - const aliceVision::matching::MatchesPerDescType& matchesPerDesc = geometricMatch.second; - - for(const auto& match: matchesPerDesc) - { - const feature::EImageDescriberType descType = match.first; - assert(descType != feature::EImageDescriberType::UNINITIALIZED); - const aliceVision::matching::IndMatches& inputMatches = match.second; - - const feature::Regions* rRegions = ®ionPerView.getRegions(indexImagePair.second, descType); - const feature::Regions* lRegions = ®ionPerView.getRegions(indexImagePair.first, descType); - - // get the regions for the current view pair: - if(rRegions && lRegions) - { - // sorting function: - aliceVision::matching::IndMatches outMatches; - sortMatches_byFeaturesScale(inputMatches, *lRegions, *rRegions, outMatches); - - if(useGridSort) - { - // TODO: rename as matchesGridOrdering - matchesGridFiltering(*lRegions, sfmData.getView(indexImagePair.first).getImgSize(), - *rRegions, sfmData.getView(indexImagePair.second).getImgSize(), - indexImagePair, outMatches); - } - if(numMatchesToKeep > 0) - { - size_t finalSize = std::min(numMatchesToKeep, outMatches.size()); - outMatches.resize(finalSize); - } - - // std::cout << "Left features: " << lRegions->Features().size() << ", right features: " << rRegions->Features().size() << ", num matches: " << inputMatches.size() << ", num filtered matches: " << outMatches.size() << std::endl; - finalMatches[indexImagePair].insert(std::make_pair(descType, outMatches)); - } - else - { - ALICEVISION_LOG_INFO("You cannot perform the grid filtering with these regions"); - } - } - } + matchesGridFilteringForAllPairs(geometricMatches, sfmData, regionPerView, useGridSort, + numMatchesToKeep, finalMatches); ALICEVISION_LOG_INFO("After grid filtering:"); - for(const auto& matchGridFiltering: finalMatches) - ALICEVISION_LOG_INFO("\t- image pair (" + std::to_string(matchGridFiltering.first.first) + ", " + std::to_string(matchGridFiltering.first.second) + ") contains " + std::to_string(matchGridFiltering.second.getNbAllMatches()) + " geometric matches."); - } + for (const auto& matchGridFiltering: finalMatches) + { + ALICEVISION_LOG_INFO("\t- image pair (" << matchGridFiltering.first.first << ", " + << matchGridFiltering.first.second << ") contains " + << matchGridFiltering.second.getNbAllMatches() + << " geometric matches."); + } // export geometric filtered matches ALICEVISION_LOG_INFO("Save geometric matches."); From 92e5acd92d333b2250ee9db9fbe5e302a9c82430 Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Wed, 17 Aug 2022 15:50:21 +0300 Subject: [PATCH 6/7] [feature] Make usage of members local in FeatureExtractor --- src/aliceVision/feature/FeatureExtractor.cpp | 19 +++++++++++-------- src/aliceVision/feature/FeatureExtractor.hpp | 2 -- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/aliceVision/feature/FeatureExtractor.cpp b/src/aliceVision/feature/FeatureExtractor.cpp index 826dd40c0f..fdccfa4ff0 100644 --- a/src/aliceVision/feature/FeatureExtractor.cpp +++ b/src/aliceVision/feature/FeatureExtractor.cpp @@ -98,6 +98,9 @@ void FeatureExtractor::process() std::size_t jobMaxMemoryConsuption = 0; + std::vector cpuJobs; + std::vector gpuJobs; + for (auto it = itViewBegin; it != itViewEnd; ++it) { const sfmData::View& view = *(it->second.get()); @@ -107,13 +110,13 @@ void FeatureExtractor::process() jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption); if (viewJob.useCPU()) - _cpuJobs.push_back(viewJob); + cpuJobs.push_back(viewJob); if (viewJob.useGPU()) - _gpuJobs.push_back(viewJob); + gpuJobs.push_back(viewJob); } - if (!_cpuJobs.empty()) + if (!cpuJobs.empty()) { system::MemoryInfo memoryInformation = system::getMemoryInfo(); @@ -173,19 +176,19 @@ void FeatureExtractor::process() nbThreads = std::min(static_cast(omp_get_num_procs()), nbThreads); // nbThreads should not be higher than the number of jobs - nbThreads = std::min(_cpuJobs.size(), nbThreads); + nbThreads = std::min(cpuJobs.size(), nbThreads); ALICEVISION_LOG_INFO("# threads for extraction: " << nbThreads); omp_set_nested(1); #pragma omp parallel for num_threads(nbThreads) - for (int i = 0; i < _cpuJobs.size(); ++i) - computeViewJob(_cpuJobs.at(i), false); + for (int i = 0; i < cpuJobs.size(); ++i) + computeViewJob(cpuJobs.at(i), false); } - if (!_gpuJobs.empty()) + if (!gpuJobs.empty()) { - for (const auto& job : _gpuJobs) + for (const auto& job : gpuJobs) computeViewJob(job, true); } } diff --git a/src/aliceVision/feature/FeatureExtractor.hpp b/src/aliceVision/feature/FeatureExtractor.hpp index b18d91a86e..aeb230fee4 100644 --- a/src/aliceVision/feature/FeatureExtractor.hpp +++ b/src/aliceVision/feature/FeatureExtractor.hpp @@ -59,8 +59,6 @@ class FeatureExtractor int _rangeStart = -1; int _rangeSize = -1; int _maxThreads = -1; - std::vector _cpuJobs; - std::vector _gpuJobs; }; } // namespace feature From 82533b0873551730d8f1bd8bd0e9219e3dd7cb18 Mon Sep 17 00:00:00 2001 From: Monika Kairaityte Date: Fri, 19 Aug 2022 21:15:29 +0300 Subject: [PATCH 7/7] [feature] Make FeatureExtractorViewJob public for further usage --- src/aliceVision/feature/FeatureExtractor.cpp | 98 +++++++------------- src/aliceVision/feature/FeatureExtractor.hpp | 61 +++++++++++- 2 files changed, 91 insertions(+), 68 deletions(-) diff --git a/src/aliceVision/feature/FeatureExtractor.cpp b/src/aliceVision/feature/FeatureExtractor.cpp index fdccfa4ff0..e67f22d4bd 100644 --- a/src/aliceVision/feature/FeatureExtractor.cpp +++ b/src/aliceVision/feature/FeatureExtractor.cpp @@ -16,64 +16,37 @@ namespace fs = boost::filesystem; namespace aliceVision { namespace feature { -struct FeatureExtractor::ViewJob -{ - const sfmData::View& view; - std::size_t memoryConsuption = 0; - std::string outputBasename; - std::vector cpuImageDescriberIndexes; - std::vector gpuImageDescriberIndexes; - - ViewJob(const sfmData::View& view, - const std::string& outputFolder) : - view(view), - outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string()) - {} - - ~ViewJob() = default; - - bool useGPU() const - { - return !gpuImageDescriberIndexes.empty(); - } - - bool useCPU() const - { - return !cpuImageDescriberIndexes.empty(); - } +FeatureExtractorViewJob::FeatureExtractorViewJob(const sfmData::View& view, + const std::string& outputFolder) : + _view(view), + _outputBasename(fs::path(fs::path(outputFolder) / fs::path(std::to_string(view.getViewId()))).string()) +{} - std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const - { - return outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".feat"; - } +FeatureExtractorViewJob::~FeatureExtractorViewJob() = default; - std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const +void FeatureExtractorViewJob::setImageDescribers( + const std::vector>& imageDescribers) +{ + for (std::size_t i = 0; i < imageDescribers.size(); ++i) { - return outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".desc"; - } + const std::shared_ptr& imageDescriber = imageDescribers.at(i); + feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); - void setImageDescribers(const std::vector>& imageDescribers) - { - for (std::size_t i = 0; i < imageDescribers.size(); ++i) + if (fs::exists(getFeaturesPath(imageDescriberType)) && + fs::exists(getDescriptorPath(imageDescriberType))) { - const std::shared_ptr& imageDescriber = imageDescribers.at(i); - feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); - - if (fs::exists(getFeaturesPath(imageDescriberType)) && - fs::exists(getDescriptorPath(imageDescriberType))) - { - continue; - } + continue; + } - memoryConsuption += imageDescriber->getMemoryConsumption(view.getWidth(), view.getHeight()); + _memoryConsuption += imageDescriber->getMemoryConsumption(_view.getWidth(), + _view.getHeight()); - if(imageDescriber->useCuda()) - gpuImageDescriberIndexes.push_back(i); - else - cpuImageDescriberIndexes.push_back(i); - } + if(imageDescriber->useCuda()) + _gpuImageDescriberIndexes.push_back(i); + else + _cpuImageDescriberIndexes.push_back(i); } -}; +} FeatureExtractor::FeatureExtractor(const sfmData::SfMData& sfmData) : @@ -98,16 +71,16 @@ void FeatureExtractor::process() std::size_t jobMaxMemoryConsuption = 0; - std::vector cpuJobs; - std::vector gpuJobs; + std::vector cpuJobs; + std::vector gpuJobs; for (auto it = itViewBegin; it != itViewEnd; ++it) { const sfmData::View& view = *(it->second.get()); - ViewJob viewJob(view, _outputFolder); + FeatureExtractorViewJob viewJob(view, _outputFolder); viewJob.setImageDescribers(_imageDescribers); - jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption); + jobMaxMemoryConsuption = std::max(jobMaxMemoryConsuption, viewJob.memoryConsuption()); if (viewJob.useCPU()) cpuJobs.push_back(viewJob); @@ -193,21 +166,21 @@ void FeatureExtractor::process() } } -void FeatureExtractor::computeViewJob(const ViewJob& job, bool useGPU) +void FeatureExtractor::computeViewJob(const FeatureExtractorViewJob& job, bool useGPU) { image::Image imageGrayFloat; image::Image imageGrayUChar; image::Image mask; - image::readImage(job.view.getImagePath(), imageGrayFloat, image::EImageColorSpace::SRGB); + image::readImage(job.view().getImagePath(), imageGrayFloat, image::EImageColorSpace::SRGB); if (!_masksFolder.empty() && fs::exists(_masksFolder)) { const auto masksFolder = fs::path(_masksFolder); const auto idMaskPath = masksFolder / - fs::path(std::to_string(job.view.getViewId())).replace_extension("png"); + fs::path(std::to_string(job.view().getViewId())).replace_extension("png"); const auto nameMaskPath = masksFolder / - fs::path(job.view.getImagePath()).filename().replace_extension("png"); + fs::path(job.view().getImagePath()).filename().replace_extension("png"); if (fs::exists(idMaskPath)) { @@ -219,10 +192,7 @@ void FeatureExtractor::computeViewJob(const ViewJob& job, bool useGPU) } } - const auto imageDescriberIndexes = useGPU ? job.gpuImageDescriberIndexes - : job.cpuImageDescriberIndexes; - - for (const auto & imageDescriberIndex : imageDescriberIndexes) + for (const auto & imageDescriberIndex : job.imageDescriberIndexes(useGPU)) { const auto& imageDescriber = _imageDescribers.at(imageDescriberIndex); const feature::EImageDescriberType imageDescriberType = imageDescriber->getDescriberType(); @@ -231,7 +201,7 @@ void FeatureExtractor::computeViewJob(const ViewJob& job, bool useGPU) // Compute features and descriptors and export them to files ALICEVISION_LOG_INFO("Extracting " << imageDescriberTypeName << " features from view '" - << job.view.getImagePath() << "' " << (useGPU ? "[gpu]" : "[cpu]")); + << job.view().getImagePath() << "' " << (useGPU ? "[gpu]" : "[cpu]")); std::unique_ptr regions; if (imageDescriber->useFloatImage()) @@ -281,7 +251,7 @@ void FeatureExtractor::computeViewJob(const ViewJob& job, bool useGPU) job.getDescriptorPath(imageDescriberType)); ALICEVISION_LOG_INFO(std::left << std::setw(6) << " " << regions->RegionCount() << " " << imageDescriberTypeName << " features extracted from view '" - << job.view.getImagePath() << "'"); + << job.view().getImagePath() << "'"); } } diff --git a/src/aliceVision/feature/FeatureExtractor.hpp b/src/aliceVision/feature/FeatureExtractor.hpp index aeb230fee4..786e3e1dc7 100644 --- a/src/aliceVision/feature/FeatureExtractor.hpp +++ b/src/aliceVision/feature/FeatureExtractor.hpp @@ -11,11 +11,64 @@ namespace aliceVision { namespace feature { -class FeatureExtractor +class FeatureExtractorViewJob { - struct ViewJob; +public: + FeatureExtractorViewJob(const sfmData::View& view, + const std::string& outputFolder); + + ~FeatureExtractorViewJob(); + + bool useGPU() const + { + return !_gpuImageDescriberIndexes.empty(); + } + + bool useCPU() const + { + return !_cpuImageDescriberIndexes.empty(); + } + + std::string getFeaturesPath(feature::EImageDescriberType imageDescriberType) const + { + return _outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".feat"; + } + + std::string getDescriptorPath(feature::EImageDescriberType imageDescriberType) const + { + return _outputBasename + "." + EImageDescriberType_enumToString(imageDescriberType) + ".desc"; + } + + void setImageDescribers( + const std::vector>& imageDescribers); - public: + const sfmData::View& view() const + { + return _view; + } + + std::size_t memoryConsuption() const + { + return _memoryConsuption; + } + + const std::vector& imageDescriberIndexes(bool useGPU) const + { + return useGPU ? _gpuImageDescriberIndexes + : _cpuImageDescriberIndexes; + } + +private: + const sfmData::View& _view; + std::size_t _memoryConsuption = 0; + std::string _outputBasename; + std::vector _cpuImageDescriberIndexes; + std::vector _gpuImageDescriberIndexes; +}; + +class FeatureExtractor +{ +public: explicit FeatureExtractor(const sfmData::SfMData& sfmData); ~FeatureExtractor(); @@ -50,7 +103,7 @@ class FeatureExtractor private: - void computeViewJob(const ViewJob& job, bool useGPU); + void computeViewJob(const FeatureExtractorViewJob& job, bool useGPU); const sfmData::SfMData& _sfmData; std::vector> _imageDescribers;