Feature detect stacks (#114)

* add min shift option

* add test for stack detection

* skip complexity check in tests

* add option to force install desktop files

Author: krupkat

CMakeLists.txt

@@ -4,6 +4,7 @@ project(Xpano)
 OPTION(BUILD_TESTING "Build tests" OFF)
 OPTION(XPANO_STATIC_VCRT "Build with static VCRT" OFF)
 OPTION(XPANO_WITH_MULTIBLEND "Build with multiblend" ON)
+OPTION(XPANO_INSTALL_DESKTOP_FILES "Install desktop files" OFF)
 
 if(XPANO_STATIC_VCRT)
   set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
@@ -167,7 +168,7 @@ install(FILES
   TYPE BIN
 )
 
-if(CMAKE_INSTALL_PREFIX MATCHES "^/usr.*|^/app.*")
+if(XPANO_INSTALL_DESKTOP_FILES OR CMAKE_INSTALL_PREFIX MATCHES "^/usr.*|^/app.*")
   install(FILES
     "misc/build/linux/xpano.desktop"
     DESTINATION "${CMAKE_INSTALL_PREFIX}/share/applications"

tests/stitcher_pipeline_test.cc

@@ -40,6 +40,9 @@ int CountNonZero(const cv::Mat& image) {
   return cv::countNonZero(image_gray);
 }
 
+// Clang-tidy doesn't like the macros
+// NOLINTBEGIN(readability-function-cognitive-complexity)
+
 TEST_CASE("Stitcher pipeline defaults") {
   xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
 
@@ -93,9 +96,6 @@ TEST_CASE("Stitcher pipeline defaults") {
   CHECK(total_pixels == non_zero_pixels);
 }
 
-// Clang-tidy doesn't like the macros
-// NOLINTBEGIN(readability-function-cognitive-complexity)
-
 TEST_CASE("Stitcher pipeline single pano matching") {
   xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
   auto loading_task = stitcher.RunLoading(
@@ -134,8 +134,6 @@ TEST_CASE("Stitcher pipeline no matching") {
   }
 }
 
-// NOLINTEND(readability-function-cognitive-complexity)
-
 const std::vector<std::filesystem::path> kShuffledInputs = {
     "data/image01.jpg",  // Pano 1
     "data/image06.jpg",  // 2
@@ -500,3 +498,37 @@ TEST_CASE("Stitcher pipeline polling") {
   CHECK_THAT(pano1->rows, WithinRel(976, eps));
   CHECK_THAT(pano1->cols, WithinRel(1335, eps));
 }
+
+const std::vector<std::filesystem::path> kInputsWithStack = {
+    "data/image01.jpg",  // Minimal shift
+    "data/image05.jpg",  // between images
+    "data/image06.jpg",
+    "data/image07.jpg",
+};
+
+TEST_CASE("Stitcher pipeline stack detection") {
+  const float min_shift = 0.2f;
+  xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
+  auto loading_task =
+      stitcher.RunLoading(kInputsWithStack, {}, {.min_shift = min_shift});
+  auto result = loading_task.future.get();
+  auto progress = loading_task.progress->Report();
+  CHECK(progress.tasks_done == progress.num_tasks);
+
+  std::vector<xpano::algorithm::Match> good_matches;
+  std::copy_if(result.matches.begin(), result.matches.end(),
+               std::back_inserter(good_matches), [](const auto& match) {
+                 return match.matches.size() >= xpano::kDefaultMatchThreshold;
+               });
+
+  CHECK(result.images.size() == 4);
+
+  REQUIRE(good_matches.size() == 2);
+  CHECK(good_matches[0].avg_shift < min_shift);
+  CHECK(good_matches[1].avg_shift >= min_shift);
+
+  REQUIRE(result.panos.size() == 1);
+  CHECK_THAT(result.panos[0].ids, Equals<int>({2, 3}));
+}
+
+// NOLINTEND(readability-function-cognitive-complexity)

xpano/algorithm/algorithm.cc

@@ -140,8 +140,8 @@ cv::Ptr<cv::detail::Blender> PickBlender(BlendingMethod blending_method,
 
 }  // namespace
 
-std::vector<cv::DMatch> MatchImages(const Image& img1, const Image& img2,
-                                    float match_conf) {
+Match MatchImages(int img1_id, int img2_id, const Image& img1,
+                  const Image& img2, float match_conf) {
   if (img1.GetKeypoints().empty() || img2.GetKeypoints().empty()) {
     return {};
   }
@@ -183,24 +183,35 @@ std::vector<cv::DMatch> MatchImages(const Image& img1, const Image& img2,
 
   // FILTER OUTLIERS
   std::vector<cv::DMatch> inliers;
+  double total_shift = 0.0f;
+
   for (int i = 0; i < good_matches.size(); i++) {
-    const cv::Vec2f diff =
+    const cv::Vec2f proj_diff =
         dst_points.at<cv::Vec2f>(0, i) - dst_points_proj.at<cv::Vec2f>(0, i);
-    if (norm(diff) < 3) {
+    if (norm(proj_diff) < 3) {
       inliers.push_back(good_matches[i]);
+
+      const cv::Vec2f diff =
+          dst_points.at<cv::Vec2f>(0, i) - src_points.at<cv::Vec2f>(0, i);
+      total_shift += norm(diff);
     }
   }
 
-  return inliers;
+  const int max_size =
+      std::max(img1.GetPreviewLongerSide(), img2.GetPreviewLongerSide());
+  const auto avg_shift = static_cast<float>(
+      total_shift / static_cast<double>(inliers.size()) / max_size);
+  return {img1_id, img2_id, inliers, avg_shift};
 }
 
 std::vector<Pano> FindPanos(const std::vector<Match>& matches,
-                            int match_threshold) {
+                            int match_threshold, float min_shift) {
   auto pano_ds = utils::DisjointSet();
 
   std::unordered_set<int> images_in_panos;
   for (const auto& match : matches) {
-    if (match.matches.size() > match_threshold) {
+    if (match.matches.size() >= match_threshold &&
+        match.avg_shift >= min_shift) {
       pano_ds.Union(match.id1, match.id2);
       images_in_panos.insert(match.id1);
       images_in_panos.insert(match.id2);

xpano/algorithm/algorithm.h

@@ -32,15 +32,16 @@ struct Match {
   int id1;
   int id2;
   std::vector<cv::DMatch> matches;
+  float avg_shift = 0.0f;
 };
 
 Pano SinglePano(int size);
 
-std::vector<cv::DMatch> MatchImages(const Image& img1, const Image& img2,
-                                    float match_conf);
+Match MatchImages(int img1_id, int img2_id, const Image& img1,
+                  const Image& img2, float match_conf);
 
 std::vector<Pano> FindPanos(const std::vector<Match>& matches,
-                            int match_threshold);
+                            int match_threshold, float min_shift);
 
 struct StitchResult {
   stitcher::Status status;

xpano/algorithm/image.cc

@@ -84,6 +84,10 @@ cv::Mat Image::GetFullRes() const { return cv::imread(path_.string()); }
 cv::Mat Image::GetThumbnail() const { return thumbnail_; }
 cv::Mat Image::GetPreview() const { return preview_; }
 
+int Image::GetPreviewLongerSide() const {
+  return std::max(preview_.size[1], preview_.size[0]);
+}
+
 float Image::GetAspect() const {
   auto width = static_cast<float>(preview_.size[1]);
   auto height = static_cast<float>(preview_.size[0]);

xpano/algorithm/image.h

@@ -26,6 +26,7 @@ class Image {
   [[nodiscard]] cv::Mat GetFullRes() const;
   [[nodiscard]] cv::Mat GetThumbnail() const;
   [[nodiscard]] cv::Mat GetPreview() const;
+  [[nodiscard]] int GetPreviewLongerSide() const;
   [[nodiscard]] float GetAspect() const;
   [[nodiscard]] cv::Mat Draw(bool show_debug) const;
   [[nodiscard]] const std::vector<cv::KeyPoint>& GetKeypoints() const;

xpano/constants.h

@@ -17,6 +17,10 @@ constexpr int kMinMatchThreshold = 4;
 constexpr int kDefaultMatchThreshold = 70;
 constexpr int kMaxMatchThreshold = 250;
 
+constexpr float kMinShiftInPano = 0.0f;
+constexpr float kDefaultShiftInPano = 0.1f;
+constexpr float kMaxShiftInPano = 1.0f;
+
 constexpr int kWindowWidth = 1280;
 constexpr int kWindowHeight = 800;
 constexpr int kMinWindowSize = 200;

xpano/gui/panels/sidebar.cc

@@ -193,9 +193,9 @@ void DrawMatchingOptionsMenu(pipeline::MatchingOptions* matching_options,
       ImGui::Spacing();
       ImGui::Text(
           "Experiment with this if the app cannot find the panoramas you "
-          "want.");
+          "want.\nThese options are applied only after reloading images.");
       ImGui::Spacing();
-      ImGui::SliderInt("Matching distance",
+      ImGui::SliderInt("Matching neighbors",
                        &matching_options->neighborhood_search_size, 0,
                        kMaxNeighborhoodSearchSize);
       ImGui::SameLine();
@@ -210,6 +210,15 @@ void DrawMatchingOptionsMenu(pipeline::MatchingOptions* matching_options,
                                "Number of keypoints that need to match in "
                                "order to include the two "
                                "images in a panorama.");
+      ImGui::SliderFloat("Minimum shift", &matching_options->min_shift,
+                         kMinShiftInPano, kMaxShiftInPano, "%.2f");
+      ImGui::SameLine();
+      utils::imgui::InfoMarker(
+          "(?)",
+          "Minimum shift between images for it to be considered a part of a "
+          "panorama.\nUseful to filter out burst shots / focus stacks that "
+          "shouldn't be handled by Xpano.\nThe value is specified in relative "
+          "terms to the size of the image.");
       if (debug_enabled) {
         ImGui::SeparatorText("Debug");
         DrawMatchConf(&matching_options->match_conf);

xpano/gui/pano_gui.cc

@@ -433,6 +433,7 @@ Action PanoGui::PerformAction(const Action& action) {
       selection_ = {SelectionType::kMatch, action.target_id};
       spdlog::info("Clicked match {}", action.target_id);
       const auto& match = stitcher_data_->matches[action.target_id];
+      spdlog::info("Match distance {}", match.avg_shift);
       auto img = DrawMatches(match, stitcher_data_->images);
       plot_pane_.Load(img, ImageType::kMatch);
       thumbnail_pane_.SetScrollX(match.id1, match.id2);

xpano/pipeline/options.h

@@ -59,6 +59,7 @@ struct MatchingOptions {
   MatchingType type = MatchingType::kAuto;
   int neighborhood_search_size = kDefaultNeighborhoodSearchSize;
   int match_threshold = kDefaultMatchThreshold;
+  float min_shift = kDefaultShiftInPano;
   float match_conf = kDefaultMatchConf;
 };
 

xpano/pipeline/stitcher_pipeline.cc

@@ -175,8 +175,7 @@ StitcherData RunMatchingPipeline(std::vector<algorithm::Image> images,
       matches_future.push_back(
           pool->submit([i, j, left = images[i], right = images[j],
                         match_conf = options.match_conf, progress]() {
-            auto match =
-                algorithm::Match{i, j, MatchImages(left, right, match_conf)};
+            auto match = algorithm::MatchImages(i, j, left, right, match_conf);
             progress->NotifyTaskDone();
             return match;
           }));
@@ -188,7 +187,7 @@ StitcherData RunMatchingPipeline(std::vector<algorithm::Image> images,
   }
   auto matches = matches_future.get();
 
-  auto panos = FindPanos(matches, options.match_threshold);
+  auto panos = FindPanos(matches, options.match_threshold, options.min_shift);
   progress->NotifyTaskDone();
   return StitcherData{images, matches, panos};
 }