Projection adjustments (#118)

* prepare for saving computed camera parameters

* enable saving camera parameters per pano

* prepare for pano rotation

* basic rotation implementation

* add helpers for (un)projecting points

* add rotate mode in sidebar

* wip rotation widget

* add horizontal and vertical handles to rotation widget

* wip code for rotation by dragging

* wip mouse cursor selection for rotation widget

* better algorithm for distance to polyline

* fix for filtering points that cannot be unwarped

* add auto detection of middle image for rotation widget

* working pitch/yaw rotation

* wip roll implementation

* filter out polylines outside viewport

* apply rotation when exiting rotate mode

* auto apply rotation after dragging

* version 0.18.0 prerelease

* fix for build on ubuntu 20

* gitignore + nix shell update

* compute roll axis at the image center

* fix vertical projections

* allow cropping on previews

* save crop rects with each pano

* switching between crop / rotate

* move full res button

* fix roll axis computation

* better flow rotate -> crop

* improve rotation widget handles positioning

* fix crop interactions

* add pitch and yaw axis computation

* add roll handle

* add pitch + yaw rotation speed computation

* fix mouse cursor selection when rotating

* add reset rotation action

* add reset crop action

* split preview panel and widgets code (drag+rotate)

* update help message

* add rotate + crop menu options + shortcuts

* split widgets (rotate + crop)

* tidy

* format

* more tidy

* add tests

* better drag directions + selecting mouse cursor

Author: krupkat

.gitignore

@@ -16,4 +16,6 @@ CMakeSettings.json
 _site
 exiv2/
 Xpano.app
-.cache
+.cache
+.envrc
+.direnv

CMakeLists.txt

@@ -67,12 +67,15 @@ set(XPANO_SOURCES
   "xpano/gui/panels/warning_pane.cc"
   "xpano/gui/pano_gui.cc"
   "xpano/gui/shortcut.cc"
+  "xpano/gui/widgets/drag.cc"
+  "xpano/gui/widgets/rotate.cc"
   "xpano/pipeline/options.cc"
   "xpano/pipeline/stitcher_pipeline.cc"
   "xpano/utils/config.cc"
   "xpano/utils/disjoint_set.cc"
   "xpano/utils/exiv2.cc"
   "xpano/utils/imgui_.cc"
+  "xpano/utils/opencv.cc"
   "xpano/utils/path.cc"
   "xpano/utils/resource.cc"
   "xpano/utils/sdl_.cc"

misc/build/nix/default.nix

@@ -13,5 +13,6 @@ pkgs.mkShell {
     exiv2
     dbus
     (python3.withPackages (pkgs: with pkgs; [ pyyaml ]))
+    clang-tools_15
   ];
 }

tests/CMakeLists.txt

@@ -32,6 +32,7 @@ add_executable(StitcherTest
   ../xpano/pipeline/stitcher_pipeline.cc
   ../xpano/utils/disjoint_set.cc
   ../xpano/utils/exiv2.cc
+  ../xpano/utils/opencv.cc
   ../xpano/utils/path.cc)
 
 target_link_libraries(StitcherTest 

tests/stitcher_pipeline_test.cc

@@ -96,6 +96,49 @@ TEST_CASE("Stitcher pipeline defaults") {
   CHECK(total_pixels == non_zero_pixels);
 }
 
+TEST_CASE("Stitcher pipeline defaults [extra results]") {
+  xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
+
+  auto loading_task = stitcher.RunLoading(kInputs, {}, {});
+  auto result = loading_task.future.get();
+  auto progress = loading_task.progress->Report();
+  CHECK(progress.tasks_done == progress.num_tasks);
+
+  CHECK(result.images.size() == 10);
+  CHECK(result.matches.size() == 17);
+  REQUIRE(result.panos.size() == 2);
+  REQUIRE_THAT(result.panos[0].ids, Equals<int>({1, 2, 3, 4, 5}));
+  REQUIRE_THAT(result.panos[1].ids, Equals<int>({6, 7, 8}));
+
+  // preview
+  auto stitching_task0 = stitcher.RunStitching(result, {.pano_id = 0});
+
+  auto stitch_result0 = stitching_task0.future.get();
+  progress = stitching_task0.progress->Report();
+  CHECK(progress.tasks_done == progress.num_tasks);
+
+  CHECK(stitch_result0.pano.has_value());
+  CHECK(stitch_result0.auto_crop.has_value());
+  CHECK(stitch_result0.mask.has_value());
+  CHECK_FALSE(stitch_result0.export_path.has_value());
+  REQUIRE(stitch_result0.cameras.has_value());
+  CHECK(stitch_result0.cameras->cameras.size() == 5);
+
+  // full resolution
+  auto stitching_task1 =
+      stitcher.RunStitching(result, {.pano_id = 1, .full_res = true});
+  auto stitch_result1 = stitching_task1.future.get();
+  progress = stitching_task1.progress->Report();
+  CHECK(progress.tasks_done == progress.num_tasks);
+
+  CHECK(stitch_result1.pano.has_value());
+  CHECK(stitch_result1.auto_crop.has_value());
+  CHECK(stitch_result1.mask.has_value());
+  CHECK_FALSE(stitch_result1.export_path.has_value());
+  REQUIRE(stitch_result1.cameras.has_value());
+  CHECK(stitch_result1.cameras->cameras.size() == 3);
+}
+
 TEST_CASE("Stitcher pipeline single pano matching") {
   xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
   auto loading_task = stitcher.RunLoading(
@@ -280,6 +323,29 @@ const std::vector<std::filesystem::path> kInputsWithExifMetadata = {
     "data/image08.jpg",
 };
 
+TEST_CASE("Export [extra results]") {
+  const std::filesystem::path tmp_path =
+      xpano::tests::TmpPath().replace_extension("jpg");
+
+  xpano::pipeline::StitcherPipeline<kReturnFuture> stitcher;
+  auto loading_task = stitcher.RunLoading(kInputsWithExifMetadata, {}, {});
+  auto data = loading_task.future.get();
+  REQUIRE(data.panos.size() == 1);
+  auto stitch_result =
+      stitcher.RunStitching(data, {.pano_id = 0, .export_path = tmp_path})
+          .future.get();
+
+  CHECK(stitch_result.pano.has_value());
+  CHECK(stitch_result.auto_crop.has_value());
+  CHECK(stitch_result.mask.has_value());
+  CHECK(stitch_result.export_path.has_value());
+  REQUIRE(stitch_result.cameras.has_value());
+  CHECK(stitch_result.cameras->cameras.size() == 3);
+
+  REQUIRE(std::filesystem::exists(tmp_path));
+  std::filesystem::remove(tmp_path);
+}
+
 TEST_CASE("ExportWithMetadata") {
   const std::filesystem::path tmp_path =
       xpano::tests::TmpPath().replace_extension("jpg");

xpano/algorithm/algorithm.cc

@@ -21,11 +21,14 @@
 #include <opencv2/photo.hpp>
 #include <opencv2/stitching.hpp>
 #include <opencv2/stitching/detail/matchers.hpp>
+#include <opencv2/stitching/detail/warpers.hpp>
+#include <opencv2/stitching/warpers.hpp>
 
 #include "xpano/algorithm/auto_crop.h"
 #include "xpano/algorithm/blenders.h"
 #include "xpano/algorithm/image.h"
 #include "xpano/algorithm/stitcher.h"
+#include "xpano/algorithm/warpers.h"
 #include "xpano/utils/disjoint_set.h"
 #include "xpano/utils/rect.h"
 #include "xpano/utils/threadpool.h"
@@ -38,6 +41,7 @@ void InsertInOrder(int value, std::vector<int>* vec) {
   auto iter = std::lower_bound(vec->begin(), vec->end(), value);
   vec->insert(iter, value);
 }
+
 cv::Ptr<cv::WarperCreator> PickWarper(ProjectionOptions options) {
   cv::Ptr<cv::WarperCreator> warper_creator;
   switch (options.type) {
@@ -74,25 +78,32 @@ cv::Ptr<cv::WarperCreator> PickWarper(ProjectionOptions options) {
   return warper_creator;
 }
 
-std::optional<cv::RotateFlags> GetRotationFlags(
-    WaveCorrectionType wave_correction,
-    cv::detail::WaveCorrectKind wave_correction_auto) {
-  // have to rotate clockwise in case vertical wave correction was either
-  // selected by user or autodetected
-  switch (wave_correction) {
-    case WaveCorrectionType::kOff:
-      [[fallthrough]];
-    case WaveCorrectionType::kHorizontal:
-      return {};
-    case WaveCorrectionType::kVertical:
-      return cv::ROTATE_90_CLOCKWISE;
-    case WaveCorrectionType::kAuto:
+cv::Ptr<cv::WarperCreator> PickWarperPortrait(ProjectionOptions options) {
+  // When vertical wave correction is detected / selected, the portrait
+  // variants of projections are used (if implemented)
+  cv::Ptr<cv::WarperCreator> warper_creator;
+  switch (options.type) {
+    case ProjectionType::kPerspective:
+      warper_creator = cv::makePtr<stitcher::PlanePortraitWarper>();
+      break;
+    case ProjectionType::kCylindrical:
+      warper_creator = cv::makePtr<stitcher::CylindricalPortraitWarper>();
+      break;
+    case ProjectionType::kSpherical:
+      warper_creator = cv::makePtr<stitcher::SphericalPortraitWarper>();
+      break;
+    case ProjectionType::kCompressedRectilinear:
+      warper_creator = cv::makePtr<cv::CompressedRectilinearPortraitWarper>(
+          options.a_param, options.b_param);
+      break;
+    case ProjectionType::kPanini:
+      warper_creator = cv::makePtr<cv::PaniniPortraitWarper>(options.a_param,
+                                                             options.b_param);
+      break;
+    default:
       break;
   }
-  if (wave_correction_auto == cv::detail::WAVE_CORRECT_VERT) {
-    return cv::ROTATE_90_CLOCKWISE;
-  }
-  return {};
+  return warper_creator;
 }
 
 cv::Ptr<cv::FeatureDetector> PickFeaturesFinder(FeatureType feature) {
@@ -242,9 +253,11 @@ std::vector<Pano> FindPanos(const std::vector<Match>& matches,
 }
 
 StitchResult Stitch(const std::vector<cv::Mat>& images,
+                    const std::optional<Cameras>& cameras,
                     StitchUserOptions user_options, StitchOptions options) {
   auto stitcher = stitcher::Stitcher::Create(cv::Stitcher::PANORAMA);
   stitcher->SetWarper(PickWarper(user_options.projection));
+  stitcher->SetPortraitWarper(PickWarperPortrait(user_options.projection));
   stitcher->SetFeaturesFinder(PickFeaturesFinder(user_options.feature));
   stitcher->SetFeaturesMatcher(cv::makePtr<cv::detail::BestOf2NearestMatcher>(
       false, user_options.match_conf));
@@ -259,7 +272,17 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
   stitcher->SetProgressMonitor(options.progress_monitor);
 
   cv::Mat pano;
-  auto status = stitcher->Stitch(images, pano);
+  stitcher::Status status;
+
+  if (cameras &&
+      cameras->wave_correction_user == user_options.wave_correction &&
+      cameras->cameras.size() == images.size()) {
+    stitcher->SetWaveCorrectKind(cameras->wave_correction_auto);
+    stitcher->SetTransform(images, cameras->cameras);
+    status = stitcher->ComposePanorama(pano);
+  } else {
+    status = stitcher->Stitch(images, pano);
+  }
 
   if (status != stitcher::Status::kSuccess) {
     return {status, {}, {}};
@@ -270,16 +293,10 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
     stitcher->ResultMask().copyTo(mask);
   }
 
-  if (auto rotate = GetRotationFlags(user_options.wave_correction,
-                                     stitcher->WaveCorrectKind());
-      rotate) {
-    cv::rotate(pano, pano, *rotate);
-    if (options.return_pano_mask) {
-      cv::rotate(mask, mask, *rotate);
-    }
-  }
-
-  return {status, pano, mask};
+  auto result_cameras =
+      Cameras{stitcher->Cameras(), user_options.wave_correction,
+              stitcher->WaveCorrectKind(), stitcher->GetWarpHelper()};
+  return {status, pano, mask, std::move(result_cameras)};
 }
 
 int StitchTasksCount(int num_images) {
@@ -338,4 +355,13 @@ Pano SinglePano(int size) {
   return pano;
 }
 
+Cameras Rotate(const Cameras& cameras, const cv::Mat& rotation_matrix) {
+  Cameras rotated = cameras;
+  for (auto& camera : rotated.cameras) {
+    camera.R = rotation_matrix * camera.R;
+  }
+
+  return rotated;
+}
+
 }  // namespace xpano::algorithm

xpano/algorithm/algorithm.h

@@ -23,9 +23,20 @@
 
 namespace xpano::algorithm {
 
+struct Cameras {
+  std::vector<cv::detail::CameraParams> cameras;
+  WaveCorrectionType wave_correction_user;           // set by user
+  cv::detail::WaveCorrectKind wave_correction_auto;  // computed by OpenCV
+  stitcher::WarpHelper warp_helper;
+};
+
 struct Pano {
   std::vector<int> ids;
   bool exported = false;
+  std::optional<utils::RectRRf> crop;
+  std::optional<utils::RectRRf> auto_crop;
+  std::optional<Cameras> cameras;
+  std::optional<Cameras> backup_cameras;
 };
 
 struct Match {
@@ -47,6 +58,7 @@ struct StitchResult {
   stitcher::Status status;
   cv::Mat pano;
   cv::Mat mask;
+  Cameras cameras;
 };
 
 struct StitchOptions {
@@ -56,6 +68,7 @@ struct StitchOptions {
 };
 
 StitchResult Stitch(const std::vector<cv::Mat>& images,
+                    const std::optional<Cameras>& cameras,
                     StitchUserOptions user_options, StitchOptions options);
 
 int StitchTasksCount(int num_images);
@@ -67,4 +80,6 @@ std::optional<utils::RectRRf> FindLargestCrop(const cv::Mat& mask);
 cv::Mat Inpaint(const cv::Mat& pano, const cv::Mat& mask,
                 InpaintingOptions options);
 
+Cameras Rotate(const Cameras& cameras, const cv::Mat& rotation_matrix);
+
 }  // namespace xpano::algorithm