Panorama downscaling (#134)

* cap max pano size in MPx

* warn about resolution limit reached, show full res mpx message

* fix test case, progress bar with recomputed pano

* increase max pano size to 100MPx, add info message

* switch to clang-format-18

* fix clang-tidy

* more deterministic test

.github/workflows/clang-format-check.yml

@@ -16,7 +16,7 @@ permissions:
 
 jobs:
   clang-format-check:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
 

misc/scripts/format.sh

@@ -1,2 +1,2 @@
-clang-format-15 -i `find xpano -name *.cc -or -name *.h`
-clang-format-15 -i `find tests -name *.cc -or -name *.h`
+clang-format-18 -i `find xpano -name *.cc -or -name *.h`
+clang-format-18 -i `find tests -name *.cc -or -name *.h`

tests/stitcher_pipeline_test.cc

@@ -26,6 +26,7 @@
 #include "xpano/algorithm/options.h"
 #include "xpano/algorithm/stitcher.h"
 #include "xpano/constants.h"
+#include "xpano/utils/opencv.h"
 #include "xpano/utils/rect.h"
 #include "xpano/utils/vec_opencv.h"
 
@@ -164,10 +165,12 @@ TEST_CASE("Pano too large") {
   REQUIRE_THAT(stitch_data.panos[0].ids, Equals<int>({0, 1, 2, 3, 4}));
 
   const float eps = 0.02;
+  const int max_pano_mpx = 16;
 
   // stitch for the 1st time
   auto proj_options = xpano::algorithm::StitchUserOptions{
-      .projection = {.type = xpano::algorithm::ProjectionType::kPerspective}};
+      .projection = {.type = xpano::algorithm::ProjectionType::kPerspective},
+      .max_pano_mpx = max_pano_mpx};
   auto stitching_task0 = stitcher.RunStitching(
       stitch_data, {.pano_id = 0, .stitch_algorithm = proj_options});
 
@@ -195,10 +198,13 @@ TEST_CASE("Pano too large") {
   auto stitch_result1 = stitching_task1.future.get();
   progress = stitching_task1.progress->Report();
 
-  REQUIRE(progress.tasks_done != progress.num_tasks);
-  REQUIRE_FALSE(stitch_result1.pano.has_value());
+  REQUIRE(progress.tasks_done == progress.num_tasks);
+  REQUIRE(stitch_result1.pano.has_value());
   REQUIRE(stitch_result1.status ==
-          xpano::algorithm::stitcher::Status::kErrPanoTooLarge);
+          xpano::algorithm::stitcher::Status::kSuccessResolutionCapped);
+
+  auto pano_mpx = xpano::utils::opencv::MPx(*stitch_result1.pano);
+  CHECK_THAT(pano_mpx, WithinRel(max_pano_mpx, eps));
 }
 
 const std::vector<std::filesystem::path> kInputsIncomplete = {

tests/vec_test.cc

@@ -87,8 +87,8 @@ TEST_CASE("Add") {
 
 template <typename TLeft, typename TRight, typename TResult = void>
 concept Addable = requires(TLeft lhs, TRight rhs, TResult result) {
-                    { lhs + rhs } -> std::same_as<TResult>;
-                  };
+  { lhs + rhs } -> std::same_as<TResult>;
+};
 
 TEST_CASE("Add type checks") {
   REQUIRE(Addable<Vec2f, Vec2f, Vec2f>);
@@ -124,8 +124,8 @@ TEST_CASE("Subtract") {
 
 template <typename TLeft, typename TRight, typename TResult = void>
 concept Subtractable = requires(TLeft lhs, TRight rhs, TResult result) {
-                         { lhs - rhs } -> std::same_as<TResult>;
-                       };
+  { lhs - rhs } -> std::same_as<TResult>;
+};
 
 TEST_CASE("Subtract type checks") {
   REQUIRE(Subtractable<Vec2f, Vec2f, Vec2f>);
@@ -191,8 +191,8 @@ TEST_CASE("Divide by Vec") {
 
 template <typename TLeft, typename TRight, typename TResult = void>
 concept Divisible = requires(TLeft lhs, TRight rhs, TResult result) {
-                      { lhs / rhs } -> std::same_as<TResult>;
-                    };
+  { lhs / rhs } -> std::same_as<TResult>;
+};
 
 TEST_CASE("Divide type checks") {
   SECTION("Divide by constant") {
@@ -280,8 +280,8 @@ TEST_CASE("Multiply by Ratio") {
 
 template <typename TLeft, typename TRight, typename TResult = void>
 concept Multiplicable = requires(TLeft lhs, TRight rhs, TResult result) {
-                          { lhs* rhs } -> std::same_as<TResult>;
-                        };
+  { lhs* rhs } -> std::same_as<TResult>;
+};
 
 TEST_CASE("Multiply type checks") {
   SECTION("Multiply by constant") {

xpano/algorithm/algorithm.cc

@@ -260,7 +260,7 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
       false, user_options.match_conf));
   stitcher->SetWaveCorrection(user_options.wave_correction !=
                               WaveCorrectionType::kOff);
-  stitcher->SetMaxPanoSize(user_options.max_pano_size);
+  stitcher->SetMaxPanoMpx(user_options.max_pano_mpx);
   if (stitcher->WaveCorrection()) {
     stitcher->SetWaveCorrectKind(
         PickWaveCorrectKind(user_options.wave_correction));
@@ -281,7 +281,7 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
     status = stitcher->Stitch(images, pano);
   }
 
-  if (status != stitcher::Status::kSuccess) {
+  if (!IsSuccess(status)) {
     return {status, {}, {}};
   }
 
@@ -296,12 +296,15 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
   return {status, pano, mask, std::move(result_cameras)};
 }
 
-int StitchTasksCount(int num_images) {
-  return 1 +           // find features
-         1 +           // match features
-         1 +           // estimate homography
-         1 +           // bundle adjustment
-         1 +           // compute pano size
+int StitchTasksCount(int num_images, bool cameras_precomputed) {
+  int tasks = 0;
+  if (!cameras_precomputed) {
+    tasks += 1 +  // find features
+             1 +  // match features
+             1 +  // estimate homography
+             1;   // bundle adjustment
+  }
+  return tasks + 1 +   // compute pano size
          1 +           // prepare seams
          1 +           // find seams
          num_images +  // compose
@@ -313,6 +316,8 @@ std::string ToString(stitcher::Status& status) {
   switch (status) {
     case stitcher::Status::kSuccess:
       return "OK";
+    case stitcher::Status::kSuccessResolutionCapped:
+      return "OK_resolution_capped";
     case stitcher::Status::kCancelled:
       return "Cancelled";
     case stitcher::Status::kErrNeedMoreImgs:
@@ -321,8 +326,6 @@ std::string ToString(stitcher::Status& status) {
       return "ERR_HOMOGRAPHY_EST_FAIL";
     case stitcher::Status::kErrCameraParamsAdjustFail:
       return "ERR_CAMERA_PARAMS_ADJUST_FAIL";
-    case stitcher::Status::kErrPanoTooLarge:
-      return "ERR_PANO_TOO_LARGE\nReset the adjustments through the edit menu.";
     default:
       return "ERR_UNKNOWN";
   }

xpano/algorithm/algorithm.h

@@ -69,7 +69,7 @@ StitchResult Stitch(const std::vector<cv::Mat>& images,
                     const std::optional<Cameras>& cameras,
                     StitchUserOptions user_options, StitchOptions options);
 
-int StitchTasksCount(int num_images);
+int StitchTasksCount(int num_images, bool cameras_precomputed);
 
 std::string ToString(stitcher::Status& status);
 

xpano/algorithm/options.h

@@ -84,7 +84,7 @@ struct StitchUserOptions {
   FeatureType feature = FeatureType::kSift;
   WaveCorrectionType wave_correction = WaveCorrectionType::kAuto;
   float match_conf = kDefaultMatchConf;
-  int max_pano_size = kMaxPanoSize;
+  int max_pano_mpx = kMaxPanoMpx;
   BlendingMethod blending_method = kDefaultBlendingMethod;
 };
 

xpano/algorithm/stitcher.cc

@@ -132,8 +132,47 @@ std::vector<TType> Index(const std::vector<TType> &vec,
   return subset;
 }
 
+struct Roi {
+  std::vector<cv::Point> corners;
+  std::vector<cv::Size> sizes;
+  cv::Ptr<cv::detail::RotationWarper> warper;
+  cv::Rect rect;
+};
+
+Roi ComputeRoi(const std::vector<cv::detail::CameraParams> &cameras_scaled,
+               std::vector<cv::Size> full_img_sizes,
+               const cv::Ptr<cv::WarperCreator> &warper_creater,
+               float warp_scale) {
+  spdlog::info("Calculating pano size... ");
+  auto compute_roi_timer = Timer();
+
+  std::vector<cv::Point> corners(cameras_scaled.size());
+  std::vector<cv::Size> sizes(cameras_scaled.size());
+
+  auto warper = warper_creater->create(warp_scale);
+
+  // Update corners and sizes
+  for (size_t i = 0; i < cameras_scaled.size(); ++i) {
+    auto k_float = utils::opencv::ToFloat(cameras_scaled[i].K());
+    const cv::Rect roi =
+        warper->warpRoi(full_img_sizes[i], k_float, cameras_scaled[i].R);
+    corners[i] = roi.tl();
+    sizes[i] = roi.size();
+  }
+
+  auto dst_roi = cv::detail::resultRoi(corners, sizes);
+
+  compute_roi_timer.Report(" compute pano size time");
+  return {corners, sizes, warper, dst_roi};
+}
+
 }  // namespace
 
+bool IsSuccess(Status status) {
+  return status == Status::kSuccess ||
+         status == Status::kSuccessResolutionCapped;
+}
+
 cv::Ptr<Stitcher> Stitcher::Create(Mode mode) {
   cv::Ptr<Stitcher> stitcher = cv::makePtr<Stitcher>();
 
@@ -265,36 +304,32 @@ Status Stitcher::ComposePanorama(cv::OutputArray pano) {
   auto compose_work_aspect = 1.0 / work_scale_;
   auto cameras_scaled = utils::opencv::Scale(cameras_, compose_work_aspect);
 
-  std::vector<cv::Point> corners(imgs_.size());
-  std::vector<cv::Size> sizes(imgs_.size());
+  NextTask(ProgressType::kStitchComputeRoi);
 
-  cv::Ptr<cv::detail::RotationWarper> warper;
-  {
-    spdlog::info("Calculating pano size... ");
-    NextTask(ProgressType::kStitchComputeRoi);
-    auto compute_roi_timer = Timer();
+  bool resolution_capped = false;
+  auto warp_scale =
+      static_cast<float>(warped_image_scale_ * compose_work_aspect);
+  auto roi =
+      ComputeRoi(cameras_scaled, full_img_sizes_, warper_creater_, warp_scale);
+  auto pano_mpx = utils::opencv::MPx(roi.rect);
 
-    // Update warped image scale
-    auto warp_scale =
+  if (pano_mpx > max_pano_mpx_) {
+    const float downscale_ratio = std::sqrt(max_pano_mpx_ / pano_mpx);
+    warped_image_scale_ *= downscale_ratio;
+
+    spdlog::warn(
+        "Panorama is too large to compute: {}x{} ({:.2f} Mpx), max size is {} "
+        "MPx",
+        roi.rect.width, roi.rect.height, pano_mpx, max_pano_mpx_);
+
+    auto smaller_warp_scale =
         static_cast<float>(warped_image_scale_ * compose_work_aspect);
-    warper = warper_creater_->create(warp_scale);
-
-    // Update corners and sizes
-    for (size_t i = 0; i < imgs_.size(); ++i) {
-      auto k_float = utils::opencv::ToFloat(cameras_scaled[i].K());
-      const cv::Rect roi =
-          warper->warpRoi(full_img_sizes_[i], k_float, cameras_scaled[i].R);
-      corners[i] = roi.tl();
-      sizes[i] = roi.size();
-    }
-    compute_roi_timer.Report(" compute pano size time");
-  }
-  auto dst_roi = cv::detail::resultRoi(corners, sizes);
+    roi = ComputeRoi(cameras_scaled, full_img_sizes_, warper_creater_,
+                     smaller_warp_scale);
+    spdlog::warn("Limiting panorama size to {}x{}", roi.rect.width,
+                 roi.rect.height);
 
-  if (dst_roi.width >= max_pano_size_ || dst_roi.height >= max_pano_size_) {
-    spdlog::error("Panorama is too large to compute: {}x{}, max size is {}",
-                  dst_roi.width, dst_roi.height, max_pano_size_);
-    return Status::kErrPanoTooLarge;
+    resolution_capped = true;
   }
 
   std::vector<cv::UMat> masks_warped;
@@ -317,7 +352,7 @@ Status Stitcher::ComposePanorama(cv::OutputArray pano) {
   spdlog::info("Compositing...");
   auto compositing_total_timer = Timer();
 
-  blender_->prepare(dst_roi);
+  blender_->prepare(roi.rect);
   for (size_t img_idx = 0; img_idx < imgs_.size(); ++img_idx) {
     NextTask(ProgressType::kStitchCompose);
     if (auto non_zero = cv::countNonZero(masks_warped[img_idx]);
@@ -337,19 +372,19 @@ Status Stitcher::ComposePanorama(cv::OutputArray pano) {
     auto timer = Timer();
 
     // Warp the current image
-    warper->warp(img, k_float, cameras_[img_idx].R, interp_flags_,
-                 cv::BORDER_REFLECT, img_warped);
+    roi.warper->warp(img, k_float, cameras_[img_idx].R, interp_flags_,
+                     cv::BORDER_REFLECT, img_warped);
     timer.Report(" warp the current image");
 
     // Warp the current image mask
     mask.create(img_size, CV_8U);
     mask.setTo(cv::Scalar::all(kMaskValueOn));
-    warper->warp(mask, k_float, cameras_[img_idx].R, cv::INTER_NEAREST,
-                 cv::BORDER_CONSTANT, mask_warped);
+    roi.warper->warp(mask, k_float, cameras_[img_idx].R, cv::INTER_NEAREST,
+                     cv::BORDER_CONSTANT, mask_warped);
     timer.Report(" warp the current image mask");
 
     // Compensate exposure
-    exposure_comp_->apply(static_cast<int>(img_idx), corners[img_idx],
+    exposure_comp_->apply(static_cast<int>(img_idx), roi.corners[img_idx],
                           img_warped, mask_warped);
     timer.Report(" compensate exposure");
 
@@ -365,7 +400,7 @@ Status Stitcher::ComposePanorama(cv::OutputArray pano) {
     timer.Report(" other");
 
     // Blend the current image
-    blender_->feed(img_warped, mask_warped, corners[img_idx]);
+    blender_->feed(img_warped, mask_warped, roi.corners[img_idx]);
     timer.Report(" feed time");
 
     compositing_timer.Report("Compositing ## time");
@@ -386,11 +421,12 @@ Status Stitcher::ComposePanorama(cv::OutputArray pano) {
 
   pano.assign(result);
 
-  warp_helper_ = {work_scale_, corners, sizes, full_img_sizes_,
-                  std::move(warper)};
+  warp_helper_ = {work_scale_, roi.corners, roi.sizes, full_img_sizes_,
+                  std::move(roi.warper)};
 
   EndMonitoring();
-  return Status::kSuccess;
+  return (resolution_capped) ? Status::kSuccessResolutionCapped
+                             : Status::kSuccess;
 }
 
 Status Stitcher::Stitch(cv::InputArrayOfArrays images, cv::OutputArray pano) {

xpano/algorithm/stitcher.h

@@ -61,13 +61,15 @@ namespace xpano::algorithm::stitcher {
 
 enum class Status {
   kSuccess,
+  kSuccessResolutionCapped,
   kCancelled,
   kErrNeedMoreImgs,
   kErrHomographyEstFail,
-  kErrCameraParamsAdjustFail,
-  kErrPanoTooLarge
+  kErrCameraParamsAdjustFail
 };
 
+bool IsSuccess(Status status);
+
 struct WarpHelper {
   double work_scale;
   std::vector<cv::Point> corners;
@@ -135,8 +137,9 @@ class Stitcher {
     features_matcher_ = features_matcher;
   }
 
-  [[nodiscard]] int MaxPanoSize() const { return max_pano_size_; }
-  void SetMaxPanoSize(int max_pano_size) { max_pano_size_ = max_pano_size; }
+  void SetMaxPanoMpx(int max_pano_mpx) {
+    max_pano_mpx_ = static_cast<float>(max_pano_mpx);
+  }
 
   [[nodiscard]] const cv::UMat& MatchingMask() const { return matching_mask_; }
   void SetMatchingMask(const cv::UMat& mask) {
@@ -275,7 +278,7 @@ class Stitcher {
 
   ProgressMonitor* monitor_ = nullptr;
   WarpHelper warp_helper_ = {};
-  int max_pano_size_;
+  float max_pano_mpx_;
 };
 
 }  // namespace xpano::algorithm::stitcher

xpano/constants.h

@@ -103,7 +103,6 @@ constexpr int kCancelAnimationFrameDuration = 128;
 const std::string kGithubIssuesLink = "https://github.com/krupkat/xpano/issues";
 const std::string kAuthorEmail = "[email protected]";
 
-constexpr int kMaxPanoSize = 16384;
-constexpr int kMaxPanoSizeStep = 256;
+constexpr int kMaxPanoMpx = 100;
 
 }  // namespace xpano

xpano/gui/action.h

@@ -34,6 +34,7 @@ enum class ActionType {
   kShowPano,
   kModifyPano,
   kRecomputePano,
+  kRecomputePanoFullRes,
   kQuit,
   kToggleDebugLog,
   kWarnInputConversion,