From 4ab43864ff201c22660a8e2e00f56b33be8d742f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Wed, 6 Feb 2019 12:03:54 +0100 Subject: [PATCH 1/3] [depthMap] Add similarity volume debug export --- .../depthMap/SemiGlobalMatchingRc.cpp | 9 +++ .../depthMap/SemiGlobalMatchingVolume.cpp | 73 +++++++++++++++++++ .../depthMap/SemiGlobalMatchingVolume.hpp | 3 + src/aliceVision/fuseCut/DelaunayGraphCut.cpp | 2 +- src/aliceVision/mvsUtils/MultiViewParams.hpp | 5 ++ 5 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp index 7495847617..3a68f97b89 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp @@ -475,9 +475,15 @@ bool SemiGlobalMatchingRc::sgmrc(bool checkIfExists) delete simVolume; } + if(_sp->exportIntermediateResults) + svol->exportVolume(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_beforeReduction.abc"); + // reduction of 'volume' (X, Y, Z) into 'volumeStepZ' (X, Y, Z/step) svol->cloneVolumeSecondStepZ(); + if(_sp->exportIntermediateResults) + svol->exportVolumeStep(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_afterReduction.abc"); + // filter on the 3D volume to weight voxels based on their neighborhood strongness. // so it downweights local minimums that are not supported by their neighborhood. // this is here for experimental reason ... to show how SGGC work on non @@ -485,6 +491,9 @@ bool SemiGlobalMatchingRc::sgmrc(bool checkIfExists) if(_sp->doSGMoptimizeVolume) svol->SGMoptimizeVolumeStepZ(_rc, _step, 0, 0, _scale); + if(_sp->exportIntermediateResults) + svol->exportVolumeStep(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_afterFiltering.abc"); + // for each pixel: choose the voxel with the minimal similarity value const int zborder = 2; _volumeBestIdVal = svol->getOrigVolumeBestIdValFromVolumeStepZ(zborder); diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp index 529075d268..e77ac6ef1b 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp @@ -7,8 +7,12 @@ #include "SemiGlobalMatchingVolume.hpp" #include #include +#include #include +#include +#include + namespace aliceVision { namespace depthMap { @@ -137,6 +141,75 @@ void SemiGlobalMatchingVolume::cloneVolumeSecondStepZ() mvsUtils::printfElapsedTime(tall, "SemiGlobalMatchingVolume::cloneVolumeSecondStepZ "); } +void SemiGlobalMatchingVolume::exportVolume(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const +{ + sfmData::SfMData pointCloud; + const unsigned char* _volumeSecondBestPtr = _volumeSecondBest->getData().data(); + const mvsUtils::MultiViewParams* mp = sp->mp; + + IndexT landmarkId; + for(int z = 0; z < volDimZ; ++z) + { + for(int y = 0; y < volDimY; ++y) + { + for(int x = 0; x < volDimX; ++x) + { + const double planeDepth = depths[z]; + const Point3d planen = (mp->iRArr[camIndex] * Point3d(0.0f, 0.0f, 1.0f)).normalize(); + const Point3d planep = mp->CArr[camIndex] + planen * planeDepth; + const Point3d v = (mp->iCamArr[camIndex] * Point2d(x * scale * step, y * scale * step)).normalize(); + const Point3d p = linePlaneIntersect(mp->CArr[camIndex], v, planep, planen); + + const int index = z * volDimX * volDimY + y * volDimX + x; + const int maxValue = 80; + if(_volumeSecondBestPtr[index] > maxValue) + continue; + const rgb c = getRGBFromJetColorMap(static_cast(_volumeSecondBestPtr[index]) / double(maxValue)); + pointCloud.getLandmarks()[landmarkId] = sfmData::Landmark(Vec3(p.x, p.y, p.z), feature::EImageDescriberType::UNKNOWN, sfmData::Observations(), image::RGBColor(c.r, c.g, c.b)); + + ++landmarkId; + } + } + } + + sfmDataIO::Save(pointCloud, filepath, sfmDataIO::ESfMData::STRUCTURE); +} + +void SemiGlobalMatchingVolume::exportVolumeStep(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const +{ + sfmData::SfMData pointCloud; + const unsigned char* volumePtr = _volumeStepZ->getData().data(); + const mvsUtils::MultiViewParams* mp = sp->mp; + + IndexT landmarkId; + for(int stepZ = 0; stepZ < (volDimZ / volStepZ); ++stepZ) + { + for(int y = 0; y < volDimY; ++y) + { + for(int x = 0; x < volDimX; ++x) + { + const int z = (*_volumeBestZ)[stepZ * volDimX * volDimY + y * volDimX + x]; + const double planeDepth = depths[z]; + const Point3d planen = (mp->iRArr[camIndex] * Point3d(0.0f, 0.0f, 1.0f)).normalize(); + const Point3d planep = mp->CArr[camIndex] + planen * planeDepth; + const Point3d v = (mp->iCamArr[camIndex] * Point2d(x * scale * step, y * scale * step)).normalize(); + const Point3d p = linePlaneIntersect(mp->CArr[camIndex], v, planep, planen); + + const int index = stepZ * volDimX * volDimY + y * volDimX + x; + const int maxValue = 80; + if(volumePtr[index] > maxValue) + continue; + const rgb c = getRGBFromJetColorMap(static_cast(volumePtr[index]) / double(maxValue)); + pointCloud.getLandmarks()[landmarkId] = sfmData::Landmark(Vec3(p.x, p.y, p.z), feature::EImageDescriberType::UNKNOWN, sfmData::Observations(), image::RGBColor(c.r, c.g, c.b)); + + ++landmarkId; + } + } + } + + sfmDataIO::Save(pointCloud, filepath, sfmDataIO::ESfMData::STRUCTURE); +} + /** * @param[in] volStepXY step in the image space */ diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp index f4fa69c7b1..eb5d91b991 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp @@ -30,6 +30,9 @@ class SemiGlobalMatchingVolume void SGMoptimizeVolumeStepZ(int rc, int volStepXY, int volLUX, int volLUY, int scale); StaticVector* getOrigVolumeBestIdValFromVolumeStepZ(int zborder); + void exportVolume(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const; + void exportVolumeStep(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const; + private: SemiGlobalMatchingParams* sp; diff --git a/src/aliceVision/fuseCut/DelaunayGraphCut.cpp b/src/aliceVision/fuseCut/DelaunayGraphCut.cpp index b0bedd5385..13fe5002db 100644 --- a/src/aliceVision/fuseCut/DelaunayGraphCut.cpp +++ b/src/aliceVision/fuseCut/DelaunayGraphCut.cpp @@ -325,7 +325,7 @@ void createVerticesWithVisibilities(const StaticVector& cams, std::vector

CArr[c] + (mp->iCamArr[c] * Point2d((float)x, (float)y)).normalize() * depth; + const Point3d p = mp->backproject(c, Point2d(x, y), depth); const double pixSize = mp->getCamPixelSize(p, c); #ifdef USE_GEOGRAM_KDTREE const std::size_t nearestVertexIndex = kdTree.get_nearest_neighbor(p.m); diff --git a/src/aliceVision/mvsUtils/MultiViewParams.hpp b/src/aliceVision/mvsUtils/MultiViewParams.hpp index ac5fb3c075..21ebc3b73a 100644 --- a/src/aliceVision/mvsUtils/MultiViewParams.hpp +++ b/src/aliceVision/mvsUtils/MultiViewParams.hpp @@ -109,6 +109,11 @@ class MultiViewParams ~MultiViewParams(); + inline Point3d backproject(const int camIndex, const Point2d& pix, double depth) const + { + const Point3d p = CArr[camIndex] + (iCamArr[camIndex] * pix).normalize() * depth; + return p; + } inline const std::string& getImagePath(int index) const { return _imagesParams.at(index).path; From 97fde86da34e0ed746e2dbd695a3aecbc93265b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gr=C3=A9goire=20De=20Lillo?= Date: Tue, 12 Feb 2019 14:09:46 +0100 Subject: [PATCH 2/3] [depthMap] Add debug 9 points volume CSV exportation --- .../depthMap/SemiGlobalMatchingRc.cpp | 6 +++ .../depthMap/SemiGlobalMatchingVolume.cpp | 45 +++++++++++++++++++ .../depthMap/SemiGlobalMatchingVolume.hpp | 1 + 3 files changed, 52 insertions(+) diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp index 3a68f97b89..2f5c066d34 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp @@ -482,7 +482,10 @@ bool SemiGlobalMatchingRc::sgmrc(bool checkIfExists) svol->cloneVolumeSecondStepZ(); if(_sp->exportIntermediateResults) + { svol->exportVolumeStep(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_afterReduction.abc"); + svol->export9PCSV(*_depths, _rc, _scale, _step, "afterReduction", _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_9p.csv"); + } // filter on the 3D volume to weight voxels based on their neighborhood strongness. // so it downweights local minimums that are not supported by their neighborhood. @@ -492,7 +495,10 @@ bool SemiGlobalMatchingRc::sgmrc(bool checkIfExists) svol->SGMoptimizeVolumeStepZ(_rc, _step, 0, 0, _scale); if(_sp->exportIntermediateResults) + { svol->exportVolumeStep(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_afterFiltering.abc"); + svol->export9PCSV(*_depths, _rc, _scale, _step, "afterFiltering", _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_9p.csv"); + } // for each pixel: choose the voxel with the minimal similarity value const int zborder = 2; diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp index e77ac6ef1b..876b84ba88 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp @@ -210,6 +210,51 @@ void SemiGlobalMatchingVolume::exportVolumeStep(StaticVector& depths, int sfmDataIO::Save(pointCloud, filepath, sfmDataIO::ESfMData::STRUCTURE); } +void SemiGlobalMatchingVolume::export9PCSV(StaticVector& depths, int camIndex, int scale, int step, const std::string& name, const std::string& filepath) const +{ + const unsigned char* volumePtr = _volumeStepZ->getData().data(); + + const int xOffset = std::floor(volDimX / 4.0f); + const int yOffset = std::floor(volDimY / 4.0f); + + std::array, 9> ptsDepths; + + for(int iy = 0; iy < 3; ++iy) + { + for(int ix = 0; ix < 3; ++ix) + { + const int x = (ix + 1) * xOffset; + const int y = (iy + 1) * yOffset; + + std::vector& pDepths = ptsDepths.at(iy * 3 + ix); + + for(int stepZ = 0; stepZ < (volDimZ / volStepZ); ++stepZ) + { + pDepths.push_back(volumePtr[stepZ * volDimX * volDimY + y * volDimX + x]); + } + } + } + + std::stringstream ss; + { + ss << name << "\n"; + int ptId = 1; + for(const std::vector& pDepths : ptsDepths) + { + ss << "p" << ptId << ";"; + for(const double& depth : pDepths) + ss << depth << ";"; + ss << "\n"; + ++ptId; + } + } + + std::ofstream file; + file.open(filepath, std::ios_base::app); + if(file.is_open()) + file << ss.str(); +} + /** * @param[in] volStepXY step in the image space */ diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp index eb5d91b991..06b495b2a8 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.hpp @@ -32,6 +32,7 @@ class SemiGlobalMatchingVolume void exportVolume(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const; void exportVolumeStep(StaticVector& depths, int camIndex, int scale, int step, const std::string& filepath) const; + void export9PCSV(StaticVector& depths, int camIndex, int scale, int step, const std::string& name, const std::string& filepath) const; private: SemiGlobalMatchingParams* sp; From 243fee2c733ed2a7620fa856183a887f8b728559 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Thu, 28 Feb 2019 16:19:11 +0100 Subject: [PATCH 3/3] [depthMap] debug export: reduce amount of data --- src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp | 4 ++-- src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp index 2f5c066d34..c2acd78c0e 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingRc.cpp @@ -475,8 +475,8 @@ bool SemiGlobalMatchingRc::sgmrc(bool checkIfExists) delete simVolume; } - if(_sp->exportIntermediateResults) - svol->exportVolume(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_beforeReduction.abc"); + //if(_sp->exportIntermediateResults) + // svol->exportVolume(*_depths, _rc, _scale, _step, _sp->mp->getDepthMapsFolder() + std::to_string(_sp->mp->getViewId(_rc)) + "_vol_beforeReduction.abc"); // reduction of 'volume' (X, Y, Z) into 'volumeStepZ' (X, Y, Z/step) svol->cloneVolumeSecondStepZ(); diff --git a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp index 876b84ba88..f153e50d42 100644 --- a/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp +++ b/src/aliceVision/depthMap/SemiGlobalMatchingVolume.cpp @@ -146,13 +146,14 @@ void SemiGlobalMatchingVolume::exportVolume(StaticVector& depths, int cam sfmData::SfMData pointCloud; const unsigned char* _volumeSecondBestPtr = _volumeSecondBest->getData().data(); const mvsUtils::MultiViewParams* mp = sp->mp; + int eStep = 10; IndexT landmarkId; for(int z = 0; z < volDimZ; ++z) { - for(int y = 0; y < volDimY; ++y) + for(int y = 0; y < volDimY; y+=eStep) { - for(int x = 0; x < volDimX; ++x) + for(int x = 0; x < volDimX; x+=eStep) { const double planeDepth = depths[z]; const Point3d planen = (mp->iRArr[camIndex] * Point3d(0.0f, 0.0f, 1.0f)).normalize(); @@ -180,13 +181,14 @@ void SemiGlobalMatchingVolume::exportVolumeStep(StaticVector& depths, int sfmData::SfMData pointCloud; const unsigned char* volumePtr = _volumeStepZ->getData().data(); const mvsUtils::MultiViewParams* mp = sp->mp; + int eStep = 10; IndexT landmarkId; for(int stepZ = 0; stepZ < (volDimZ / volStepZ); ++stepZ) { - for(int y = 0; y < volDimY; ++y) + for(int y = 0; y < volDimY; y+=eStep) { - for(int x = 0; x < volDimX; ++x) + for(int x = 0; x < volDimX; x+=eStep) { const int z = (*_volumeBestZ)[stepZ * volDimX * volDimY + y * volDimX + x]; const double planeDepth = depths[z];