add python interface

CMakeLists.txt

@@ -42,6 +42,19 @@ foreach(file ${files})
 	install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
 endforeach()
 
+#copy Dlib Landmark model
+file(GLOB files "lib/local/LandmarkDetector/model/*.dat")
+foreach(file ${files})
+	if (MSVC)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/Debug/model)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/Release/model)
+	else(MSVC)
+		file(COPY ${file} DESTINATION ${CMAKE_BINARY_DIR}/bin/model)
+	endif(MSVC)
+
+	install(FILES ${file} DESTINATION ${CMAKE_CONFIG_DIR}/model)
+endforeach()
+
 # Move the hierarchical LandmarkDetector models
 file(GLOB files "lib/local/LandmarkDetector/model/model*")
 foreach(file ${files})
@@ -178,3 +191,4 @@ add_subdirectory(exe/FaceLandmarkImg)
 add_subdirectory(exe/FaceLandmarkVid)
 add_subdirectory(exe/FaceLandmarkVidMulti)
 add_subdirectory(exe/FeatureExtraction)
+add_subdirectory(python)

lib/3rdParty/dlib/CMakeLists.txt

@@ -361,6 +361,16 @@ if (NOT TARGET dlib)
 
       add_library(dlib STATIC ${source_files} )
       target_link_libraries(dlib ${dlib_needed_libraries} )
+      if (UNIX AND NOT DLIB_IN_PROJECT_BUILD)
+         if (DLIB_USE_CUDA)
+            cuda_add_library(dlib_shared SHARED ${source_files} )
+         else()
+            add_library(dlib_shared SHARED ${source_files} )
+         endif()
+         target_link_libraries(dlib_shared ${dlib_needed_libraries} )
+         install(TARGETS dlib_shared LIBRARY DESTINATION lib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
+
+      endif()
 
    endif ()  ##### end of if NOT DLIB_ISO_CPP_ONLY ##########################################################
 
@@ -429,4 +439,4 @@ if (NOT TARGET dlib)
    endif()
 
 
-endif()
+endif()

lib/local/LandmarkDetector/CMakeLists.txt

@@ -1,6 +1,5 @@
 #TBB library
 include_directories(${TBB_ROOT_DIR}/include)
-
 include_directories(${BOOST_INCLUDE_DIR})
 
 SET(SOURCE
@@ -36,6 +35,9 @@ include_directories(./include)
 include_directories(${LandmarkDetector_SOURCE_DIR}/include)
 
 add_library( LandmarkDetector ${SOURCE} ${HEADERS} )
+add_library( LandmarkDetectorShared SHARED ${SOURCE} ${HEADERS} )
+target_link_libraries( LandmarkDetectorShared ${OpenCV_LIBS} ${Boost_LIBRARIES} ${TBB_LIBRARIES} )
 
 install (TARGETS LandmarkDetector DESTINATION lib)
+install (TARGETS  LandmarkDetectorShared LIBRARY DESTINATION lib PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
 install (FILES ${HEADERS} DESTINATION include/OpenFace)

python/CMakeLists.txt

@@ -0,0 +1,29 @@
+# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC")
+
+find_package( Boost 1.5.9 REQUIRED COMPONENTS filesystem system python)
+
+# Python stuff.
+find_package(PythonLibs 2.7 REQUIRED)
+include_directories(${PYTHON_INCLUDE_DIRS})
+
+# TBB library.
+include_directories(${TBB_ROOT_DIR}/include)
+
+# Local libraries.
+include_directories(${LandmarkDetector_SOURCE_DIR}/include)
+
+include_directories(../lib/local/LandmarkDetector/include)
+include_directories(../lib/local/FaceAnalyser/include)  
+
+add_library(pyopenface SHARED pyopenface.cpp detector.cpp tracker.cpp)
+target_link_libraries(pyopenface LandmarkDetectorShared)
+target_link_libraries(pyopenface ${OpenCV_LIBS} ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} ${TBB_LIBRARIES})
+target_link_libraries(pyopenface dlib_shared)
+
+set_property(TARGET pyopenface PROPERTY CXX_STANDARD 11)
+set_property(TARGET pyopenface PROPERTY SUFFIX .so)
+set_property(TARGET pyopenface PROPERTY PREFIX "")
+
+install(TARGETS pyopenface LIBRARY DESTINATION /usr/lib/python2.7/dist-packages PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ GROUP_WRITE WORLD_WRITE WORLD_EXECUTE WORLD_READ)
+

python/README.md

@@ -0,0 +1,31 @@
+# Depend Boost-python and numpy
+### On Ubuntu 14.04
+```
+    sudo apt-get install libboost-all-dev python-dev
+```
+### On Mac
+```
+   TODO
+```
+
+# PyOpenFace
+## Detector
+### class instance
+    detector = pyopenface.Detector(base_path + "OpenFace/lib/local/LandmarkDetector/model/main_clnf_general.txt")
+### get face rect
+    max_face_rect = detector.detect(img_gray)
+### get face landmark
+    re = detector.landmark(img, max_face_rect)
+    face_keypoints = [(int(kp[0]), int(kp[1])) for kp in zip(re[0], re[1])]
+
+## Tracker()
+### class instance
+    tracker = pyopenface.Tracker()
+### tracking with frame
+    tr_results = tracker.tracking(img_gray)
+### get face rect
+    face_rects = tr_results[0]
+### get face landmark
+    face_keypoints = tr_results[1]
+    face_keypoints = [(int(kp[0]), int(kp[1])) for kp in zip(face_keypoints[0], face_keypoints[1])]
+

python/detector.cpp

@@ -0,0 +1,111 @@
+#include "detector.hpp"
+
+#include <iostream>
+#include <string>
+#include <utility>
+#include <vector>
+#include <LandmarkCoreIncludes.h>
+#include <dlib/image_processing/frontal_face_detector.h>
+
+using std::cout;
+using std::string;
+using std::vector;
+
+Detector * Detector::Create(const char *binary_path) {
+
+  vector<string> arguments;
+  arguments.push_back(string("  ")); // if CPP this is the application name
+  arguments.push_back(string("-mloc"));
+  arguments.push_back(string(binary_path));
+
+  LandmarkDetector::FaceModelParameters det_parameters(arguments);
+  det_parameters.track_gaze = false;
+  // No need to validate detections, as we're not doing tracking.
+  det_parameters.validate_detections = false;
+
+  // Grab camera parameters, if they are not defined 
+  // (approximate values will be used).
+  float fx = 0, fy = 0, cx = 0, cy = 0;
+  int device = -1;
+  // Get camera parameters
+  LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments);
+  // If cx (optical axis centre) is undefined will use the image size/2 as 
+  // an estimate.
+  bool cx_undefined = false;
+  bool fx_undefined = false;
+  if (cx == 0 || cy == 0) {
+    cx_undefined = true;
+  }
+  if (fx == 0 || fy == 0) {
+    fx_undefined = true;
+  }
+
+  // The modules that are being used for tracking.
+  LandmarkDetector::CLNF clnf_model(det_parameters.model_location);
+
+  cv::CascadeClassifier classifier(det_parameters.face_detector_location);
+  dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector();
+
+  return new Detector(det_parameters, clnf_model, classifier, face_detector_hog);
+
+}
+
+Detector::Detector(LandmarkDetector::FaceModelParameters &det_parameters,
+                   LandmarkDetector::CLNF &clnf_model,
+                   cv::CascadeClassifier &classifier,
+                   dlib::frontal_face_detector &face_detector_hog) : det_parameters_(std::move(det_parameters)), clnf_model_(std::move(clnf_model)),
+                                                                     classifier_(std::move(classifier)),
+                                                                     face_detector_hog_(std::move(face_detector_hog)) {}
+
+bool CompareRect(cv::Rect_<double> r1, cv::Rect_<double> r2) {
+
+  return r1.height < r2.height;
+
+}
+
+cv::Rect_<double> Detector::DetectFace(const cv::Mat &grayscale_frame) {
+
+  vector<cv::Rect_<double>> face_detections;
+
+  if(det_parameters_.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) {
+    vector<double> confidences;
+    //cout << "  DetectFacesHOG" << std::endl;
+    LandmarkDetector::DetectFacesHOG( face_detections, grayscale_frame, face_detector_hog_, confidences);
+  } else {
+    //cout << "  DetectFaces" << std::endl;
+    LandmarkDetector::DetectFaces( face_detections, grayscale_frame, classifier_);
+  }
+
+  // Finding the biggest face among the detected ones.
+  // cout << "  Find biggest face" << std::endl;
+  if (face_detections.empty()) {
+      std::cout<< " No faces detected " << std::endl;
+    //throw std::invalid_argument("No faces detected");
+  }
+  cv::Rect_<double> face_l = *max_element( face_detections.begin(), face_detections.end(), CompareRect);
+
+  // Return the biggest face.
+  return face_l;
+
+}
+
+cv::Mat_<int> Detector::GetVisibilities() {
+  int idx = clnf_model_.patch_experts.GetViewIdx(clnf_model_.params_global, 0);
+  return clnf_model_.patch_experts.visibilities[0][idx];
+}
+
+cv::Mat_<double> Detector::Run(const cv::Mat &grayscale_frame, const cv::Rect_<double> &face_rect){
+
+  cv::Mat_<float> depth_image;
+  bool success = LandmarkDetector::DetectLandmarksInImage( grayscale_frame, depth_image, face_rect, clnf_model_, det_parameters_);
+
+  if (!success) {
+    throw std::runtime_error("Unable to detect landmarks");
+  }
+
+  cv::Mat_<double> landmarks_2d = clnf_model_.detected_landmarks;
+  landmarks_2d = landmarks_2d.reshape(1, 2);
+
+  return landmarks_2d;
+
+}

python/detector.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/core.hpp>
+#include <LandmarkCoreIncludes.h>
+
+using std::unique_ptr;
+
+class Detector {
+
+public:
+  static Detector * Create(const char *binary_path);
+  cv::Mat_<double> Run(const cv::Mat &input_frame, const cv::Rect_<double> &face_rect);
+  cv::Rect_<double> DetectFace(const cv::Mat &grayscale_frame);
+
+  cv::Mat_<uchar> grayscale_frame_;
+
+private:
+  Detector(LandmarkDetector::FaceModelParameters &det_parameters,
+           LandmarkDetector::CLNF &clnf_model,
+           cv::CascadeClassifier &classifier,
+           dlib::frontal_face_detector &face_detector_hog);
+
+  cv::Mat_<int> GetVisibilities();
+
+  LandmarkDetector::FaceModelParameters det_parameters_;
+  LandmarkDetector::CLNF clnf_model_;
+  cv::CascadeClassifier classifier_;
+  dlib::frontal_face_detector face_detector_hog_;
+
+};