diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 4df2a4a5..1f81f320 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -157,7 +157,7 @@ int main(int argc, char **argv) } // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + cv::Vec6f pose_estimate = LandmarkDetector::GetPose(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); // Keeping track of FPS fps_tracker.AddFrame(); diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index c95e7a5d..63220baf 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -100,8 +100,8 @@ namespace LandmarkDetector vector CalculateVisibleLandmarks(const CLNF& clnf_model); vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); - vector CalculateAllLandmarks(const cv::Mat_& shape2D); - vector CalculateAllLandmarks(const CLNF& clnf_model); + vector CalculateAllLandmarks(const cv::Mat_& shape2D); + vector CalculateAllLandmarks(const CLNF& clnf_model); vector CalculateAllEyeLandmarks(const CLNF& clnf_model); vector Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 3f5d7de5..6b4eefc7 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -588,42 +588,37 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa // Do the hierarchical models in parallel tbb::parallel_for(0, (int)hierarchical_models.size(), [&](int part_model){ { - // Only do the synthetic eye models if we're doing gaze - if (!((hierarchical_model_names[part_model].compare("right_eye_28") == 0 || - hierarchical_model_names[part_model].compare("left_eye_28") == 0))) + + int n_part_points = hierarchical_models[part_model].pdm.NumberOfPoints(); + + vector> mappings = this->hierarchical_mapping[part_model]; + + cv::Mat_ part_model_locs(n_part_points * 2, 1, 0.0f); + + // Extract the corresponding landmarks + for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) { + part_model_locs.at(mappings[mapping_ind].second) = detected_landmarks.at(mappings[mapping_ind].first); + part_model_locs.at(mappings[mapping_ind].second + n_part_points) = detected_landmarks.at(mappings[mapping_ind].first + this->pdm.NumberOfPoints()); + } - int n_part_points = hierarchical_models[part_model].pdm.NumberOfPoints(); + // Fit the part based model PDM + hierarchical_models[part_model].pdm.CalcParams(hierarchical_models[part_model].params_global, hierarchical_models[part_model].params_local, part_model_locs); - vector> mappings = this->hierarchical_mapping[part_model]; + // Only do this if we don't need to upsample + if (params_global[0] > 0.9 * hierarchical_models[part_model].patch_experts.patch_scaling[0]) + { + parts_used = true; - cv::Mat_ part_model_locs(n_part_points * 2, 1, 0.0f); + this->hierarchical_params[part_model].window_sizes_current = this->hierarchical_params[part_model].window_sizes_init; - // Extract the corresponding landmarks - for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) - { - part_model_locs.at(mappings[mapping_ind].second) = detected_landmarks.at(mappings[mapping_ind].first); - part_model_locs.at(mappings[mapping_ind].second + n_part_points) = detected_landmarks.at(mappings[mapping_ind].first + this->pdm.NumberOfPoints()); - } + // Do the actual landmark detection + hierarchical_models[part_model].DetectLandmarks(image, hierarchical_params[part_model]); - // Fit the part based model PDM - hierarchical_models[part_model].pdm.CalcParams(hierarchical_models[part_model].params_global, hierarchical_models[part_model].params_local, part_model_locs); - - // Only do this if we don't need to upsample - if (params_global[0] > 0.9 * hierarchical_models[part_model].patch_experts.patch_scaling[0]) - { - parts_used = true; - - this->hierarchical_params[part_model].window_sizes_current = this->hierarchical_params[part_model].window_sizes_init; - - // Do the actual landmark detection - hierarchical_models[part_model].DetectLandmarks(image, hierarchical_params[part_model]); - - } - else - { - hierarchical_models[part_model].pdm.CalcShape2D(hierarchical_models[part_model].detected_landmarks, hierarchical_models[part_model].params_local, hierarchical_models[part_model].params_global); - } + } + else + { + hierarchical_models[part_model].pdm.CalcShape2D(hierarchical_models[part_model].detected_landmarks, hierarchical_models[part_model].params_local, hierarchical_models[part_model].params_global); } } }); @@ -636,15 +631,11 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa { vector> mappings = this->hierarchical_mapping[part_model]; - if (!((hierarchical_model_names[part_model].compare("right_eye_28") == 0 || - hierarchical_model_names[part_model].compare("left_eye_28") == 0))) + // Reincorporate the models into main tracker + for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) { - // Reincorporate the models into main tracker - for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) - { - detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); - detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); - } + detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); + detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); } } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 7f67088e..32d509c2 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -1118,11 +1118,11 @@ namespace LandmarkDetector } // Computing landmarks (to be drawn later possibly) - vector CalculateAllLandmarks(const cv::Mat_& shape2D) + vector CalculateAllLandmarks(const cv::Mat_& shape2D) { int n; - vector landmarks; + vector landmarks; if (shape2D.cols == 2) { @@ -1135,14 +1135,14 @@ namespace LandmarkDetector for (int i = 0; i < n; ++i) { - cv::Point2d featurePoint; + cv::Point2f featurePoint; if (shape2D.cols == 1) { - featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i + n)); + featurePoint = cv::Point2f(shape2D.at(i), shape2D.at(i + n)); } else { - featurePoint = cv::Point2d(shape2D.at(i, 0), shape2D.at(i, 1)); + featurePoint = cv::Point2f(shape2D.at(i, 0), shape2D.at(i, 1)); } landmarks.push_back(featurePoint); @@ -1152,7 +1152,7 @@ namespace LandmarkDetector } // Computing landmarks (to be drawn later possibly) - vector CalculateAllLandmarks(const CLNF& clnf_model) + vector CalculateAllLandmarks(const CLNF& clnf_model) { return CalculateAllLandmarks(clnf_model.detected_landmarks); } diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h index e40e0068..71cc6876 100644 --- a/lib/local/Utilities/include/VisualizationUtils.h +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -45,7 +45,7 @@ namespace Utilities // TODO draw AU results // Drawing a bounding box around the face in an image - void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); + void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); void DrawBox(const std::vector>& lines, cv::Mat image, cv::Scalar color, int thickness); // Computing a bounding box to be drawn diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp index b9e980b2..684487e2 100644 --- a/lib/local/Utilities/src/VisualizationUtils.cpp +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -77,7 +77,7 @@ namespace Utilities frame_times.pop(); } - void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) + void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) { auto edge_lines = CalculateBox(pose, fx, fy, cx, cy); DrawBox(edge_lines, image, color, thickness); @@ -109,7 +109,7 @@ namespace Utilities edges.push_back(std::pair(7, 6)); // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100.0f; + cv::Mat_ box = cv::Mat(8, 3, CV_32F, boxVerts).clone() * 100.0f; cv::Matx33f rot = Euler2RotationMatrix(cv::Vec3f(pose[3], pose[4], pose[5])); cv::Mat_ rotBox;