ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2026-04-17 01:30:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Continuing towards visualizer integration and 3D eye landmark inclusion in output. Cleanup of rotation conversion in unilities

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2017-11-11 21:13:29 +00:00
parent e636b4ca7c
commit e1c260eee6
10 changed files with 195 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
lib/local/Utilities/src/Visualizer.cpp
View File

@@ -34,8 +34,15 @@
#include "Visualizer.h"
#include "VisualizationUtils.h"
// For drawing on images
#include <opencv2/imgproc.hpp>
using namespace Utilities;
// For subpixel accuracy drawing
const int draw_shiftbits = 4;
const int draw_multiplier = 1 << 4;
Visualizer::Visualizer(std::vector<std::string> arguments)
{
// By default not visualizing anything
@@ -97,7 +104,26 @@ void Visualizer::SetObservationHOG(const cv::Mat_<double>& hog_descriptor, int n
void Visualizer::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, double confidence, bool success, const cv::Mat_<int>& visibilities)
{
DrawLandmarkDetResults(captured_image, landmarks_2D, visibilities);
// Draw 2D landmarks on the image
int n = landmarks_2D.rows / 2;
// Drawing feature points
for (int i = 0; i < n; ++i)
{
if (visibilities.empty() || visibilities.at<int>(i))
{
cv::Point featurePoint(cvRound(landmarks_2D.at<double>(i) * (double)draw_multiplier), cvRound(landmarks_2D.at<double>(i + n) * (double)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0);
int thickness_2 = (int)std::ceil(1.0* ((double)captured_image.cols) / 640.0);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits);
cv::circle(captured_image, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
}
}
}
void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
@@ -123,18 +149,103 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
}
}
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1,
const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks)
// TODO add 3D eye landmark locations
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const cv::Vec2d& gaze_angle, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d)
{
// TODO actual drawing
// TODO actual drawing, first of eye landmarks then of gaze
if (det_parameters.track_gaze && detection_success && face_model.eye_model)
if (eye_landmarks.size() > 0)
{
GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy);
// FIrst draw the eye region landmarks
for (size_t i = 0; i < eye_landmarks.size(); ++i)
{
cv::Point featurePoint(cvRound(eye_landmarks[i].x * (double)draw_multiplier), eye_landmarks[i].y * (double)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = 1.0;
int thickness_2 = 1.0;
int next_point = i + 1;
if (i == 7)
next_point = 0;
if (i == 19)
next_point = 8;
if (i == 27)
next_point = 20;
cv::Point nextFeaturePoint(cvRound(eye_landmarks[next_point].x * (double)draw_multiplier), cvRound(eye_landmarks[next_point].y * (double)draw_multiplier));
if (i < 8 || i > 19)
cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits);
else
cv::line(captured_image, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits);
}
// Now draw the gaze lines themselves
cv::Mat cameraMat = (cv::Mat_<double>(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0);
int part_left = -1;
int part_right = -1;
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
{
if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0)
{
part_left = i;
}
if (clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0)
{
part_right = i;
}
}
cv::Mat eyeLdmks3d_left = clnf_model.hierarchical_models[part_left].GetShape(fx, fy, cx, cy);
cv::Point3f pupil_left = GetPupilPosition(eyeLdmks3d_left);
cv::Mat_<double> irisLdmks3d_left = eyeLdmks3d_left.rowRange(0, 8);
cv::Point3f pupil_left(cv::mean(irisLdmks3d_left.col(0))[0], cv::mean(irisLdmks3d_left.col(1))[0], cv::mean(irisLdmks3d_left.col(2))[0]);
cv::Mat eyeLdmks3d_right = clnf_model.hierarchical_models[part_right].GetShape(fx, fy, cx, cy);
cv::Point3f pupil_right = GetPupilPosition(eyeLdmks3d_right);
std::vector<cv::Point3d> points_left;
points_left.push_back(cv::Point3d(pupil_left));
points_left.push_back(cv::Point3d(pupil_left + gaze_direction0*50.0));
std::vector<cv::Point3d> points_right;
points_right.push_back(cv::Point3d(pupil_right));
points_right.push_back(cv::Point3d(pupil_right + gaze_direction1*50.0));
cv::Mat_<double> proj_points;
cv::Mat_<double> mesh_0 = (cv::Mat_<double>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
Project(proj_points, mesh_0, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<double>(0, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)draw_multiplier)),
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
cv::Mat_<double> mesh_1 = (cv::Mat_<double>(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z);
Project(proj_points, mesh_1, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<double>(0, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)draw_multiplier)),
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
}
}
void Visualizer::ShowObservation()
{
if (vis_track)
{
cv::namedWindow("tracking_result", 1);
cv::imshow("tracking_result", captured_image);
cv::waitKey(1);
}
if (vis_align)
{
cv::imshow("sim_warp", aligned_face_image);
}
if (vis_hog)
{
cv::imshow("hog", hog_image);
}
}