From da29fb402c5e2da122f3a604695cae047b645d4f Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 12 Feb 2018 08:08:14 +0000 Subject: [PATCH] Merge preparations for CE-CLM and master --- .gitignore | 1 + exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 2 +- lib/3rdParty/OpenCV3.4/openCV3.4.props | 4 +- .../opencv_world340.lib} | Bin .../lib/{ => Release}/opencv_world340.lib | Bin .../opencv_world340.lib} | Bin .../lib/{ => Release}/opencv_world340.lib | Bin lib/local/GazeAnalyser/src/GazeEstimation.cpp | 8 +- .../LandmarkDetector/LandmarkDetector.vcxproj | 6 +- .../include/LandmarkDetectorUtils.h | 5 +- lib/local/LandmarkDetector/src/CNN_utils.cpp | 8 +- .../src/LandmarkDetectorUtils.cpp | 10 +-- lib/local/Utilities/include/RotationHelpers.h | 69 +++++++++--------- .../Utilities/include/VisualizationUtils.h | 2 +- lib/local/Utilities/include/Visualizer.h | 6 +- .../Utilities/src/VisualizationUtils.cpp | 20 ++--- lib/local/Utilities/src/Visualizer.cpp | 44 ++++++----- 17 files changed, 96 insertions(+), 89 deletions(-) rename lib/3rdParty/OpenCV3.4/x64/v140/lib/{opencv_world340d.lib => Debug/opencv_world340.lib} (100%) rename lib/3rdParty/OpenCV3.4/x64/v140/lib/{ => Release}/opencv_world340.lib (100%) rename lib/3rdParty/OpenCV3.4/x86/v140/lib/{opencv_world340d.lib => Debug/opencv_world340.lib} (100%) rename lib/3rdParty/OpenCV3.4/x86/v140/lib/{ => Release}/opencv_world340.lib (100%) diff --git a/.gitignore b/.gitignore index 4a16329c..50d8ec1b 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ matlab_runners/Gaze Experiments/mpii_out/ *.vspx matlab_version/AU_training/experiments/full_model_training/mat_models_o/ lib/local/Utilities/x64/Release/ +lib/local/Utilities/x64/ diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 00229647..4df2a4a5 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -143,7 +143,7 @@ int main(int argc, char **argv) cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); // The actual facial landmark detection / tracking - bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); diff --git a/lib/3rdParty/OpenCV3.4/openCV3.4.props b/lib/3rdParty/OpenCV3.4/openCV3.4.props index c0c74987..f604b26a 100644 --- a/lib/3rdParty/OpenCV3.4/openCV3.4.props +++ b/lib/3rdParty/OpenCV3.4/openCV3.4.props @@ -8,8 +8,8 @@ $(SolutionDir)lib\3rdParty\OpenCV3.4\include\opencv;$(SolutionDir)lib\3rdParty\OpenCV3.4\include;%(AdditionalIncludeDirectories) - $(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib;%(AdditionalLibraryDirectories) - opencv_world340.lib;opencv_world340d.lib;%(AdditionalDependencies) + $(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib\$(Configuration);%(AdditionalLibraryDirectories) + opencv_world340.lib;%(AdditionalDependencies) xcopy /I /E /Y /D /C "$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\bin\$(Configuration)" "$(OutDir)" diff --git a/lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340d.lib b/lib/3rdParty/OpenCV3.4/x64/v140/lib/Debug/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340d.lib rename to lib/3rdParty/OpenCV3.4/x64/v140/lib/Debug/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340.lib b/lib/3rdParty/OpenCV3.4/x64/v140/lib/Release/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340.lib rename to lib/3rdParty/OpenCV3.4/x64/v140/lib/Release/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340d.lib b/lib/3rdParty/OpenCV3.4/x86/v140/lib/Debug/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340d.lib rename to lib/3rdParty/OpenCV3.4/x86/v140/lib/Debug/opencv_world340.lib diff --git a/lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340.lib b/lib/3rdParty/OpenCV3.4/x86/v140/lib/Release/opencv_world340.lib similarity index 100% rename from lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340.lib rename to lib/3rdParty/OpenCV3.4/x86/v140/lib/Release/opencv_world340.lib diff --git a/lib/local/GazeAnalyser/src/GazeEstimation.cpp b/lib/local/GazeAnalyser/src/GazeEstimation.cpp index 37cfbab9..ed499edf 100644 --- a/lib/local/GazeAnalyser/src/GazeEstimation.cpp +++ b/lib/local/GazeAnalyser/src/GazeEstimation.cpp @@ -88,9 +88,9 @@ cv::Point3f GazeAnalysis::GetPupilPosition(cv::Mat_ eyeLdmks3d){ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye) { - cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); - cv::Vec3d eulerAngles(headPose(3), headPose(4), headPose(5)); - cv::Matx33d rotMat = Utilities::Euler2RotationMatrix(eulerAngles); + cv::Vec6f headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); + cv::Vec3f eulerAngles(headPose(3), headPose(4), headPose(5)); + cv::Matx33f rotMat = Utilities::Euler2RotationMatrix(eulerAngles); int part = -1; for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) @@ -118,7 +118,7 @@ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po cv::Mat faceLdmks3d = clnf_model.GetShape(fx, fy, cx, cy); faceLdmks3d = faceLdmks3d.t(); - cv::Mat offset = (cv::Mat_(3, 1) << 0, -3.5, 7.0); + cv::Mat offset = (cv::Mat_(3, 1) << 0, -3.5, 7.0); int eyeIdx = 1; if (left_eye) diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj index db8c2c03..37f7806a 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj @@ -111,7 +111,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model" -xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers" +xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers" @@ -134,7 +134,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model" -xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers" +xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers" @@ -159,7 +159,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model" -xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers" +xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers" diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 3945a44b..c95e7a5d 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -95,14 +95,15 @@ namespace LandmarkDetector vector> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy); void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); + // TODO all these should return floats rather than doubles vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); vector CalculateVisibleLandmarks(const CLNF& clnf_model); vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); vector CalculateAllLandmarks(const cv::Mat_& shape2D); vector CalculateAllLandmarks(const CLNF& clnf_model); - vector CalculateAllEyeLandmarks(const CLNF& clnf_model); - vector Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy); + vector CalculateAllEyeLandmarks(const CLNF& clnf_model); + vector Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy); void DrawLandmarks(cv::Mat img, vector landmarks); diff --git a/lib/local/LandmarkDetector/src/CNN_utils.cpp b/lib/local/LandmarkDetector/src/CNN_utils.cpp index 25a45b57..343b0ef4 100644 --- a/lib/local/LandmarkDetector/src/CNN_utils.cpp +++ b/lib/local/LandmarkDetector/src/CNN_utils.cpp @@ -85,7 +85,13 @@ namespace LandmarkDetector { float in_val = *iter; // Apply the PReLU - *iter++ = in_val >= 0 ? in_val : in_val * neg_mult; + *iter = in_val >= 0 ? in_val : in_val * neg_mult; + + // To deal with OpenCV 3.4s debug mode not allowing to go over iteration boundaries + if(i + 1 < w) + { + iter++; + } } } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 022c9edc..7f67088e 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -1196,10 +1196,10 @@ namespace LandmarkDetector return to_return; } // Computing the 3D eye landmarks - vector Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy) + vector Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy) { - vector to_return; + vector to_return; for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) { @@ -1214,7 +1214,7 @@ namespace LandmarkDetector for (int lmk = 0; lmk < num_landmarks; ++lmk) { - cv::Point3d curr_lmk(lmks.at(0, lmk), lmks.at(1, lmk), lmks.at(2, lmk)); + cv::Point3f curr_lmk(lmks.at(0, lmk), lmks.at(1, lmk), lmks.at(2, lmk)); to_return.push_back(curr_lmk); } } @@ -1223,10 +1223,10 @@ namespace LandmarkDetector } // Computing eye landmarks (to be drawn later or in different interfaces) - vector CalculateAllEyeLandmarks(const CLNF& clnf_model) + vector CalculateAllEyeLandmarks(const CLNF& clnf_model) { - vector to_return; + vector to_return; // If the model has hierarchical updates draw those too for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) { diff --git a/lib/local/Utilities/include/RotationHelpers.h b/lib/local/Utilities/include/RotationHelpers.h index 242a0a1d..34829a57 100644 --- a/lib/local/Utilities/include/RotationHelpers.h +++ b/lib/local/Utilities/include/RotationHelpers.h @@ -44,17 +44,17 @@ namespace Utilities //=========================================================================== // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - static cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) + static cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles) { - cv::Matx33d rotation_matrix; + cv::Matx33f rotation_matrix; - double s1 = sin(eulerAngles[0]); - double s2 = sin(eulerAngles[1]); - double s3 = sin(eulerAngles[2]); + float s1 = sin(eulerAngles[0]); + float s2 = sin(eulerAngles[1]); + float s3 = sin(eulerAngles[2]); - double c1 = cos(eulerAngles[0]); - double c2 = cos(eulerAngles[1]); - double c3 = cos(eulerAngles[2]); + float c1 = cos(eulerAngles[0]); + float c2 = cos(eulerAngles[1]); + float c3 = cos(eulerAngles[2]); rotation_matrix(0, 0) = c2 * c3; rotation_matrix(0, 1) = -c2 *s3; @@ -70,62 +70,63 @@ namespace Utilities } // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - static cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) + static cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix) { - double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0; - double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0); - double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0); - double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0); + float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f; + float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0); + float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0); + float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0); - double t1 = 2.0 * (q0*q2 + q1*q3); + float t1 = 2.0f * (q0*q2 + q1*q3); - double yaw = asin(2.0 * (q0*q2 + q1*q3)); - double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); - double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); + float yaw = asin(2.0 * (q0*q2 + q1*q3)); + float pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); + float roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); - return cv::Vec3d(pitch, yaw, roll); + return cv::Vec3f(pitch, yaw, roll); } - static cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) + static cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler) { - cv::Matx33d rotMatrix = Euler2RotationMatrix(euler); - cv::Vec3d axis_angle; + cv::Matx33f rotMatrix = Euler2RotationMatrix(euler); + cv::Vec3f axis_angle; cv::Rodrigues(rotMatrix, axis_angle); return axis_angle; } - static cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) + static cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle) { - cv::Matx33d rotation_matrix; + cv::Matx33f rotation_matrix; cv::Rodrigues(axis_angle, rotation_matrix); return RotationMatrix2Euler(rotation_matrix); } - static cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) + static cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle) { - cv::Matx33d rotation_matrix; + cv::Matx33f rotation_matrix; cv::Rodrigues(axis_angle, rotation_matrix); return rotation_matrix; } - static cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) + static cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix) { - cv::Vec3d axis_angle; + cv::Vec3f axis_angle; cv::Rodrigues(rotation_matrix, axis_angle); return axis_angle; } // Generally useful 3D functions - static void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) + static void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy) { - dest = cv::Mat_(mesh.rows, 2, 0.0); + dest = cv::Mat_(mesh.rows, 2, 0.0); int num_points = mesh.rows; - double X, Y, Z; + float X, Y, Z; - cv::Mat_::const_iterator mData = mesh.begin(); - cv::Mat_::iterator projected = dest.begin(); + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); for (int i = 0; i < num_points; i++) { @@ -134,8 +135,8 @@ namespace Utilities Y = *(mData++); Z = *(mData++); - double x; - double y; + float x; + float y; // if depth is 0 the projection is different if (Z != 0) diff --git a/lib/local/Utilities/include/VisualizationUtils.h b/lib/local/Utilities/include/VisualizationUtils.h index b04348b4..e40e0068 100644 --- a/lib/local/Utilities/include/VisualizationUtils.h +++ b/lib/local/Utilities/include/VisualizationUtils.h @@ -49,7 +49,7 @@ namespace Utilities void DrawBox(const std::vector>& lines, cv::Mat image, cv::Scalar color, int thickness); // Computing a bounding box to be drawn - std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); + std::vector> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy); void Visualise_FHOG(const cv::Mat_& descriptor, int num_rows, int num_cols, cv::Mat& visualisation); diff --git a/lib/local/Utilities/include/Visualizer.h b/lib/local/Utilities/include/Visualizer.h index 813a5e40..31180355 100644 --- a/lib/local/Utilities/include/Visualizer.h +++ b/lib/local/Utilities/include/Visualizer.h @@ -62,13 +62,13 @@ namespace Utilities void SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy); // All observations relevant to facial landmarks (optional visibilities parameter to not display all landmarks) - void SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, const cv::Mat_& visibilities = cv::Mat_()); + void SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, const cv::Mat_& visibilities = cv::Mat_()); // Pose related observations - void SetObservationPose(const cv::Vec6d& pose, double confidence); + void SetObservationPose(const cv::Vec6f& pose, double confidence); // Gaze related observations - void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector& eye_landmarks, const std::vector& eye_landmarks3d, double confidence); + void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector& eye_landmarks, const std::vector& eye_landmarks3d, double confidence); // Face alignment related observations void SetObservationFaceAlign(const cv::Mat& aligned_face); diff --git a/lib/local/Utilities/src/VisualizationUtils.cpp b/lib/local/Utilities/src/VisualizationUtils.cpp index 8b9f5bca..b9e980b2 100644 --- a/lib/local/Utilities/src/VisualizationUtils.cpp +++ b/lib/local/Utilities/src/VisualizationUtils.cpp @@ -83,9 +83,9 @@ namespace Utilities DrawBox(edge_lines, image, color, thickness); } - std::vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) + std::vector> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy) { - double boxVerts[] = { -1, 1, -1, + float boxVerts[] = { -1, 1, -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, @@ -109,10 +109,10 @@ namespace Utilities edges.push_back(std::pair(7, 6)); // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; + cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100.0f; - cv::Matx33d rot = Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); - cv::Mat_ rotBox; + cv::Matx33f rot = Euler2RotationMatrix(cv::Vec3f(pose[3], pose[4], pose[5])); + cv::Mat_ rotBox; // Rotate the box rotBox = cv::Mat(rot) * box.t(); @@ -124,21 +124,21 @@ namespace Utilities rotBox.col(2) = rotBox.col(2) + pose[2]; // draw the lines - cv::Mat_ rotBoxProj; + cv::Mat_ rotBoxProj; Project(rotBoxProj, rotBox, fx, fy, cx, cy); std::vector> lines; for (size_t i = 0; i < edges.size(); ++i) { - cv::Mat_ begin; - cv::Mat_ end; + cv::Mat_ begin; + cv::Mat_ end; rotBoxProj.row(edges[i].first).copyTo(begin); rotBoxProj.row(edges[i].second).copyTo(end); - cv::Point2d p1(begin.at(0), begin.at(1)); - cv::Point2d p2(end.at(0), end.at(1)); + cv::Point2d p1(begin.at(0), begin.at(1)); + cv::Point2d p2(end.at(0), end.at(1)); lines.push_back(std::pair(p1, p2)); diff --git a/lib/local/Utilities/src/Visualizer.cpp b/lib/local/Utilities/src/Visualizer.cpp index 16f1ba59..3a2ba80f 100644 --- a/lib/local/Utilities/src/Visualizer.cpp +++ b/lib/local/Utilities/src/Visualizer.cpp @@ -134,7 +134,7 @@ void Visualizer::SetObservationHOG(const cv::Mat_& hog_descriptor, int n } -void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, const cv::Mat_& visibilities) +void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, double confidence, const cv::Mat_& visibilities) { if(confidence > visualisation_boundary) @@ -147,7 +147,7 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, d { if (visibilities.empty() || visibilities.at(i)) { - cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (float)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (float)draw_multiplier)); // A rough heuristic for drawn point size int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0); @@ -160,7 +160,7 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, d else { // Draw a fainter point if the landmark is self occluded - cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(landmarks_2D.at(i) * (double)draw_multiplier), cvRound(landmarks_2D.at(i + n) * (double)draw_multiplier)); // A rough heuristic for drawn point size int thickness = (int)std::ceil(2.5* ((double)captured_image.cols) / 640.0); @@ -174,11 +174,9 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_& landmarks_2D, d } } -void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) +void Visualizer::SetObservationPose(const cv::Vec6f& pose, double confidence) { - - // Only draw if the reliability is reasonable, the value is slightly ad-hoc if (confidence > visualisation_boundary) { @@ -198,7 +196,7 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence) } // Eye gaze infomration drawing, first of eye landmarks then of gaze -void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, double confidence) +void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector& eye_landmarks2d, const std::vector& eye_landmarks3d, double confidence) { if(confidence > visualisation_boundary) { @@ -237,11 +235,11 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv } // Now draw the gaze lines themselves - cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); + cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); // Grabbing the pupil location, to draw eye gaze need to know where the pupil is - cv::Point3d pupil_left(0, 0, 0); - cv::Point3d pupil_right(0, 0, 0); + cv::Point3f pupil_left(0, 0, 0); + cv::Point3f pupil_right(0, 0, 0); for (size_t i = 0; i < 8; ++i) { pupil_left = pupil_left + eye_landmarks3d[i]; @@ -250,24 +248,24 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv pupil_left = pupil_left / 8; pupil_right = pupil_right / 8; - std::vector points_left; - points_left.push_back(cv::Point3d(pupil_left)); - points_left.push_back(cv::Point3d(pupil_left + cv::Point3d(gaze_direction0)*50.0)); + std::vector points_left; + points_left.push_back(cv::Point3f(pupil_left)); + points_left.push_back(cv::Point3f(pupil_left) + cv::Point3f(gaze_direction0)*50.0); - std::vector points_right; - points_right.push_back(cv::Point3d(pupil_right)); - points_right.push_back(cv::Point3d(pupil_right + cv::Point3d(gaze_direction1)*50.0)); + std::vector points_right; + points_right.push_back(cv::Point3f(pupil_right)); + points_right.push_back(cv::Point3f(pupil_right) + cv::Point3f(gaze_direction1)*50.0); - cv::Mat_ proj_points; - cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); + cv::Mat_ proj_points; + cv::Mat_ mesh_0 = (cv::Mat_(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z); Project(proj_points, mesh_0, fx, fy, cx, cy); - cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), - cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (float)draw_multiplier), cvRound(proj_points.at(0, 1) * (float)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (float)draw_multiplier), cvRound(proj_points.at(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); - cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); + cv::Mat_ mesh_1 = (cv::Mat_(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z); Project(proj_points, mesh_1, fx, fy, cx, cy); - cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (double)draw_multiplier), cvRound(proj_points.at(0, 1) * (double)draw_multiplier)), - cv::Point(cvRound(proj_points.at(1, 0) * (double)draw_multiplier), cvRound(proj_points.at(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); + cv::line(captured_image, cv::Point(cvRound(proj_points.at(0, 0) * (float)draw_multiplier), cvRound(proj_points.at(0, 1) * (float)draw_multiplier)), + cv::Point(cvRound(proj_points.at(1, 0) * (float)draw_multiplier), cvRound(proj_points.at(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits); } }