ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2026-09-02 05:27:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge preparations for CE-CLM and master

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2018-02-12 08:08:14 +00:00
parent 50ea71169e
commit da29fb402c
17 changed files with 96 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -65,3 +65,4 @@ matlab_runners/Gaze Experiments/mpii_out/
*.vspx
matlab_version/AU_training/experiments/full_model_training/mat_models_o/
lib/local/Utilities/x64/Release/
lib/local/Utilities/x64/

2
exe/FaceLandmarkVid/FaceLandmarkVid.cpp
View File

@@ -143,7 +143,7 @@ int main(int argc, char **argv)
cv::Mat_<uchar> grayscale_image = sequence_reader.GetGrayFrame();
// The actual facial landmark detection / tracking
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters);
bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters);
// Gaze tracking, absolute gaze direction
cv::Point3f gazeDirection0(0, 0, -1);

4
lib/3rdParty/OpenCV3.4/openCV3.4.props vendored
View File

@@ -8,8 +8,8 @@
<AdditionalIncludeDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\include\opencv;$(SolutionDir)lib\3rdParty\OpenCV3.4\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<AdditionalLibraryDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>opencv_world340.lib;opencv_world340d.lib;%(AdditionalDependencies)</AdditionalDependencies>
<AdditionalLibraryDirectories>$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\lib\$(Configuration);%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
<AdditionalDependencies>opencv_world340.lib;%(AdditionalDependencies)</AdditionalDependencies>
</Link>
<PostBuildEvent>
<Command>xcopy /I /E /Y /D /C "$(SolutionDir)lib\3rdParty\OpenCV3.4\$(PlatformTarget)\$(PlatformToolset)\bin\$(Configuration)" "$(OutDir)"

0
lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340d.lib → lib/3rdParty/OpenCV3.4/x64/v140/lib/Debug/opencv_world340.lib vendored
View File

0
lib/3rdParty/OpenCV3.4/x64/v140/lib/opencv_world340.lib → lib/3rdParty/OpenCV3.4/x64/v140/lib/Release/opencv_world340.lib vendored
View File

0
lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340d.lib → lib/3rdParty/OpenCV3.4/x86/v140/lib/Debug/opencv_world340.lib vendored
View File

0
lib/3rdParty/OpenCV3.4/x86/v140/lib/opencv_world340.lib → lib/3rdParty/OpenCV3.4/x86/v140/lib/Release/opencv_world340.lib vendored
View File

8
lib/local/GazeAnalyser/src/GazeEstimation.cpp
View File

@@ -88,9 +88,9 @@ cv::Point3f GazeAnalysis::GetPupilPosition(cv::Mat_<float> eyeLdmks3d){
void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye)
{
cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy);
cv::Vec3d eulerAngles(headPose(3), headPose(4), headPose(5));
cv::Matx33d rotMat = Utilities::Euler2RotationMatrix(eulerAngles);
cv::Vec6f headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy);
cv::Vec3f eulerAngles(headPose(3), headPose(4), headPose(5));
cv::Matx33f rotMat = Utilities::Euler2RotationMatrix(eulerAngles);
int part = -1;
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
@@ -118,7 +118,7 @@ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po
cv::Mat faceLdmks3d = clnf_model.GetShape(fx, fy, cx, cy);
faceLdmks3d = faceLdmks3d.t();
cv::Mat offset = (cv::Mat_<double>(3, 1) << 0, -3.5, 7.0);
cv::Mat offset = (cv::Mat_<float>(3, 1) << 0, -3.5, 7.0);
int eyeIdx = 1;
if (left_eye)

6
lib/local/LandmarkDetector/LandmarkDetector.vcxproj
View File

@@ -111,7 +111,7 @@
</Lib>
<PostBuildEvent>
<Command>xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model"
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers"</Command>
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers"</Command>
</PostBuildEvent>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
@@ -134,7 +134,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c
</Lib>
<PostBuildEvent>
<Command>xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model"
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers"</Command>
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers"</Command>
</PostBuildEvent>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
@@ -159,7 +159,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c
</Lib>
<PostBuildEvent>
<Command>xcopy /I /E /Y /D "$(SolutionDir)lib\local\LandmarkDetector\model" "$(OutDir)model"
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)classifiers"</Command>
xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)classifiers"</Command>
</PostBuildEvent>
</ItemDefinitionGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">

5
lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h
View File

@@ -95,14 +95,15 @@ namespace LandmarkDetector
vector<std::pair<cv::Point2f, cv::Point2f>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy);
void DrawBox(vector<pair<cv::Point, cv::Point>> lines, cv::Mat image, cv::Scalar color, int thickness);
// TODO all these should return floats rather than doubles
vector<cv::Point2d> CalculateVisibleLandmarks(const cv::Mat_<float>& shape2D, const cv::Mat_<int>& visibilities);
vector<cv::Point2d> CalculateVisibleLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateVisibleEyeLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateAllLandmarks(const cv::Mat_<float>& shape2D);
vector<cv::Point2d> CalculateAllLandmarks(const CLNF& clnf_model);
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model);
vector<cv::Point3d> Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy);
vector<cv::Point2f> CalculateAllEyeLandmarks(const CLNF& clnf_model);
vector<cv::Point3f> Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy);
void DrawLandmarks(cv::Mat img, vector<cv::Point> landmarks);

8
lib/local/LandmarkDetector/src/CNN_utils.cpp
View File

@@ -85,7 +85,13 @@ namespace LandmarkDetector
{
float in_val = *iter;
// Apply the PReLU
*iter++ = in_val >= 0 ? in_val : in_val * neg_mult;
*iter = in_val >= 0 ? in_val : in_val * neg_mult;
// To deal with OpenCV 3.4s debug mode not allowing to go over iteration boundaries
if(i + 1 < w)
{
iter++;
}
}
}

10
lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp
View File

@@ -1196,10 +1196,10 @@ namespace LandmarkDetector
return to_return;
}
// Computing the 3D eye landmarks
vector<cv::Point3d> Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy)
vector<cv::Point3f> Calculate3DEyeLandmarks(const CLNF& clnf_model, float fx, float fy, float cx, float cy)
{
vector<cv::Point3d> to_return;
vector<cv::Point3f> to_return;
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
{
@@ -1214,7 +1214,7 @@ namespace LandmarkDetector
for (int lmk = 0; lmk < num_landmarks; ++lmk)
{
cv::Point3d curr_lmk(lmks.at<float>(0, lmk), lmks.at<float>(1, lmk), lmks.at<float>(2, lmk));
cv::Point3f curr_lmk(lmks.at<float>(0, lmk), lmks.at<float>(1, lmk), lmks.at<float>(2, lmk));
to_return.push_back(curr_lmk);
}
}
@@ -1223,10 +1223,10 @@ namespace LandmarkDetector
}
// Computing eye landmarks (to be drawn later or in different interfaces)
vector<cv::Point2d> CalculateAllEyeLandmarks(const CLNF& clnf_model)
vector<cv::Point2f> CalculateAllEyeLandmarks(const CLNF& clnf_model)
{
vector<cv::Point2d> to_return;
vector<cv::Point2f> to_return;
// If the model has hierarchical updates draw those too
for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i)
{

69
lib/local/Utilities/include/RotationHelpers.h
View File

@@ -44,17 +44,17 @@ namespace Utilities
//===========================================================================
// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
static cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles)
static cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles)
{
cv::Matx33d rotation_matrix;
cv::Matx33f rotation_matrix;
double s1 = sin(eulerAngles[0]);
double s2 = sin(eulerAngles[1]);
double s3 = sin(eulerAngles[2]);
float s1 = sin(eulerAngles[0]);
float s2 = sin(eulerAngles[1]);
float s3 = sin(eulerAngles[2]);
double c1 = cos(eulerAngles[0]);
double c2 = cos(eulerAngles[1]);
double c3 = cos(eulerAngles[2]);
float c1 = cos(eulerAngles[0]);
float c2 = cos(eulerAngles[1]);
float c3 = cos(eulerAngles[2]);
rotation_matrix(0, 0) = c2 * c3;
rotation_matrix(0, 1) = -c2 *s3;
@@ -70,62 +70,63 @@ namespace Utilities
}
// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign
static cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix)
static cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix)
{
double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0;
double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0);
double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0);
double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0);
float q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0f;
float q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0f*q0);
float q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0f*q0);
float q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0f*q0);
double t1 = 2.0 * (q0*q2 + q1*q3);
float t1 = 2.0f * (q0*q2 + q1*q3);
double yaw = asin(2.0 * (q0*q2 + q1*q3));
double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3);
double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3);
float yaw = asin(2.0 * (q0*q2 + q1*q3));
float pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3);
float roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3);
return cv::Vec3d(pitch, yaw, roll);
return cv::Vec3f(pitch, yaw, roll);
}
static cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler)
static cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler)
{
cv::Matx33d rotMatrix = Euler2RotationMatrix(euler);
cv::Vec3d axis_angle;
cv::Matx33f rotMatrix = Euler2RotationMatrix(euler);
cv::Vec3f axis_angle;
cv::Rodrigues(rotMatrix, axis_angle);
return axis_angle;
}
static cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle)
static cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle)
{
cv::Matx33d rotation_matrix;
cv::Matx33f rotation_matrix;
cv::Rodrigues(axis_angle, rotation_matrix);
return RotationMatrix2Euler(rotation_matrix);
}
static cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle)
static cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle)
{
cv::Matx33d rotation_matrix;
cv::Matx33f rotation_matrix;
cv::Rodrigues(axis_angle, rotation_matrix);
return rotation_matrix;
}
static cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix)
static cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix)
{
cv::Vec3d axis_angle;
cv::Vec3f axis_angle;
cv::Rodrigues(rotation_matrix, axis_angle);
return axis_angle;
}
// Generally useful 3D functions
static void Project(cv::Mat_<double>& dest, const cv::Mat_<double>& mesh, double fx, double fy, double cx, double cy)
static void Project(cv::Mat_<float>& dest, const cv::Mat_<float>& mesh, float fx, float fy, float cx, float cy)
{
dest = cv::Mat_<double>(mesh.rows, 2, 0.0);
dest = cv::Mat_<float>(mesh.rows, 2, 0.0);
int num_points = mesh.rows;
double X, Y, Z;
float X, Y, Z;
cv::Mat_<double>::const_iterator mData = mesh.begin();
cv::Mat_<double>::iterator projected = dest.begin();
cv::Mat_<float>::const_iterator mData = mesh.begin();
cv::Mat_<float>::iterator projected = dest.begin();
for (int i = 0; i < num_points; i++)
{
@@ -134,8 +135,8 @@ namespace Utilities
Y = *(mData++);
Z = *(mData++);
double x;
double y;
float x;
float y;
// if depth is 0 the projection is different
if (Z != 0)

2
lib/local/Utilities/include/VisualizationUtils.h
View File

@@ -49,7 +49,7 @@ namespace Utilities
void DrawBox(const std::vector<std::pair<cv::Point2d, cv::Point2d>>& lines, cv::Mat image, cv::Scalar color, int thickness);
// Computing a bounding box to be drawn
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy);
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy);
void Visualise_FHOG(const cv::Mat_<double>& descriptor, int num_rows, int num_cols, cv::Mat& visualisation);

6
lib/local/Utilities/include/Visualizer.h
View File

@@ -62,13 +62,13 @@ namespace Utilities
void SetImage(const cv::Mat& canvas, float fx, float fy, float cx, float cy);
// All observations relevant to facial landmarks (optional visibilities parameter to not display all landmarks)
void SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities = cv::Mat_<int>());
void SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities = cv::Mat_<int>());
// Pose related observations
void SetObservationPose(const cv::Vec6d& pose, double confidence);
void SetObservationPose(const cv::Vec6f& pose, double confidence);
// Gaze related observations
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2d>& eye_landmarks, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence);
void SetObservationGaze(const cv::Point3f& gazeDirection0, const cv::Point3f& gazeDirection1, const std::vector<cv::Point2f>& eye_landmarks, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence);
// Face alignment related observations
void SetObservationFaceAlign(const cv::Mat& aligned_face);

20
lib/local/Utilities/src/VisualizationUtils.cpp
View File

@@ -83,9 +83,9 @@ namespace Utilities
DrawBox(edge_lines, image, color, thickness);
}
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy)
std::vector<std::pair<cv::Point2d, cv::Point2d>> CalculateBox(cv::Vec6f pose, float fx, float fy, float cx, float cy)
{
double boxVerts[] = { -1, 1, -1,
float boxVerts[] = { -1, 1, -1,
1, 1, -1,
1, 1, 1,
-1, 1, 1,
@@ -109,10 +109,10 @@ namespace Utilities
edges.push_back(std::pair<int, int>(7, 6));
// The size of the head is roughly 200mm x 200mm x 200mm
cv::Mat_<double> box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100;
cv::Mat_<float> box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100.0f;
cv::Matx33d rot = Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5]));
cv::Mat_<double> rotBox;
cv::Matx33f rot = Euler2RotationMatrix(cv::Vec3f(pose[3], pose[4], pose[5]));
cv::Mat_<float> rotBox;
// Rotate the box
rotBox = cv::Mat(rot) * box.t();
@@ -124,21 +124,21 @@ namespace Utilities
rotBox.col(2) = rotBox.col(2) + pose[2];
// draw the lines
cv::Mat_<double> rotBoxProj;
cv::Mat_<float> rotBoxProj;
Project(rotBoxProj, rotBox, fx, fy, cx, cy);
std::vector<std::pair<cv::Point2d, cv::Point2d>> lines;
for (size_t i = 0; i < edges.size(); ++i)
{
cv::Mat_<double> begin;
cv::Mat_<double> end;
cv::Mat_<float> begin;
cv::Mat_<float> end;
rotBoxProj.row(edges[i].first).copyTo(begin);
rotBoxProj.row(edges[i].second).copyTo(end);
cv::Point2d p1(begin.at<double>(0), begin.at<double>(1));
cv::Point2d p2(end.at<double>(0), end.at<double>(1));
cv::Point2d p1(begin.at<float>(0), begin.at<float>(1));
cv::Point2d p2(end.at<float>(0), end.at<float>(1));
lines.push_back(std::pair<cv::Point2d, cv::Point2d>(p1, p2));

44
lib/local/Utilities/src/Visualizer.cpp
View File

@@ -134,7 +134,7 @@ void Visualizer::SetObservationHOG(const cv::Mat_<double>& hog_descriptor, int n
}
void Visualizer::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities)
void Visualizer::SetObservationLandmarks(const cv::Mat_<float>& landmarks_2D, double confidence, const cv::Mat_<int>& visibilities)
{
if(confidence > visualisation_boundary)
@@ -147,7 +147,7 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, d
{
if (visibilities.empty() || visibilities.at<int>(i))
{
cv::Point featurePoint(cvRound(landmarks_2D.at<double>(i) * (double)draw_multiplier), cvRound(landmarks_2D.at<double>(i + n) * (double)draw_multiplier));
cv::Point featurePoint(cvRound(landmarks_2D.at<float>(i) * (float)draw_multiplier), cvRound(landmarks_2D.at<float>(i + n) * (float)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = (int)std::ceil(3.0* ((double)captured_image.cols) / 640.0);
@@ -160,7 +160,7 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, d
else
{
// Draw a fainter point if the landmark is self occluded
cv::Point featurePoint(cvRound(landmarks_2D.at<double>(i) * (double)draw_multiplier), cvRound(landmarks_2D.at<double>(i + n) * (double)draw_multiplier));
cv::Point featurePoint(cvRound(landmarks_2D.at<float>(i) * (double)draw_multiplier), cvRound(landmarks_2D.at<float>(i + n) * (double)draw_multiplier));
// A rough heuristic for drawn point size
int thickness = (int)std::ceil(2.5* ((double)captured_image.cols) / 640.0);
@@ -174,11 +174,9 @@ void Visualizer::SetObservationLandmarks(const cv::Mat_<double>& landmarks_2D, d
}
}
void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
void Visualizer::SetObservationPose(const cv::Vec6f& pose, double confidence)
{
// Only draw if the reliability is reasonable, the value is slightly ad-hoc
if (confidence > visualisation_boundary)
{
@@ -198,7 +196,7 @@ void Visualizer::SetObservationPose(const cv::Vec6d& pose, double confidence)
}
// Eye gaze infomration drawing, first of eye landmarks then of gaze
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2d>& eye_landmarks2d, const std::vector<cv::Point3d>& eye_landmarks3d, double confidence)
void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv::Point3f& gaze_direction1, const std::vector<cv::Point2f>& eye_landmarks2d, const std::vector<cv::Point3f>& eye_landmarks3d, double confidence)
{
if(confidence > visualisation_boundary)
{
@@ -237,11 +235,11 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv
}
// Now draw the gaze lines themselves
cv::Mat cameraMat = (cv::Mat_<double>(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0);
cv::Mat cameraMat = (cv::Mat_<float>(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0);
// Grabbing the pupil location, to draw eye gaze need to know where the pupil is
cv::Point3d pupil_left(0, 0, 0);
cv::Point3d pupil_right(0, 0, 0);
cv::Point3f pupil_left(0, 0, 0);
cv::Point3f pupil_right(0, 0, 0);
for (size_t i = 0; i < 8; ++i)
{
pupil_left = pupil_left + eye_landmarks3d[i];
@@ -250,24 +248,24 @@ void Visualizer::SetObservationGaze(const cv::Point3f& gaze_direction0, const cv
pupil_left = pupil_left / 8;
pupil_right = pupil_right / 8;
std::vector<cv::Point3d> points_left;
points_left.push_back(cv::Point3d(pupil_left));
points_left.push_back(cv::Point3d(pupil_left + cv::Point3d(gaze_direction0)*50.0));
std::vector<cv::Point3f> points_left;
points_left.push_back(cv::Point3f(pupil_left));
points_left.push_back(cv::Point3f(pupil_left) + cv::Point3f(gaze_direction0)*50.0);
std::vector<cv::Point3d> points_right;
points_right.push_back(cv::Point3d(pupil_right));
points_right.push_back(cv::Point3d(pupil_right + cv::Point3d(gaze_direction1)*50.0));
std::vector<cv::Point3f> points_right;
points_right.push_back(cv::Point3f(pupil_right));
points_right.push_back(cv::Point3f(pupil_right) + cv::Point3f(gaze_direction1)*50.0);
cv::Mat_<double> proj_points;
cv::Mat_<double> mesh_0 = (cv::Mat_<double>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
cv::Mat_<float> proj_points;
cv::Mat_<float> mesh_0 = (cv::Mat_<float>(2, 3) << points_left[0].x, points_left[0].y, points_left[0].z, points_left[1].x, points_left[1].y, points_left[1].z);
Project(proj_points, mesh_0, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<double>(0, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)draw_multiplier)),
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<float>(0, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(0, 1) * (float)draw_multiplier)),
cv::Point(cvRound(proj_points.at<float>(1, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
cv::Mat_<double> mesh_1 = (cv::Mat_<double>(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z);
cv::Mat_<float> mesh_1 = (cv::Mat_<float>(2, 3) << points_right[0].x, points_right[0].y, points_right[0].z, points_right[1].x, points_right[1].y, points_right[1].z);
Project(proj_points, mesh_1, fx, fy, cx, cy);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<double>(0, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(0, 1) * (double)draw_multiplier)),
cv::Point(cvRound(proj_points.at<double>(1, 0) * (double)draw_multiplier), cvRound(proj_points.at<double>(1, 1) * (double)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
cv::line(captured_image, cv::Point(cvRound(proj_points.at<float>(0, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(0, 1) * (float)draw_multiplier)),
cv::Point(cvRound(proj_points.at<float>(1, 0) * (float)draw_multiplier), cvRound(proj_points.at<float>(1, 1) * (float)draw_multiplier)), cv::Scalar(110, 220, 0), 2, CV_AA, draw_shiftbits);
}
}