Adding gaze models to all landmark models, a small stability fix for gaze (not returning junk when no model is available)

.gitignore

@@ -81,3 +81,4 @@ gui/HeadPose-live/obj/Debug/
 lib/3rdParty/CameraEnumerator/x64/
 matlab_runners/Action Unit Experiments/out_bosph/
 matlab_runners/Action Unit Experiments/out_bp4d/
+matlab_runners/Feature Point Experiments/300VW_experiment/

lib/local/CppInerop/LandmarkDetectorInterop.h

@@ -282,7 +282,6 @@ namespace CppInterop {
 			}
 
 			bool DetectLandmarksInVideo(OpenCVWrappers::RawImage^ rgb_image, FaceModelParameters^ modelParams, OpenCVWrappers::RawImage^ gray_image) {
-				//cout << modelParams->getParams()->curr_face_detector << endl;
 				return ::LandmarkDetector::DetectLandmarksInVideo(rgb_image->Mat, *clnf, *modelParams->getParams(), gray_image->Mat);
 			}
 

lib/local/GazeAnalyser/src/GazeEstimation.cpp

@@ -108,6 +108,8 @@ void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po
 	if (part == -1)
 	{
 		std::cout << "Couldn't find the eye model, something wrong" << std::endl;
+		gaze_absolute = cv::Point3f(0, 0, 0);
+		return;
 	}
 
 	cv::Mat eyeLdmks3d = clnf_model.hierarchical_models[part].GetShape(fx, fy, cx, cy);

lib/local/LandmarkDetector/model/main_clm_general.txt

@@ -1,3 +1,5 @@
 LandmarkDetector clm_general.txt
+LandmarkDetector_part model_eye/main_clnf_synth_left.txt left_eye_28 36 8 37 10 38 12 39 14 40 16 41 18
+LandmarkDetector_part model_eye/main_clnf_synth_right.txt right_eye_28 42 8 43 10 44 12 45 14 46 16 47 18
 FaceDetConversion haarAlign.txt
 DetectionValidator detection_validation/validator_cnn.txt

lib/local/LandmarkDetector/model/main_clm_wild.txt

@@ -1,3 +1,5 @@
 LandmarkDetector clm_wild.txt
+LandmarkDetector_part model_eye/main_clnf_synth_left.txt left_eye_28 36 8 37 10 38 12 39 14 40 16 41 18
+LandmarkDetector_part model_eye/main_clnf_synth_right.txt right_eye_28 42 8 43 10 44 12 45 14 46 16 47 18
 FaceDetConversion haarAlign.txt
 DetectionValidator detection_validation/validator_cnn.txt

lib/local/LandmarkDetector/model/main_clnf_multi_pie.txt

@@ -1,3 +1,5 @@
 LandmarkDetector clnf_multi_pie.txt
+LandmarkDetector_part model_eye/main_clnf_synth_left.txt left_eye_28 36 8 37 10 38 12 39 14 40 16 41 18
+LandmarkDetector_part model_eye/main_clnf_synth_right.txt right_eye_28 42 8 43 10 44 12 45 14 46 16 47 18
 FaceDetConversion haarAlign.txt
 DetectionValidator detection_validation/validator_cnn.txt

matlab_runners/Action Unit Experiments/results/BP4D_valid_res_class.txt

@@ -1,11 +1,11 @@
-AU1 class, Precision - 0.521, Recall - 0.518, F1 - 0.519
-AU2 class, Precision - 0.429, Recall - 0.412, F1 - 0.420
-AU4 class, Precision - 0.550, Recall - 0.495, F1 - 0.521
+AU1 class, Precision - 0.520, Recall - 0.518, F1 - 0.519
+AU2 class, Precision - 0.431, Recall - 0.412, F1 - 0.421
+AU4 class, Precision - 0.547, Recall - 0.493, F1 - 0.519
 AU6 class, Precision - 0.758, Recall - 0.773, F1 - 0.765
-AU7 class, Precision - 0.748, Recall - 0.696, F1 - 0.721
-AU10 class, Precision - 0.865, Recall - 0.838, F1 - 0.852
+AU7 class, Precision - 0.747, Recall - 0.698, F1 - 0.722
+AU10 class, Precision - 0.866, Recall - 0.839, F1 - 0.852
 AU12 class, Precision - 0.907, Recall - 0.812, F1 - 0.857
-AU14 class, Precision - 0.562, Recall - 0.763, F1 - 0.647
-AU15 class, Precision - 0.396, Recall - 0.434, F1 - 0.414
-AU17 class, Precision - 0.634, Recall - 0.603, F1 - 0.619
-AU23 class, Precision - 0.342, Recall - 0.612, F1 - 0.439
+AU14 class, Precision - 0.564, Recall - 0.763, F1 - 0.648
+AU15 class, Precision - 0.397, Recall - 0.435, F1 - 0.415
+AU17 class, Precision - 0.635, Recall - 0.604, F1 - 0.619
+AU23 class, Precision - 0.341, Recall - 0.611, F1 - 0.438

matlab_runners/Action Unit Experiments/results/Bosphorus_res_class.txt

@@ -1,17 +1,17 @@
-AU1 class, Precision - 0.334, Recall - 0.810, F1 - 0.473
-AU2 class, Precision - 0.285, Recall - 0.794, F1 - 0.420
-AU4 class, Precision - 0.502, Recall - 0.815, F1 - 0.621
-AU5 class, Precision - 0.351, Recall - 0.851, F1 - 0.497
-AU6 class, Precision - 0.373, Recall - 0.814, F1 - 0.512
-AU7 class, Precision - 0.791, Recall - 0.773, F1 - 0.782
-AU9 class, Precision - 0.400, Recall - 0.953, F1 - 0.563
-AU10 class, Precision - 0.293, Recall - 0.769, F1 - 0.425
-AU12 class, Precision - 0.706, Recall - 0.843, F1 - 0.769
-AU14 class, Precision - 0.188, Recall - 0.858, F1 - 0.308
-AU15 class, Precision - 0.159, Recall - 0.901, F1 - 0.270
-AU17 class, Precision - 0.302, Recall - 0.906, F1 - 0.452
-AU20 class, Precision - 0.115, Recall - 0.912, F1 - 0.204
-AU23 class, Precision - 0.112, Recall - 0.807, F1 - 0.197
-AU25 class, Precision - 0.934, Recall - 0.828, F1 - 0.878
-AU26 class, Precision - 0.385, Recall - 0.798, F1 - 0.520
-AU45 class, Precision - 0.373, Recall - 0.811, F1 - 0.511
+AU1 class, Precision - 0.334, Recall - 0.812, F1 - 0.474
+AU2 class, Precision - 0.286, Recall - 0.797, F1 - 0.421
+AU4 class, Precision - 0.501, Recall - 0.820, F1 - 0.622
+AU5 class, Precision - 0.351, Recall - 0.853, F1 - 0.498
+AU6 class, Precision - 0.374, Recall - 0.817, F1 - 0.513
+AU7 class, Precision - 0.791, Recall - 0.776, F1 - 0.784
+AU9 class, Precision - 0.399, Recall - 0.960, F1 - 0.564
+AU10 class, Precision - 0.294, Recall - 0.776, F1 - 0.427
+AU12 class, Precision - 0.707, Recall - 0.849, F1 - 0.771
+AU14 class, Precision - 0.187, Recall - 0.863, F1 - 0.307
+AU15 class, Precision - 0.158, Recall - 0.901, F1 - 0.269
+AU17 class, Precision - 0.302, Recall - 0.912, F1 - 0.453
+AU20 class, Precision - 0.114, Recall - 0.912, F1 - 0.202
+AU23 class, Precision - 0.113, Recall - 0.815, F1 - 0.199
+AU25 class, Precision - 0.934, Recall - 0.831, F1 - 0.879
+AU26 class, Precision - 0.385, Recall - 0.800, F1 - 0.520
+AU45 class, Precision - 0.375, Recall - 0.819, F1 - 0.515

matlab_runners/Action Unit Experiments/results/Bosphorus_res_int.txt

@@ -1,17 +1,17 @@
-AU1 intensity, Corr - 0.683, RMS - 0.830, CCC - 0.580
-AU2 intensity, Corr - 0.654, RMS - 0.870, CCC - 0.593
-AU4 intensity, Corr - 0.774, RMS - 0.655, CCC - 0.740
-AU5 intensity, Corr - 0.750, RMS - 0.747, CCC - 0.683
-AU6 intensity, Corr - 0.555, RMS - 0.776, CCC - 0.523
-AU7 intensity, Corr - 0.830, RMS - 0.748, CCC - 0.813
-AU9 intensity, Corr - 0.762, RMS - 0.608, CCC - 0.709
-AU10 intensity, Corr - 0.478, RMS - 0.812, CCC - 0.439
-AU12 intensity, Corr - 0.816, RMS - 0.695, CCC - 0.778
-AU14 intensity, Corr - 0.346, RMS - 0.933, CCC - 0.270
-AU15 intensity, Corr - 0.510, RMS - 0.763, CCC - 0.390
-AU17 intensity, Corr - 0.566, RMS - 0.934, CCC - 0.477
-AU20 intensity, Corr - 0.440, RMS - 0.860, CCC - 0.306
-AU23 intensity, Corr - 0.347, RMS - 0.677, CCC - 0.282
-AU25 intensity, Corr - 0.789, RMS - 0.932, CCC - 0.752
-AU26 intensity, Corr - 0.520, RMS - 0.973, CCC - 0.455
-AU45 intensity, Corr - 0.847, RMS - 0.628, CCC - 0.806
+AU1 intensity, Corr - 0.688, RMS - 0.826, CCC - 0.585
+AU2 intensity, Corr - 0.659, RMS - 0.866, CCC - 0.598
+AU4 intensity, Corr - 0.778, RMS - 0.651, CCC - 0.744
+AU5 intensity, Corr - 0.751, RMS - 0.747, CCC - 0.683
+AU6 intensity, Corr - 0.556, RMS - 0.776, CCC - 0.524
+AU7 intensity, Corr - 0.833, RMS - 0.742, CCC - 0.816
+AU9 intensity, Corr - 0.764, RMS - 0.607, CCC - 0.710
+AU10 intensity, Corr - 0.479, RMS - 0.812, CCC - 0.439
+AU12 intensity, Corr - 0.820, RMS - 0.689, CCC - 0.782
+AU14 intensity, Corr - 0.345, RMS - 0.936, CCC - 0.268
+AU15 intensity, Corr - 0.510, RMS - 0.764, CCC - 0.390
+AU17 intensity, Corr - 0.567, RMS - 0.935, CCC - 0.476
+AU20 intensity, Corr - 0.440, RMS - 0.861, CCC - 0.305
+AU23 intensity, Corr - 0.350, RMS - 0.677, CCC - 0.283
+AU25 intensity, Corr - 0.790, RMS - 0.928, CCC - 0.755
+AU26 intensity, Corr - 0.519, RMS - 0.975, CCC - 0.453
+AU45 intensity, Corr - 0.853, RMS - 0.620, CCC - 0.812

matlab_runners/Action Unit Experiments/results/DISFA_valid_res.txt

@@ -1,12 +1,12 @@
-AU1 results - corr 0.847, rms 0.385, ccc - 0.826
-AU2 results - corr 0.755, rms 0.447, ccc - 0.658
+AU1 results - corr 0.848, rms 0.384, ccc - 0.827
+AU2 results - corr 0.759, rms 0.445, ccc - 0.660
 AU4 results - corr 0.868, rms 0.576, ccc - 0.840
-AU5 results - corr 0.768, rms 0.173, ccc - 0.731
-AU6 results - corr 0.724, rms 0.581, ccc - 0.684
-AU9 results - corr 0.735, rms 0.391, ccc - 0.653
-AU12 results - corr 0.868, rms 0.501, ccc - 0.854
-AU15 results - corr 0.752, rms 0.269, ccc - 0.719
-AU17 results - corr 0.610, rms 0.567, ccc - 0.527
-AU20 results - corr 0.624, rms 0.310, ccc - 0.585
+AU5 results - corr 0.768, rms 0.173, ccc - 0.732
+AU6 results - corr 0.724, rms 0.582, ccc - 0.684
+AU9 results - corr 0.734, rms 0.392, ccc - 0.652
+AU12 results - corr 0.868, rms 0.502, ccc - 0.854
+AU15 results - corr 0.751, rms 0.269, ccc - 0.718
+AU17 results - corr 0.610, rms 0.567, ccc - 0.528
+AU20 results - corr 0.623, rms 0.310, ccc - 0.585
 AU25 results - corr 0.929, rms 0.497, ccc - 0.919
 AU26 results - corr 0.817, rms 0.437, ccc - 0.787

matlab_runners/Action Unit Experiments/results/FERA2011_res_class.txt

@@ -1,11 +1,11 @@
-AU1 class, Precision - 0.456, Recall - 0.856, F1 - 0.595
-AU2 class, Precision - 0.410, Recall - 0.708, F1 - 0.520
-AU4 class, Precision - 0.488, Recall - 0.694, F1 - 0.573
-AU6 class, Precision - 0.754, Recall - 0.708, F1 - 0.730
-AU7 class, Precision - 0.668, Recall - 0.790, F1 - 0.724
-AU10 class, Precision - 0.524, Recall - 0.635, F1 - 0.574
-AU12 class, Precision - 0.894, Recall - 0.734, F1 - 0.806
-AU15 class, Precision - 0.341, Recall - 0.579, F1 - 0.430
-AU17 class, Precision - 0.212, Recall - 0.202, F1 - 0.207
-AU25 class, Precision - 0.204, Recall - 0.835, F1 - 0.328
-AU26 class, Precision - 0.118, Recall - 0.975, F1 - 0.210
+AU1 class, Precision - 0.461, Recall - 0.875, F1 - 0.604
+AU2 class, Precision - 0.416, Recall - 0.720, F1 - 0.527
+AU4 class, Precision - 0.493, Recall - 0.697, F1 - 0.577
+AU6 class, Precision - 0.754, Recall - 0.709, F1 - 0.731
+AU7 class, Precision - 0.678, Recall - 0.790, F1 - 0.730
+AU10 class, Precision - 0.522, Recall - 0.631, F1 - 0.571
+AU12 class, Precision - 0.895, Recall - 0.733, F1 - 0.806
+AU15 class, Precision - 0.345, Recall - 0.591, F1 - 0.435
+AU17 class, Precision - 0.217, Recall - 0.208, F1 - 0.213
+AU25 class, Precision - 0.204, Recall - 0.834, F1 - 0.328
+AU26 class, Precision - 0.118, Recall - 0.977, F1 - 0.210

matlab_runners/Action Unit Experiments/results/SEMAINE_valid_res.txt

@@ -1,6 +1,6 @@
-AU2 class, Precision - 0.326, Recall - 0.760, F1 - 0.457
-AU12 class, Precision - 0.448, Recall - 0.764, F1 - 0.565
-AU17 class, Precision - 0.119, Recall - 0.843, F1 - 0.209
-AU25 class, Precision - 0.331, Recall - 0.511, F1 - 0.402
-AU28 class, Precision - 0.584, Recall - 0.603, F1 - 0.594
-AU45 class, Precision - 0.294, Recall - 0.581, F1 - 0.391
+AU2 class, Precision - 0.327, Recall - 0.760, F1 - 0.457
+AU12 class, Precision - 0.450, Recall - 0.765, F1 - 0.567
+AU17 class, Precision - 0.118, Recall - 0.840, F1 - 0.206
+AU25 class, Precision - 0.331, Recall - 0.510, F1 - 0.401
+AU28 class, Precision - 0.583, Recall - 0.603, F1 - 0.593
+AU45 class, Precision - 0.294, Recall - 0.582, F1 - 0.391

matlab_runners/Feature Point Experiments/results/300VW_49.txt

@@ -7,5 +7,5 @@ DRMF	3.33	3.55	5.27
 PO-CR	2.95	3.08	3.68
 CFAN	3.17	3.72	4.94
 ------------------------------
-OpenFace CLNF	2.83	3.38	3.82
-OpenFace CE-CLM	2.41	2.68	3.21
+OpenFace CLNF	2.83	3.39	3.82
+OpenFace CE-CLM	2.41	2.67	3.21

matlab_runners/Feature Point Experiments/results/300VW_66.txt

@@ -6,4 +6,4 @@ ICCR	3.37	3.45	4.46
 CFAN	3.93	4.43	6.39
 ------------------------------
 OpenFace CLNF	3.79	4.24	4.78
-OpenFace CE-CLM	3.15	3.56	4.18
+OpenFace CE-CLM	3.14	3.55	4.18

matlab_runners/Feature Point Experiments/results/fps_yt.txt

@@ -1,4 +1,4 @@
 Model, mean,  median
-OpenFace (CE-CLM):  0.0499,   0.0473
-OpenFace (CLNF):  0.0553,   0.0515
-CLM:   0.0631,   0.0588
+OpenFace (CE-CLM):  0.0499,   0.0474
+OpenFace (CLNF):  0.0553,   0.0514
+CLM:   0.0632,   0.0588

matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt

@@ -1,4 +1,4 @@
 Dataset and model,        pitch,  yaw,  roll,  mean,  median
-biwi error:  6.272,   4.942, 3.635,  4.950,  2.424
-bu error:    2.405,   3.246, 2.410,  2.687,  1.841
-ict error:   3.149,   3.510, 3.137,  3.265,  1.870
+biwi error:  6.109,   5.117, 3.701,  4.976,  2.419
+bu error:    2.396,   3.246, 2.408,  2.683,  1.834
+ict error:   3.133,   3.511, 3.136,  3.260,  1.867

matlab_runners/Head Pose Experiments/results/Pose_OF_CLNF.txt

@@ -1,4 +1,4 @@
 Dataset and model,        pitch,  yaw,  roll,  mean,  median
-biwi error:  7.410,   5.174, 4.534,  5.706,  2.572
-bu error:    2.785,   3.346, 2.456,  2.862,  1.998
-ict error:   3.843,   3.716, 3.560,  3.706,  1.925
+biwi error:  7.454,   5.115, 4.747,  5.772,  2.557
+bu error:    2.779,   3.347, 2.448,  2.858,  1.985
+ict error:   3.878,   3.765, 3.597,  3.747,  1.932