From 4fe4fa0b24382f8d3290627a70f649eb83ec13da Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Thu, 15 Feb 2018 08:24:24 +0000 Subject: [PATCH] Removing redundant functions (rotation conversions) that have been defined in utilities now. --- exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 18 ++-- exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 12 +-- .../FaceLandmarkVidMulti.cpp | 25 ++---- exe/FeatureExtraction/FeatureExtraction.cpp | 14 ++- .../LandmarkDetector/LandmarkDetector.vcxproj | 11 ++- .../include/LandmarkDetectorFunc.h | 10 ++- .../include/LandmarkDetectorUtils.h | 30 +------ .../src/FaceDetectorMTCNN.cpp | 1 + .../src/LandmarkDetectorFunc.cpp | 52 ++++++----- .../src/LandmarkDetectorModel.cpp | 3 +- .../src/LandmarkDetectorUtils.cpp | 88 +------------------ lib/local/LandmarkDetector/src/PDM.cpp | 19 ++-- matlab_runners/Full_test_suite.m | 6 +- .../results/Pose_OF_CECLM.txt | 6 +- ....m => run_head_pose_tests_OpenFace_CLNF.m} | 4 +- 15 files changed, 97 insertions(+), 202 deletions(-) rename matlab_runners/Head Pose Experiments/{run_head_pose_tests_OpenFace.m => run_head_pose_tests_OpenFace_CLNF.m} (96%) diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 404a4e30..d2a44cdb 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -116,24 +116,24 @@ int main(int argc, char **argv) // A utility for visualizing the results Utilities::Visualizer visualizer(arguments); - cv::Mat captured_image; + cv::Mat rgb_image; - captured_image = image_reader.GetNextImage(); + rgb_image = image_reader.GetNextImage(); cout << "Starting tracking" << endl; - while (!captured_image.empty()) + while (!rgb_image.empty()) { Utilities::RecorderOpenFaceParameters recording_params(arguments, false, false, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); Utilities::RecorderOpenFace open_face_rec(image_reader.name, recording_params, arguments); - visualizer.SetImage(captured_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); + visualizer.SetImage(rgb_image, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); if (recording_params.outputGaze() && !face_model.eye_model) cout << "WARNING: no eye model defined, but outputting gaze" << endl; - // Making sure the image is in uchar grayscale + // Making sure the image is in uchar grayscale (some face detectors use RGB, landmark detector uses grayscale) cv::Mat_ grayscale_image = image_reader.GetGrayFrame(); // Detect faces in an image @@ -157,7 +157,7 @@ int main(int argc, char **argv) else { vector confidences; - LandmarkDetector::DetectFacesMTCNN(face_detections, captured_image, face_detector_mtcnn, confidences); + LandmarkDetector::DetectFacesMTCNN(face_detections, rgb_image, face_detector_mtcnn, confidences); } } @@ -167,7 +167,7 @@ int main(int argc, char **argv) for (size_t face = 0; face < face_detections.size(); ++face) { // if there are multiple detections go through them - bool success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_detections[face], face_model, det_parameters); + bool success = LandmarkDetector::DetectLandmarksInImage(rgb_image, face_detections[face], face_model, det_parameters, grayscale_image); // Estimate head pose and eye gaze cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, image_reader.fx, image_reader.fy, image_reader.cx, image_reader.cy); @@ -190,7 +190,7 @@ int main(int argc, char **argv) // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - face_analyser.PredictStaticAUsAndComputeFeatures(captured_image, face_model.detected_landmarks); + face_analyser.PredictStaticAUsAndComputeFeatures(rgb_image, face_model.detected_landmarks); face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } @@ -220,7 +220,7 @@ int main(int argc, char **argv) } // Grabbing the next frame in the sequence - captured_image = image_reader.GetNextImage(); + rgb_image = image_reader.GetNextImage(); } diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 1f81f320..9c436336 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -133,17 +133,17 @@ int main(int argc, char **argv) } INFO_STREAM("Device or file opened"); - cv::Mat captured_image = sequence_reader.GetNextFrame(); + cv::Mat rgb_image = sequence_reader.GetNextFrame(); INFO_STREAM("Starting tracking"); - while (!captured_image.empty()) // this is not a for loop as we might also be reading from a webcam + while (!rgb_image.empty()) // this is not a for loop as we might also be reading from a webcam { - // Reading the images + // Reading the grayscale image as well (face detection is done in RGB, landmark detection in grayscale) cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); // The actual facial landmark detection / tracking - bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters); + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_model, det_parameters, grayscale_image); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); @@ -163,7 +163,7 @@ int main(int argc, char **argv) fps_tracker.AddFrame(); // Displaying the tracking visualizations - visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); visualizer.SetObservationPose(pose_estimate, face_model.detection_certainty); visualizer.SetObservationGaze(gazeDirection0, gazeDirection1, LandmarkDetector::CalculateAllEyeLandmarks(face_model), LandmarkDetector::Calculate3DEyeLandmarks(face_model, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy), face_model.detection_certainty); @@ -183,7 +183,7 @@ int main(int argc, char **argv) } // Grabbing the next frame in the sequence - captured_image = sequence_reader.GetNextFrame(); + rgb_image = sequence_reader.GetNextFrame(); } diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 9974fe48..cf72623a 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -182,27 +182,16 @@ int main(int argc, char **argv) } INFO_STREAM("Device or file opened"); - cv::Mat captured_image = sequence_reader.GetNextFrame(); + cv::Mat rgb_image = sequence_reader.GetNextFrame(); int frame_count = 0; INFO_STREAM("Starting tracking"); - while (!captured_image.empty()) + while (!rgb_image.empty()) { // Reading the images - cv::Mat_ grayscale_image; - - cv::Mat disp_image = captured_image.clone(); - - if (captured_image.channels() == 3) - { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); - } - else - { - grayscale_image = captured_image.clone(); - } + cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); vector > face_detections; @@ -230,7 +219,7 @@ int main(int argc, char **argv) else { vector confidences; - LandmarkDetector::DetectFacesMTCNN(face_detections, captured_image, face_models[0].face_detector_MTCNN, confidences); + LandmarkDetector::DetectFacesMTCNN(face_detections, rgb_image, face_models[0].face_detector_MTCNN, confidences); } } @@ -269,7 +258,7 @@ int main(int argc, char **argv) // This ensures that a wider window is used for the initial landmark localisation face_models[model].detection_success = false; - detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_detections[detection_ind], face_models[model], det_parameters[model]); + detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_detections[detection_ind], face_models[model], det_parameters[model], grayscale_image); // This activates the model active_models[model] = true; @@ -291,7 +280,7 @@ int main(int argc, char **argv) // Keeping track of FPS fps_tracker.AddFrame(); - visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); // Go through every model and visualise the results for (size_t model = 0; model < face_models.size(); ++model) @@ -327,7 +316,7 @@ int main(int argc, char **argv) frame_count++; // Grabbing the next frame in the sequence - captured_image = sequence_reader.GetNextFrame(); + rgb_image = sequence_reader.GetNextFrame(); } diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 2e7e66ef..1d0710a5 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -138,8 +138,6 @@ int main(int argc, char **argv) visualizer.vis_track = true; } - cv::Mat captured_image; - Utilities::RecorderOpenFaceParameters recording_params(arguments, true, sequence_reader.IsWebcam(), sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy, sequence_reader.fps); Utilities::RecorderOpenFace open_face_rec(sequence_reader.name, recording_params, arguments); @@ -147,20 +145,20 @@ int main(int argc, char **argv) if (recording_params.outputGaze() && !face_model.eye_model) cout << "WARNING: no eye model defined, but outputting gaze" << endl; - captured_image = sequence_reader.GetNextFrame(); + cv::Mat rgb_image = sequence_reader.GetNextFrame(); // For reporting progress double reported_completion = 0; INFO_STREAM("Starting tracking"); - while (!captured_image.empty()) + while (!rgb_image.empty()) { // Converting to grayscale cv::Mat_ grayscale_image = sequence_reader.GetGrayFrame(); // The actual facial landmark detection / tracking - bool detection_success = LandmarkDetector::DetectLandmarksInVideo(captured_image, face_model, det_parameters); + bool detection_success = LandmarkDetector::DetectLandmarksInVideo(rgb_image, face_model, det_parameters, grayscale_image); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, 0); cv::Point3f gazeDirection1(0, 0, 0); cv::Vec2d gazeAngle(0, 0); @@ -179,7 +177,7 @@ int main(int argc, char **argv) // Perform AU detection and HOG feature extraction, as this can be expensive only compute it if needed by output or visualization if (recording_params.outputAlignedFaces() || recording_params.outputHOG() || recording_params.outputAUs() || visualizer.vis_align || visualizer.vis_hog) { - face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); + face_analyser.AddNextFrame(rgb_image, face_model.detected_landmarks, face_model.detection_success, sequence_reader.time_stamp, sequence_reader.IsWebcam()); face_analyser.GetLatestAlignedFace(sim_warped_img); face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); } @@ -191,7 +189,7 @@ int main(int argc, char **argv) fps_tracker.AddFrame(); // Displaying the tracking visualizations - visualizer.SetImage(captured_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); + visualizer.SetImage(rgb_image, sequence_reader.fx, sequence_reader.fy, sequence_reader.cx, sequence_reader.cy); visualizer.SetObservationFaceAlign(sim_warped_img); visualizer.SetObservationHOG(hog_descriptor, num_hog_rows, num_hog_cols); visualizer.SetObservationLandmarks(face_model.detected_landmarks, face_model.detection_certainty, face_model.GetVisibilities()); @@ -232,7 +230,7 @@ int main(int argc, char **argv) } // Grabbing the next frame in the sequence - captured_image = sequence_reader.GetNextFrame(); + rgb_image = sequence_reader.GetNextFrame(); } diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj index 37f7806a..3b4c343a 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj @@ -93,7 +93,7 @@ Disabled - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN32;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) true EnableFastChecks @@ -117,7 +117,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)c Disabled - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebugDLL @@ -141,7 +141,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)c Full true - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) MultiThreadedDLL false @@ -166,7 +166,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)c Full true - ./include;%(AdditionalIncludeDirectories) + ./include;$(SolutionDir)lib\local\Utilities\include;%(AdditionalIncludeDirectories) WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) MultiThreadedDLL false @@ -274,6 +274,9 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.4\classifiers" "$(OutDir)c {b47a5f12-2567-44e9-ae49-35763ec82149} + + {8e741ea2-9386-4cf2-815e-6f9b08991eac} + diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h b/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h index 17607f46..dc15854a 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h @@ -53,17 +53,19 @@ namespace LandmarkDetector //================================================================================================================ // Landmark detection in videos, need to provide an image and model parameters (default values work well) // Optionally can provide a bounding box from which to start tracking + // Can also optionally pass a grayscale image if it has already been computed to speed things up a bit //================================================================================================================ - bool DetectLandmarksInVideo(const cv::Mat &image, CLNF& clnf_model, FaceModelParameters& params); - bool DetectLandmarksInVideo(const cv::Mat &image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params); + bool DetectLandmarksInVideo(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image = cv::Mat_()); + bool DetectLandmarksInVideo(const cv::Mat &rgb_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image = cv::Mat_()); //================================================================================================================ // Landmark detection in image, need to provide an image and optionally CLNF model together with parameters (default values work well) // Optionally can provide a bounding box in which detection is performed (this is useful if multiple faces are to be detected in images) + // Can also optionally pass a grayscale image if it has already been computed to speed things up a bit //================================================================================================================ - bool DetectLandmarksInImage(const cv::Mat &image, CLNF& clnf_model, FaceModelParameters& params); + bool DetectLandmarksInImage(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image = cv::Mat_()); // Providing a bounding box - bool DetectLandmarksInImage(const cv::Mat &image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params); + bool DetectLandmarksInImage(const cv::Mat &rgb_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image = cv::Mat_()); //================================================================ // Helper function for getting head pose from CLNF parameters diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 63220baf..c1e135bc 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -52,7 +52,7 @@ namespace LandmarkDetector //============================================================================================= - // Helper functions for parsing the inputs + // Helper functions for parsing the inputs, TODO rem //============================================================================================= void get_video_input_output_params(vector &input_video_file, vector &output_files, vector &output_video_files, string &output_codec, vector &arguments); @@ -86,7 +86,7 @@ namespace LandmarkDetector void ExtractBoundingBox(const cv::Mat_& landmarks, float &min_x, float &max_x, float &min_y, float &max_y); //=========================================================================== - // Visualisation functions + // Visualisation functions, TODO rem //=========================================================================== void Project(cv::Mat_& dest, const cv::Mat_& mesh, float fx, float fy, float cx, float cy); void DrawBox(cv::Mat image, cv::Vec6f pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy); @@ -111,23 +111,6 @@ namespace LandmarkDetector void Draw(cv::Mat img, const cv::Mat_& shape2D); void Draw(cv::Mat img, const CLNF& clnf_model); - - //=========================================================================== - // Angle representation conversion helpers - //=========================================================================== - cv::Matx33f Euler2RotationMatrix(const cv::Vec3f& eulerAngles); - - // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign - cv::Vec3f RotationMatrix2Euler(const cv::Matx33f& rotation_matrix); - - cv::Vec3f Euler2AxisAngle(const cv::Vec3f& euler); - - cv::Vec3f AxisAngle2Euler(const cv::Vec3f& axis_angle); - - cv::Matx33f AxisAngle2RotationMatrix(const cv::Vec3f& axis_angle); - - cv::Vec3f RotationMatrix2AxisAngle(const cv::Matx33f& rotation_matrix); - //============================================================================ // Face detection helpers //============================================================================ @@ -162,15 +145,6 @@ namespace LandmarkDetector // Skipping comments (lines starting with # symbol) void SkipComments(std::ifstream& stream); - //============================================================================ - // General utilty functions - //============================================================================ - - // Convert an image to grayscale - void convert_to_grayscale(const cv::Mat& in, cv::Mat& out); - - // Convert an image to an 8bit one with one channel (Grayscale) or three channels (BGR), depending on number of original channels - void convert_to_8bit_bgr_or_grayscale(cv::Mat& in_out); } #endif diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index 2a1d0bfe..8b85c88e 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -676,6 +676,7 @@ bool FaceDetectorMTCNN::DetectFaces(vector >& o_regions, const cv::Mat input_img; + // Force the image to three channels if (img_in.channels() == 1) { cv::cvtColor(img_in, input_img, CV_GRAY2RGB); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index 419ac806..96f288f1 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -34,7 +34,9 @@ #include "stdafx.h" -#include +#include "LandmarkDetectorFunc.h" +#include "RotationHelpers.h" +#include "ImageManipulationHelpers.h" // OpenCV includes #include @@ -84,7 +86,7 @@ cv::Vec6f LandmarkDetector::GetPose(const CLNF& clnf_model, float fx, float fy, cv::solvePnP(landmarks_3D, landmarks_2D, camera_matrix, cv::Mat(), vec_rot, vec_trans, true); - cv::Vec3f euler = LandmarkDetector::AxisAngle2Euler(vec_rot); + cv::Vec3f euler = Utilities::AxisAngle2Euler(vec_rot); return cv::Vec6f(vec_trans[0], vec_trans[1], vec_trans[2], euler[0], euler[1], euler[2]); } @@ -137,12 +139,12 @@ cv::Vec6f LandmarkDetector::GetPoseWRTCamera(const CLNF& clnf_model, float fx, f float z_y = cv::sqrt(vec_trans[1] * vec_trans[1] + vec_trans[2] * vec_trans[2]); float eul_y = -atan2(vec_trans[0], z_y); - cv::Matx33f camera_rotation = LandmarkDetector::Euler2RotationMatrix(cv::Vec3f(eul_x, eul_y, 0)); - cv::Matx33f head_rotation = LandmarkDetector::AxisAngle2RotationMatrix(vec_rot); + cv::Matx33f camera_rotation = Utilities::Euler2RotationMatrix(cv::Vec3f(eul_x, eul_y, 0)); + cv::Matx33f head_rotation = Utilities::AxisAngle2RotationMatrix(vec_rot); cv::Matx33f corrected_rotation = camera_rotation * head_rotation; - cv::Vec3f euler_corrected = LandmarkDetector::RotationMatrix2Euler(corrected_rotation); + cv::Vec3f euler_corrected = Utilities::RotationMatrix2Euler(corrected_rotation); return cv::Vec6f(vec_trans[0], vec_trans[1], vec_trans[2], euler_corrected[0], euler_corrected[1], euler_corrected[2]); } @@ -210,14 +212,16 @@ void CorrectGlobalParametersVideo(const cv::Mat_ &grayscale_image, CLNF& } -bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, CLNF& clnf_model, FaceModelParameters& params) +bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_& grayscale_image) { // First need to decide if the landmarks should be "detected" or "tracked" // Detected means running face detection and a larger search area, tracked means initialising from previous step // and using a smaller search area - cv::Mat grayscale_image; - convert_to_grayscale(image, grayscale_image); + if(grayscale_image.empty()) + { + Utilities::ConvertToGrayscale_8bit(rgb_image, grayscale_image); + } // Indicating that this is a first detection in video sequence or after restart bool initial_detection = !clnf_model.tracking_initialised; @@ -298,7 +302,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, CLNF& clnf_m else if (params.curr_face_detector == FaceModelParameters::MTCNN_DETECTOR) { float confidence; - face_detection_success = LandmarkDetector::DetectSingleFaceMTCNN(bounding_box, image, clnf_model.face_detector_MTCNN, confidence, preference_det); + face_detection_success = LandmarkDetector::DetectSingleFaceMTCNN(bounding_box, rgb_image, clnf_model.face_detector_MTCNN, confidence, preference_det); } // Attempt to detect landmarks using the detected face (if unseccessful the detection will be ignored) @@ -372,7 +376,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, CLNF& clnf_m } -bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) +bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &rgb_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image) { if(bounding_box.width > 0) { @@ -384,7 +388,7 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, const cv::Re clnf_model.tracking_initialised = true; } - return DetectLandmarksInVideo(image, clnf_model, params); + return DetectLandmarksInVideo(rgb_image, clnf_model, params, grayscale_image); } @@ -638,11 +642,13 @@ bool DetectLandmarksInImageMultiHypEarlyTerm(const cv::Mat_ &grayscale_im // This is the one where the actual work gets done, other DetectLandmarksInImage calls lead to this one -bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) +bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &rgb_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image) { - cv::Mat grayscale_image; - convert_to_grayscale(image, grayscale_image); + if (grayscale_image.empty()) + { + Utilities::ConvertToGrayscale_8bit(rgb_image, grayscale_image); + } // Can have multiple hypotheses vector rotation_hypotheses; @@ -674,19 +680,21 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &image, const cv::Re // Either use basic multi-hypothesis testing or clever testing if early termination parameters are present if(clnf_model.patch_experts.early_term_biases.size() == 0) { - success = DetectLandmarksInImageMultiHypBasic(image, rotation_hypotheses, bounding_box, clnf_model, params); + success = DetectLandmarksInImageMultiHypBasic(grayscale_image, rotation_hypotheses, bounding_box, clnf_model, params); } else { - success = DetectLandmarksInImageMultiHypEarlyTerm(image, rotation_hypotheses, bounding_box, clnf_model, params); + success = DetectLandmarksInImageMultiHypEarlyTerm(grayscale_image, rotation_hypotheses, bounding_box, clnf_model, params); } return success; } -bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &image, CLNF& clnf_model, FaceModelParameters& params) +bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &rgb_image, CLNF& clnf_model, FaceModelParameters& params, cv::Mat_ &grayscale_image) { - cv::Mat grayscale_image; - convert_to_grayscale(image, grayscale_image); + if (grayscale_image.empty()) + { + Utilities::ConvertToGrayscale_8bit(rgb_image, grayscale_image); + } cv::Rect_ bounding_box; @@ -710,12 +718,12 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &image, CLNF& clnf_m } else if(params.curr_face_detector == FaceModelParameters::HAAR_DETECTOR) { - LandmarkDetector::DetectSingleFace(bounding_box, image, clnf_model.face_detector_HAAR); + LandmarkDetector::DetectSingleFace(bounding_box, rgb_image, clnf_model.face_detector_HAAR); } else if (params.curr_face_detector == FaceModelParameters::MTCNN_DETECTOR) { float confidence; - LandmarkDetector::DetectSingleFaceMTCNN(bounding_box, image, clnf_model.face_detector_MTCNN, confidence); + LandmarkDetector::DetectSingleFaceMTCNN(bounding_box, rgb_image, clnf_model.face_detector_MTCNN, confidence); } if(bounding_box.width == 0) @@ -724,6 +732,6 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat &image, CLNF& clnf_m } else { - return DetectLandmarksInImage(image, bounding_box, clnf_model, params); + return DetectLandmarksInImage(rgb_image, bounding_box, clnf_model, params, grayscale_image); } } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 6b4eefc7..fd247324 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -49,6 +49,7 @@ // Local includes #include +#include using namespace LandmarkDetector; @@ -1141,7 +1142,7 @@ cv::Mat_ CLNF::GetShape(float fx, float fy, float cx, float cy) const // Need to rotate the shape to get the actual 3D representation // get the rotation matrix from the euler angles - cv::Matx33f R = LandmarkDetector::Euler2RotationMatrix(cv::Vec3f((float)params_global[1], (float)params_global[2], (float)params_global[3])); + cv::Matx33f R = Utilities::Euler2RotationMatrix(cv::Vec3f((float)params_global[1], (float)params_global[2], (float)params_global[3])); shape3d = shape3d.reshape(1, 3); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 32d509c2..63d042e4 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -35,6 +35,7 @@ #include "stdafx.h" #include +#include // OpenCV includes #include @@ -976,7 +977,7 @@ namespace LandmarkDetector // The size of the head is roughly 200mm x 200mm x 200mm cv::Mat_ box = cv::Mat(8, 3, CV_32F, boxVerts).clone() * 100; - cv::Matx33f rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3f((float)pose[3], (float)pose[4], (float)pose[5])); + cv::Matx33f rot = Utilities::Euler2RotationMatrix(cv::Vec3f((float)pose[3], (float)pose[4], (float)pose[5])); cv::Mat_ rotBox; // Rotate the box @@ -1044,7 +1045,7 @@ namespace LandmarkDetector // The size of the head is roughly 200mm x 200mm x 200mm cv::Mat_ box = cv::Mat(8, 3, CV_32F, boxVerts).clone() * 100; - cv::Matx33f rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d((float) pose[3], (float)pose[4], (float)pose[5])); + cv::Matx33f rot = Utilities::Euler2RotationMatrix(cv::Vec3d((float) pose[3], (float)pose[4], (float)pose[5])); cv::Mat_ rotBox; // Rotate the box @@ -1893,87 +1894,4 @@ void SkipComments(std::ifstream& stream) } } -// Some other utility functions -void convert_to_grayscale(const cv::Mat& in, cv::Mat& out) -{ - if (in.channels() == 3) - { - // Make sure it's in a correct format - if (in.depth() != CV_8U) - { - if (in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - tmp.convertTo(tmp, CV_8U); - cv::cvtColor(tmp, out, CV_BGR2GRAY); - } - } - else - { - cv::cvtColor(in, out, CV_BGR2GRAY); - } - } - else if (in.channels() == 4) - { - cv::cvtColor(in, out, CV_BGRA2GRAY); - } - else - { - if (in.depth() == CV_16U) - { - cv::Mat tmp = in / 256; - out = tmp.clone(); - } - else if (in.depth() != CV_8U) - { - in.convertTo(out, CV_8U); - } - else - { - out = in.clone(); - } - } -} - -void convert_to_8bit_bgr_or_grayscale(cv::Mat& in_out) -{ - if (in_out.channels() == 3) - { - // Make sure it's in a correct format - if (in_out.depth() != CV_8U) - { - if (in_out.depth() == CV_16U) - { - in_out = in_out / 256; - in_out.convertTo(in_out, CV_8UC3); - } - else if (in_out.depth() != CV_8U) - { - in_out.convertTo(in_out, CV_8U); - } - } - } - else if (in_out.channels() == 4) - { - cv::cvtColor(in_out, in_out, CV_BGRA2BGR); - - if (in_out.depth() == CV_16U) - { - in_out = in_out / 256; - in_out.convertTo(in_out, CV_8UC3); - } - } - else - { - if (in_out.depth() == CV_16U) - { - in_out = in_out / 256; - } - else if (in_out.depth() != CV_8U) - { - in_out.convertTo(in_out, CV_8U); - } - } -} - } diff --git a/lib/local/LandmarkDetector/src/PDM.cpp b/lib/local/LandmarkDetector/src/PDM.cpp index 4b3507ff..daa80d0e 100644 --- a/lib/local/LandmarkDetector/src/PDM.cpp +++ b/lib/local/LandmarkDetector/src/PDM.cpp @@ -35,6 +35,7 @@ #include "stdafx.h" #include +#include // OpenCV include #include @@ -158,7 +159,7 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_ // get the rotation matrix from the euler angles cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); // get the 3D shape of the object cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; @@ -189,7 +190,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, const cv::Rect_& bound CalcShape3D(current_shape, params_local); // rotate the shape - cv::Matx33f rotation_matrix = Euler2RotationMatrix(rotation); + cv::Matx33f rotation_matrix = Utilities::Euler2RotationMatrix(rotation); cv::Mat_ reshaped = current_shape.reshape(1, 3); @@ -259,7 +260,7 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6f& // Get the rotation matrix cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); float r11 = currRot(0,0); float r12 = currRot(0,1); @@ -352,7 +353,7 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6f& this->CalcShape3D(shape_3D, params_local); cv::Vec3f euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33f currRot = Euler2RotationMatrix(euler); + cv::Matx33f currRot = Utilities::Euler2RotationMatrix(euler); float r11 = currRot(0,0); float r12 = currRot(0,1); @@ -449,7 +450,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& // get the original rotation matrix cv::Vec3f eulerGlobal(params_global[1], params_global[2], params_global[3]); - cv::Matx33f R1 = Euler2RotationMatrix(eulerGlobal); + cv::Matx33f R1 = Utilities::Euler2RotationMatrix(eulerGlobal); // construct R' = [1, -wz, wy // wz, 1, -wx @@ -467,8 +468,8 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& cv::Matx33f R3 = R1 *R2; // Extract euler angle (through axis angle first to make sure it's legal) - cv::Vec3f axis_angle = RotationMatrix2AxisAngle(R3); - cv::Vec3f euler = AxisAngle2Euler(axis_angle); + cv::Vec3f axis_angle = Utilities::RotationMatrix2AxisAngle(R3); + cv::Vec3f euler = Utilities::AxisAngle2Euler(axis_angle); params_global[1] = euler[0]; params_global[2] = euler[1]; @@ -558,7 +559,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_l float scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2.0f; cv::Vec3f rotation_init = rotation; - cv::Matx33f R = Euler2RotationMatrix(rotation_init); + cv::Matx33f R = Utilities::Euler2RotationMatrix(rotation_init); cv::Vec2f translation((min_x + max_x) / 2.0f, (min_y + max_y) / 2.0f); cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); @@ -645,7 +646,7 @@ void PDM::CalcParams(cv::Vec6f& out_params_global, cv::Mat_& out_params_l translation[0] = glob_params[4]; translation[1] = glob_params[5]; - R = Euler2RotationMatrix(rotation_init); + R = Utilities::Euler2RotationMatrix(rotation_init); R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); diff --git a/matlab_runners/Full_test_suite.m b/matlab_runners/Full_test_suite.m index d96cc80f..3b3d1dd1 100644 --- a/matlab_runners/Full_test_suite.m +++ b/matlab_runners/Full_test_suite.m @@ -5,16 +5,16 @@ tic %% Head pose cd('Head Pose Experiments'); run_head_pose_tests_OpenFace_CECLM; -assert(median(all_errors_biwi_OF(:)) < 2.7); +assert(median(all_errors_biwi_OF(:)) < 2.45); assert(median(all_errors_bu_OF(:)) < 1.85); assert(median(all_errors_ict_OF(:)) < 1.9); -run_head_pose_tests_OpenFace; +run_head_pose_tests_OpenFace_CLNF; assert(median(all_errors_biwi_OF(:)) < 2.7); assert(median(all_errors_bu_OF(:)) < 2.0); assert(median(all_errors_ict_OF(:)) < 2.0); cd('../'); -%% Features +%% Landmarks cd('Feature Point Experiments'); run_OpenFace_feature_point_tests_300W; assert(median(err_ceclm) < 0.035); diff --git a/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt b/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt index 3fefa14d..719624db 100644 --- a/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt +++ b/matlab_runners/Head Pose Experiments/results/Pose_OF_CECLM.txt @@ -1,4 +1,4 @@ Dataset and model, pitch, yaw, roll, mean, median -biwi error: 6.924, 6.471, 3.678, 5.691, 2.629 -bu error: 2.398, 3.247, 2.409, 2.685, 1.834 -ict error: 3.130, 3.510, 3.136, 3.259, 1.867 +biwi error: 6.094, 5.120, 3.716, 4.977, 2.417 +bu error: 2.396, 3.246, 2.408, 2.684, 1.834 +ict error: 3.132, 3.512, 3.136, 3.260, 1.867 diff --git a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CLNF.m similarity index 96% rename from matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m rename to matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CLNF.m index 27224c18..156de44c 100644 --- a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace.m +++ b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CLNF.m @@ -38,11 +38,11 @@ ict_dir = ['/ict/']; [ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]); %% Save the results -filename = 'results/Pose_OF'; +filename = 'results/Pose_OF_CLNF'; save(filename); % Also save them in a reasonable .txt format for easy comparison -f = fopen('results/Pose_OF.txt', 'w'); +f = fopen('results/Pose_OF_CLNF.txt', 'w'); fprintf(f, 'Dataset and model, pitch, yaw, roll, mean, median\n'); fprintf(f, 'biwi error: %.3f, %.3f, %.3f, %.3f, %.3f\n', biwi_error_OF, mean(all_errors_biwi_OF(:)), median(all_errors_biwi_OF(:))); fprintf(f, 'bu error: %.3f, %.3f, %.3f, %.3f, %.3f\n', bu_error_OF, mean(all_errors_bu_OF(:)), median(all_errors_bu_OF(:)));