From 165a708796996944e04bb6814fb4aff5ef8f700e Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Tue, 20 Jun 2017 14:55:59 -0400 Subject: [PATCH] Decoupling C++ code so that AU detection works properly with a new PDM. --- .gitignore | 2 + OpenFace.sln | 11 + exe/FaceLandmarkImg/FaceLandmarkImg.cpp | 208 +- exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj | 11 +- exe/FaceLandmarkVid/FaceLandmarkVid.cpp | 154 +- exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj | 11 +- .../FaceLandmarkVidMulti.cpp | 249 +- exe/FeatureExtraction/FeatureExtraction.cpp | 418 +-- .../FeatureExtraction.vcxproj | 11 +- .../In-the-wild_aligned_PDM_68.txt | 421 +++ .../AU_predictors/main_dynamic_svms.txt | 3 + .../AU_predictors/main_static_svms.txt | 3 + .../AU_predictors/tris_68_full.txt | 114 + lib/local/FaceAnalyser/FaceAnalyser.vcxproj | 25 +- .../FaceAnalyser/FaceAnalyser.vcxproj.filters | 16 +- lib/local/FaceAnalyser/include/FaceAnalyser.h | 336 +- .../include/FaceAnalyserParameters.h | 100 + lib/local/FaceAnalyser/include/Face_utils.h | 91 +- .../FaceAnalyser/include/GazeEstimation.h | 72 - lib/local/FaceAnalyser/include/PAW.h | 138 + lib/local/FaceAnalyser/include/PDM.h | 105 + .../FaceAnalyser/include/SVM_dynamic_lin.h | 39 +- .../FaceAnalyser/include/SVM_static_lin.h | 39 +- .../include/SVR_dynamic_lin_regressors.h | 39 +- .../include/SVR_static_lin_regressors.h | 39 +- lib/local/FaceAnalyser/src/FaceAnalyser.cpp | 341 +- .../src/FaceAnalyserParameters.cpp | 254 ++ lib/local/FaceAnalyser/src/Face_utils.cpp | 363 ++- lib/local/FaceAnalyser/src/PAW.cpp | 500 +++ lib/local/FaceAnalyser/src/PDM.cpp | 613 ++++ .../FaceAnalyser/src/SVM_dynamic_lin.cpp | 49 +- lib/local/FaceAnalyser/src/SVM_static_lin.cpp | 49 +- .../src/SVR_dynamic_lin_regressors.cpp | 50 +- .../src/SVR_static_lin_regressors.cpp | 50 +- lib/local/GazeAnalyser/GazeAnalyser.vcxproj | 158 + .../GazeAnalyser/GazeAnalyser.vcxproj.filters | 23 + .../GazeAnalyser/include/GazeEstimation.h | 55 + .../src/GazeEstimation.cpp | 68 +- .../include/LandmarkDetectorFunc.h | 56 +- .../include/LandmarkDetectorUtils.h | 66 +- .../LandmarkDetector/model/cen_general.txt | 2 +- .../src/LandmarkDetectorFunc.cpp | 104 +- .../src/LandmarkDetectorModel.cpp | 10 +- .../src/LandmarkDetectorParameters.cpp | 2 +- .../src/LandmarkDetectorUtils.cpp | 2880 +++++++++-------- 45 files changed, 5309 insertions(+), 3039 deletions(-) create mode 100644 lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt create mode 100644 lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt create mode 100644 lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt create mode 100644 lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt create mode 100644 lib/local/FaceAnalyser/include/FaceAnalyserParameters.h delete mode 100644 lib/local/FaceAnalyser/include/GazeEstimation.h create mode 100644 lib/local/FaceAnalyser/include/PAW.h create mode 100644 lib/local/FaceAnalyser/include/PDM.h create mode 100644 lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp create mode 100644 lib/local/FaceAnalyser/src/PAW.cpp create mode 100644 lib/local/FaceAnalyser/src/PDM.cpp create mode 100644 lib/local/GazeAnalyser/GazeAnalyser.vcxproj create mode 100644 lib/local/GazeAnalyser/GazeAnalyser.vcxproj.filters create mode 100644 lib/local/GazeAnalyser/include/GazeEstimation.h rename lib/local/{FaceAnalyser => GazeAnalyser}/src/GazeEstimation.cpp (69%) diff --git a/.gitignore b/.gitignore index a3898fa0..2def8ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ matlab_version/experiments_300VW/CLNF_res_general/ matlab_version/experiments_300VW/CECLM_res_general/ matlab_version/experiments_300VW/CECLM_res_general_mtcnn/ matlab_version/experiments_300VW/CLNF_res_general_mtcnn/ +matlab_runners/Feature Point Experiments/out_ceclm/ +lib/local/GazeAnalyser/x64/ diff --git a/OpenFace.sln b/OpenFace.sln index 84732ad4..ee4836e5 100644 --- a/OpenFace.sln +++ b/OpenFace.sln @@ -23,6 +23,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkVid", "exe\Face EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FaceLandmarkImg", "exe\FaceLandmarkImg\FaceLandmarkImg.vcxproj", "{DDC3535E-526C-44EC-9DF4-739E2D3A323B}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "GazeAnalyser", "lib\local\GazeAnalyser\GazeAnalyser.vcxproj", "{5F915541-F531-434F-9C81-79F5DB58012B}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -95,6 +97,14 @@ Global {DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|Win32.Build.0 = Release|Win32 {DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|x64.ActiveCfg = Release|x64 {DDC3535E-526C-44EC-9DF4-739E2D3A323B}.Release|x64.Build.0 = Release|x64 + {5F915541-F531-434F-9C81-79F5DB58012B}.Debug|Win32.ActiveCfg = Debug|Win32 + {5F915541-F531-434F-9C81-79F5DB58012B}.Debug|Win32.Build.0 = Debug|Win32 + {5F915541-F531-434F-9C81-79F5DB58012B}.Debug|x64.ActiveCfg = Debug|x64 + {5F915541-F531-434F-9C81-79F5DB58012B}.Debug|x64.Build.0 = Debug|x64 + {5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.ActiveCfg = Release|Win32 + {5F915541-F531-434F-9C81-79F5DB58012B}.Release|Win32.Build.0 = Release|Win32 + {5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.ActiveCfg = Release|x64 + {5F915541-F531-434F-9C81-79F5DB58012B}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -108,5 +118,6 @@ Global {2D80FA0B-2DE8-4475-BA5A-C08A9E1EDAAC} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631} {34032CF2-1B99-4A25-9050-E9C13DD4CD0A} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631} {DDC3535E-526C-44EC-9DF4-739E2D3A323B} = {9961DDAC-BE6E-4A6E-8EEF-FFC7D67BD631} + {5F915541-F531-434F-9C81-79F5DB58012B} = {99FEBA13-BDDF-4076-B57E-D8EF4076E20D} EndGlobalSection EndGlobal diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index bf9f716c..07e0db4e 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -1,37 +1,13 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical @@ -89,7 +65,7 @@ vector get_arguments(int argc, char **argv) vector arguments; - for(int i = 0; i < argc; ++i) + for (int i = 0; i < argc; ++i) { arguments.push_back(string(argv[i])); } @@ -98,12 +74,12 @@ vector get_arguments(int argc, char **argv) void convert_to_grayscale(const cv::Mat& in, cv::Mat& out) { - if(in.channels() == 3) + if (in.channels() == 3) { // Make sure it's in a correct format - if(in.depth() != CV_8U) + if (in.depth() != CV_8U) { - if(in.depth() == CV_16U) + if (in.depth() == CV_16U) { cv::Mat tmp = in / 256; tmp.convertTo(tmp, CV_8U); @@ -115,18 +91,18 @@ void convert_to_grayscale(const cv::Mat& in, cv::Mat& out) cv::cvtColor(in, out, CV_BGR2GRAY); } } - else if(in.channels() == 4) + else if (in.channels() == 4) { cv::cvtColor(in, out, CV_BGRA2GRAY); } else { - if(in.depth() == CV_16U) + if (in.depth() == CV_16U) { cv::Mat tmp = in / 256; out = tmp.clone(); } - else if(in.depth() != CV_8U) + else if (in.depth() != CV_8U) { in.convertTo(out, CV_8U); } @@ -173,7 +149,7 @@ void write_out_pose_landmarks(const string& outfeatures, const cv::Mat_& for (int i = 0; i < n; ++i) { // Use matlab format, so + 1 - featuresFile << shape3D.at(i) << " " << shape3D.at(i + n) << " " << shape3D.at(i + 2*n) << endl; + featuresFile << shape3D.at(i) << " " << shape3D.at(i + n) << " " << shape3D.at(i + 2 * n) << endl; } featuresFile << "}" << endl; @@ -255,15 +231,15 @@ void write_out_landmarks(const string& outfeatures, const LandmarkDetector::CLNF void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkDetector::CLNF& clnf_model) { - + // Draw head pose if present and draw eye gaze as well // preparing the visualisation image - display_image = orig.clone(); + display_image = orig.clone(); // Creating a display image - cv::Mat xs = clnf_model.detected_landmarks(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2)); - cv::Mat ys = clnf_model.detected_landmarks(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2)); + cv::Mat xs = clnf_model.detected_landmarks(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows / 2)); + cv::Mat ys = clnf_model.detected_landmarks(cv::Rect(0, clnf_model.detected_landmarks.rows / 2, 1, clnf_model.detected_landmarks.rows / 2)); double min_x, max_x, min_y, max_y; cv::minMaxLoc(xs, &min_x, &max_x); @@ -272,17 +248,17 @@ void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkD double width = max_x - min_x; double height = max_y - min_y; - int minCropX = max((int)(min_x-width/3.0),0); - int minCropY = max((int)(min_y-height/3.0),0); + int minCropX = max((int)(min_x - width / 3.0), 0); + int minCropY = max((int)(min_y - height / 3.0), 0); - int widthCrop = min((int)(width*5.0/3.0), display_image.cols - minCropX - 1); - int heightCrop = min((int)(height*5.0/3.0), display_image.rows - minCropY - 1); + int widthCrop = min((int)(width*5.0 / 3.0), display_image.cols - minCropX - 1); + int heightCrop = min((int)(height*5.0 / 3.0), display_image.rows - minCropY - 1); + + double scaling = 350.0 / widthCrop; - double scaling = 350.0/widthCrop; - // first crop the image display_image = display_image(cv::Rect((int)(minCropX), (int)(minCropY), (int)(widthCrop), (int)(heightCrop))); - + // now scale it cv::resize(display_image.clone(), display_image, cv::Size(), scaling, scaling); @@ -292,8 +268,8 @@ void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkD cv::Mat shape = clnf_model.detected_landmarks.clone(); - xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows/2))); - ys.copyTo(shape(cv::Rect(0, clnf_model.detected_landmarks.rows/2, 1, clnf_model.detected_landmarks.rows/2))); + xs.copyTo(shape(cv::Rect(0, 0, 1, clnf_model.detected_landmarks.rows / 2))); + ys.copyTo(shape(cv::Rect(0, clnf_model.detected_landmarks.rows / 2, 1, clnf_model.detected_landmarks.rows / 2))); // Do the shifting for the hierarchical models as well for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) @@ -312,12 +288,12 @@ void create_display_image(const cv::Mat& orig, cv::Mat& display_image, LandmarkD } LandmarkDetector::Draw(display_image, clnf_model); - + } -int main (int argc, char **argv) +int main(int argc, char **argv) { - + //Convert arguments to more convenient vector form vector arguments = get_arguments(argc, argv); @@ -330,9 +306,9 @@ int main (int argc, char **argv) // Bounding boxes for a face in each image (optional) vector > bounding_boxes; - + LandmarkDetector::get_image_input_output_params(files, depth_files, output_landmark_locations, output_pose_locations, output_images, bounding_boxes, arguments); - LandmarkDetector::FaceModelParameters det_parameters(arguments); + LandmarkDetector::FaceModelParameters det_parameters(arguments); // No need to validate detections, as we're not doing tracking det_parameters.validate_detections = false; @@ -357,59 +333,19 @@ int main (int argc, char **argv) cout << "Loading the model" << endl; LandmarkDetector::CLNF clnf_model(det_parameters.model_location); cout << "Model loaded" << endl; - + cv::CascadeClassifier classifier(det_parameters.face_detector_location); dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector(); - // Loading the AU prediction models - string au_loc = "AU_predictors/AU_all_static.txt"; - - boost::filesystem::path au_loc_path = boost::filesystem::path(au_loc); - if (boost::filesystem::exists(au_loc_path)) - { - au_loc = au_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/au_loc_path)) - { - au_loc = (parent_path/au_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/au_loc_path)) - { - au_loc = (config_path/au_loc_path).string(); - } - else - { - cout << "Can't find AU prediction files, exiting" << endl; - return 1; - } - - // Used for image masking for AUs - string tri_loc; - boost::filesystem::path tri_loc_path = boost::filesystem::path("model/tris_68_full.txt"); - if (boost::filesystem::exists(tri_loc_path)) - { - tri_loc = tri_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/tri_loc_path)) - { - tri_loc = (parent_path/tri_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/tri_loc_path)) - { - tri_loc = (config_path/tri_loc_path).string(); - } - else - { - cout << "Can't find triangulation files, exiting" << endl; - return 1; - } - - FaceAnalysis::FaceAnalyser face_analyser(vector(), 0.7, 112, 112, au_loc, tri_loc); + // Load facial feature extractor and AU analyser (make sure it is static) + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + face_analysis_params.OptimizeForImages(); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); bool visualise = !det_parameters.quiet_mode; // Do some image loading - for(size_t i = 0; i < files.size(); i++) + for (size_t i = 0; i < files.size(); i++) { string file = files.at(i); @@ -425,7 +361,7 @@ int main (int argc, char **argv) // Loading depth file if exists (optional) cv::Mat_ depth_image; - if(depth_files.size() > 0) + if (depth_files.size() > 0) { string dFile = depth_files.at(i); cv::Mat dTemp = cv::imread(dFile, -1); @@ -435,7 +371,7 @@ int main (int argc, char **argv) // Making sure the image is in uchar grayscale cv::Mat_ grayscale_image; convert_to_grayscale(read_image, grayscale_image); - + // If optical centers are not defined just use center of image if (cx_undefined) @@ -455,13 +391,13 @@ int main (int argc, char **argv) // if no pose defined we just use a face detector - if(bounding_boxes.empty()) + if (bounding_boxes.empty()) { - + // Detect faces in an image vector > face_detections; - if(det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) + if (det_parameters.curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, face_detector_hog, confidences); @@ -474,29 +410,29 @@ int main (int argc, char **argv) // Detect landmarks around detected faces int face_det = 0; // perform landmark detection for every face detected - for(size_t face=0; face < face_detections.size(); ++face) + for (size_t face = 0; face < face_detections.size(); ++face) { // if there are multiple detections go through them bool success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, depth_image, face_detections[face], clnf_model, det_parameters); // Estimate head pose and eye gaze - cv::Vec6d headPose = LandmarkDetector::GetCorrectedPoseWorld(clnf_model, fx, fy, cx, cy); + cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); if (success && det_parameters.track_gaze) - { - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + { + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); } - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); + auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); // Writing out the detected landmarks (in an OS independent manner) - if(!output_landmark_locations.empty()) + if (!output_landmark_locations.empty()) { char name[100]; // append detection number (in case multiple faces are detected) @@ -535,28 +471,28 @@ int main (int argc, char **argv) if (det_parameters.track_gaze) { - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetCorrectedPoseWorld(clnf_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); // Draw it in reddish if uncertain, blueish if certain LandmarkDetector::DrawBox(read_image, pose_estimate_to_draw, cv::Scalar(255.0, 0, 0), 3, fx, fy, cx, cy); - FaceAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); + GazeAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); } // displaying detected landmarks cv::Mat display_image; create_display_image(read_image, display_image, clnf_model); - if(visualise && success) + if (visualise && success) { imshow("colour", display_image); cv::waitKey(1); } // Saving the display images (in an OS independent manner) - if(!output_images.empty() && success) + if (!output_images.empty() && success) { string outimage = output_images.at(i); - if(!outimage.empty()) + if (!outimage.empty()) { char name[100]; sprintf(name, "_det_%d", face_det); @@ -571,8 +507,8 @@ int main (int argc, char **argv) boost::filesystem::path ext = out_feat_path.extension(); outimage = dir.string() + preferredSlash + fname.string() + string(name) + ext.string(); create_directory_from_file(outimage); - bool write_success = cv::imwrite(outimage, display_image); - + bool write_success = cv::imwrite(outimage, display_image); + if (!write_success) { cout << "Could not output a processed image" << endl; @@ -583,7 +519,7 @@ int main (int argc, char **argv) } - if(success) + if (success) { face_det++; } @@ -596,22 +532,22 @@ int main (int argc, char **argv) LandmarkDetector::DetectLandmarksInImage(grayscale_image, bounding_boxes[i], clnf_model, det_parameters); // Estimate head pose and eye gaze - cv::Vec6d headPose = LandmarkDetector::GetCorrectedPoseWorld(clnf_model, fx, fy, cx, cy); + cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); - + if (det_parameters.track_gaze) { - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); } - auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model, false); + auto ActionUnits = face_analyser.PredictStaticAUs(read_image, clnf_model.detected_landmarks, false); // Writing out the detected landmarks - if(!output_landmark_locations.empty()) + if (!output_landmark_locations.empty()) { string outfeatures = output_landmark_locations.at(i); write_out_landmarks(outfeatures, clnf_model, headPose, gazeDirection0, gazeDirection1, ActionUnits.first, ActionUnits.second); @@ -629,28 +565,28 @@ int main (int argc, char **argv) if (det_parameters.track_gaze) { - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetCorrectedPoseWorld(clnf_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); // Draw it in reddish if uncertain, blueish if certain LandmarkDetector::DrawBox(read_image, pose_estimate_to_draw, cv::Scalar(255.0, 0, 0), 3, fx, fy, cx, cy); - FaceAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); + GazeAnalysis::DrawGaze(read_image, clnf_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); } create_display_image(read_image, display_image, clnf_model); - if(visualise) + if (visualise) { imshow("colour", display_image); cv::waitKey(1); } - if(!output_images.empty()) + if (!output_images.empty()) { string outimage = output_images.at(i); - if(!outimage.empty()) + if (!outimage.empty()) { create_directory_from_file(outimage); - bool write_success = imwrite(outimage, display_image); + bool write_success = imwrite(outimage, display_image); if (!write_success) { @@ -659,10 +595,10 @@ int main (int argc, char **argv) } } } - } + } } - + return 0; } diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj index 140b29c9..08de70ad 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.vcxproj @@ -112,7 +112,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -127,7 +127,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true @@ -144,7 +144,7 @@ true true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -164,7 +164,7 @@ true true WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -183,6 +183,9 @@ {0e7fc556-0e80-45ea-a876-dde4c2fedcd7} + + {5f915541-f531-434f-9c81-79f5db58012b} + {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp index 056bfd4e..e54f5913 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.cpp +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.cpp @@ -1,37 +1,13 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical @@ -74,9 +50,6 @@ #include #include -// OpenBLAS -#include - #define INFO_STREAM( stream ) \ std::cout << stream << std::endl @@ -86,10 +59,10 @@ std::cout << "Warning: " << stream << std::endl #define ERROR_STREAM( stream ) \ std::cout << "Error: " << stream << std::endl -static void printErrorAndAbort( const std::string & error ) +static void printErrorAndAbort(const std::string & error) { - std::cout << error << std::endl; - abort(); + std::cout << error << std::endl; + abort(); } #define FATAL_STREAM( stream ) \ @@ -102,7 +75,7 @@ vector get_arguments(int argc, char **argv) vector arguments; - for(int i = 0; i < argc; ++i) + for (int i = 0; i < argc; ++i) { arguments.push_back(string(argv[i])); } @@ -139,14 +112,14 @@ void visualise_tracking(cv::Mat& captured_image, cv::Mat_& depth_image, c // A rough heuristic for box around the face width int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetCorrectedPoseWorld(face_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); // Draw it in reddish if uncertain, blueish if certain LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); - + if (det_parameters.track_gaze && detection_success && face_model.eye_model) { - FaceAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); + GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); } } @@ -179,28 +152,27 @@ void visualise_tracking(cv::Mat& captured_image, cv::Mat_& depth_image, c } } -int main (int argc, char **argv) +int main(int argc, char **argv) { vector arguments = get_arguments(argc, argv); // Some initial parameters that can be overriden from command line vector files, depth_directories, output_video_files, out_dummy; - + // By default try webcam 0 int device = 0; LandmarkDetector::FaceModelParameters det_parameters(arguments); // Get the input output file parameters - + // Indicates that rotation should be with respect to world or camera coordinates - bool u; string output_codec; - LandmarkDetector::get_video_input_output_params(files, depth_directories, out_dummy, output_video_files, u, output_codec, arguments); - + LandmarkDetector::get_video_input_output_params(files, depth_directories, out_dummy, output_video_files, output_codec, arguments); + // The modules that are being used for tracking - LandmarkDetector::CLNF clnf_model(det_parameters.model_location); + LandmarkDetector::CLNF clnf_model(det_parameters.model_location); // Grab camera parameters, if they are not defined (approximate values will be used) float fx = 0, fy = 0, cx = 0, cy = 0; @@ -220,33 +192,33 @@ int main (int argc, char **argv) } // If multiple video files are tracked, use this to indicate if we are done - bool done = false; + bool done = false; int f_n = -1; - + det_parameters.track_gaze = true; - while(!done) // this is not a for loop as we might also be reading from a webcam + while (!done) // this is not a for loop as we might also be reading from a webcam { - + string current_file; // We might specify multiple video files as arguments - if(files.size() > 0) + if (files.size() > 0) { - f_n++; - current_file = files[f_n]; + f_n++; + current_file = files[f_n]; } else { // If we want to write out from webcam f_n = 0; } - - bool use_depth = !depth_directories.empty(); + + bool use_depth = !depth_directories.empty(); // Do some grabbing cv::VideoCapture video_capture; - if( current_file.size() > 0 ) + if (current_file.size() > 0) { if (!boost::filesystem::exists(current_file)) { @@ -256,13 +228,13 @@ int main (int argc, char **argv) current_file = boost::filesystem::path(current_file).generic_string(); - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); + INFO_STREAM("Attempting to read from file: " << current_file); + video_capture = cv::VideoCapture(current_file); } else { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); + INFO_STREAM("Attempting to capture from device: " << device); + video_capture = cv::VideoCapture(device); // Read a first frame often empty in camera cv::Mat captured_image; @@ -274,10 +246,10 @@ int main (int argc, char **argv) FATAL_STREAM("Failed to open video source"); return 1; } - else INFO_STREAM( "Device or file opened"); + else INFO_STREAM("Device or file opened"); cv::Mat captured_image; - video_capture >> captured_image; + video_capture >> captured_image; // If optical centers are not defined just use center of image if (cx_undefined) @@ -293,46 +265,46 @@ int main (int argc, char **argv) fx = (fx + fy) / 2.0; fy = fx; - } - + } + int frame_count = 0; - + // saving the videos cv::VideoWriter writerFace; if (!output_video_files.empty()) { try - { + { writerFace = cv::VideoWriter(output_video_files[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true); } - catch(cv::Exception e) + catch (cv::Exception e) { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); } } // Use for timestamping if using a webcam int64 t_initial = cv::getTickCount(); - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) + { // Reading the images cv::Mat_ depth_image; cv::Mat_ grayscale_image; - if(captured_image.channels() == 3) + if (captured_image.channels() == 3) { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); + cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); } else { - grayscale_image = captured_image.clone(); + grayscale_image = captured_image.clone(); } - + // Get depth image - if(use_depth) + if (use_depth) { char* dst = new char[100]; std::stringstream sstream; @@ -343,19 +315,19 @@ int main (int argc, char **argv) cv::Mat_ depth_image_16_bit = cv::imread(string(dst), -1); // Convert to a floating point depth image - if(!depth_image_16_bit.empty()) + if (!depth_image_16_bit.empty()) { depth_image_16_bit.convertTo(depth_image, CV_32F); } else { - WARN_STREAM( "Can't find depth image" ); + WARN_STREAM("Can't find depth image"); } } - + // The actual facial landmark detection / tracking bool detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, depth_image, clnf_model, det_parameters); - + // Visualising the results // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised double detection_certainty = clnf_model.detection_certainty; @@ -366,12 +338,12 @@ int main (int argc, char **argv) if (det_parameters.track_gaze && detection_success && clnf_model.eye_model) { - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); - FaceAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection0, fx, fy, cx, cy, true); + GazeAnalysis::EstimateGaze(clnf_model, gazeDirection1, fx, fy, cx, cy, false); } visualise_tracking(captured_image, depth_image, clnf_model, det_parameters, gazeDirection0, gazeDirection1, frame_count, fx, fy, cx, cy); - + // output the tracked video if (!output_video_files.empty()) { @@ -380,17 +352,17 @@ int main (int argc, char **argv) video_capture >> captured_image; - + // detect key presses char character_press = cv::waitKey(1); - + // restart the tracker - if(character_press == 'r') + if (character_press == 'r') { clnf_model.Reset(); } // quit the application - else if(character_press=='q') + else if (character_press == 'q') { return(0); } @@ -399,14 +371,14 @@ int main (int argc, char **argv) frame_count++; } - + frame_count = 0; // Reset the model, for the next video clnf_model.Reset(); - + // break out of the loop if done with all the files (or using a webcam) - if(f_n == files.size() -1 || files.empty()) + if (f_n == files.size() - 1 || files.empty()) { done = true; } diff --git a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj index fc666363..858265cf 100644 --- a/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj +++ b/exe/FaceLandmarkVid/FaceLandmarkVid.vcxproj @@ -112,7 +112,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -128,7 +128,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -147,7 +147,7 @@ true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -169,7 +169,7 @@ true WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions @@ -189,6 +189,9 @@ {0e7fc556-0e80-45ea-a876-dde4c2fedcd7} + + {5f915541-f531-434f-9c81-79f5db58012b} + {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index 47889790..7af23306 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -1,37 +1,13 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical @@ -68,8 +44,6 @@ #include // Video write #include #include -// OpenBLAS -#include #define INFO_STREAM( stream ) \ std::cout << stream << std::endl @@ -80,10 +54,10 @@ std::cout << "Warning: " << stream << std::endl #define ERROR_STREAM( stream ) \ std::cout << "Error: " << stream << std::endl -static void printErrorAndAbort( const std::string & error ) +static void printErrorAndAbort(const std::string & error) { - std::cout << error << std::endl; - abort(); + std::cout << error << std::endl; + abort(); } #define FATAL_STREAM( stream ) \ @@ -96,7 +70,7 @@ vector get_arguments(int argc, char **argv) vector arguments; - for(int i = 0; i < argc; ++i) + for (int i = 0; i < argc; ++i) { arguments.push_back(string(argv[i])); } @@ -107,19 +81,19 @@ void NonOverlapingDetections(const vector& clnf_models, { // Go over the model and eliminate detections that are not informative (there already is a tracker there) - for(size_t model = 0; model < clnf_models.size(); ++model) + for (size_t model = 0; model < clnf_models.size(); ++model) { // See if the detections intersect cv::Rect_ model_rect = clnf_models[model].GetBoundingBox(); - - for(int detection = face_detections.size()-1; detection >=0; --detection) + + for (int detection = face_detections.size() - 1; detection >= 0; --detection) { double intersection_area = (model_rect & face_detections[detection]).area(); double union_area = model_rect.area() + face_detections[detection].area() - 2 * intersection_area; // If the model is already tracking what we're detecting ignore the detection, this is determined by amount of overlap - if( intersection_area/union_area > 0.5) + if (intersection_area / union_area > 0.5) { face_detections.erase(face_detections.begin() + detection); } @@ -127,20 +101,20 @@ void NonOverlapingDetections(const vector& clnf_models, } } -int main (int argc, char **argv) +int main(int argc, char **argv) { vector arguments = get_arguments(argc, argv); // Some initial parameters that can be overriden from command line vector files, depth_directories, tracked_videos_output, dummy_out; - + // By default try webcam 0 int device = 0; // cx and cy aren't necessarilly in the image center, so need to be able to override it (start with unit vals and init them if none specified) - float fx = 600, fy = 600, cx = 0, cy = 0; - + float fx = 600, fy = 600, cx = 0, cy = 0; + LandmarkDetector::FaceModelParameters det_params(arguments); det_params.use_face_template = true; // This is so that the model would not try re-initialising itself @@ -152,12 +126,11 @@ int main (int argc, char **argv) det_parameters.push_back(det_params); // Get the input output file parameters - bool u; string output_codec; - LandmarkDetector::get_video_input_output_params(files, depth_directories, dummy_out, tracked_videos_output, u, output_codec, arguments); + LandmarkDetector::get_video_input_output_params(files, depth_directories, dummy_out, tracked_videos_output, output_codec, arguments); // Get camera parameters LandmarkDetector::get_camera_params(device, fx, fy, cx, cy, arguments); - + // The modules that are being used for tracking vector clnf_models; vector active_models; @@ -167,7 +140,7 @@ int main (int argc, char **argv) LandmarkDetector::CLNF clnf_model(det_parameters[0].model_location); clnf_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); clnf_model.face_detector_location = det_parameters[0].face_detector_location; - + clnf_models.reserve(num_faces_max); clnf_models.push_back(clnf_model); @@ -179,43 +152,43 @@ int main (int argc, char **argv) active_models.push_back(false); det_parameters.push_back(det_params); } - + // If multiple video files are tracked, use this to indicate if we are done - bool done = false; + bool done = false; int f_n = -1; // If cx (optical axis centre) is undefined will use the image size/2 as an estimate bool cx_undefined = false; - if(cx == 0 || cy == 0) + if (cx == 0 || cy == 0) { cx_undefined = true; - } - - while(!done) // this is not a for loop as we might also be reading from a webcam + } + + while (!done) // this is not a for loop as we might also be reading from a webcam { - + string current_file; // We might specify multiple video files as arguments - if(files.size() > 0) + if (files.size() > 0) { - f_n++; - current_file = files[f_n]; + f_n++; + current_file = files[f_n]; } - bool use_depth = !depth_directories.empty(); + bool use_depth = !depth_directories.empty(); // Do some grabbing cv::VideoCapture video_capture; - if( current_file.size() > 0 ) + if (current_file.size() > 0) { - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); + INFO_STREAM("Attempting to read from file: " << current_file); + video_capture = cv::VideoCapture(current_file); } else { - INFO_STREAM( "Attempting to capture from device: " << device ); - video_capture = cv::VideoCapture( device ); + INFO_STREAM("Attempting to capture from device: " << device); + video_capture = cv::VideoCapture(device); // Read a first frame often empty in camera cv::Mat captured_image; @@ -227,43 +200,43 @@ int main (int argc, char **argv) FATAL_STREAM("Failed to open video source"); return 1; } - else INFO_STREAM( "Device or file opened"); + else INFO_STREAM("Device or file opened"); cv::Mat captured_image; - video_capture >> captured_image; - + video_capture >> captured_image; + // If optical centers are not defined just use center of image - if(cx_undefined) + if (cx_undefined) { cx = captured_image.cols / 2.0f; cy = captured_image.rows / 2.0f; } - + int frame_count = 0; - + // saving the videos cv::VideoWriter writerFace; - if(!tracked_videos_output.empty()) + if (!tracked_videos_output.empty()) { try { - writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0],output_codec[1],output_codec[2],output_codec[3]), 30, captured_image.size(), true); + writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), 30, captured_image.size(), true); } - catch(cv::Exception e) + catch (cv::Exception e) { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); } } - + // For measuring the timings - int64 t1,t0 = cv::getTickCount(); + int64 t1, t0 = cv::getTickCount(); double fps = 10; - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) + { // Reading the images cv::Mat_ depth_image; @@ -271,17 +244,17 @@ int main (int argc, char **argv) cv::Mat disp_image = captured_image.clone(); - if(captured_image.channels() == 3) + if (captured_image.channels() == 3) { - cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); + cv::cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); } else { - grayscale_image = captured_image.clone(); + grayscale_image = captured_image.clone(); } - + // Get depth image - if(use_depth) + if (use_depth) { char* dst = new char[100]; std::stringstream sstream; @@ -292,31 +265,31 @@ int main (int argc, char **argv) cv::Mat_ depth_image_16_bit = cv::imread(string(dst), -1); // Convert to a floating point depth image - if(!depth_image_16_bit.empty()) + if (!depth_image_16_bit.empty()) { depth_image_16_bit.convertTo(depth_image, CV_32F); } else { - WARN_STREAM( "Can't find depth image" ); + WARN_STREAM("Can't find depth image"); } } vector > face_detections; bool all_models_active = true; - for(unsigned int model = 0; model < clnf_models.size(); ++model) + for (unsigned int model = 0; model < clnf_models.size(); ++model) { - if(!active_models[model]) + if (!active_models[model]) { all_models_active = false; } } - + // Get the detections (every 8th frame and when there are free models available for tracking) - if(frame_count % 8 == 0 && !all_models_active) - { - if(det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) + if (frame_count % 8 == 0 && !all_models_active) + { + if (det_parameters[0].curr_face_detector == LandmarkDetector::FaceModelParameters::HOG_SVM_DETECTOR) { vector confidences; LandmarkDetector::DetectFacesHOG(face_detections, grayscale_image, clnf_models[0].face_detector_HOG, confidences); @@ -334,37 +307,37 @@ int main (int argc, char **argv) vector > face_detections_used(face_detections.size()); // Go through every model and update the tracking - tbb::parallel_for(0, (int)clnf_models.size(), [&](int model){ - //for(unsigned int model = 0; model < clnf_models.size(); ++model) - //{ + tbb::parallel_for(0, (int)clnf_models.size(), [&](int model) { + //for(unsigned int model = 0; model < clnf_models.size(); ++model) + //{ bool detection_success = false; // If the current model has failed more than 4 times in a row, remove it - if(clnf_models[model].failures_in_a_row > 4) - { + if (clnf_models[model].failures_in_a_row > 4) + { active_models[model] = false; clnf_models[model].Reset(); } // If the model is inactive reactivate it with new detections - if(!active_models[model]) + if (!active_models[model]) { - - for(size_t detection_ind = 0; detection_ind < face_detections.size(); ++detection_ind) + + for (size_t detection_ind = 0; detection_ind < face_detections.size(); ++detection_ind) { // if it was not taken by another tracker take it (if it is false swap it to true and enter detection, this makes it parallel safe) - if(face_detections_used[detection_ind].compare_and_swap(true, false) == false) + if (face_detections_used[detection_ind].compare_and_swap(true, false) == false) { - + // Reinitialise the model clnf_models[model].Reset(); // This ensures that a wider window is used for the initial landmark localisation clnf_models[model].detection_success = false; detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, depth_image, face_detections[detection_ind], clnf_models[model], det_parameters[model]); - + // This activates the model active_models[model] = true; @@ -380,59 +353,59 @@ int main (int argc, char **argv) detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, depth_image, clnf_models[model], det_parameters[model]); } }); - + // Go through every model and visualise the results - for(size_t model = 0; model < clnf_models.size(); ++model) + for (size_t model = 0; model < clnf_models.size(); ++model) { // Visualising the results // Drawing the facial landmarks on the face and the bounding box around it if tracking is successful and initialised double detection_certainty = clnf_models[model].detection_certainty; double visualisation_boundary = -0.1; - + // Only draw if the reliability is reasonable, the value is slightly ad-hoc - if(detection_certainty < visualisation_boundary) + if (detection_certainty < visualisation_boundary) { LandmarkDetector::Draw(disp_image, clnf_models[model]); - if(detection_certainty > 1) + if (detection_certainty > 1) detection_certainty = 1; - if(detection_certainty < -1) + if (detection_certainty < -1) detection_certainty = -1; - detection_certainty = (detection_certainty + 1)/(visualisation_boundary +1); + detection_certainty = (detection_certainty + 1) / (visualisation_boundary + 1); // A rough heuristic for box around the face width int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - + // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate = LandmarkDetector::GetCorrectedPoseWorld(clnf_models[model], fx, fy, cx, cy); - + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(clnf_models[model], fx, fy, cx, cy); + // Draw it in reddish if uncertain, blueish if certain - LandmarkDetector::DrawBox(disp_image, pose_estimate, cv::Scalar((1-detection_certainty)*255.0,0, detection_certainty*255), thickness, fx, fy, cx, cy); + LandmarkDetector::DrawBox(disp_image, pose_estimate, cv::Scalar((1 - detection_certainty)*255.0, 0, detection_certainty * 255), thickness, fx, fy, cx, cy); } } // Work out the framerate - if(frame_count % 10 == 0) - { + if (frame_count % 10 == 0) + { t1 = cv::getTickCount(); - fps = 10.0 / (double(t1-t0)/cv::getTickFrequency()); + fps = 10.0 / (double(t1 - t0) / cv::getTickFrequency()); t0 = t1; } - + // Write out the framerate on the image before displaying it char fpsC[255]; sprintf(fpsC, "%d", (int)fps); string fpsSt("FPS:"); fpsSt += fpsC; - cv::putText(disp_image, fpsSt, cv::Point(10,20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - + cv::putText(disp_image, fpsSt, cv::Point(10, 20), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + int num_active_models = 0; - for( size_t active_model = 0; active_model < active_models.size(); active_model++) + for (size_t active_model = 0; active_model < active_models.size(); active_model++) { - if(active_models[active_model]) + if (active_models[active_model]) { num_active_models++; } @@ -442,42 +415,42 @@ int main (int argc, char **argv) sprintf(active_m_C, "%d", num_active_models); string active_models_st("Active models:"); active_models_st += active_m_C; - cv::putText(disp_image, active_models_st, cv::Point(10,60), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255,0,0), 1, CV_AA); - - if(!det_parameters[0].quiet_mode) + cv::putText(disp_image, active_models_st, cv::Point(10, 60), CV_FONT_HERSHEY_SIMPLEX, 0.5, CV_RGB(255, 0, 0), 1, CV_AA); + + if (!det_parameters[0].quiet_mode) { - cv::namedWindow("tracking_result",1); + cv::namedWindow("tracking_result", 1); cv::imshow("tracking_result", disp_image); - if(!depth_image.empty()) + if (!depth_image.empty()) { // Division needed for visualisation purposes - imshow("depth", depth_image/2000.0); + imshow("depth", depth_image / 2000.0); } } // output the tracked video - if(!tracked_videos_output.empty()) - { + if (!tracked_videos_output.empty()) + { writerFace << disp_image; } video_capture >> captured_image; - + // detect key presses char character_press = cv::waitKey(1); - + // restart the trackers - if(character_press == 'r') + if (character_press == 'r') { - for(size_t i=0; i < clnf_models.size(); ++i) + for (size_t i = 0; i < clnf_models.size(); ++i) { clnf_models[i].Reset(); active_models[i] = false; } } // quit the application - else if(character_press=='q') + else if (character_press == 'q') { return(0); } @@ -485,18 +458,18 @@ int main (int argc, char **argv) // Update the frame count frame_count++; } - + frame_count = 0; // Reset the model, for the next video - for(size_t model=0; model < clnf_models.size(); ++model) + for (size_t model = 0; model < clnf_models.size(); ++model) { clnf_models[model].Reset(); active_models[model] = false; } // break out of the loop if done with all the files - if(f_n == files.size() -1) + if (f_n == files.size() - 1) { done = true; } diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index ca1901c6..e165bdfa 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -1,37 +1,13 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical @@ -79,11 +55,9 @@ #include #include +#include #include -// OpenBlas include -#include - #ifndef CONFIG_DIR #define CONFIG_DIR "~" #endif @@ -97,9 +71,9 @@ std::cout << "Warning: " << stream << std::endl #define ERROR_STREAM( stream ) \ std::cout << "Error: " << stream << std::endl -static void printErrorAndAbort( const std::string & error ) +static void printErrorAndAbort(const std::string & error) { - std::cout << error << std::endl; + std::cout << error << std::endl; } #define FATAL_STREAM( stream ) \ @@ -115,7 +89,7 @@ vector get_arguments(int argc, char **argv) vector arguments; // First argument is reserved for the name of the executable - for(int i = 0; i < argc; ++i) + for (int i = 0; i < argc; ++i) { arguments.push_back(string(argv[i])); } @@ -127,14 +101,14 @@ void create_directory_from_file(string output_path) { // Creating the right directory structure - + // First get rid of the file auto p = path(path(output_path).parent_path()); - if(!p.empty() && !boost::filesystem::exists(p)) + if (!p.empty() && !boost::filesystem::exists(p)) { bool success = boost::filesystem::create_directories(p); - if(!success) + if (!success) { cout << "Failed to create a directory... " << p.string() << endl; } @@ -147,20 +121,19 @@ void create_directory(string output_path) // Creating the right directory structure auto p = path(output_path); - if(!boost::filesystem::exists(p)) + if (!boost::filesystem::exists(p)) { bool success = boost::filesystem::create_directories(p); - - if(!success) + + if (!success) { cout << "Failed to create a directory..." << p.string() << endl; } } } -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, double &similarity_scale, - int &similarity_size, bool &grayscale, bool& verbose, bool& dynamic, bool &output_2D_landmarks, bool &output_3D_landmarks, - bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments); +void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& verbose, + bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments); void get_image_input_output_params_feats(vector > &input_image_files, bool& as_video, vector &arguments); @@ -196,14 +169,14 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f // A rough heuristic for box around the face width int thickness = (int)std::ceil(2.0* ((double)captured_image.cols) / 640.0); - cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetCorrectedPoseWorld(face_model, fx, fy, cx, cy); + cv::Vec6d pose_estimate_to_draw = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); // Draw it in reddish if uncertain, blueish if certain LandmarkDetector::DrawBox(captured_image, pose_estimate_to_draw, cv::Scalar((1 - vis_certainty)*255.0, 0, vis_certainty * 255), thickness, fx, fy, cx, cy); if (det_parameters.track_gaze && detection_success && face_model.eye_model) { - FaceAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); + GazeAnalysis::DrawGaze(captured_image, face_model, gazeDirection0, gazeDirection1, fx, fy, cx, cy); } } @@ -231,40 +204,28 @@ void visualise_tracking(cv::Mat& captured_image, const LandmarkDetector::CLNF& f void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_landmarks, int num_model_modes, vector au_names_class, vector au_names_reg); + int num_landmarks, int num_eye_lmks, int num_model_modes, vector au_names_class, vector au_names_reg); // Output all of the information into one file in one go (quite a few parameters, but simplifies the flow) void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, + cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser); -void post_process_output_file(FaceAnalysis::FaceAnalyser& face_analyser, string output_file, bool dynamic); - - -int main (int argc, char **argv) +int main(int argc, char **argv) { vector arguments = get_arguments(argc, argv); - // Search paths - boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); - boost::filesystem::path parent_path = boost::filesystem::path(arguments[0]).parent_path(); - // Some initial parameters that can be overriden from command line vector input_files, depth_directories, output_files, tracked_videos_output; - - LandmarkDetector::FaceModelParameters det_parameters(arguments); - // Always track gaze in feature extraction - det_parameters.track_gaze = true; // Get the input output file parameters - + // Indicates that rotation should be with respect to camera or world coordinates - bool use_world_coordinates; string output_codec; //not used but should - LandmarkDetector::get_video_input_output_params(input_files, depth_directories, output_files, tracked_videos_output, use_world_coordinates, output_codec, arguments); + LandmarkDetector::get_video_input_output_params(input_files, depth_directories, output_files, tracked_videos_output, output_codec, arguments); bool video_input = true; bool verbose = true; @@ -273,26 +234,25 @@ int main (int argc, char **argv) vector > input_image_files; // Adding image support for reading in the files - if(input_files.empty()) + if (input_files.empty()) { vector d_files; vector o_img; vector> bboxes; - get_image_input_output_params_feats(input_image_files, images_as_video, arguments); + get_image_input_output_params_feats(input_image_files, images_as_video, arguments); - if(!input_image_files.empty()) + if (!input_image_files.empty()) { video_input = false; } - } // Grab camera parameters, if they are not defined (approximate values will be used) float fx = 0, fy = 0, cx = 0, cy = 0; int d = 0; // Get camera parameters - LandmarkDetector::get_camera_params(d, fx, fy, cx, cy, arguments); - + LandmarkDetector::get_camera_params(d, fx, fy, cx, cy, arguments); + // If cx (optical axis centre) is undefined will use the image size/2 as an estimate bool cx_undefined = false; bool fx_undefined = false; @@ -305,19 +265,10 @@ int main (int argc, char **argv) fx_undefined = true; } - // The modules that are being used for tracking - LandmarkDetector::CLNF face_model(det_parameters.model_location); - vector output_similarity_align; vector output_hog_align_files; - double sim_scale = -1; - int sim_size = 112; - bool grayscale = false; bool video_output = false; - bool dynamic = true; // Indicates if a dynamic AU model should be used (dynamic is useful if the video is long enough to include neutral expressions) - int num_hog_rows; - int num_hog_cols; // By default output all parameters, but these can be turned off to get smaller files or slightly faster processing times // use -no2Dfp, -no3Dfp, -noMparams, -noPose, -noAUs, -noGaze to turn them off @@ -328,96 +279,47 @@ int main (int argc, char **argv) bool output_AUs = true; bool output_gaze = true; - get_output_feature_params(output_similarity_align, output_hog_align_files, sim_scale, sim_size, grayscale, verbose, dynamic, + get_output_feature_params(output_similarity_align, output_hog_align_files, verbose, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, arguments); - // Used for image masking - string tri_loc; - boost::filesystem::path tri_loc_path = boost::filesystem::path("model/tris_68_full.txt"); - if (boost::filesystem::exists(tri_loc_path)) - { - tri_loc = tri_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/tri_loc_path)) - { - tri_loc = (parent_path/tri_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/tri_loc_path)) - { - tri_loc = (config_path/tri_loc_path).string(); - } - else - { - cout << "Can't find triangulation files, exiting" << endl; - return 1; - } - - // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = face_model.pdm.mean_shape * sim_scale; - // Discard the z component - similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); // If multiple video files are tracked, use this to indicate if we are done - bool done = false; + bool done = false; int f_n = -1; int curr_img = -1; - string au_loc; - string au_loc_local; - if (dynamic) - { - au_loc_local = "AU_predictors/AU_all_best.txt"; - } - else - { - au_loc_local = "AU_predictors/AU_all_static.txt"; - } + // Load the modules that are being used for tracking and face analysis - boost::filesystem::path au_loc_path = boost::filesystem::path(au_loc_local); - if (boost::filesystem::exists(au_loc_path)) - { - au_loc = au_loc_path.string(); - } - else if (boost::filesystem::exists(parent_path/au_loc_path)) - { - au_loc = (parent_path/au_loc_path).string(); - } - else if (boost::filesystem::exists(config_path/au_loc_path)) - { - au_loc = (config_path/au_loc_path).string(); - } - else - { - cout << "Can't find AU prediction files, exiting" << endl; - return 1; - } + // Load face landmark detector + LandmarkDetector::FaceModelParameters det_parameters(arguments); + // Always track gaze in feature extraction + det_parameters.track_gaze = true; + LandmarkDetector::CLNF face_model(det_parameters.model_location); - // Creating a face analyser that will be used for AU extraction - // Make sure sim_scale is proportional to sim_size if not set - if (sim_scale == -1) sim_scale = sim_size * (0.7 / 112.0); + // Load facial feature extractor and AU analyser + FaceAnalysis::FaceAnalyserParameters face_analysis_params(arguments); + FaceAnalysis::FaceAnalyser face_analyser(face_analysis_params); - FaceAnalysis::FaceAnalyser face_analyser(vector(), sim_scale, sim_size, sim_size, au_loc, tri_loc); - - while(!done) // this is not a for loop as we might also be reading from a webcam + while (!done) // this is not a for loop as we might also be reading from a webcam { - + string current_file; - + cv::VideoCapture video_capture; - + cv::Mat captured_image; int total_frames = -1; int reported_completion = 0; double fps_vid_in = -1.0; - if(video_input) + if (video_input) { // We might specify multiple video files as arguments - if(input_files.size() > 0) + if (input_files.size() > 0) { - f_n++; + f_n++; current_file = input_files[f_n]; } else @@ -426,10 +328,10 @@ int main (int argc, char **argv) f_n = 0; } // Do some grabbing - if( current_file.size() > 0 ) + if (current_file.size() > 0) { - INFO_STREAM( "Attempting to read from file: " << current_file ); - video_capture = cv::VideoCapture( current_file ); + INFO_STREAM("Attempting to read from file: " << current_file); + video_capture = cv::VideoCapture(current_file); total_frames = (int)video_capture.get(CV_CAP_PROP_FRAME_COUNT); fps_vid_in = video_capture.get(CV_CAP_PROP_FPS); @@ -451,27 +353,27 @@ int main (int argc, char **argv) INFO_STREAM("Device or file opened"); } - video_capture >> captured_image; + video_capture >> captured_image; } else { - f_n++; + f_n++; curr_img++; - if(!input_image_files[f_n].empty()) + if (!input_image_files[f_n].empty()) { string curr_img_file = input_image_files[f_n][curr_img]; captured_image = cv::imread(curr_img_file, -1); } else { - FATAL_STREAM( "No .jpg or .png images in a specified drectory, exiting" ); + FATAL_STREAM("No .jpg or .png images in a specified drectory, exiting"); return 1; } - } - + } + // If optical centers are not defined just use center of image - if(cx_undefined) + if (cx_undefined) { cx = captured_image.cols / 2.0f; cy = captured_image.rows / 2.0f; @@ -485,47 +387,46 @@ int main (int argc, char **argv) fx = (fx + fy) / 2.0; fy = fx; } - + // Creating output files std::ofstream output_file; if (!output_files.empty()) { output_file.open(output_files[f_n], ios_base::out); - prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); + prepareOutputFile(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, face_model.pdm.NumberOfPoints(), + LandmarkDetector::CalculateAllEyeLandmarks(face_model).size(), face_model.pdm.NumberOfModes(), face_analyser.GetAUClassNames(), face_analyser.GetAURegNames()); } // Saving the HOG features std::ofstream hog_output_file; - if(!output_hog_align_files.empty()) + if (!output_hog_align_files.empty()) { hog_output_file.open(output_hog_align_files[f_n], ios_base::out | ios_base::binary); } // saving the videos cv::VideoWriter writerFace; - if(!tracked_videos_output.empty()) + if (!tracked_videos_output.empty()) { try { - writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0],output_codec[1],output_codec[2],output_codec[3]), fps_vid_in, captured_image.size(), true); + writerFace = cv::VideoWriter(tracked_videos_output[f_n], CV_FOURCC(output_codec[0], output_codec[1], output_codec[2], output_codec[3]), fps_vid_in, captured_image.size(), true); } - catch(cv::Exception e) + catch (cv::Exception e) { - WARN_STREAM( "Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); + WARN_STREAM("Could not open VideoWriter, OUTPUT FILE WILL NOT BE WRITTEN. Currently using codec " << output_codec << ", try using an other one (-oc option)"); } - - } int frame_count = 0; - + // This is useful for a second pass run (if want AU predictions) vector params_global_video; vector successes_video; vector> params_local_video; vector> detected_landmarks_video; - + // Use for timestamping if using a webcam int64 t_initial = cv::getTickCount(); @@ -534,14 +435,14 @@ int main (int argc, char **argv) // Timestamp in seconds of current processing double time_stamp = 0; - INFO_STREAM( "Starting tracking"); - while(!captured_image.empty()) - { + INFO_STREAM("Starting tracking"); + while (!captured_image.empty()) + { // Grab the timestamp first if (video_input) { - time_stamp = (double)frame_count * (1.0 / fps_vid_in); + time_stamp = (double)frame_count * (1.0 / fps_vid_in); } else { @@ -552,19 +453,19 @@ int main (int argc, char **argv) // Reading the images cv::Mat_ grayscale_image; - if(captured_image.channels() == 3) + if (captured_image.channels() == 3) { - cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); + cvtColor(captured_image, grayscale_image, CV_BGR2GRAY); } else { - grayscale_image = captured_image.clone(); + grayscale_image = captured_image.clone(); } - + // The actual facial landmark detection / tracking bool detection_success; - - if(video_input || images_as_video) + + if (video_input || images_as_video) { detection_success = LandmarkDetector::DetectLandmarksInVideo(grayscale_image, face_model, det_parameters); } @@ -572,54 +473,51 @@ int main (int argc, char **argv) { detection_success = LandmarkDetector::DetectLandmarksInImage(grayscale_image, face_model, det_parameters); } - + + + // Work out the pose of the head from the tracked model + cv::Vec6d pose_estimate = LandmarkDetector::GetPose(face_model, fx, fy, cx, cy); + // Gaze tracking, absolute gaze direction cv::Point3f gazeDirection0(0, 0, -1); cv::Point3f gazeDirection1(0, 0, -1); + cv::Vec2d gazeAngle(0, 0); if (det_parameters.track_gaze && detection_success && face_model.eye_model) { - FaceAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true); - FaceAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false); + GazeAnalysis::EstimateGaze(face_model, gazeDirection0, fx, fy, cx, cy, true); + GazeAnalysis::EstimateGaze(face_model, gazeDirection1, fx, fy, cx, cy, false); + gazeAngle = GazeAnalysis::GetGazeAngle(gazeDirection0, gazeDirection1, pose_estimate); } // Do face alignment cv::Mat sim_warped_img; cv::Mat_ hog_descriptor; + int num_hog_rows, num_hog_cols; // But only if needed in output - if(!output_similarity_align.empty() || hog_output_file.is_open() || output_AUs) + if (!output_similarity_align.empty() || hog_output_file.is_open() || output_AUs) { - face_analyser.AddNextFrame(captured_image, face_model, time_stamp, false, !det_parameters.quiet_mode); + face_analyser.AddNextFrame(captured_image, face_model.detected_landmarks, face_model.detection_success, time_stamp, false, !det_parameters.quiet_mode); face_analyser.GetLatestAlignedFace(sim_warped_img); - if(!det_parameters.quiet_mode) + if (!det_parameters.quiet_mode) { - cv::imshow("sim_warp", sim_warped_img); + cv::imshow("sim_warp", sim_warped_img); } - if(hog_output_file.is_open()) + if (hog_output_file.is_open()) { face_analyser.GetLatestHOG(hog_descriptor, num_hog_rows, num_hog_cols); - if(visualise_hog && !det_parameters.quiet_mode) + if (visualise_hog && !det_parameters.quiet_mode) { cv::Mat_ hog_descriptor_vis; FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_vis); - cv::imshow("hog", hog_descriptor_vis); + cv::imshow("hog", hog_descriptor_vis); } } } - // Work out the pose of the head from the tracked model - cv::Vec6d pose_estimate; - if(use_world_coordinates) - { - pose_estimate = LandmarkDetector::GetCorrectedPoseWorld(face_model, fx, fy, cx, cy); - } - else - { - pose_estimate = LandmarkDetector::GetCorrectedPoseCamera(face_model, fx, fy, cx, cy); - } if (hog_output_file.is_open()) { @@ -630,11 +528,6 @@ int main (int argc, char **argv) if (!output_similarity_align.empty()) { - if (sim_warped_img.channels() == 3 && grayscale) - { - cvtColor(sim_warped_img, sim_warped_img, CV_BGR2GRAY); - } - char name[100]; // Filename is based on frame number @@ -647,7 +540,7 @@ int main (int argc, char **argv) string out_file = output_similarity_align[f_n] + preferredSlash + string(name); bool write_success = imwrite(out_file, sim_warped_img); - + if (!write_success) { cout << "Could not output similarity aligned image image" << endl; @@ -660,23 +553,23 @@ int main (int argc, char **argv) // Output the landmarks, pose, gaze, parameters and AUs outputAllFeatures(&output_file, output_2D_landmarks, output_3D_landmarks, output_model_params, output_pose, output_AUs, output_gaze, - face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, + face_model, frame_count, time_stamp, detection_success, gazeDirection0, gazeDirection1, gazeAngle, pose_estimate, fx, fy, cx, cy, face_analyser); // output the tracked video - if(!tracked_videos_output.empty()) - { + if (!tracked_videos_output.empty()) + { writerFace << captured_image; } - if(video_input) + if (video_input) { video_capture >> captured_image; } else { curr_img++; - if(curr_img < (int)input_image_files[f_n].size()) + if (curr_img < (int)input_image_files[f_n].size()) { string curr_img_file = input_image_files[f_n][curr_img]; captured_image = cv::imread(curr_img_file, -1); @@ -686,30 +579,30 @@ int main (int argc, char **argv) captured_image = cv::Mat(); } } - + if (!det_parameters.quiet_mode) { // detect key presses char character_press = cv::waitKey(1); - + // restart the tracker - if(character_press == 'r') + if (character_press == 'r') { face_model.Reset(); } // quit the application - else if(character_press=='q') + else if (character_press == 'q') { return(0); } } - + // Update the frame count frame_count++; - if(total_frames != -1) + if (total_frames != -1) { - if((double)frame_count/(double)total_frames >= reported_completion / 10.0) + if ((double)frame_count / (double)total_frames >= reported_completion / 10.0) { cout << reported_completion * 10 << "% "; reported_completion = reported_completion + 1; @@ -717,13 +610,13 @@ int main (int argc, char **argv) } } - + output_file.close(); if (output_files.size() > 0 && output_AUs) { cout << "Postprocessing the Action Unit predictions" << endl; - face_analyser.PostprocessOutputFile(output_files[f_n], dynamic); + face_analyser.PostprocessOutputFile(output_files[f_n]); } // Reset the models for the next video face_analyser.Reset(); @@ -738,7 +631,7 @@ int main (int argc, char **argv) } // break out of the loop if done with all the files (or using a webcam) - if((video_input && f_n == input_files.size() -1) || (!video_input && f_n == input_image_files.size() - 1)) + if ((video_input && f_n == input_files.size() - 1) || (!video_input && f_n == input_image_files.size() - 1)) { done = true; } @@ -749,14 +642,24 @@ int main (int argc, char **argv) void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, - int num_landmarks, int num_model_modes, vector au_names_class, vector au_names_reg) + int num_landmarks, int num_eye_lmks, int num_model_modes, vector au_names_class, vector au_names_reg) { *output_file << "frame, timestamp, confidence, success"; if (output_gaze) { - *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z"; + *output_file << ", gaze_0_x, gaze_0_y, gaze_0_z, gaze_1_x, gaze_1_y, gaze_1_z, gaze_angle_x, gaze_angle_y"; + + // Also output eye-landmarks as they are needed for gaze visualization etc. + for (int i = 0; i < num_eye_lmks; ++i) + { + *output_file << ", eye_lmk_x_" << i; + } + for (int i = 0; i < num_eye_lmks; ++i) + { + *output_file << ", eye_lmk_y_" << i; + } } if (output_pose) @@ -825,25 +728,58 @@ void prepareOutputFile(std::ofstream* output_file, bool output_2D_landmarks, boo void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, bool output_3D_landmarks, bool output_model_params, bool output_pose, bool output_AUs, bool output_gaze, const LandmarkDetector::CLNF& face_model, int frame_count, double time_stamp, bool detection_success, - cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, + cv::Point3f gazeDirection0, cv::Point3f gazeDirection1, cv::Vec2d gaze_angle, const cv::Vec6d& pose_estimate, double fx, double fy, double cx, double cy, const FaceAnalysis::FaceAnalyser& face_analyser) { double confidence = 0.5 * (1 - face_model.detection_certainty); - *output_file << frame_count + 1 << ", " << time_stamp << ", " << confidence << ", " << detection_success; + *output_file << std::setprecision(9); + *output_file << frame_count + 1 << ", " << time_stamp << ", "; + + *output_file << std::setprecision(2); + *output_file << confidence << ", " << detection_success; + + *output_file << std::setprecision(5); // Output the estimated gaze if (output_gaze) { *output_file << ", " << gazeDirection0.x << ", " << gazeDirection0.y << ", " << gazeDirection0.z - << ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z; + << ", " << gazeDirection1.x << ", " << gazeDirection1.y << ", " << gazeDirection1.z + << ", " << gaze_angle[0] << ", " << gaze_angle[1]; + + // Output gaze landmarks + vector eye_lmks = LandmarkDetector::CalculateAllEyeLandmarks(face_model); + for (size_t i = 0; i < eye_lmks.size(); ++i) + { + if (face_model.tracking_initialised) + { + *output_file << ", " << eye_lmks[i].x; + } + else + { + *output_file << ", 0"; + } + } + for (size_t i = 0; i < eye_lmks.size(); ++i) + { + if (face_model.tracking_initialised) + { + *output_file << ", " << eye_lmks[i].y; + } + else + { + *output_file << ", 0"; + } + } } + *output_file << std::setprecision(4); // Output the estimated head pose if (output_pose) { - if(face_model.tracking_initialised) + if (face_model.tracking_initialised) { *output_file << ", " << pose_estimate[0] << ", " << pose_estimate[1] << ", " << pose_estimate[2] << ", " << pose_estimate[3] << ", " << pose_estimate[4] << ", " << pose_estimate[5]; @@ -854,12 +790,13 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo } } + *output_file << std::setprecision(4); // Output the detected 2D facial landmarks if (output_2D_landmarks) { for (int i = 0; i < face_model.pdm.NumberOfPoints() * 2; ++i) { - if(face_model.tracking_initialised) + if (face_model.tracking_initialised) { *output_file << ", " << face_model.detected_landmarks.at(i); } @@ -902,7 +839,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo } for (int i = 0; i < face_model.pdm.NumberOfModes(); ++i) { - if(face_model.tracking_initialised) + if (face_model.tracking_initialised) { *output_file << ", " << face_model.params_local.at(i, 0); } @@ -973,8 +910,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo } -void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, double &similarity_scale, - int &similarity_size, bool &grayscale, bool& verbose, bool& dynamic, +void get_output_feature_params(vector &output_similarity_aligned, vector &output_hog_aligned_files, bool& verbose, bool &output_2D_landmarks, bool &output_3D_landmarks, bool &output_model_params, bool &output_pose, bool &output_AUs, bool &output_gaze, vector &arguments) { @@ -990,9 +926,6 @@ void get_output_feature_params(vector &output_similarity_aligned, vector string output_root = ""; - // By default the model is dynamic - dynamic = true; - string separator = string(1, boost::filesystem::path::preferred_separator); // First check if there is a root argument (so that videos and outputs could be defined more easilly) @@ -1032,29 +965,6 @@ void get_output_feature_params(vector &output_similarity_aligned, vector { verbose = true; } - else if (arguments[i].compare("-au_static") == 0) - { - dynamic = false; - } - else if (arguments[i].compare("-g") == 0) - { - grayscale = true; - valid[i] = false; - } - else if (arguments[i].compare("-simscale") == 0) - { - similarity_scale = stod(arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-simsize") == 0) - { - similarity_size = stoi(arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } else if (arguments[i].compare("-no2Dfp") == 0) { output_2D_landmarks = false; diff --git a/exe/FeatureExtraction/FeatureExtraction.vcxproj b/exe/FeatureExtraction/FeatureExtraction.vcxproj index d289ccb6..12df2bc0 100644 --- a/exe/FeatureExtraction/FeatureExtraction.vcxproj +++ b/exe/FeatureExtraction/FeatureExtraction.vcxproj @@ -111,7 +111,7 @@ Level3 Disabled WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false StreamingSIMDExtensions2 true @@ -127,7 +127,7 @@ Level3 Disabled WIN64;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false AdvancedVectorExtensions true @@ -145,7 +145,7 @@ false true WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false Speed StreamingSIMDExtensions2 @@ -167,7 +167,7 @@ false true WIN64;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + $(SolutionDir)\lib\local\FaceAnalyser\include;$(SolutionDir)\lib\local\LandmarkDetector\include;$(SolutionDir)\lib\local\GazeAnalyser\include;%(AdditionalIncludeDirectories) false Speed AdvancedVectorExtensions @@ -188,6 +188,9 @@ {0e7fc556-0e80-45ea-a876-dde4c2fedcd7} + + {5f915541-f531-434f-9c81-79f5db58012b} + {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} diff --git a/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt b/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt new file mode 100644 index 00000000..d538397c --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/In-the-wild_aligned_PDM_68.txt @@ -0,0 +1,421 @@ +# The mean values of the components (in mm) +204 +1 +6 +-73.393523 +-72.775014 +-70.533638 +-66.850058 +-59.790187 +-48.368973 +-34.121101 +-17.875411 +0.098749 +17.477031 +32.648966 +46.372358 +57.343480 +64.388482 +68.212038 +70.486405 +71.375822 +-61.119406 +-51.287588 +-37.804800 +-24.022754 +-11.635713 +12.056636 +25.106256 +38.338588 +51.191007 +60.053851 +0.653940 +0.804809 +0.992204 +1.226783 +-14.772472 +-7.180239 +0.555920 +8.272499 +15.214351 +-46.047290 +-37.674688 +-27.883856 +-19.648268 +-28.272965 +-38.082418 +19.265868 +27.894191 +37.437529 +45.170805 +38.196454 +28.764989 +-28.916267 +-17.533194 +-6.684590 +0.381001 +8.375443 +18.876618 +28.794412 +19.057574 +8.956375 +0.381549 +-7.428895 +-18.160634 +-24.377490 +-6.897633 +0.340663 +8.444722 +24.474473 +8.449166 +0.205322 +-7.198266 +-29.801432 +-10.949766 +7.929818 +26.074280 +42.564390 +56.481080 +67.246992 +75.056892 +77.061286 +74.758448 +66.929021 +56.311389 +42.419126 +25.455880 +6.990805 +-11.666193 +-30.365191 +-49.361602 +-58.769795 +-61.996155 +-61.033399 +-56.686759 +-57.391033 +-61.902186 +-62.777713 +-59.302347 +-50.190255 +-42.193790 +-30.993721 +-19.944596 +-8.414541 +2.598255 +4.751589 +6.562900 +4.661005 +2.643046 +-37.471411 +-42.730510 +-42.711517 +-36.754742 +-35.134493 +-34.919043 +-37.032306 +-43.342445 +-43.110822 +-38.086515 +-35.532024 +-35.484289 +28.612716 +22.172187 +19.029051 +20.721118 +19.035460 +22.394109 +28.079924 +36.298248 +39.634575 +40.395647 +39.836405 +36.677899 +28.677771 +25.475976 +26.014269 +25.326198 +28.323008 +30.596216 +31.408738 +30.844876 +47.667532 +45.909403 +44.842580 +43.141114 +38.635298 +30.750622 +18.456453 +3.609035 +-0.881698 +5.181201 +19.176563 +30.770570 +37.628629 +40.886309 +42.281449 +44.142567 +47.140426 +14.254422 +7.268147 +0.442051 +-6.606501 +-11.967398 +-12.051204 +-7.315098 +-1.022953 +5.349435 +11.615746 +-13.380835 +-21.150853 +-29.284036 +-36.948060 +-20.132003 +-23.536684 +-25.944448 +-23.695741 +-20.858157 +7.037989 +3.021217 +1.353629 +-0.111088 +-0.147273 +1.476612 +-0.665746 +0.247660 +1.696435 +4.894163 +0.282961 +-1.172675 +-2.240310 +-15.934335 +-22.611355 +-23.748437 +-22.721995 +-15.610679 +-3.217393 +-14.987997 +-22.554245 +-23.591626 +-22.406106 +-15.121907 +-4.785684 +-20.893742 +-22.220479 +-21.025520 +-5.712776 +-20.671489 +-21.903670 +-20.328022 +# The principal components (eigenvectors) of identity or combined identity and expression model +204 +34 +6 +-0.007395 -0.093690 0.039362 0.204443 -0.104698 0.033568 -0.090173 0.010170 -0.042341 0.104375 0.032695 0.038750 0.064385 -0.037025 0.058377 0.032457 -0.100005 -0.082042 -0.053440 0.008782 -0.027164 -0.000368 -0.129614 0.035436 -0.062685 -0.075349 0.140764 -0.032290 -0.115829 -0.037865 0.068590 0.008886 -0.066442 0.259211 +-0.002553 -0.075344 0.037127 0.201967 -0.069124 0.041645 -0.100939 0.018352 0.006598 0.081569 0.004894 0.038886 0.050631 0.056656 0.055353 -0.028457 -0.022518 -0.089693 -0.079196 0.010851 -0.017583 0.015095 -0.077834 -0.000944 -0.104255 -0.060338 0.106169 -0.065799 -0.067068 -0.051588 0.080490 0.063152 -0.047531 0.178216 +-0.002097 -0.055385 0.025705 0.215785 -0.026274 0.053548 -0.086364 0.041926 0.035784 0.033485 -0.014602 0.049999 0.031950 0.125201 0.045372 -0.064200 0.070073 -0.074313 -0.083942 -0.007566 -0.003666 0.034226 -0.018354 -0.011639 -0.084633 -0.039431 0.064057 -0.056515 -0.046926 -0.074699 0.073477 0.074109 -0.054882 0.046675 +-0.005533 -0.035821 0.019787 0.226957 0.030415 0.059508 -0.051058 0.057386 0.023240 -0.025487 -0.013088 0.043330 0.025141 0.113119 0.017370 -0.071064 0.108140 -0.049138 -0.043417 -0.065375 0.012959 0.070483 0.075536 -0.035685 -0.005847 0.012618 0.040297 -0.049743 -0.061354 -0.087164 0.033762 0.048771 -0.096563 -0.045189 +-0.006914 0.011987 0.030022 0.205560 0.099633 0.072384 -0.008313 0.060244 -0.006584 -0.104943 -0.018036 0.032806 0.029879 0.068434 -0.043721 -0.052564 0.077505 -0.062192 0.026865 -0.122834 -0.001529 0.064443 0.160557 -0.036750 0.124476 0.072641 0.048609 -0.043443 -0.060729 -0.040443 -0.050540 0.050509 -0.118886 -0.146867 +-0.004182 0.064263 0.036886 0.149341 0.158261 0.075501 0.030568 0.067074 -0.040268 -0.167116 -0.020230 0.029360 0.046726 0.033410 -0.121685 -0.004528 -0.017505 -0.050390 0.063237 -0.133203 -0.070794 0.005133 0.178032 -0.002768 0.153446 0.105966 0.061693 0.011010 -0.027442 0.001944 -0.103967 0.045708 -0.083197 -0.172889 +0.016954 0.088125 0.034199 0.081242 0.170872 0.048964 0.063434 0.083842 -0.058188 -0.164769 -0.012135 0.002128 0.084787 -0.009691 -0.136231 0.068411 -0.091616 0.042046 0.053339 -0.068859 -0.141834 -0.040710 0.017020 0.034007 0.053187 0.096749 0.068416 0.075079 0.038728 0.063588 -0.109581 0.007737 0.033105 -0.152075 +0.021733 0.081042 0.015310 0.029492 0.112212 0.012959 0.065003 0.095565 -0.047228 -0.120559 -0.022904 -0.051461 0.036411 -0.036990 -0.101765 0.150918 -0.102123 0.163128 0.047199 0.037063 -0.223718 -0.065303 -0.149191 -0.017823 -0.134138 0.084419 0.010718 0.087970 0.043030 0.040099 -0.068487 -0.008773 0.077035 -0.082281 +0.024713 0.084952 0.002319 -0.002792 0.028194 -0.016856 0.036083 0.087381 -0.020743 -0.024386 -0.027268 -0.091654 -0.009025 -0.052124 -0.033407 0.168353 -0.061447 0.244847 0.024587 0.064221 -0.161138 0.020993 -0.164584 -0.068179 -0.181775 0.087509 -0.068202 0.059078 -0.004250 -0.001472 -0.031248 -0.049011 -0.019415 0.022163 +0.025404 0.095704 -0.013447 -0.036864 -0.064015 -0.035962 0.001144 0.050101 0.006172 0.084003 -0.015860 -0.102295 -0.028308 -0.053076 0.017520 0.110370 -0.009494 0.248830 -0.017769 0.064655 -0.062487 0.133872 -0.077193 -0.077102 -0.104610 0.084923 -0.116710 -0.000424 -0.001453 -0.001504 -0.004986 -0.072792 -0.151917 0.063339 +0.027234 0.088446 -0.037011 -0.090962 -0.124015 -0.052844 -0.020061 0.017974 0.031824 0.138013 -0.009523 -0.093892 -0.061812 -0.016350 0.043025 0.034106 0.036403 0.177560 -0.066137 0.039288 0.041972 0.145398 -0.004524 -0.021113 0.025045 0.065780 -0.102128 -0.061053 0.010043 -0.018688 0.018733 -0.002990 -0.240304 0.007274 +0.033301 0.051083 -0.050161 -0.137239 -0.130410 -0.078352 -0.030467 -0.016581 0.042916 0.153230 -0.008295 -0.048519 -0.056020 -0.017449 0.051644 -0.021078 0.049456 0.042549 -0.075780 0.051205 0.057200 0.117965 0.058462 0.078890 0.121853 0.027311 -0.037081 -0.082100 0.013948 0.019188 0.029703 0.076102 -0.243643 -0.067711 +0.034561 -0.024817 -0.037181 -0.171731 -0.091823 -0.080914 -0.018022 -0.019280 0.055534 0.112038 -0.006388 0.000380 -0.027767 -0.031689 0.027915 -0.012644 0.004351 -0.072270 -0.053990 0.028620 0.011428 0.077628 0.073278 0.159948 0.096141 0.006855 0.036568 -0.013290 -0.001587 0.033748 0.052795 0.164287 -0.166678 -0.098685 +0.031311 -0.094373 -0.015266 -0.181991 -0.032285 -0.067979 0.020822 -0.028457 0.050194 0.060652 0.006169 0.060618 0.005717 -0.047184 0.006512 0.012425 -0.059877 -0.141325 -0.029145 -0.010906 -0.069894 0.032082 0.054382 0.151384 0.041519 0.026929 0.076049 0.067107 -0.027885 0.040313 0.041582 0.194576 -0.086531 -0.080338 +0.030327 -0.127857 -0.008012 -0.166922 0.025066 -0.062660 0.063753 -0.029802 0.044933 0.028298 0.025152 0.109179 0.013585 -0.043385 0.003918 0.007538 -0.080941 -0.141969 0.011067 -0.055727 -0.099322 -0.018025 0.009874 0.065510 -0.019589 0.084273 0.059580 0.099232 -0.026102 0.043042 0.023322 0.151827 -0.031702 -0.023000 +0.030239 -0.141766 -0.020490 -0.154724 0.073176 -0.066098 0.076339 -0.045120 0.049403 -0.014474 0.024782 0.109864 0.005943 -0.008921 0.001482 -0.008304 -0.060453 -0.100507 0.030769 -0.093919 -0.042422 -0.045051 -0.014060 -0.020183 -0.093279 0.134797 0.006048 0.040717 -0.038709 0.039630 0.006137 0.076303 0.021244 0.014680 +0.030493 -0.158938 -0.031460 -0.156567 0.120023 -0.070626 0.084879 -0.065759 0.063746 -0.047728 -0.002344 0.094855 -0.013995 0.014159 0.011125 -0.034394 -0.064702 -0.055871 0.041269 -0.110361 0.045722 -0.064289 -0.013213 -0.104140 -0.133226 0.183607 -0.037284 0.002311 -0.041808 0.033510 -0.034315 0.012710 0.065609 0.046794 +-0.029307 -0.000612 0.006003 0.066897 -0.126329 0.008282 0.106455 -0.058070 -0.127455 0.112217 0.028721 -0.008227 0.023123 -0.184660 -0.000131 0.060814 -0.038192 0.061802 0.055924 -0.086767 -0.011372 0.116214 0.039681 -0.047083 0.046626 0.042705 0.011194 -0.002237 0.023832 0.021378 0.074731 -0.040608 -0.025003 0.203421 +-0.038458 0.020813 -0.004500 0.021759 -0.086203 0.017004 0.123118 -0.051398 -0.154096 0.048594 0.160783 0.035847 -0.105209 -0.131283 -0.034334 0.088202 -0.022053 0.065794 0.043514 -0.164773 0.038562 0.221823 -0.008980 0.076251 -0.067806 0.014541 0.027230 -0.014379 -0.016754 -0.010835 0.026126 -0.010414 0.029183 0.058614 +-0.021490 0.036216 -0.005865 0.020024 -0.060804 0.026512 0.126241 -0.083197 -0.120419 0.032472 0.187630 -0.015555 -0.159327 -0.011903 -0.087082 0.079345 -0.061540 0.015194 0.062573 -0.039469 0.032277 0.191416 0.026454 0.082398 -0.056523 -0.017131 0.027510 -0.008610 0.004494 -0.034043 -0.017450 0.006765 0.078634 -0.036122 +0.000867 0.042636 0.003104 0.029552 -0.036234 0.016402 0.123844 -0.126971 -0.075012 0.060293 0.148833 -0.081953 -0.173218 0.088994 -0.136465 0.063644 -0.092909 -0.034989 0.083554 0.107321 0.022452 0.121649 0.095422 0.064093 -0.044127 -0.041514 -0.001734 -0.007934 0.040275 -0.042178 -0.037676 0.021143 0.091871 -0.056272 +0.024120 0.058528 0.006018 0.028610 -0.012243 0.008813 0.097121 -0.159602 -0.017056 0.077811 0.087002 -0.120528 -0.150197 0.184860 -0.180434 0.040369 -0.097894 -0.042266 0.075094 0.203623 0.039492 0.033806 0.132455 0.059185 -0.102173 -0.071020 -0.051542 0.002257 0.073697 -0.021845 -0.035627 0.009661 0.123071 -0.065039 +-0.056420 0.059125 -0.030321 -0.057038 0.055493 -0.007116 -0.126529 0.019949 -0.058614 -0.078223 0.002145 0.050900 0.167834 -0.173333 0.181067 -0.004047 0.063488 0.151506 -0.125850 -0.136067 0.023547 0.034287 -0.056834 0.003824 0.134406 -0.055792 0.087938 0.068804 -0.033240 0.076407 -0.033551 -0.097730 0.084249 -0.079917 +-0.038591 0.041658 -0.021071 -0.054808 0.085131 -0.020672 -0.150923 -0.028119 -0.005051 -0.082123 -0.027470 0.004437 0.151800 -0.139134 0.205779 0.055682 0.024516 0.096154 -0.038417 -0.113849 0.045637 0.075165 -0.039525 0.011140 0.058496 -0.061165 0.070389 0.031381 -0.020956 0.065917 -0.033395 -0.096762 0.067056 0.005758 +-0.007940 0.026461 -0.003035 -0.037585 0.112114 -0.032320 -0.154598 -0.070183 0.034104 -0.066949 -0.048103 -0.076036 0.108505 -0.081056 0.189128 0.100712 0.001290 0.009722 0.046728 -0.023127 0.013390 0.076873 0.012671 0.031037 0.075639 -0.064473 0.061414 -0.019564 0.014597 0.046786 -0.011485 -0.084927 -0.000334 0.061175 +0.018946 -0.000145 0.011179 -0.029672 0.136914 -0.030086 -0.147824 -0.105721 0.066935 -0.079902 -0.036968 -0.110909 0.052283 0.004188 0.126896 0.134626 -0.006676 -0.052490 0.091685 0.063396 -0.013036 0.002970 0.053049 0.019284 0.038579 -0.025497 0.025340 -0.089926 0.050450 -0.006666 -0.013118 -0.030505 -0.058633 0.064143 +0.026629 -0.059165 -0.000823 -0.058599 0.151849 -0.022073 -0.130501 -0.084385 0.071751 -0.134872 0.039773 -0.057794 -0.044027 0.076494 0.035989 0.101529 0.005315 -0.029779 0.032498 0.028803 0.005618 -0.063529 -0.004924 -0.022183 -0.113427 0.055922 -0.008186 -0.140830 0.090960 -0.114112 -0.009417 0.083947 -0.041032 -0.006371 +-0.010951 0.075350 -0.001817 -0.012876 0.013096 0.004579 -0.016153 -0.047823 -0.029850 0.016150 0.033334 0.015682 0.035621 0.037191 -0.019504 -0.019256 0.044839 0.020295 -0.073131 0.039780 -0.036879 -0.040970 0.053655 -0.001482 -0.002821 -0.032902 -0.000066 0.005729 0.001116 0.056517 -0.017058 -0.021947 0.037761 -0.039276 +-0.012100 0.064207 -0.001096 -0.013941 0.006814 0.008710 -0.012277 -0.037790 -0.022203 0.014245 0.023934 0.012607 0.028185 0.041440 -0.015444 -0.022646 0.048805 0.020521 -0.080918 0.029123 -0.081415 -0.044517 0.073681 0.021481 -0.045782 -0.048161 0.003254 0.036460 -0.020011 0.065280 0.002563 -0.013150 0.045393 -0.015692 +-0.013403 0.059276 0.001032 -0.014545 0.001798 0.010646 -0.008624 -0.026081 -0.017773 0.015405 0.015492 0.014040 0.022709 0.044120 -0.010399 -0.025661 0.051172 0.017503 -0.087720 0.021120 -0.123991 -0.047505 0.087387 0.047337 -0.088646 -0.065851 0.004667 0.069448 -0.040053 0.076718 0.023178 -0.004565 0.059432 0.003138 +-0.013951 0.058265 0.004661 -0.015235 -0.000197 0.012932 -0.005151 -0.015313 -0.012719 0.014418 0.008297 0.018443 0.016678 0.051541 -0.004867 -0.028055 0.052224 0.011609 -0.091004 0.016980 -0.158439 -0.046922 0.104617 0.077207 -0.137148 -0.078657 0.005486 0.103939 -0.062683 0.082257 0.041975 0.000801 0.070429 0.027527 +-0.012210 0.028100 -0.049457 -0.000857 0.000872 -0.019030 0.038293 -0.002178 0.022197 0.008799 -0.017828 -0.005589 -0.010209 -0.026452 0.021247 -0.052723 0.074659 0.058170 -0.117953 -0.020708 -0.092993 -0.079196 0.037883 0.239302 -0.027228 -0.100648 0.058909 0.082147 0.014782 0.033906 0.058190 -0.035069 -0.021313 -0.027376 +-0.009373 0.029412 -0.026531 -0.009199 -0.003610 -0.013154 0.015847 0.006945 0.008617 0.011675 -0.022788 0.000421 -0.007697 0.005751 0.004594 -0.044864 0.063024 0.031416 -0.086743 -0.005786 -0.098907 -0.064158 0.049203 0.156108 -0.042964 -0.089257 0.038446 0.080608 -0.015415 0.071399 0.069731 -0.012170 0.005798 -0.005172 +-0.007108 0.026011 0.001392 -0.008785 -0.008676 0.005534 -0.002323 0.008151 -0.006538 0.012718 -0.016466 0.016828 0.006599 0.032792 -0.008449 -0.031395 0.045979 0.010267 -0.064263 0.010528 -0.087206 -0.035954 0.070434 0.037452 -0.075439 -0.048176 -0.000307 0.074329 -0.047401 0.093968 0.057921 0.003554 0.025206 0.029137 +-0.008616 0.022966 0.025314 -0.006647 -0.012574 0.026313 -0.017855 0.005573 -0.012147 0.015962 -0.009681 0.035391 0.015389 0.061577 -0.023327 -0.005465 0.025007 -0.014906 -0.045808 0.023055 -0.069940 -0.012007 0.101986 -0.066696 -0.085795 -0.002255 -0.051891 0.079767 -0.057766 0.078189 0.009062 0.022309 0.030125 0.051138 +-0.007559 0.013536 0.044396 -0.007128 -0.015567 0.038755 -0.032699 0.009439 -0.018290 0.012901 -0.008295 0.047013 0.018435 0.076580 -0.045586 0.016721 0.010805 -0.037831 -0.013755 0.026798 -0.057710 -0.005862 0.113804 -0.159457 -0.085242 0.031892 -0.090113 0.066240 -0.055402 0.053706 -0.034191 0.047815 0.018971 0.071248 +-0.009091 0.030428 0.013028 0.035270 -0.081955 0.036276 0.100983 -0.087388 -0.023476 0.093938 -0.074920 -0.042748 0.009283 -0.052713 0.069944 0.007487 0.044433 0.011768 -0.036858 -0.036562 0.024165 -0.141056 -0.039714 -0.075826 0.052239 0.062199 -0.048408 -0.033798 0.070343 0.124688 -0.003466 0.051026 0.026850 -0.026589 +-0.006597 0.030631 0.019664 0.021478 -0.061211 0.024677 0.092608 -0.095386 -0.041650 0.068021 -0.054620 -0.036136 0.003440 -0.021689 0.046307 0.007803 0.056620 0.018329 -0.048106 -0.037355 0.044779 -0.170111 -0.043462 -0.076020 0.064547 0.063396 -0.047873 -0.054226 0.076392 0.081326 0.000747 0.074683 0.030657 -0.067772 +-0.006641 0.024694 0.004169 0.027776 -0.027517 0.013511 0.064805 -0.082940 -0.047464 0.039102 -0.047983 -0.025260 -0.012175 0.006766 0.039254 0.004585 0.108825 0.031769 -0.009359 -0.008230 0.016421 -0.164640 -0.117072 -0.007971 0.069640 0.013253 -0.033817 -0.074767 0.071010 0.026436 0.003056 0.090554 0.005886 -0.126966 +0.003124 0.009371 0.014768 0.046412 -0.023874 0.007554 0.036147 -0.057698 -0.058069 0.021256 -0.033315 -0.028152 -0.018259 0.027895 0.014161 0.004788 0.111066 0.042408 -0.022688 -0.017329 0.008180 -0.162840 -0.121148 0.012000 0.040787 0.005472 -0.028935 -0.070284 0.051087 0.002654 0.026039 0.077896 -0.025666 -0.138535 +-0.000470 0.012891 0.009958 0.038234 -0.036776 0.012638 0.068663 -0.077589 -0.036998 0.050930 -0.057174 -0.042770 -0.006013 -0.005012 0.037131 0.011314 0.092853 0.040593 -0.014258 -0.022855 0.028109 -0.170083 -0.092356 -0.017012 0.053071 0.021873 -0.039264 -0.061002 0.057296 0.043219 0.021993 0.064114 -0.018287 -0.100373 +-0.004849 0.015938 0.022983 0.026902 -0.073622 0.023429 0.089584 -0.088285 -0.026052 0.080286 -0.059955 -0.060351 0.007903 -0.035545 0.046797 0.007563 0.038369 0.030247 -0.059021 -0.056175 0.056872 -0.166273 -0.011250 -0.093711 0.035468 0.071037 -0.053530 -0.042127 0.065314 0.098363 0.023179 0.056227 0.018585 -0.031417 +-0.022520 -0.008135 -0.028919 -0.050592 0.028229 -0.006670 -0.060855 -0.025108 0.032576 -0.018823 0.066255 0.018170 0.050211 0.001974 -0.062666 -0.057057 -0.027695 0.045343 -0.016243 0.061633 0.033008 0.024487 0.074017 -0.133643 0.041960 -0.008005 0.017760 -0.020807 0.009949 -0.031184 -0.022929 0.014085 0.078201 0.073117 +-0.008675 -0.004691 -0.013081 -0.031226 0.031157 -0.009924 -0.096969 -0.029153 0.000787 -0.038194 0.087791 0.000669 0.047391 0.025733 -0.093797 -0.072336 -0.039133 0.081739 -0.024145 0.067113 0.036711 0.015968 0.050245 -0.135056 0.022818 0.008747 0.011110 -0.044613 0.004409 -0.030744 -0.014206 -0.016314 0.026864 0.033661 +-0.008350 -0.011949 -0.026508 -0.026911 0.063836 -0.024789 -0.126394 -0.011390 -0.004841 -0.063477 0.084207 0.008974 0.040823 0.048228 -0.106669 -0.072520 0.012682 0.093989 0.007098 0.087810 0.011179 0.012357 -0.005468 -0.074058 0.051617 -0.038525 0.027206 -0.052944 -0.001957 -0.069244 -0.001150 -0.012855 0.002275 -0.012184 +0.001527 -0.031469 -0.019959 -0.032691 0.084353 -0.036934 -0.139297 0.005872 -0.012615 -0.087659 0.090429 0.021235 0.029692 0.064764 -0.125564 -0.055872 0.037072 0.084147 -0.001494 0.073344 0.027889 -0.000113 -0.020299 -0.040534 0.015971 -0.035466 0.033553 -0.071962 -0.021193 -0.078379 0.024814 -0.018801 -0.021044 -0.038763 +-0.009394 -0.028927 -0.028906 -0.024460 0.073718 -0.026898 -0.120720 -0.003723 -0.012295 -0.070463 0.075708 0.021766 0.032103 0.053181 -0.101806 -0.052394 0.030694 0.112096 0.015064 0.098439 -0.005609 0.016172 -0.039627 -0.035390 0.062895 -0.064759 0.040731 -0.023003 -0.019984 -0.078158 0.013167 -0.008582 -0.012839 -0.026034 +-0.014806 -0.020586 -0.019304 -0.037248 0.040627 -0.009079 -0.093808 -0.017684 0.001926 -0.042956 0.076953 0.012620 0.038220 0.029003 -0.084598 -0.051706 -0.017717 0.096814 -0.013796 0.075862 0.017984 0.023149 0.017010 -0.093650 0.035446 -0.023135 0.026297 -0.011769 -0.011753 -0.043351 0.000060 -0.001953 0.028135 0.023804 +0.029937 -0.018527 -0.150727 0.081029 0.034186 -0.056299 0.148788 0.006526 0.091516 -0.080441 -0.035735 0.001997 0.019030 -0.077643 0.107984 -0.056258 0.031181 -0.051535 -0.027580 0.051815 0.012036 0.081437 0.046704 -0.059784 0.100904 0.025934 0.021662 -0.016937 0.039461 -0.046583 -0.092257 -0.006300 0.025170 0.075511 +0.027077 -0.012351 -0.112038 0.035893 0.028873 -0.001260 0.084827 -0.000242 0.048683 -0.054899 -0.029997 0.032628 -0.039785 -0.108273 0.045755 -0.052769 0.008202 -0.062941 -0.003272 0.105699 0.039987 0.020061 0.023151 0.020974 0.008953 0.010896 0.031997 -0.023033 0.010634 -0.071553 -0.062459 -0.034568 0.012175 0.063440 +0.008543 0.000575 -0.036234 0.011338 0.007897 -0.004413 0.042000 0.015857 0.024113 -0.029744 -0.022255 0.053502 -0.038865 -0.097944 0.017588 -0.046984 0.033087 -0.053276 0.001529 0.130264 0.069392 -0.067173 0.042421 0.083647 -0.052001 0.027113 0.009719 0.006872 -0.038341 -0.067952 0.010640 -0.078205 -0.015510 0.044074 +-0.004854 -0.002651 -0.001199 -0.004656 -0.017881 0.006502 0.011106 0.033451 0.012608 -0.007254 -0.021085 0.015936 -0.015926 -0.005386 0.006492 -0.032176 -0.004622 -0.056644 0.008831 0.017207 0.029211 -0.031234 -0.017190 0.017370 -0.014199 -0.016837 0.012242 0.028844 0.024051 -0.033052 0.005809 -0.026851 -0.003806 0.029867 +-0.015813 -0.010424 0.049608 -0.025236 -0.047180 0.033566 -0.017722 0.037312 0.003478 0.019989 -0.017899 -0.019776 0.013111 0.105673 -0.000689 -0.029181 -0.055355 -0.059296 0.016272 -0.126448 -0.003426 0.027239 -0.080562 -0.048943 0.063203 -0.071188 0.018085 0.055066 0.103002 0.015990 -0.024552 0.027166 0.005359 -0.008937 +-0.030164 -0.027591 0.112130 -0.046006 -0.063466 0.021257 -0.061882 0.075039 -0.009550 0.046188 -0.025234 -0.036147 0.006893 0.099786 -0.046972 -0.003808 -0.042536 -0.034124 0.072153 -0.146706 -0.007001 0.011713 -0.032354 -0.013285 0.064616 -0.076540 -0.019793 0.045941 0.057997 -0.041981 -0.013777 -0.007784 -0.005500 0.063654 +-0.019028 -0.051789 0.153268 -0.072220 -0.068001 0.053218 -0.114782 0.100188 -0.032458 0.065327 -0.041993 0.000529 -0.049657 0.064999 -0.113438 0.028145 -0.026783 -0.004495 0.083060 -0.067097 0.025177 -0.054527 0.012232 0.061208 0.035559 -0.041029 -0.084866 0.011647 -0.034824 -0.100697 -0.069165 -0.000768 0.005575 0.110722 +-0.028944 -0.046124 0.115567 -0.042613 -0.067724 0.014225 -0.060491 0.103470 0.006232 0.039413 -0.031752 -0.037167 -0.006997 0.085995 -0.038657 0.010688 -0.040086 -0.042208 0.098912 -0.130371 0.037304 0.026656 -0.077987 -0.030978 0.062407 -0.072535 -0.073939 0.000909 0.019295 -0.034458 0.004467 -0.047183 0.050673 0.035176 +-0.010823 -0.035975 0.055356 -0.020108 -0.053716 0.014079 -0.008891 0.083913 0.022598 0.005824 -0.031201 -0.038542 -0.005753 0.089595 0.015986 -0.004255 -0.071317 -0.065118 0.079317 -0.106829 0.072446 0.039057 -0.148855 -0.072824 0.059724 -0.064924 -0.048338 0.013216 0.059644 0.037032 0.035514 -0.058015 0.052789 -0.068904 +0.002749 -0.025782 0.001067 -0.000955 -0.021760 -0.004910 0.022410 0.073031 0.031948 -0.017623 -0.036578 0.002064 -0.036635 -0.028107 0.022568 -0.013938 -0.026049 -0.064352 0.076825 0.032919 0.104873 -0.018636 -0.086478 -0.011360 -0.025080 -0.005483 -0.057313 -0.019218 -0.018251 -0.006270 0.058547 -0.109717 0.043965 -0.042968 +0.019557 -0.022238 -0.040271 0.015531 -0.000923 -0.011734 0.050924 0.059961 0.044488 -0.037512 -0.047698 0.041131 -0.067325 -0.125904 0.038185 -0.026749 0.003161 -0.063752 0.059945 0.149873 0.138916 -0.043073 -0.038810 0.040424 -0.060299 0.036249 -0.057345 -0.030897 -0.069820 -0.039359 0.051438 -0.140784 0.040745 -0.023033 +0.038295 -0.025011 -0.119022 0.039087 0.023303 -0.007108 0.097748 0.031451 0.066314 -0.064618 -0.049757 0.022319 -0.054682 -0.131953 0.060803 -0.041005 -0.002889 -0.065952 0.048990 0.118731 0.087371 0.038178 -0.008116 -0.019859 0.013619 0.027323 -0.017437 -0.041034 -0.017123 -0.056194 -0.033365 -0.067470 0.076076 0.024548 +0.027485 -0.031004 -0.155083 0.076967 0.030238 -0.066616 0.146021 0.018614 0.095435 -0.090351 -0.026858 -0.001195 0.004065 -0.062969 0.102157 -0.046214 0.025250 -0.047454 0.004391 0.045916 0.011457 0.054752 0.064190 -0.063966 0.092363 0.019011 0.007010 -0.017645 0.023346 -0.068486 -0.085984 -0.024607 0.046082 0.096832 +0.011663 0.006518 -0.034439 0.013241 0.007617 -0.009766 0.044050 0.033840 0.029582 -0.028393 -0.028713 0.055641 -0.043922 -0.099413 0.025141 -0.040381 0.025727 -0.058214 0.010448 0.139032 0.097498 -0.059681 0.018053 0.084934 -0.052343 0.037634 -0.003643 0.010254 -0.050500 -0.038878 0.012009 -0.092562 -0.007898 0.022313 +-0.002313 0.000470 0.000002 -0.004007 -0.016215 0.003976 0.010717 0.048028 0.016739 -0.006186 -0.024483 0.020048 -0.016876 -0.003548 0.010717 -0.026953 -0.010998 -0.055979 0.019518 0.025395 0.050851 -0.031442 -0.027433 0.021960 -0.018106 -0.009491 0.000531 0.037647 0.007463 -0.008452 0.013288 -0.048925 0.006064 0.024257 +-0.012568 -0.006592 0.049539 -0.024277 -0.046609 0.030396 -0.019942 0.053337 0.007757 0.021468 -0.022706 -0.013044 0.011272 0.109312 0.003377 -0.022628 -0.060616 -0.061807 0.030161 -0.121278 0.019402 0.026839 -0.095660 -0.042434 0.064863 -0.064156 0.007525 0.057258 0.096057 0.035786 -0.019042 0.010115 0.006232 -0.015055 +-0.023212 -0.060246 0.157807 -0.069886 -0.065658 0.063342 -0.104423 0.082001 -0.030749 0.079114 -0.044460 0.004171 -0.030429 0.042545 -0.096287 0.021890 -0.044874 -0.001170 0.067071 -0.085689 0.021431 -0.035122 0.002907 0.064224 0.017922 -0.037758 -0.083689 0.004437 -0.037603 -0.087459 -0.078257 -0.011921 0.002713 0.130484 +-0.010961 -0.010595 0.057062 -0.023094 -0.047200 0.021346 -0.011516 0.068656 0.015527 0.014810 -0.022984 -0.021018 0.006940 0.106574 0.011226 -0.023889 -0.062806 -0.069001 0.044159 -0.108003 0.050370 0.047099 -0.112640 -0.068669 0.081751 -0.068521 -0.039172 0.022972 0.052837 0.036443 0.008144 -0.028320 0.041313 -0.064570 +0.001631 -0.003250 0.002325 -0.002118 -0.016181 0.000274 0.016759 0.060910 0.024499 -0.012519 -0.030588 0.015271 -0.023100 -0.011757 0.016440 -0.028638 -0.012990 -0.064844 0.039739 0.031429 0.079984 -0.015321 -0.052897 -0.006078 -0.010170 -0.011420 -0.050449 -0.009900 -0.025391 -0.012909 0.032748 -0.085004 0.038699 -0.043162 +0.017805 0.004146 -0.038887 0.014222 0.006581 -0.009683 0.047461 0.046887 0.038038 -0.032345 -0.032763 0.056280 -0.049485 -0.108966 0.031645 -0.041431 0.020138 -0.064023 0.031810 0.142533 0.126216 -0.050995 -0.002245 0.056266 -0.050403 0.037812 -0.053115 -0.030572 -0.081976 -0.050982 0.028974 -0.127634 0.029303 -0.029574 +0.012632 -0.105919 0.042832 -0.056614 0.095894 -0.023994 -0.019214 -0.026746 -0.070577 0.202269 0.218185 -0.141116 0.012637 -0.191347 0.155533 -0.244488 0.121779 -0.013130 0.138472 -0.059114 -0.216544 -0.050477 0.066853 -0.149560 -0.157390 -0.095098 -0.011175 -0.068427 0.210754 -0.054577 0.001436 0.040342 -0.004236 -0.006205 +-0.009740 -0.029067 0.036475 -0.046866 0.118228 0.012225 -0.006405 -0.060398 -0.086556 0.108100 0.186918 -0.112111 0.050136 -0.100126 0.105268 -0.169540 0.136663 -0.073114 0.067763 -0.014521 -0.082623 -0.012863 0.033765 -0.100967 -0.086562 -0.092416 -0.077945 0.022729 0.114331 -0.008651 -0.069342 0.005683 0.051571 -0.009558 +-0.031301 0.046217 0.033243 -0.035388 0.123785 0.052248 0.021922 -0.090547 -0.125667 0.007581 0.182378 -0.055823 0.074368 -0.029074 0.053632 -0.049594 0.107463 -0.137761 0.005821 -0.001673 0.066449 0.006118 0.023528 0.013804 -0.007269 -0.081388 -0.113434 0.070675 0.027780 0.045024 -0.106739 -0.011678 0.031635 0.044212 +-0.047441 0.105188 0.031918 -0.024384 0.114098 0.082601 0.048491 -0.112125 -0.170704 -0.059301 0.164752 -0.009947 0.090739 0.006616 0.010369 0.042357 0.050623 -0.158918 -0.042210 0.014508 0.136027 -0.007670 -0.034495 0.072312 0.050835 -0.067209 -0.075955 0.023597 0.006577 0.084867 -0.082590 0.024191 -0.041988 0.068711 +-0.060936 0.133902 0.031530 -0.004574 0.064304 0.088650 0.058070 -0.124018 -0.171917 -0.102988 0.148276 0.058157 0.076201 0.044152 0.002505 0.090897 -0.015203 -0.110439 -0.090091 0.069662 0.172459 -0.039446 -0.067579 0.050794 0.093654 -0.082964 -0.026363 0.019254 -0.002660 0.016061 0.024545 0.043909 -0.092051 0.131171 +-0.074646 0.131695 0.040257 0.018490 -0.004794 0.081236 0.061589 -0.133348 -0.147784 -0.119212 0.094712 0.084045 0.033173 0.036408 0.031797 0.094617 -0.048706 0.003825 -0.075566 0.078798 0.127307 -0.065285 -0.109546 -0.039558 0.030185 -0.072564 0.052974 -0.008825 -0.088317 -0.022192 0.091494 0.087785 -0.062428 0.116086 +-0.090236 0.103626 0.066939 0.057319 -0.079163 0.060686 0.053354 -0.131920 -0.087102 -0.115977 0.028577 0.074951 -0.040556 0.035491 0.087538 0.018752 -0.081645 0.116761 0.007481 0.045592 0.001466 -0.094081 -0.126988 -0.146376 -0.033849 -0.021719 0.127080 -0.004981 -0.202755 -0.119973 0.081963 0.106640 -0.105281 -0.008628 +-0.101059 0.055177 0.092659 0.069983 -0.150193 0.026469 0.028574 -0.115258 -0.008902 -0.088879 -0.036594 0.022392 -0.109572 -0.005240 0.149355 -0.115056 -0.119773 0.122550 0.083092 0.011822 -0.092288 -0.074034 -0.034558 -0.115984 -0.044921 0.007854 0.164244 0.014803 -0.170980 -0.127570 0.019750 0.095694 0.000371 -0.041291 +-0.101563 0.024376 0.106668 0.071318 -0.187231 0.012534 0.023991 -0.099795 0.030129 -0.076617 -0.069392 -0.016558 -0.116462 -0.024845 0.158655 -0.199665 -0.116228 0.076206 0.060656 -0.024832 -0.096681 -0.049299 0.082420 -0.042295 -0.032862 0.018749 0.125343 0.039212 -0.037558 -0.047906 -0.025870 -0.049529 0.088169 -0.088273 +-0.101840 -0.013119 0.089419 0.075166 -0.182579 0.048658 0.037188 -0.067749 0.031914 -0.101814 -0.073274 -0.026481 -0.109668 -0.000858 0.110793 -0.202483 -0.060253 0.007145 0.019590 -0.069464 -0.056870 0.023689 0.132006 0.031411 0.013119 -0.007012 0.038285 0.031656 0.001551 -0.025447 -0.065416 -0.114616 0.017261 -0.086359 +-0.084252 -0.076412 0.065909 0.060881 -0.142931 0.092286 0.073191 -0.013071 0.017189 -0.159961 -0.092115 -0.031651 -0.101432 -0.005721 0.027741 -0.106024 0.050453 -0.034217 -0.067543 -0.069923 -0.011090 0.116035 0.100208 0.041462 -0.024840 -0.044467 -0.108854 -0.009354 0.073244 0.020455 -0.063103 -0.129646 -0.010099 -0.129133 +-0.053651 -0.123308 0.046845 0.020630 -0.076671 0.113112 0.110897 0.086910 0.003931 -0.182808 -0.094838 -0.023499 -0.059496 -0.018019 -0.032692 -0.004557 0.053504 -0.066417 -0.110231 -0.014603 -0.024896 0.113252 0.016412 0.021225 -0.025860 -0.075046 -0.145044 -0.030890 0.110020 0.003382 0.007818 -0.098244 -0.037896 -0.009802 +-0.023018 -0.146263 0.038114 -0.004028 -0.020148 0.111274 0.136014 0.170400 0.002465 -0.171044 -0.075670 -0.007542 -0.006424 -0.025856 -0.031416 0.083827 0.010568 -0.069980 -0.109310 0.034905 -0.065427 0.031177 -0.044852 0.019905 -0.066999 -0.093510 -0.115084 -0.095158 0.120015 0.040204 0.030019 -0.030543 -0.096811 0.096700 +0.002563 -0.127886 0.038658 -0.012071 0.033255 0.099861 0.164565 0.238246 0.001567 -0.105923 -0.057783 0.008664 0.027296 -0.030573 -0.009853 0.117020 -0.032701 -0.000283 -0.057331 0.051644 -0.084379 -0.039476 -0.031627 -0.008417 -0.048362 -0.080557 -0.044505 -0.140295 0.129312 0.090377 0.020906 0.080368 -0.088585 0.136852 +0.022743 -0.067070 0.035835 -0.011617 0.073365 0.063325 0.146213 0.262278 -0.002566 -0.007656 -0.026090 0.035984 0.031533 -0.009645 0.044980 0.084848 -0.046015 0.086576 -0.001782 0.061428 0.003570 -0.047340 0.031674 0.003656 -0.006269 -0.074342 -0.011328 -0.102136 0.042874 0.026755 -0.027391 0.130301 -0.059867 0.018942 +0.044270 0.008816 0.045389 -0.021660 0.105411 0.011416 0.109320 0.275577 -0.009832 0.108035 0.004491 0.064390 0.012428 0.006669 0.095357 -0.012238 -0.068972 0.152817 0.075822 0.056884 0.138813 -0.051472 0.080042 0.012909 0.015075 -0.053459 0.028833 0.077070 0.026517 -0.070675 -0.045298 0.143809 -0.084532 -0.083487 +0.062375 0.088371 0.056930 -0.031399 0.124205 -0.042608 0.070908 0.302850 -0.030713 0.221806 0.047883 0.094536 -0.004204 0.009990 0.148794 -0.108520 -0.096997 0.187564 0.140322 0.044017 0.262141 -0.062547 0.161527 -0.013297 0.037476 -0.028287 0.047541 0.256555 -0.040101 -0.087736 0.006879 0.108453 -0.022617 -0.092782 +0.101351 -0.059896 0.089671 -0.007831 -0.037385 0.008829 -0.001672 -0.180136 0.189267 0.006458 -0.232031 -0.060867 0.181227 0.130852 -0.046776 0.035489 0.001493 0.160017 0.051721 0.209411 0.059181 -0.083851 0.207104 -0.102355 0.039265 -0.194572 -0.024321 -0.005820 -0.022768 0.136793 -0.107813 -0.023127 -0.048810 0.065417 +0.081528 -0.068502 0.070887 -0.055525 -0.045082 0.007139 -0.005518 -0.118178 0.156592 -0.033074 -0.063860 0.040561 0.111628 0.029568 0.001045 0.060952 0.022298 0.185051 0.009521 -0.033300 0.094005 0.013600 0.105380 0.040334 -0.061870 -0.114470 0.021331 0.005113 0.002683 0.024141 -0.026811 -0.015689 0.020153 0.092373 +0.068413 -0.083706 0.052305 -0.086296 -0.055880 0.021816 0.004492 -0.068120 0.145758 -0.102567 0.092434 0.075123 0.007851 0.008520 0.005845 0.082068 0.031453 0.180413 -0.035339 -0.133303 0.088724 0.067980 0.008976 0.111553 -0.060575 -0.053669 0.037053 -0.004075 0.049025 0.024206 0.031083 0.032459 0.049141 0.038812 +0.072230 -0.065201 0.036058 -0.094849 -0.081347 0.028568 0.008797 -0.040623 0.140707 -0.126779 0.170257 0.059293 -0.052776 0.003182 -0.000677 0.081645 0.026914 0.131691 -0.055588 -0.098342 0.031136 -0.006790 0.015095 0.065597 0.078600 0.017709 0.045791 -0.034864 0.076622 0.013773 0.067413 -0.028287 0.059081 0.012658 +0.085169 -0.029802 0.042368 -0.082336 -0.097939 0.016608 0.019748 -0.012989 0.121625 -0.087612 0.209630 0.002191 -0.126582 0.050998 -0.030648 0.060010 0.000332 0.046455 -0.023080 0.055862 -0.053376 -0.135321 0.065824 0.029844 0.265716 0.072053 0.059330 -0.027794 0.089039 -0.055593 0.088900 -0.113375 0.061857 -0.009200 +0.083666 0.022002 0.040552 -0.085899 -0.099865 0.015644 0.005068 0.015547 0.156486 -0.077980 0.202180 0.014951 -0.087338 0.065851 -0.063459 -0.028938 0.043400 -0.024140 -0.005938 0.066480 -0.072232 -0.179693 -0.014319 -0.032909 0.209669 0.122452 -0.039648 0.049486 0.011409 -0.011759 -0.007885 -0.033417 -0.079718 0.113528 +0.083042 0.057031 0.040177 -0.099114 -0.076680 0.009479 -0.003028 -0.000359 0.165022 -0.119377 0.166816 0.098159 -0.004888 0.020345 -0.028135 -0.047425 0.054928 -0.014242 -0.021031 -0.044814 0.032783 -0.054961 -0.131702 -0.039012 -0.016037 0.136114 -0.052413 0.121523 -0.038968 0.020495 -0.008329 -0.024960 -0.081805 0.022575 +0.083252 0.078107 0.056261 -0.095219 -0.059524 0.003298 -0.006615 -0.018114 0.165684 -0.099090 0.091701 0.078091 0.033441 -0.005544 -0.024775 -0.060616 0.054683 -0.025820 0.034770 -0.091304 0.061625 0.040350 -0.178139 0.001904 -0.150915 0.076588 -0.034127 0.096746 -0.053605 0.008977 -0.035470 0.025028 -0.047913 -0.054413 +0.104239 0.086773 0.093349 -0.055643 -0.033261 -0.013246 -0.005449 -0.083837 0.158899 -0.044047 -0.029467 -0.026012 0.082055 -0.049138 0.021797 0.008367 0.031893 -0.087107 0.189897 -0.063379 0.017963 0.128801 -0.143253 0.036421 -0.143812 -0.013284 0.024955 0.056737 -0.069906 0.014591 -0.040838 0.026539 -0.056752 -0.047314 +0.135218 0.100220 0.124814 0.000991 -0.008239 -0.012542 -0.007248 -0.156811 0.151583 -0.017420 -0.147099 -0.155713 0.101870 -0.021523 0.028922 0.110680 0.026595 -0.179864 0.346267 0.114265 -0.070904 0.161901 -0.031710 0.014850 -0.024451 -0.120325 0.049384 -0.013378 -0.123593 0.072623 -0.129874 0.065687 -0.065814 -0.083173 +0.059343 -0.012441 -0.000702 -0.081890 -0.008092 -0.077276 -0.028878 0.037295 -0.029467 -0.011411 -0.007775 0.009769 -0.027726 0.005769 0.053610 0.007873 -0.041172 -0.106420 -0.027219 0.033050 -0.102197 -0.048661 -0.066981 -0.013190 0.109772 0.030705 0.044138 0.003928 0.036674 0.004169 0.127857 0.033634 0.266703 0.060249 +0.041616 -0.014486 -0.010645 -0.063688 0.002398 0.007759 -0.030515 0.045130 -0.060320 -0.012907 -0.003144 -0.002022 0.026184 0.062737 0.093489 -0.012212 -0.012731 -0.061123 0.017645 0.059738 -0.070951 0.028276 -0.008200 -0.013282 0.059036 0.130001 0.007378 -0.026986 0.024631 0.046745 0.129132 0.029581 0.168902 0.047167 +0.022209 -0.018785 -0.021705 -0.047282 0.007684 0.096519 -0.037379 0.046370 -0.086593 -0.016226 0.004486 -0.016435 0.077777 0.123311 0.134030 -0.031013 0.005692 -0.004530 0.059412 0.082817 -0.053324 0.097630 0.047516 -0.011465 0.006049 0.229178 -0.021088 -0.047042 0.006557 0.091702 0.122335 0.003955 0.066979 0.045888 +0.003588 -0.021043 -0.030263 -0.030409 0.008371 0.181691 -0.044518 0.046051 -0.107937 -0.022794 0.009934 -0.032445 0.119848 0.179589 0.164240 -0.043652 0.020389 0.047092 0.094236 0.116267 -0.035936 0.155206 0.107422 -0.005142 -0.059549 0.322080 -0.057313 -0.069079 -0.015886 0.152295 0.127300 -0.018228 -0.013766 0.025152 +0.010241 -0.012175 -0.071132 -0.010497 0.020024 0.062831 -0.040007 0.039616 -0.072378 0.035685 -0.031690 0.021640 0.001871 0.085038 0.053829 -0.017137 0.002122 0.049195 0.087728 -0.000013 -0.026558 0.022211 -0.058457 0.166112 0.018559 -0.012076 -0.001099 0.016641 -0.019024 0.008824 0.091310 -0.040634 0.020518 0.051538 +0.005987 -0.016904 -0.065972 -0.010944 0.009374 0.084910 -0.052019 0.031735 -0.073636 0.036433 -0.016286 0.003295 0.010276 0.085334 0.068926 -0.031102 -0.004326 0.037220 0.078248 0.028015 -0.030765 0.043535 -0.024633 0.098897 0.024402 0.055196 -0.037672 -0.019356 0.027747 -0.077052 0.014090 -0.032531 0.010393 0.037211 +0.000485 -0.009792 -0.056592 -0.011803 0.001583 0.104774 -0.061755 0.023501 -0.079032 0.039804 -0.011369 -0.011280 0.024653 0.096826 0.073638 -0.046150 -0.008896 0.030450 0.076726 0.038805 -0.033667 0.064954 0.000900 0.056790 0.020232 0.116360 -0.065525 -0.045163 0.037248 -0.094933 -0.027418 -0.028737 -0.007585 -0.011135 +0.006045 -0.001260 -0.070490 -0.009554 0.009933 0.083302 -0.054060 0.013771 -0.080073 0.047390 -0.001836 -0.009542 0.020186 0.082461 0.052009 -0.037023 0.003870 0.028911 0.066158 0.019377 -0.061487 0.042407 -0.017089 0.104383 0.041935 0.073886 -0.019325 -0.089932 0.029586 -0.070428 0.024956 -0.057337 0.001123 -0.019238 +0.011242 -0.005679 -0.074412 -0.010055 0.015755 0.060586 -0.047635 0.003394 -0.091558 0.044380 -0.005161 -0.022962 0.015788 0.070458 0.028218 -0.036979 0.017605 0.028455 0.076360 -0.013275 -0.085415 0.018537 -0.048259 0.163691 0.057659 0.026190 0.028125 -0.133961 0.018631 -0.042571 0.090055 -0.092227 0.027934 -0.057308 +0.036225 -0.071534 0.000459 -0.046818 0.009337 -0.083866 -0.046178 -0.015273 -0.060642 -0.002822 -0.089257 0.012351 -0.039814 -0.035023 -0.019465 0.040371 -0.061960 0.039386 -0.094567 -0.024135 0.027052 0.059938 0.091591 0.001979 -0.040503 -0.107158 0.083539 -0.101314 0.013352 -0.110938 0.043372 0.129228 0.064399 -0.008668 +0.044367 -0.059525 0.015036 -0.064121 -0.018004 -0.082253 -0.045795 -0.026919 -0.070584 -0.018065 -0.070391 -0.010245 -0.029751 -0.046835 -0.028796 0.042482 -0.143375 0.002483 -0.181009 -0.070951 0.071994 -0.003359 0.199769 -0.097579 -0.087064 0.021258 0.050995 -0.107355 -0.017802 0.053633 0.017380 0.028065 0.073668 -0.008871 +0.043601 -0.049111 0.016832 -0.062722 -0.017690 -0.081388 -0.042392 -0.021628 -0.089830 -0.013515 -0.066851 -0.008770 -0.034597 -0.040346 -0.030496 0.037791 -0.131468 -0.012470 -0.162906 -0.049426 0.049875 -0.008340 0.154951 -0.079562 -0.052199 0.026980 0.057094 -0.092234 -0.013567 0.075758 0.017100 -0.027829 0.047598 -0.032337 +0.048275 -0.038263 0.001498 -0.051199 0.023312 -0.088975 -0.041336 0.005000 -0.095004 -0.003208 -0.060932 0.022466 -0.042236 0.005051 -0.023424 0.039786 -0.019435 -0.003108 -0.056234 0.026974 -0.047324 0.009209 -0.052575 0.062674 0.041095 -0.089544 0.071103 -0.050484 0.024541 0.021517 -0.001177 -0.034659 0.017784 -0.145986 +0.041785 -0.047523 -0.029972 -0.046078 0.035909 -0.097136 -0.056514 0.005071 -0.089750 -0.024247 -0.085315 0.019769 -0.065103 -0.012747 -0.033756 0.024734 -0.014857 0.011973 -0.055355 0.034763 -0.048361 0.024915 -0.036487 0.063963 0.019368 -0.114169 0.095381 -0.092460 0.033839 -0.059458 -0.018405 0.004700 0.059549 -0.091842 +0.044692 -0.060996 -0.030403 -0.035242 0.035271 -0.092818 -0.058749 -0.002830 -0.079239 -0.025316 -0.092148 0.019888 -0.062782 -0.020325 -0.029536 0.030127 -0.024623 0.028330 -0.061266 0.010928 -0.030107 0.035277 -0.006948 0.059755 -0.013675 -0.115893 0.086825 -0.110997 0.020747 -0.090179 -0.023240 0.046759 0.032906 -0.074567 +0.053635 0.024179 0.003976 -0.054942 0.021919 -0.085630 -0.040517 0.019222 -0.098845 -0.002500 -0.058854 0.005767 -0.030366 0.004222 -0.039686 -0.024247 -0.003591 -0.045881 -0.051159 0.028487 -0.059601 0.002317 -0.116937 0.048023 0.058109 -0.042187 -0.022689 0.101089 -0.085146 -0.047060 0.001347 -0.071920 -0.053922 -0.029344 +0.050654 0.043811 0.016440 -0.071448 -0.018825 -0.083915 -0.050552 0.001389 -0.099731 -0.011661 -0.055724 -0.038660 -0.011181 -0.044498 -0.046592 -0.061600 -0.110975 -0.092581 -0.146722 -0.054465 0.022640 -0.027495 0.052884 -0.124647 -0.044100 0.090185 -0.056166 0.049620 -0.094099 0.024798 -0.007076 -0.078109 -0.056294 0.024809 +0.056727 0.056174 0.016328 -0.067723 -0.018273 -0.089164 -0.053246 -0.004474 -0.088775 -0.016960 -0.055755 -0.049357 -0.003393 -0.057029 -0.057944 -0.078602 -0.116320 -0.116419 -0.135346 -0.065112 0.031714 -0.026089 0.053926 -0.146501 -0.088218 0.112238 -0.073090 0.019614 -0.073331 -0.010391 -0.016316 -0.081091 -0.101346 0.033206 +0.053863 0.070189 0.002551 -0.055065 0.015300 -0.088786 -0.052051 0.006288 -0.079362 0.003819 -0.069097 -0.028654 -0.012574 -0.043552 -0.060415 -0.082841 -0.030205 -0.090290 -0.056654 -0.010515 -0.026259 0.018692 -0.060457 -0.049003 -0.020092 0.013016 -0.079443 0.055920 -0.044095 -0.132379 -0.069462 -0.028557 -0.081805 -0.014368 +0.054487 0.052156 -0.025662 -0.043672 0.038234 -0.097652 -0.061550 0.009141 -0.091552 -0.018906 -0.073348 -0.023368 -0.035182 -0.019426 -0.067143 -0.077651 0.000840 -0.064016 -0.050794 0.012015 -0.070276 -0.009742 -0.116913 0.009872 0.004210 -0.020941 -0.054064 0.033929 -0.041700 -0.124288 -0.068147 -0.077671 -0.106441 -0.047888 +0.053071 0.036339 -0.025735 -0.047996 0.041005 -0.100444 -0.057829 0.013845 -0.101921 -0.018419 -0.077645 -0.013513 -0.041059 -0.010444 -0.059771 -0.061024 0.005044 -0.056949 -0.052826 0.026317 -0.078409 -0.007588 -0.129995 0.033848 0.029133 -0.047594 -0.035022 0.047235 -0.058100 -0.091996 -0.054179 -0.066075 -0.078103 -0.046683 +-0.031460 -0.013987 -0.149818 0.080350 0.020681 -0.057818 0.011712 -0.001008 0.061313 0.017201 0.005085 0.074818 -0.047190 0.015967 -0.029569 0.136405 0.070269 -0.039349 0.090760 -0.058563 0.065843 0.078259 -0.020924 -0.163454 -0.040312 -0.104189 0.069961 0.035861 0.064290 0.042196 0.239137 -0.024848 0.084332 -0.100068 +-0.002327 -0.017521 -0.139425 0.070865 0.048305 0.079434 -0.035213 -0.026484 0.062642 0.070813 -0.027307 0.065759 -0.050332 -0.068535 -0.050488 0.065260 -0.034808 0.001184 0.022065 -0.018875 0.057119 -0.010911 -0.054658 -0.020400 -0.028406 -0.025774 0.004056 0.020749 -0.040770 0.067283 0.024925 0.111530 0.079035 -0.073362 +-0.000584 -0.011118 -0.122177 0.051211 0.037283 0.151535 -0.057755 -0.043066 0.075106 0.105702 -0.045195 0.033969 -0.051730 -0.086152 -0.035502 0.008973 -0.086193 0.022393 -0.032203 -0.010625 0.037487 0.012267 -0.063710 0.003409 0.012534 0.018225 -0.069447 -0.045926 -0.057301 0.086177 -0.125110 0.108154 0.062487 -0.033929 +0.000480 -0.003086 -0.124476 0.057127 0.033887 0.149034 -0.061112 -0.048529 0.070950 0.110661 -0.040134 0.025767 -0.039288 -0.067746 -0.045629 0.005133 -0.081370 0.017086 -0.037104 -0.021039 0.008034 -0.013678 -0.051069 0.002511 0.032016 0.008503 -0.097044 -0.037501 -0.056407 0.051508 -0.144414 0.076356 0.059066 -0.021945 +-0.000559 0.003997 -0.124027 0.053465 0.037579 0.149996 -0.049979 -0.055144 0.070483 0.113771 -0.040946 0.023671 -0.042610 -0.080241 -0.054001 0.002411 -0.078521 0.019903 -0.057589 -0.015049 -0.008310 -0.005005 -0.034476 0.006235 0.033286 0.024151 -0.120019 0.018377 -0.072419 0.051602 -0.134832 0.064370 0.037586 0.038711 +-0.000565 0.012494 -0.144971 0.071663 0.048838 0.081814 -0.029624 -0.048941 0.058314 0.080172 -0.010279 0.030880 -0.020393 -0.055948 -0.087529 0.040786 -0.005034 0.001085 -0.046251 -0.044062 -0.050526 -0.012975 0.003798 -0.023311 0.028814 -0.002381 -0.096432 0.041930 -0.029305 0.030971 -0.027834 0.007430 -0.084175 0.004963 +-0.024116 0.030827 -0.152081 0.079655 0.025107 -0.047327 0.011967 -0.026416 0.054183 0.040100 0.018945 0.030843 -0.004122 0.017941 -0.093164 0.086158 0.096071 -0.024027 0.018254 -0.066233 -0.099375 0.036822 0.073623 -0.184943 0.021773 -0.066513 -0.059088 0.109738 0.066223 -0.037492 0.178023 -0.104653 -0.133898 -0.056362 +-0.091307 0.021185 0.005452 0.093079 0.004462 -0.137494 0.027255 0.016193 0.035027 0.043270 0.017597 -0.052594 0.050678 -0.017621 -0.100552 0.045091 0.137042 -0.020604 0.004831 0.003501 -0.076423 -0.058265 0.083175 0.046838 -0.041851 0.066991 0.022783 0.025688 0.023265 -0.031676 0.073439 -0.072413 -0.208999 0.025261 +-0.117606 0.016526 0.081068 0.093572 -0.000820 -0.179734 0.034199 0.055206 0.036610 0.045268 0.036467 -0.091173 0.086052 0.019906 -0.077335 0.021613 0.112294 -0.024854 0.001051 -0.024213 0.006556 -0.070798 0.044003 0.109037 -0.026106 0.108415 0.052460 -0.013149 -0.012349 -0.024464 -0.073565 -0.035321 -0.006109 0.085182 +-0.119255 0.013200 0.094025 0.092040 -0.002146 -0.181868 0.034095 0.066850 0.038987 0.042822 0.035705 -0.088591 0.084897 0.040278 -0.058173 0.015843 0.087899 -0.013908 0.006996 -0.042503 0.059924 -0.066579 0.004384 0.103210 -0.024232 0.105557 0.087612 -0.040335 0.005558 -0.016772 -0.091889 -0.007373 0.029448 0.052985 +-0.116290 0.010377 0.087191 0.091107 -0.002629 -0.185149 0.030103 0.072827 0.039401 0.035673 0.032489 -0.068554 0.067030 0.041143 -0.052476 0.026493 0.086061 -0.007730 0.013083 -0.014025 0.109392 -0.074283 -0.017676 0.099392 -0.055753 0.103876 0.090079 -0.044614 -0.019969 0.012981 -0.056802 -0.005694 0.062207 0.013923 +-0.088956 0.002770 0.014073 0.083735 0.000259 -0.146666 0.024624 0.057239 0.040658 0.024938 0.022137 -0.006530 0.016834 0.033547 -0.054639 0.066908 0.066243 -0.006261 0.048043 0.020086 0.160704 -0.046164 -0.050282 0.007988 -0.125443 0.055461 0.087275 -0.006875 -0.007782 0.054046 0.083770 0.004746 0.021062 -0.066056 +-0.035780 -0.013571 -0.129727 0.075906 0.019981 -0.033424 -0.002959 -0.000563 0.050482 0.021935 0.002944 0.075011 -0.045732 0.001801 -0.047564 0.120066 0.054278 -0.033545 0.090304 -0.063131 0.074862 0.062176 -0.020091 -0.117017 -0.059383 -0.080761 0.087965 0.081966 0.076590 0.065730 0.239584 -0.057628 0.009591 -0.117312 +-0.008820 -0.007627 -0.158056 0.055203 0.017956 0.077099 -0.075129 -0.007191 0.032083 0.085832 -0.010026 0.034463 -0.026295 -0.045494 -0.041653 0.027555 -0.049879 0.009335 0.083404 -0.078297 0.040478 -0.117472 0.024219 0.074484 -0.004484 -0.034318 -0.038895 -0.061118 -0.048430 -0.005360 -0.051358 0.010804 0.052828 0.036864 +-0.007737 -0.003256 -0.151791 0.055268 0.019966 0.087559 -0.078741 -0.018450 0.030698 0.091713 -0.015450 0.020864 -0.015028 -0.032981 -0.052983 0.014209 -0.059863 0.016094 0.069445 -0.092531 0.010370 -0.126361 0.028344 0.071716 0.009341 -0.033236 -0.067826 -0.044003 -0.045043 -0.018573 -0.090383 0.001302 0.035380 0.033187 +-0.007412 0.002548 -0.158757 0.061181 0.018661 0.073976 -0.071983 -0.025639 0.028395 0.091940 0.000206 0.022257 -0.009808 -0.034953 -0.058116 0.017318 -0.038657 0.005313 0.049993 -0.087077 -0.010529 -0.130728 0.062698 0.069370 0.019104 -0.031545 -0.097670 0.007100 -0.077453 -0.016369 -0.071504 -0.037358 0.022404 0.093558 +-0.032512 0.028427 -0.133433 0.078989 0.025468 -0.028533 0.004691 -0.021006 0.046417 0.047024 0.018770 0.037960 0.000477 0.007323 -0.092459 0.082200 0.086554 -0.011680 0.010444 -0.068859 -0.057549 0.014886 0.067619 -0.129370 0.001837 -0.039609 -0.014336 0.136787 0.056466 0.005406 0.166161 -0.086177 -0.153931 -0.065626 +-0.106396 0.012082 0.093804 0.073458 0.006325 -0.122138 0.053986 0.020569 0.052258 0.097040 -0.030117 -0.063935 0.029007 -0.048173 -0.073480 0.006487 0.079422 -0.018318 -0.098277 0.085998 -0.045678 0.124596 -0.056062 -0.009915 0.115255 0.082543 0.043762 0.055274 0.033651 0.018777 -0.097186 0.025652 0.107246 0.082007 +-0.108266 0.013196 0.107352 0.073267 0.005501 -0.118182 0.053900 0.026988 0.059629 0.098678 -0.033463 -0.059244 0.030748 -0.021010 -0.048874 -0.003174 0.046149 -0.004115 -0.096188 0.073125 -0.013743 0.137679 -0.084938 -0.025411 0.119975 0.081742 0.078842 0.041866 0.054237 0.044745 -0.113358 0.068330 0.125398 0.071168 +-0.105677 0.013856 0.100276 0.068541 0.007179 -0.120756 0.052194 0.033207 0.056095 0.092773 -0.030246 -0.037497 0.018805 -0.020756 -0.042271 0.009948 0.041864 0.000654 -0.101674 0.096004 0.022386 0.126403 -0.085055 -0.027232 0.087129 0.079991 0.091934 0.037413 0.035609 0.080869 -0.077055 0.068696 0.147811 0.051447 +0.200300 0.206381 0.037666 -0.062044 0.105603 0.039287 0.129443 -0.026108 -0.006145 0.008167 -0.180506 -0.005344 -0.154896 0.029728 -0.078524 -0.093022 0.282292 0.094500 -0.012038 -0.075772 -0.007254 0.075631 -0.021364 -0.032209 0.106197 -0.033075 -0.057528 -0.036911 -0.079279 -0.070271 0.134924 0.050103 0.176129 0.174669 +0.174909 0.262626 0.001650 -0.050760 0.056767 0.046800 0.116679 0.016697 0.036953 -0.001587 -0.129708 -0.018479 -0.077744 0.017170 -0.006678 -0.084015 0.149428 0.003802 0.023948 -0.034466 -0.037293 0.015672 -0.028566 0.018799 0.029636 -0.058638 0.022368 -0.067504 -0.036674 -0.038199 0.101980 -0.006688 0.114199 0.092127 +0.145419 0.271569 -0.034873 -0.044417 -0.005370 0.054598 0.106804 0.058877 0.047845 -0.006222 -0.038196 -0.028624 0.051275 0.021585 -0.019610 -0.032440 -0.042228 -0.040054 0.010931 -0.080129 -0.032979 -0.037624 0.046956 0.005674 -0.043045 -0.070160 0.084897 -0.059861 0.011866 -0.020727 0.016502 -0.105536 0.054309 0.077813 +0.121784 0.205203 -0.040615 -0.025224 -0.078037 0.055910 0.087096 0.101567 0.032281 0.047114 0.067450 -0.029754 0.115044 0.004978 -0.002689 0.033261 -0.183060 -0.118272 -0.032756 -0.083449 -0.036678 -0.109978 0.089839 -0.018425 -0.047398 -0.071371 0.132785 -0.079312 0.053450 -0.016013 -0.087251 -0.136778 0.010646 0.048749 +0.133914 0.102310 -0.020287 0.004986 -0.141709 0.003779 0.009098 0.120102 -0.047440 0.060693 0.150862 0.032856 0.123342 0.036340 0.053349 0.081323 -0.106337 -0.089787 -0.055918 0.076288 0.022085 -0.032767 0.044374 -0.049343 -0.054487 -0.077724 0.094554 -0.099233 0.070220 0.032782 -0.134028 -0.176615 0.058256 -0.110604 +0.136156 0.012237 0.028350 0.055205 -0.144687 -0.046150 -0.025253 0.036940 -0.082417 0.046931 0.140621 0.125137 0.070684 0.035326 0.117788 0.115735 0.014819 -0.011779 -0.095812 0.103471 0.027698 0.015841 -0.045732 -0.086235 -0.100680 -0.049399 -0.026388 -0.094643 0.100905 0.044076 -0.187605 -0.094315 0.003539 -0.202719 +0.096825 -0.052123 0.057933 0.114355 -0.098038 -0.121617 -0.055895 -0.022068 -0.059251 -0.000028 0.053843 0.302609 -0.066475 0.059366 0.163306 0.029775 0.086102 0.037617 -0.004275 0.136346 -0.030689 0.070476 -0.032601 -0.031896 -0.043260 -0.110607 -0.181097 0.120911 0.098080 0.036017 -0.155625 -0.014093 -0.023635 -0.063091 +0.053280 -0.060018 0.037506 0.137972 0.011258 -0.178318 -0.023102 -0.051378 -0.072595 -0.086067 0.000313 0.285580 -0.051029 -0.025127 0.085024 -0.036423 0.014116 0.013590 0.100603 0.056762 -0.064465 -0.011526 0.015876 -0.005954 0.118192 -0.107172 -0.166406 0.124543 0.111421 0.028985 -0.074010 0.076955 -0.032659 0.141702 +0.045574 -0.013216 0.032142 0.093571 0.050167 -0.171049 0.052834 -0.067576 -0.122586 -0.148784 -0.054492 0.096339 -0.018259 -0.062274 -0.040250 -0.143567 -0.086938 0.063209 0.139416 -0.029860 -0.021312 -0.063067 0.038636 0.074870 0.004951 -0.015358 -0.118334 -0.004443 0.136427 0.102426 0.070536 0.141456 -0.079749 0.125498 +0.059666 0.033773 0.011922 0.066781 0.019188 -0.136499 0.012310 -0.053448 -0.112842 -0.162605 -0.033516 -0.083470 0.040366 -0.073418 -0.073634 -0.134124 -0.183406 0.070520 0.140600 -0.024180 0.078113 -0.014717 -0.030918 0.055699 -0.059861 -0.024527 -0.045187 -0.018538 0.147098 0.098916 0.092276 0.126182 -0.051521 0.096536 +0.102008 0.062652 0.002559 0.033904 -0.034709 -0.028794 -0.029963 -0.001155 -0.088109 -0.139233 -0.017772 -0.206193 0.050147 -0.078321 -0.066908 -0.098039 -0.008521 0.078240 0.003116 -0.115075 0.170119 0.063555 0.042319 0.085075 -0.085497 -0.000332 -0.058397 -0.013971 0.095261 -0.001791 0.088311 0.169500 -0.076125 -0.011915 +0.130156 0.030796 -0.028981 0.037504 -0.110897 0.045667 -0.073986 0.097997 -0.071465 -0.114585 0.024864 -0.164295 0.078517 -0.068363 -0.064291 -0.015782 0.086141 -0.030675 -0.057981 -0.071696 0.160759 0.037941 0.106627 -0.011196 0.036540 -0.017581 -0.051521 0.093436 -0.051622 -0.102102 0.046722 0.156601 0.065388 -0.029184 +0.168973 -0.048346 -0.054128 0.028599 -0.159253 0.088824 -0.040072 0.101158 -0.015747 -0.066487 0.069651 -0.136555 0.131030 -0.095612 0.005098 0.051930 0.141634 -0.059902 -0.077737 -0.001406 0.060549 0.016779 0.073755 -0.079590 0.025474 0.002819 -0.008349 0.092361 -0.102785 -0.130624 0.011078 0.134838 0.105198 -0.029608 +0.196868 -0.140552 -0.038053 0.030363 -0.136280 0.109791 0.082851 0.031947 -0.010593 0.024018 0.071666 -0.120218 0.158965 -0.056053 -0.005157 0.089872 0.082277 -0.037538 -0.032307 -0.004000 -0.077070 -0.047653 -0.036306 -0.059558 -0.007532 0.012245 0.064610 0.069522 -0.100870 -0.125037 -0.000483 0.105122 0.004543 -0.031563 +0.220722 -0.189143 -0.019812 0.050177 -0.022114 0.078768 0.141034 -0.059095 -0.020072 0.062754 0.030385 -0.038935 0.084110 -0.009296 0.007037 0.034972 0.032848 -0.022511 0.007807 0.025446 -0.154512 -0.055275 -0.065423 0.057532 0.001583 -0.000069 0.148740 0.038380 -0.084950 -0.055361 -0.005477 0.024714 -0.104505 -0.036786 +0.226640 -0.179498 0.049226 0.043510 0.105148 -0.008538 0.147024 -0.091442 -0.062929 0.109845 0.005074 0.058119 0.017767 0.089994 -0.044924 -0.062390 -0.007394 0.037390 0.010439 -0.015520 -0.064619 0.042369 -0.058993 0.055481 -0.000104 0.024385 0.166181 0.000132 -0.087429 0.021365 -0.001406 -0.118401 -0.061089 0.025621 +0.236683 -0.154942 0.112382 0.048015 0.188712 -0.046365 0.145337 -0.144361 -0.102570 0.165949 -0.050088 0.165649 -0.012631 0.142407 -0.082270 -0.087023 0.021488 0.106059 -0.048916 -0.157005 0.118604 0.107512 -0.049746 -0.120962 0.092471 0.115214 0.092679 -0.156215 -0.070360 0.083441 -0.097703 -0.215489 -0.072382 0.027100 +0.098748 -0.110249 0.040632 0.106294 0.034691 0.030852 -0.096378 -0.114844 0.076385 0.054995 -0.090801 -0.147677 0.060390 0.097064 -0.014503 -0.075035 -0.111119 -0.058225 -0.072715 0.091030 0.140176 -0.115472 -0.124135 0.002804 -0.034399 0.069021 -0.071552 0.126436 0.208688 -0.064774 0.167890 0.023198 0.065838 -0.115322 +0.045293 -0.089374 0.018089 0.085862 0.012661 -0.008385 -0.067383 -0.079743 -0.026056 0.045936 -0.103235 -0.087935 0.099554 0.036145 -0.034997 -0.024818 -0.088738 0.005542 -0.088960 0.117580 0.051157 -0.047818 -0.031283 0.050240 -0.019133 0.069263 -0.073513 0.173658 0.080920 -0.038285 0.140359 -0.084856 0.069031 -0.038620 +-0.004362 -0.070045 -0.005441 0.028721 -0.035581 0.008714 -0.023807 -0.055672 -0.056772 0.019390 -0.051096 -0.030542 0.150701 -0.048870 -0.050577 0.013535 -0.059832 0.055515 -0.059955 0.062922 0.041041 0.043294 -0.006914 0.139734 -0.004182 0.051132 -0.044750 0.176739 0.008876 -0.029887 0.077004 -0.117767 0.056034 -0.030393 +-0.046908 -0.055997 -0.030801 -0.012931 -0.095239 0.007226 0.007382 -0.049966 -0.065128 -0.036849 -0.026556 0.024951 0.158839 -0.114967 -0.061207 0.047650 -0.045883 0.050290 0.011314 0.022440 -0.012041 0.017932 0.042840 0.074523 0.063977 0.076758 -0.024231 0.082162 -0.002699 -0.024259 0.019286 -0.035404 0.016136 -0.056954 +-0.084798 -0.042180 -0.051749 -0.022690 -0.163003 -0.027359 0.032238 -0.030906 -0.064720 -0.018269 -0.007672 0.068355 0.116840 -0.135774 -0.081552 0.048596 -0.080784 0.017814 0.146383 0.045115 -0.164446 -0.039490 0.025936 0.042829 0.216153 0.010996 0.006383 0.024871 0.002893 -0.034911 -0.009591 0.024318 -0.029688 -0.025116 +-0.106381 0.037472 -0.062633 -0.029746 -0.165792 -0.010239 0.012509 -0.033520 0.075020 0.048760 0.008410 0.182188 0.245250 0.004609 -0.187742 -0.114282 -0.018821 0.047174 0.096399 0.074864 -0.071890 0.002693 0.007320 0.000897 0.070591 0.012206 -0.053249 -0.072801 0.098886 -0.111274 0.027638 0.030143 0.068139 0.024639 +-0.081730 0.078091 -0.032084 -0.016937 -0.094799 -0.034978 -0.035412 -0.016776 0.073337 0.010988 -0.014905 0.221079 0.199761 -0.067149 -0.096254 -0.041963 0.032786 0.009098 0.047313 -0.051221 -0.021669 0.051603 0.007726 -0.033954 -0.118397 0.100014 -0.080501 -0.162248 0.076284 -0.144372 0.005300 0.030886 0.089261 -0.018362 +-0.032591 0.093392 0.005011 0.033984 -0.033653 -0.056018 -0.045205 -0.034382 0.054241 0.038895 -0.028006 0.147818 0.118347 -0.066122 -0.022476 0.025348 -0.024972 -0.006291 0.042302 -0.124172 0.007567 0.044630 0.010710 0.049093 -0.093271 0.092758 -0.055927 -0.139825 0.084283 -0.141690 0.001749 -0.018718 0.034520 -0.013745 +0.025131 0.111171 0.052338 0.093959 0.036120 -0.076090 -0.055111 -0.055018 0.028419 0.039071 -0.021446 0.070711 -0.045120 0.015776 0.119003 0.157191 -0.056194 -0.037864 -0.019407 -0.101495 -0.025226 -0.020203 0.081456 0.053677 0.018315 0.139230 -0.090725 -0.024605 0.076010 -0.142354 0.011727 -0.028982 0.009696 0.010824 +0.086343 0.128544 0.089920 0.151910 0.118411 -0.058267 -0.085331 -0.086356 0.086411 0.023158 -0.022226 -0.036163 -0.177136 0.077261 0.277540 0.318083 -0.140120 -0.082366 -0.088998 -0.072522 -0.066982 0.005449 0.106803 0.086831 0.077782 0.142782 -0.071544 0.132130 0.111582 -0.160749 0.004996 0.042392 -0.012883 0.064412 +-0.061540 0.002620 -0.073689 0.003550 -0.071570 -0.007284 0.013967 -0.001633 -0.094799 -0.008264 -0.030895 0.022099 0.055730 0.034688 -0.025540 -0.038442 -0.002687 -0.049120 0.053353 0.026489 0.022921 0.048577 -0.071688 0.080905 0.064980 0.013782 -0.014355 -0.070959 -0.107091 0.226291 0.009386 0.025020 -0.005604 -0.092437 +-0.083267 -0.013381 -0.104616 -0.054605 -0.053350 -0.040950 0.035114 0.002258 -0.090956 -0.033881 -0.040826 0.018097 0.028479 0.074550 0.001071 0.021358 0.041581 -0.040821 0.048402 0.008007 0.052241 0.038045 -0.031619 0.029420 0.021501 0.017835 -0.011740 -0.040020 -0.113478 0.098608 -0.030604 0.032750 0.004352 -0.062761 +-0.112570 -0.029556 -0.131694 -0.107775 -0.048902 -0.072433 0.059086 0.003800 -0.085443 -0.052929 -0.057306 0.008400 0.005223 0.134826 0.035720 0.064559 0.065656 -0.055342 0.042012 0.015993 0.045317 0.026565 0.036061 -0.016830 -0.006164 0.013784 -0.005233 0.023644 -0.106323 0.001697 -0.058936 0.048287 0.020797 -0.018860 +-0.138208 -0.042880 -0.154716 -0.161145 -0.036682 -0.087614 0.078138 -0.004009 -0.085960 -0.078308 -0.075158 -0.013667 -0.009818 0.193081 0.069946 0.112698 0.091929 -0.063611 0.052582 0.024224 0.037179 0.031363 0.096072 -0.069509 -0.079675 0.025210 -0.001654 0.058541 -0.103434 -0.086341 -0.084715 0.091389 0.038254 0.015236 +-0.071614 -0.025433 -0.079555 -0.044231 -0.056001 0.003353 0.020448 0.039313 -0.034449 0.021201 -0.079939 0.028644 -0.024787 0.188439 0.060152 0.035784 0.042994 0.023862 -0.002200 -0.094819 -0.024939 0.021039 0.009160 -0.055080 -0.021112 -0.045149 -0.025621 0.216443 -0.029294 0.072591 -0.079226 -0.017757 0.067523 0.075291 +-0.072195 -0.021031 -0.121170 -0.072247 -0.036905 -0.070614 0.012662 0.040015 -0.043028 0.012293 -0.078126 -0.030670 -0.033902 0.118408 0.040616 0.048834 0.050552 0.038276 0.008483 -0.058060 -0.034078 -0.041976 -0.009228 -0.045368 -0.013261 -0.068158 0.022561 0.132560 0.004265 -0.025965 -0.059467 -0.019725 0.063179 0.097074 +-0.083844 -0.027702 -0.148124 -0.094158 -0.023167 -0.103209 0.009646 0.023882 -0.042179 0.000862 -0.043166 -0.071528 -0.042289 0.065372 0.035839 0.017436 0.064917 0.010734 0.049860 -0.058510 -0.037633 -0.085134 -0.033890 -0.026309 -0.004119 -0.070958 0.080402 -0.027002 0.048232 -0.088265 -0.053340 0.014549 0.001863 0.085118 +-0.077701 -0.032876 -0.116524 -0.069424 -0.039070 -0.071006 0.000686 0.000032 -0.060440 -0.009999 -0.033027 -0.093775 -0.041865 0.049000 0.019074 0.068912 0.042780 0.002562 0.084029 -0.024194 0.004517 -0.150073 -0.009540 -0.075583 0.001970 -0.012793 0.069308 -0.097943 0.026051 -0.040273 -0.046429 -0.037232 -0.075387 0.056425 +-0.073314 -0.025474 -0.086713 -0.034135 -0.056786 -0.033425 -0.010800 -0.006306 -0.083626 -0.019529 -0.023848 -0.103377 -0.072124 0.027925 0.038858 0.096703 0.015690 0.004385 0.117400 -0.019385 0.068925 -0.151549 0.006375 -0.118295 0.070831 0.037136 0.042043 -0.099400 -0.017748 0.020165 -0.062059 -0.092214 -0.152389 0.036151 +0.044589 -0.132278 -0.016993 0.083882 0.043778 -0.034594 -0.072011 0.039110 -0.046069 -0.040587 0.123917 -0.047575 -0.122385 -0.035400 -0.024195 -0.012518 0.013652 0.031937 0.019659 -0.001715 0.023088 -0.000309 -0.031436 0.022571 0.019611 -0.006564 0.012784 0.083317 -0.104572 0.062104 0.000647 -0.056669 0.044672 0.004338 +0.016329 -0.109007 -0.005045 0.067630 0.019108 -0.002645 -0.071902 0.025076 0.007237 -0.051092 0.091200 -0.100586 -0.101496 -0.065336 -0.015376 0.013166 0.008475 0.063623 0.047215 0.020495 -0.013691 0.030534 0.016318 -0.010397 -0.023315 0.035663 -0.023907 0.042304 -0.100402 0.062642 -0.013752 0.012255 0.032336 0.012548 +0.005789 -0.090540 0.006017 0.048376 -0.000562 -0.003770 -0.064679 0.019118 0.018765 -0.056858 0.072645 -0.105570 -0.099008 -0.063197 -0.036400 0.020176 0.006068 0.073862 0.022493 -0.014200 -0.022956 0.011319 0.028866 -0.059130 0.024068 0.043375 -0.037957 0.009386 -0.070164 0.052847 -0.054076 0.043887 0.014360 -0.001251 +0.001863 -0.067279 -0.004424 0.022537 -0.021339 0.004205 -0.062496 0.031100 0.053349 -0.011934 0.065752 -0.132376 -0.118604 -0.084354 0.018198 0.004240 -0.004340 0.003937 0.017652 -0.050848 0.024164 -0.013728 -0.034087 -0.019119 0.077051 0.006899 0.013968 -0.043247 -0.030695 0.066883 -0.075176 -0.002876 0.013993 -0.016796 +0.005001 -0.090158 -0.011976 0.055472 -0.011104 -0.019645 -0.061557 0.044266 0.027270 -0.030345 0.068012 -0.132701 -0.097672 -0.062167 -0.026502 -0.015816 -0.002765 0.014242 0.048233 0.014931 0.013361 0.021076 -0.002662 -0.047362 0.074697 -0.011056 0.032939 0.005948 -0.057100 0.111103 -0.023048 -0.033487 -0.004465 0.045128 +0.012182 -0.107450 -0.027021 0.065245 0.008961 -0.027772 -0.074458 0.050282 0.025229 -0.041151 0.086397 -0.119923 -0.099702 -0.055513 -0.011844 -0.012761 -0.000561 0.025024 0.077500 0.042190 0.004065 0.022608 -0.020626 -0.014056 0.026667 -0.025100 0.039847 0.066569 -0.072695 0.106463 0.007401 -0.072767 0.005736 0.071139 +-0.009462 0.065949 -0.034941 0.008894 -0.064655 -0.020166 -0.060698 0.028627 0.066320 0.002801 0.040050 0.031191 -0.045838 0.048249 0.022311 -0.088733 0.012498 0.012592 -0.012528 -0.038889 0.028963 -0.007368 -0.006653 0.082948 -0.006713 0.005645 -0.025498 -0.054304 -0.022462 0.175263 -0.039024 -0.042552 -0.094353 -0.013449 +0.000399 0.075432 -0.010525 0.041532 -0.020281 -0.016042 -0.094246 0.065468 0.057754 -0.013075 0.035003 0.037001 -0.063322 0.051499 0.004328 -0.061380 -0.015256 0.001025 -0.051518 0.003395 -0.032991 0.024094 0.008757 0.054130 -0.053594 0.010905 -0.022832 -0.025114 -0.048340 0.149929 -0.021867 0.081009 -0.007931 -0.009067 +0.011378 0.085467 -0.017111 0.041300 -0.001692 -0.039680 -0.104984 0.079709 0.063221 -0.028397 0.048561 0.041353 -0.084814 0.040411 0.000268 -0.038789 -0.000559 -0.008586 -0.044596 0.040237 -0.063474 -0.003204 -0.031149 0.016331 -0.050858 0.018453 -0.008099 -0.080392 -0.015774 0.091864 -0.016377 0.134277 0.049921 -0.027301 +0.029766 0.102153 -0.038143 0.053924 0.038638 -0.061103 -0.101860 0.090831 0.056649 -0.017724 0.060045 0.040849 -0.108462 0.025297 0.013784 -0.048798 0.017389 -0.036169 -0.049864 0.021916 -0.066484 0.016243 -0.106255 -0.005066 -0.033200 -0.008221 0.050207 -0.137478 0.022810 0.034341 -0.021252 0.120671 0.042481 -0.025859 +0.006889 0.087347 -0.040353 0.029840 -0.010245 -0.040820 -0.091202 0.072685 0.079209 -0.016224 0.043485 -0.001910 -0.084857 0.044870 -0.001342 -0.071705 -0.002282 -0.026485 -0.039688 0.033741 -0.055799 -0.013397 -0.029281 0.000374 -0.037317 -0.004296 0.042230 -0.104897 0.024080 0.125291 0.011644 0.079126 -0.013272 0.018850 +-0.000936 0.077182 -0.035678 0.029087 -0.033747 -0.023682 -0.089809 0.060071 0.081219 -0.000659 0.033152 -0.007048 -0.054525 0.054060 -0.010472 -0.078847 -0.000530 -0.009111 -0.033111 0.004942 -0.009483 0.002476 0.026701 0.054112 -0.050766 -0.014615 0.013950 -0.054182 -0.017408 0.176648 0.019946 -0.001332 -0.069141 0.058993 +-0.053340 -0.049736 0.202485 -0.040699 0.016891 0.098603 -0.079165 0.012530 -0.068702 0.013827 -0.018413 0.064039 -0.096530 -0.107456 -0.034620 0.115038 0.176489 0.048911 0.043125 -0.004140 0.021027 -0.096280 0.063451 0.033838 -0.029605 0.024958 -0.044214 -0.018466 -0.082116 -0.029533 0.007344 -0.061482 0.038981 -0.084020 +-0.084313 -0.038019 0.077710 -0.058798 0.076335 0.124114 -0.036329 -0.056135 -0.011941 0.019801 -0.032605 0.049799 -0.014743 -0.022984 -0.023983 0.002739 0.115499 0.010250 -0.071667 0.052221 0.028701 -0.018035 0.046885 0.065758 -0.088996 0.058902 0.045509 0.021842 0.012293 -0.058012 -0.054652 -0.028210 -0.045572 0.005237 +-0.092694 -0.031386 -0.029516 -0.066011 0.056512 0.140318 0.017759 -0.055408 0.042409 -0.001841 -0.047881 0.006271 -0.044880 0.010562 -0.008572 -0.032142 0.031237 0.011784 -0.071950 -0.017154 0.021388 0.094393 -0.059446 -0.036018 -0.074495 0.032641 0.092606 0.062712 0.134488 -0.112228 -0.109084 0.090907 -0.032517 0.032689 +-0.086068 -0.007082 -0.045041 -0.060765 0.055270 0.132259 0.009008 -0.042025 0.050070 -0.002958 -0.051816 0.001503 -0.066612 -0.017554 -0.020320 -0.047269 0.007336 0.021029 -0.065278 0.045943 0.031788 0.044150 -0.060233 -0.015205 -0.047170 0.032557 0.110315 0.067668 0.142436 -0.112995 -0.095937 0.043696 -0.053263 0.018166 +-0.089411 0.023741 -0.038912 -0.067964 0.042282 0.140245 0.010622 -0.036806 0.040610 0.006060 -0.017474 0.000163 -0.063926 0.009072 -0.059385 -0.064253 -0.040611 0.016892 -0.098455 0.064721 0.042317 0.071934 -0.068174 -0.023171 -0.006137 0.021292 0.136552 0.086574 0.159669 -0.085905 -0.090037 0.025543 -0.060574 -0.002428 +-0.078575 0.034575 0.031263 -0.073552 0.040647 0.136747 -0.043737 -0.008796 0.032331 0.016926 -0.029196 0.043721 -0.033823 -0.069883 -0.054957 0.000382 0.001785 -0.066324 -0.031587 0.075610 0.002026 0.123617 -0.016946 -0.073108 0.082295 -0.003377 0.160760 0.045913 0.138259 -0.012841 -0.021037 0.019773 -0.081271 0.003220 +-0.033366 0.059208 0.160210 -0.064990 0.028624 0.104300 -0.099598 0.045261 -0.039277 0.065556 -0.102912 0.108074 -0.081503 -0.190293 -0.052187 0.075107 0.008671 -0.121671 0.069010 0.122685 -0.035632 -0.012154 0.063158 -0.139158 0.053250 0.013035 0.078006 -0.006941 0.005533 0.101138 0.135668 0.005174 -0.004429 -0.021124 +-0.092656 0.031357 0.103505 -0.059111 0.068764 0.010312 0.008929 0.034866 0.057992 0.068053 0.014425 0.007778 0.041020 -0.040683 -0.027837 -0.022634 -0.126523 -0.071192 0.073877 0.037044 -0.024998 0.031381 0.000446 -0.063882 0.075465 -0.089846 -0.003293 -0.014744 -0.039966 0.049482 0.125205 -0.021707 -0.015263 0.035730 +-0.111708 0.016302 0.021802 -0.017327 0.098757 -0.032632 0.102916 -0.004106 0.112659 0.043944 0.086722 -0.075135 0.053748 0.008377 0.026036 -0.051027 -0.181615 0.007593 -0.029718 -0.022786 -0.013286 0.004632 0.013051 -0.001664 0.043884 -0.080356 -0.074392 -0.041220 -0.096235 -0.001700 0.105291 -0.079759 -0.034690 -0.007809 +-0.106284 -0.022433 0.016803 -0.009022 0.113956 -0.059454 0.115911 -0.012451 0.125340 0.015307 0.082184 -0.057654 0.037795 0.019407 0.065630 -0.026223 -0.104086 0.015953 -0.052260 -0.022874 -0.001840 -0.000797 -0.004680 0.054162 0.031667 -0.086636 -0.106973 -0.052172 -0.115387 -0.039377 0.098135 -0.063370 -0.010368 -0.058260 +-0.105834 -0.044185 0.028207 -0.010833 0.109847 -0.041639 0.113964 -0.035423 0.117943 0.018785 0.073258 -0.043540 0.048715 0.037758 0.075421 -0.003877 -0.043429 0.013819 -0.039051 -0.084951 -0.018739 0.005618 -0.000400 0.028961 0.006799 -0.081430 -0.141147 -0.095855 -0.105823 -0.040548 0.071096 -0.022040 -0.010334 -0.072087 +-0.085770 -0.066339 0.119689 -0.034493 0.098418 0.002001 0.025266 -0.023246 0.059661 0.027629 0.052453 0.022351 0.015313 -0.007107 -0.005840 0.096765 0.062508 -0.012640 -0.032725 -0.080090 -0.027756 -0.041996 0.055252 0.063300 -0.038983 -0.076443 -0.130005 -0.083137 -0.110099 -0.085295 0.021676 -0.004858 0.062427 -0.049960 +-0.063490 -0.034197 0.223953 -0.071609 0.017273 0.072534 -0.070046 0.008472 -0.059151 0.024538 -0.057458 0.054908 -0.069500 -0.108298 -0.024305 0.086168 0.173404 0.023995 0.062833 0.058041 0.036625 -0.084243 -0.007661 0.049316 -0.069261 0.031241 -0.034048 -0.059035 -0.063512 -0.030367 -0.002864 -0.044970 0.061520 -0.067669 +-0.080995 -0.029362 -0.020969 -0.037152 0.055714 0.092056 0.009933 -0.025303 0.020523 -0.004640 -0.020295 -0.009164 -0.005931 0.018848 0.033915 -0.030364 0.045067 0.004015 0.011173 -0.058507 0.056901 -0.042073 -0.006012 0.046935 -0.107241 0.032777 0.116424 0.038938 0.083070 0.044785 -0.024341 -0.040868 -0.054497 0.021228 +-0.079570 -0.003619 -0.038468 -0.043009 0.061929 0.088254 0.002999 -0.017741 0.034379 -0.004900 -0.032478 -0.019353 -0.018212 -0.005055 0.014637 -0.048472 -0.001520 0.022877 0.006651 -0.017392 0.069312 -0.058844 -0.004646 0.065142 -0.096265 0.026598 0.135522 0.071861 0.091550 0.032257 -0.016765 -0.075337 -0.064651 0.050908 +-0.077037 0.036024 -0.029906 -0.041175 0.045056 0.108172 0.001799 -0.013867 0.031473 0.010155 -0.003491 -0.014995 -0.021891 0.005419 -0.006514 -0.067131 -0.050711 0.002908 -0.017700 0.003424 0.063512 -0.037825 -0.008255 0.061508 -0.030670 0.015575 0.161325 0.090053 0.117788 0.091435 -0.010989 -0.096824 -0.085735 0.003260 +-0.051962 0.043127 0.185753 -0.068488 0.030906 0.083935 -0.070252 0.031192 -0.032752 0.081860 -0.058415 0.100433 -0.075819 -0.159865 -0.059411 0.067034 0.026048 -0.117863 0.048451 0.078422 -0.022589 0.005932 0.109565 -0.086305 0.028208 0.027111 0.068306 -0.018619 -0.000395 0.065584 0.129299 -0.018649 -0.057046 -0.000423 +-0.110771 0.026821 0.039896 -0.026400 0.073464 0.011366 0.088839 0.010905 0.059433 0.027762 0.038964 -0.069547 0.034358 0.017245 0.035428 -0.063296 -0.129262 0.026781 -0.092055 0.071498 -0.050266 0.045878 -0.036457 -0.040249 0.060855 -0.044906 -0.090430 -0.018843 -0.111822 -0.025148 0.069191 -0.000310 -0.018828 -0.067576 +-0.109310 -0.015148 0.033856 -0.022173 0.084027 -0.011325 0.100709 0.004099 0.070556 0.012019 0.035921 -0.046262 0.047837 0.012906 0.059339 -0.023335 -0.082080 0.034241 -0.099980 0.038508 -0.057517 0.048531 -0.049704 -0.033707 0.035725 -0.049554 -0.119832 -0.033233 -0.116399 -0.076705 0.057450 0.017861 0.007758 -0.058416 +-0.105002 -0.041286 0.052806 -0.020899 0.081333 0.007181 0.095687 -0.011183 0.063719 0.009967 0.029662 -0.037522 0.062980 0.038468 0.065127 -0.003635 -0.032995 0.016733 -0.100860 -0.006462 -0.072928 0.073195 -0.048581 -0.059580 0.018777 -0.048728 -0.138328 -0.072245 -0.108275 -0.063238 0.030578 0.066053 0.024802 -0.084883 +# The variances of the components (eigenvalues) of identity or combined identity and expression model +1 +34 +6 +826.213804 695.783596 380.584790 282.861398 209.814481 184.418561 113.076307 104.852653 81.857546 77.095662 71.081802 55.760367 49.883312 39.122009 37.668889 32.916377 31.165932 26.644275 24.233150 21.357573 17.029065 15.432317 13.812689 12.440567 10.213426 9.316734 5.327325 2.526152 1.659451 1.123615 1.015840 0.816193 0.718969 0.582660 diff --git a/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt b/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt new file mode 100644 index 00000000..49476e2b --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/main_dynamic_svms.txt @@ -0,0 +1,3 @@ +AUPredictor AU_all_best.txt +PDM In-the-wild_aligned_PDM_68.txt +Triangulation tris_68_full.txt \ No newline at end of file diff --git a/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt b/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt new file mode 100644 index 00000000..0ef644ab --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/main_static_svms.txt @@ -0,0 +1,3 @@ +AUPredictor AU_all_static.txt +PDM In-the-wild_aligned_PDM_68.txt +Triangulation tris_68_full.txt \ No newline at end of file diff --git a/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt b/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt new file mode 100644 index 00000000..25ee5c0e --- /dev/null +++ b/lib/local/FaceAnalyser/AU_predictors/tris_68_full.txt @@ -0,0 +1,114 @@ +111 +3 +4 +58 67 59 +60 49 48 +58 59 6 +34 52 35 +44 45 25 +39 40 29 +37 18 36 +27 42 22 +23 44 24 +41 36 1 +50 62 51 +56 65 66 +57 58 7 +64 53 63 +28 27 39 +52 34 51 +54 14 35 +29 42 28 +19 20 24 +35 15 46 +37 19 18 +36 0 1 +18 17 36 +37 20 19 +38 20 37 +21 20 38 +21 38 39 +24 44 25 +30 34 35 +21 39 27 +28 42 27 +39 29 28 +29 30 35 +31 30 29 +30 33 34 +31 29 40 +36 17 0 +41 31 40 +31 32 30 +31 41 1 +49 31 48 +48 2 3 +67 60 59 +4 48 3 +5 48 4 +6 59 5 +59 48 5 +60 48 59 +7 58 6 +61 49 60 +58 66 67 +31 2 48 +31 50 32 +1 2 31 +61 50 49 +52 62 63 +50 31 49 +34 33 51 +51 62 52 +32 50 51 +50 61 62 +63 53 52 +54 55 11 +57 8 9 +66 58 57 +8 57 7 +56 57 9 +66 57 56 +10 56 9 +55 56 10 +53 54 35 +53 35 52 +12 54 11 +55 10 11 +65 56 55 +64 55 54 +65 55 64 +54 53 64 +12 13 54 +14 54 13 +15 35 14 +47 35 46 +33 32 51 +30 32 33 +29 35 47 +15 45 46 +22 21 27 +20 21 23 +43 23 22 +29 47 42 +23 21 22 +24 20 23 +22 42 43 +23 43 44 +45 16 26 +15 16 45 +25 45 26 +41 36 37 +40 41 37 +38 40 37 +39 40 38 +47 42 43 +46 47 43 +46 43 44 +45 46 44 +67 60 61 +66 67 61 +62 66 61 +65 66 62 +63 65 62 +64 65 63 \ No newline at end of file diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj index db1dd2cc..e5d06371 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj @@ -59,7 +59,6 @@ - @@ -67,7 +66,6 @@ - @@ -75,7 +73,6 @@ - @@ -83,7 +80,6 @@ - @@ -98,7 +94,7 @@ Level3 Disabled false - ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + ./include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -117,9 +113,10 @@ Level3 Disabled false - ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + ./include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true + WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) true @@ -139,7 +136,7 @@ true - ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + ./include;%(AdditionalIncludeDirectories) StreamingSIMDExtensions2 true @@ -164,9 +161,10 @@ true - ./include;$(SolutionDir)lib\local\LandmarkDetector\include;%(AdditionalIncludeDirectories) + ./include;%(AdditionalIncludeDirectories) AdvancedVectorExtensions true + WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) true @@ -185,23 +183,24 @@ {b47a5f12-2567-44e9-ae49-35763ec82149} - - {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} - - + + + + CppCode - + + diff --git a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters index 3edc5a6d..65db0525 100644 --- a/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters +++ b/lib/local/FaceAnalyser/FaceAnalyser.vcxproj.filters @@ -33,7 +33,13 @@ Header Files - + + Header Files + + + Header Files + + Header Files @@ -56,7 +62,13 @@ Source Files - + + Source Files + + + Source Files + + Source Files diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index f4d262de..ca155b4f 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -59,198 +35,214 @@ #ifndef __FACEANALYSER_h_ #define __FACEANALYSER_h_ +// STL includes +#include +#include +#include + +// OpenCV includes +#include + +// Local includes #include "SVR_dynamic_lin_regressors.h" #include "SVR_static_lin_regressors.h" #include "SVM_static_lin.h" #include "SVM_dynamic_lin.h" - -#include -#include - -#include - -#include "LandmarkCoreIncludes.h" +#include "PDM.h" +#include "FaceAnalyserParameters.h" namespace FaceAnalysis { - class FaceAnalyser { +class FaceAnalyser{ - public: +public: - enum RegressorType { SVR_appearance_static_linear = 0, SVR_appearance_dynamic_linear = 1, SVR_dynamic_geom_linear = 2, SVR_combined_linear = 3, SVM_linear_stat = 4, SVM_linear_dyn = 5, SVR_linear_static_seg = 6, SVR_linear_dynamic_seg = 7 }; + enum RegressorType{ SVR_appearance_static_linear = 0, SVR_appearance_dynamic_linear = 1, SVR_dynamic_geom_linear = 2, SVR_combined_linear = 3, SVM_linear_stat = 4, SVM_linear_dyn = 5, SVR_linear_static_seg = 6, SVR_linear_dynamic_seg =7}; - // Constructor from a model file (or a default one if not provided - // TODO scale width and height should be read in as part of the model as opposed to being here? - FaceAnalyser(vector orientation_bins = vector(), double scale = 0.7, int width = 112, int height = 112, std::string au_location = "AU_predictors/AU_all_best.txt", std::string tri_location = "model/tris_68_full.txt"); + // Constructor for FaceAnalyser using the parameters structure + FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params); - void AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, double timestamp_seconds, bool online = false, bool visualise = true); + void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false, bool visualise = true); - // If the features are extracted manually (shouldn't really be used) - void PredictAUs(const cv::Mat_& hog_features, const cv::Mat_& geom_features, const LandmarkDetector::CLNF& clnf_model, bool online); + cv::Mat GetLatestHOGDescriptorVisualisation(); - cv::Mat GetLatestHOGDescriptorVisualisation(); + double GetCurrentTimeSeconds(); + + // Grab the current predictions about AUs from the face analyser + std::vector> GetCurrentAUsClass() const; // AU presence + std::vector> GetCurrentAUsReg() const; // AU intensity + std::vector> GetCurrentAUsCombined() const; // Both presense and intensity - double GetCurrentTimeSeconds(); + // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity + // This call is useful for detecting action units in images + std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise = true); - // Grab the current predictions about AUs from the face analyser - std::vector> GetCurrentAUsClass() const; // AU presence - std::vector> GetCurrentAUsReg() const; // AU intensity - std::vector> GetCurrentAUsCombined() const; // Both presense and intensity + void Reset(); - // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity - // This call is useful for detecting action units in images - std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise = true); + void GetLatestHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); + void GetLatestAlignedFace(cv::Mat& image); + + void GetLatestNeutralHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); + + cv::Mat_ GetTriangulation(); + + void GetGeomDescriptor(cv::Mat_& geom_desc); - void Reset(); + // Grab the names of AUs being predicted + std::vector GetAUClassNames() const; // Presence + std::vector GetAURegNames() const; // Intensity - void GetLatestHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); - void GetLatestAlignedFace(cv::Mat& image); - - void GetLatestNeutralHOG(cv::Mat_& hog_descriptor, int& num_rows, int& num_cols); - - cv::Mat_ GetTriangulation(); - - void GetGeomDescriptor(cv::Mat_& geom_desc); - - // Grab the names of AUs being predicted - std::vector GetAUClassNames() const; // Presence - std::vector GetAURegNames() const; // Intensity - - // Identify if models are static or dynamic (useful for correction and shifting) - std::vector GetDynamicAUClass() const; // Presence - std::vector> GetDynamicAUReg() const; // Intensity + // Identify if models are static or dynamic (useful for correction and shifting) + std::vector GetDynamicAUClass() const; // Presence + std::vector> GetDynamicAUReg() const; // Intensity - void ExtractAllPredictionsOfflineReg(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); - void ExtractAllPredictionsOfflineClass(vector>>& au_predictions, vector& confidences, vector& successes, vector& timestamps, bool dynamic); + void ExtractAllPredictionsOfflineReg(std::vector>>& au_predictions, + std::vector& confidences, std::vector& successes, std::vector& timestamps, bool dynamic); + void ExtractAllPredictionsOfflineClass(std::vector>>& au_predictions, + std::vector& confidences, std::vector& successes, std::vector& timestamps, bool dynamic); - // Helper function for post-processing AU output files - void PostprocessOutputFile(string output_file, bool dynamic); + // Helper function for post-processing AU output files + void FaceAnalyser::PostprocessOutputFile(std::string output_file); - private: +private: - // Where the predictions are kept - std::vector> AU_predictions_reg; - std::vector> AU_predictions_class; + // Point distribution model coddesponding to the current Face Analyser + FaceAnalysis::PDM pdm; - std::vector> AU_predictions_combined; + // Where the predictions are kept + std::vector> AU_predictions_reg; + std::vector> AU_predictions_class; - // Keeping track of AU predictions over time (useful for post-processing) - vector timestamps; - std::map> AU_predictions_reg_all_hist; - std::map> AU_predictions_class_all_hist; - std::vector confidences; - std::vector valid_preds; + std::vector> AU_predictions_combined; - int frames_tracking; + // Keeping track of AU predictions over time (useful for post-processing) + std::vector timestamps; + std::map> AU_predictions_reg_all_hist; + std::map> AU_predictions_class_all_hist; + std::vector valid_preds; - // Cache of intermediate images - cv::Mat aligned_face_for_au; - cv::Mat aligned_face_for_output; - cv::Mat hog_descriptor_visualisation; + int frames_tracking; - // Private members to be used for predictions - // The HOG descriptor of the last frame - cv::Mat_ hog_desc_frame; - int num_hog_rows; - int num_hog_cols; + // Is the AU model dynamic + bool dynamic; - // Keep a running median of the hog descriptors and a aligned images - cv::Mat_ hog_desc_median; - cv::Mat_ face_image_median; + // Cache of intermediate images + cv::Mat aligned_face_for_au; + cv::Mat aligned_face_for_output; + cv::Mat hog_descriptor_visualisation; + bool out_grayscale; - // Use histograms for quick (but approximate) median computation - // Use the same for - vector > hog_desc_hist; + // Private members to be used for predictions + // The HOG descriptor of the last frame + cv::Mat_ hog_desc_frame; + int num_hog_rows; + int num_hog_cols; - // This is not being used at the moment as it is a bit slow - vector > face_image_hist; - vector face_image_hist_sum; + // Keep a running median of the hog descriptors and a aligned images + cv::Mat_ hog_desc_median; + cv::Mat_ face_image_median; - vector head_orientations; + // Use histograms for quick (but approximate) median computation + // Use the same for + std::vector > hog_desc_hist; - int num_bins_hog; - double min_val_hog; - double max_val_hog; - vector hog_hist_sum; - int view_used; + // This is not being used at the moment as it is a bit slow + std::vector > face_image_hist; + std::vector face_image_hist_sum; - // The geometry descriptor (rigid followed by non-rigid shape parameters from CLNF) - cv::Mat_ geom_descriptor_frame; - cv::Mat_ geom_descriptor_median; + std::vector head_orientations; - int geom_hist_sum; - cv::Mat_ geom_desc_hist; - int num_bins_geom; - double min_val_geom; - double max_val_geom; + int num_bins_hog; + double min_val_hog; + double max_val_hog; + std::vector hog_hist_sum; + int view_used; - // Using the bounding box of previous analysed frame to determine if a reset is needed - cv::Rect_ face_bounding_box; + // The geometry descriptor (rigid followed by non-rigid shape parameters from CLNF) + cv::Mat_ geom_descriptor_frame; + cv::Mat_ geom_descriptor_median; + + int geom_hist_sum; + cv::Mat_ geom_desc_hist; + int num_bins_geom; + double min_val_geom; + double max_val_geom; + + // Using the bounding box of previous analysed frame to determine if a reset is needed + cv::Rect_ face_bounding_box; + + // The AU predictions internally + std::vector> PredictCurrentAUs(int view); + std::vector> PredictCurrentAUsClass(int view); - // The AU predictions internally - std::vector> PredictCurrentAUs(int view); - std::vector> PredictCurrentAUsClass(int view); + // special step for online (rather than offline AU prediction) + std::vector> CorrectOnlineAUs(std::vector> predictions_orig, int view, bool dyn_shift = false, bool dyn_scale = false, bool update_track = true, bool clip_values = false); - // special step for online (rather than offline AU prediction) - std::vector> CorrectOnlineAUs(std::vector> predictions_orig, int view, bool dyn_shift = false, bool dyn_scale = false, bool update_track = true, bool clip_values = false); + void Read(std::string model_loc); - void ReadAU(std::string au_location); + void ReadAU(std::string au_location); - void ReadRegressor(std::string fname, const vector& au_names); + void ReadRegressor(std::string fname, const std::vector& au_names); - // A utility function for keeping track of approximate running medians used for AU and emotion inference using a set of histograms (the histograms are evenly spaced from min_val to max_val) - // Descriptor has to be a row vector - // TODO this duplicates some other code - void UpdateRunningMedian(cv::Mat_& histogram, int& hist_sum, cv::Mat_& median, const cv::Mat_& descriptor, bool update, int num_bins, double min_val, double max_val); - void ExtractMedian(cv::Mat_& histogram, int hist_count, cv::Mat_& median, int num_bins, double min_val, double max_val); + // A utility function for keeping track of approximate running medians used for AU and emotion inference using a set of histograms (the histograms are evenly spaced from min_val to max_val) + // Descriptor has to be a row vector + // TODO this duplicates some other code + void UpdateRunningMedian(cv::Mat_& histogram, int& hist_sum, cv::Mat_& median, const cv::Mat_& descriptor, bool update, int num_bins, double min_val, double max_val); + void ExtractMedian(cv::Mat_& histogram, int hist_count, cv::Mat_& median, int num_bins, double min_val, double max_val); + + // The linear SVR regressors + SVR_static_lin_regressors AU_SVR_static_appearance_lin_regressors; + SVR_dynamic_lin_regressors AU_SVR_dynamic_appearance_lin_regressors; + + // The linear SVM classifiers + SVM_static_lin AU_SVM_static_appearance_lin; + SVM_dynamic_lin AU_SVM_dynamic_appearance_lin; - // The linear SVR regressors - SVR_static_lin_regressors AU_SVR_static_appearance_lin_regressors; - SVR_dynamic_lin_regressors AU_SVR_dynamic_appearance_lin_regressors; + // The AUs predicted by the model are not always 0 calibrated to a person. That is they don't always predict 0 for a neutral expression + // Keeping track of the predictions we can correct for this, by assuming that at least "ratio" of frames are neutral and subtract that value of prediction, only perform the correction after min_frames + void UpdatePredictionTrack(cv::Mat_& prediction_corr_histogram, int& prediction_correction_count, + std::vector& correction, const std::vector>& predictions, double ratio=0.25, int num_bins = 200, double min_val = -3, double max_val = 5, int min_frames = 10); + void GetSampleHist(cv::Mat_& prediction_corr_histogram, int prediction_correction_count, + std::vector& sample, double ratio, int num_bins = 200, double min_val = 0, double max_val = 5); - // The linear SVM classifiers - SVM_static_lin AU_SVM_static_appearance_lin; - SVM_dynamic_lin AU_SVM_dynamic_appearance_lin; + void PostprocessPredictions(); - // The AUs predicted by the model are not always 0 calibrated to a person. That is they don't always predict 0 for a neutral expression - // Keeping track of the predictions we can correct for this, by assuming that at least "ratio" of frames are neutral and subtract that value of prediction, only perform the correction after min_frames - void UpdatePredictionTrack(cv::Mat_& prediction_corr_histogram, int& prediction_correction_count, vector& correction, const vector>& predictions, double ratio = 0.25, int num_bins = 200, double min_val = -3, double max_val = 5, int min_frames = 10); - void GetSampleHist(cv::Mat_& prediction_corr_histogram, int prediction_correction_count, vector& sample, double ratio, int num_bins = 200, double min_val = 0, double max_val = 5); + std::vector> au_prediction_correction_histogram; + std::vector au_prediction_correction_count; - void PostprocessPredictions(); + // Some dynamic scaling (the logic is that before the extreme versions of expression or emotion are shown, + // it is hard to tell the boundaries, this allows us to scale the model to the most extreme seen) + // They have to be view specific + std::vector> dyn_scaling; + + // Keeping track of predictions for summary stats + cv::Mat_ AU_prediction_track; + cv::Mat_ geom_desc_track; - vector> au_prediction_correction_histogram; - vector au_prediction_correction_count; + double current_time_seconds; - // Some dynamic scaling (the logic is that before the extreme versions of expression or emotion are shown, - // it is hard to tell the boundaries, this allows us to scale the model to the most extreme seen) - // They have to be view specific - vector> dyn_scaling; + // Used for face alignment + cv::Mat_ triangulation; + double align_scale_au; + int align_width_au; + int align_height_au; - // Keeping track of predictions for summary stats - cv::Mat_ AU_prediction_track; - cv::Mat_ geom_desc_track; + double align_scale_out; + int align_width_out; + int align_height_out; - double current_time_seconds; + // Useful placeholder for renormalizing the initial frames of shorter videos + int max_init_frames = 3000; + std::vector> hog_desc_frames_init; + std::vector> geom_descriptor_frames_init; + std::vector views; + bool postprocessed = false; + int frames_tracking_succ = 0; - // Used for face alignment - cv::Mat_ triangulation; - double align_scale; - int align_width; - int align_height; - - // Useful placeholder for renormalizing the initial frames of shorter videos - int max_init_frames = 3000; - vector> hog_desc_frames_init; - vector> geom_descriptor_frames_init; - vector views; - bool postprocessed = false; - int frames_tracking_succ = 0; - - }; - //=========================================================================== +}; + //=========================================================================== } #endif diff --git a/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h new file mode 100644 index 00000000..5473225a --- /dev/null +++ b/lib/local/FaceAnalyser/include/FaceAnalyserParameters.h @@ -0,0 +1,100 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +// Parameters of the Face analyser +#ifndef __FACE_ANALYSER_PARAM_H +#define __FACE_ANALYSER_PARAM_H + +#include +#include + +// Boost includes +#include +#include + +using namespace std; + +namespace FaceAnalysis +{ + +struct FaceAnalyserParameters +{ +public: + // Constructors + FaceAnalyserParameters(); + FaceAnalyserParameters(string root_exe); + FaceAnalyserParameters(vector &arguments); + + // These are the parameters of training and will not change and are fixed + const double sim_scale_au = 0.7; + const int sim_size_au = 112; + + // Should the output aligned faces be grayscale + bool grayscale; + + // Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense + void setAlignedOutput(int output_size, double scale=-1); + // This will also change the model location + void OptimizeForVideos(); + void OptimizeForImages(); + + double getSimScaleOut() const { return sim_scale_out; } + int getSimSizeOut() const { return sim_size_out; } + bool getDynamic() const { return dynamic; } + string getModelLoc() const { return string(model_location); } + vector getOrientationBins() const { return vector(orientation_bins); } + +private: + + void init(); + + // Aligned face output size + double sim_scale_out; + int sim_size_out; + + // Should a video stream be assumed + bool dynamic; + + // Where to load the models from + string model_location; + // The location of the executable + boost::filesystem::path root; + + vector orientation_bins; + +}; + +} + +#endif // __FACE_ANALYSER_PARAM_H diff --git a/lib/local/FaceAnalyser/include/Face_utils.h b/lib/local/FaceAnalyser/include/Face_utils.h index 3783d692..51ebf9b5 100644 --- a/lib/local/FaceAnalyser/include/Face_utils.h +++ b/lib/local/FaceAnalyser/include/Face_utils.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -59,19 +35,19 @@ #ifndef __FACE_UTILS_h_ #define __FACE_UTILS_h_ -#include - #include #include +#include "PDM.h" + namespace FaceAnalysis { //=========================================================================== // Defining a set of useful utility functions to be used within FaceAnalyser // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, const cv::Mat_& triangulation, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); void Extract_FHOG_descriptor(cv::Mat_& descriptor, const cv::Mat& image, int& num_rows, int& num_cols, int cell_size = 8); @@ -81,5 +57,50 @@ namespace FaceAnalysis void ExtractSummaryStatistics(const cv::Mat_& descriptors, cv::Mat_& sum_stats, bool mean, bool stdev, bool max_min); void AddDescriptor(cv::Mat_& descriptors, cv::Mat_ new_descriptor, int curr_frame, int num_frames_to_keep = 120); + //=========================================================================== + // Point set and landmark manipulation functions + //=========================================================================== + // Using Kabsch's algorithm for aligning shapes + //This assumes that align_from and align_to are already mean normalised + cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); + + //============================================================================= + // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other + cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); + + //=========================================================================== + // Visualisation functions + //=========================================================================== + void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy); + + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles); + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix); + + cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler); + + cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle); + + cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle); + + cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix); + + //============================================================================ + // Matrix reading functionality + //============================================================================ + + // Reading a matrix written in a binary format + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat); + + // Reading in a matrix from a stream + void ReadMat(std::ifstream& stream, cv::Mat& output_matrix); + + // Skipping comments (lines starting with # symbol) + void SkipComments(std::ifstream& stream); + } #endif diff --git a/lib/local/FaceAnalyser/include/GazeEstimation.h b/lib/local/FaceAnalyser/include/GazeEstimation.h deleted file mode 100644 index ac92be75..00000000 --- a/lib/local/FaceAnalyser/include/GazeEstimation.h +++ /dev/null @@ -1,72 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, -// all rights reserved. -// -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. -// -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - -// * Any publications arising from the use of this software, including but -// not limited to academic journal and conference publications, technical -// reports and manuals, must cite at least one of the following works: -// -// OpenFace: an open source facial behavior analysis toolkit -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency -// in IEEE Winter Conference on Applications of Computer Vision, 2016 -// -// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation -// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling -// in IEEE International. Conference on Computer Vision (ICCV), 2015 -// -// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection -// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson -// in Facial Expression Recognition and Analysis Challenge, -// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 -// -// Constrained Local Neural Fields for robust facial landmark detection in the wild. -// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. -// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef __GAZEESTIMATION_h_ -#define __GAZEESTIMATION_h_ - -#include "opencv2/core/core.hpp" -#include "LandmarkCoreIncludes.h" - -namespace FaceAnalysis -{ - - void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye); - void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy); - -} -#endif \ No newline at end of file diff --git a/lib/local/FaceAnalyser/include/PAW.h b/lib/local/FaceAnalyser/include/PAW.h new file mode 100644 index 00000000..64ee14df --- /dev/null +++ b/lib/local/FaceAnalyser/include/PAW.h @@ -0,0 +1,138 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __PAW_h_ +#define __PAW_h_ + +// OpenCV includes +#include + +namespace FaceAnalysis +{ + //=========================================================================== + /** + A Piece-wise Affine Warp + The ideas for this piece-wise affine triangular warping are taken from the + Active appearance models revisited by Iain Matthews and Simon Baker in IJCV 2004 + This is used for both validation of landmark detection, and for avatar animation + + The code is based on the CLM tracker by Jason Saragih et al. + */ + +class PAW{ +public: + // Number of pixels after the warping to neutral shape + int number_of_pixels; + + // Minimum x coordinate in destination + double min_x; + + // minimum y coordinate in destination + double min_y; + + // Destination points (landmarks to be warped to) + cv::Mat_ destination_landmarks; + + // Destination points (landmarks to be warped from) + cv::Mat_ source_landmarks; + + // Triangulation, each triangle is warped using an affine transform + cv::Mat_ triangulation; + + // Triangle index, indicating which triangle each of destination pixels lies in + cv::Mat_ triangle_id; + + // Indicating if the destination warped pixels is valid (lies within a face) + cv::Mat_ pixel_mask; + + // A number of precomputed coefficients that are helpful for quick warping + + // affine coefficients for all triangles (see Matthews and Baker 2004) + // 6 coefficients for each triangle (are computed from alpha and beta) + // This is computed during each warp based on source landmarks + cv::Mat_ coefficients; + + // matrix of (c,x,y) coeffs for alpha + cv::Mat_ alpha; + + // matrix of (c,x,y) coeffs for alpha + cv::Mat_ beta; + + // x-source of warped points + cv::Mat_ map_x; + + // y-source of warped points + cv::Mat_ map_y; + + // Default constructor + PAW(){;} + + // Construct a warp from a destination shape and triangulation + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); + + // The final optional argument allows for optimisation if the triangle indices from previous frame are known (for tracking in video) + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y); + + // Copy constructor + PAW(const PAW& other); + + void Read(std::ifstream &s); + + // The actual warping + void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); + + // Compute coefficients needed for warping + void CalcCoeff(); + + // Perform the actual warping + void WarpRegion(cv::Mat_& map_x, cv::Mat_& map_y); + + inline int NumberOfLandmarks() const {return destination_landmarks.rows/2;} ; + inline int NumberOfTriangles() const {return triangulation.rows;} ; + + // The width and height of the warped image + inline int constWidth() const {return pixel_mask.cols;} + inline int Height() const {return pixel_mask.rows;} + +private: + + // Helper functions for dealing with triangles + static bool sameSide(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3); + static bool pointInTriangle(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3); + static int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1); + + }; + //=========================================================================== +} +#endif diff --git a/lib/local/FaceAnalyser/include/PDM.h b/lib/local/FaceAnalyser/include/PDM.h new file mode 100644 index 00000000..bee404ed --- /dev/null +++ b/lib/local/FaceAnalyser/include/PDM.h @@ -0,0 +1,105 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __PDMA_h_ +#define __PDMA_h_ + +// OpenCV includes +#include + +#include +#include +#include + +namespace FaceAnalysis +{ +//=========================================================================== +// A linear 3D Point Distribution Model (constructed using Non-Rigid structure from motion or PCA) +// Only describes the model but does not contain an instance of it (no local or global parameters are stored here) +// Contains the utility functions to help manipulate the model + +class PDM{ + public: + + // The 3D mean shape vector of the PDM [x1,..,xn,y1,...yn,z1,...,zn] + cv::Mat_ mean_shape; + + // Principal components or variation bases of the model, + cv::Mat_ princ_comp; + + // Eigenvalues (variances) corresponding to the bases + cv::Mat_ eigen_values; + + PDM(){;} + + // A copy constructor + PDM(const PDM& other); + + void Read(std::string location); + + // Number of vertices + inline int NumberOfPoints() const {return mean_shape.rows/3;} + + // Listing the number of modes of variation + inline int NumberOfModes() const {return princ_comp.cols;} + + // Compute shape in object space (3D) + void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; + + // Compute shape in image space (2D) + void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; + + // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box + void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; + + // Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results) + void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; + + // provided the model parameters, compute the bounding box of a face + void CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local) const; + + // Helpers for computing Jacobians, and Jacobians with the weight matrix + void ComputeRigidJacobian(const cv::Mat_& params_local, const cv::Vec6d& params_global, cv::Mat_ &Jacob, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const; + void ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& params_global, cv::Mat_ &Jacobian, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const; + + // Given the current parameters, and the computed delta_p compute the updated parameters + void UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& params_local, cv::Vec6d& params_global) const; + + // Helper utilities + private: + static void Orthonormalise(cv::Matx33d &R); + }; + //=========================================================================== +} +#endif diff --git a/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h b/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h index 1e70cd8c..5b801347 100644 --- a/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h +++ b/lib/local/FaceAnalyser/include/SVM_dynamic_lin.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -64,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVM_static_lin.h b/lib/local/FaceAnalyser/include/SVM_static_lin.h index 6de81aa4..3d58d2ab 100644 --- a/lib/local/FaceAnalyser/include/SVM_static_lin.h +++ b/lib/local/FaceAnalyser/include/SVM_static_lin.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -64,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h b/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h index ed31d432..b90b9a3c 100644 --- a/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h +++ b/lib/local/FaceAnalyser/include/SVR_dynamic_lin_regressors.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -64,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h b/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h index 9fc2896b..8f4fc3e7 100644 --- a/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h +++ b/lib/local/FaceAnalyser/include/SVR_static_lin_regressors.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -64,6 +40,7 @@ #include #include +#include #include diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index 5de97f7a..8fbd0255 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -65,6 +41,7 @@ // System includes #include #include +#include #include @@ -75,7 +52,6 @@ #include // Local includes -#include "LandmarkCoreIncludes.h" #include "Face_utils.h" using namespace FaceAnalysis; @@ -83,13 +59,17 @@ using namespace FaceAnalysis; using namespace std; // Constructor from a model file (or a default one if not provided -FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int width, int height, std::string au_location, std::string tri_location) +FaceAnalyser::FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params) { - this->ReadAU(au_location); + this->Read(face_analyser_params.getModelLoc()); - align_scale = scale; - align_width = width; - align_height = height; + align_scale_out = face_analyser_params.getSimScaleOut(); + align_width_out = face_analyser_params.getSimSizeOut(); + align_height_out = face_analyser_params.getSimSizeOut(); + + align_scale_au = face_analyser_params.sim_scale_au; + align_width_au = face_analyser_params.sim_size_au; + align_height_au = face_analyser_params.sim_size_au; // Initialise the histograms that will represent bins from 0 - 1 (as HoG values are only stored as those) num_bins_hog = 1000; @@ -103,15 +83,20 @@ FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int // Keep track for how many frames have been tracked so far frames_tracking = 0; - - if(orientation_bins.empty()) + + // If the model used is dynamic (person callibration and video correction) + dynamic = face_analyser_params.getDynamic(); + + out_grayscale = face_analyser_params.grayscale; + + if(face_analyser_params.getOrientationBins().empty()) { // Just using frontal currently head_orientations.push_back(cv::Vec3d(0,0,0)); } else { - head_orientations = orientation_bins; + head_orientations = face_analyser_params.getOrientationBins(); } hog_hist_sum.resize(head_orientations.size()); face_image_hist_sum.resize(head_orientations.size()); @@ -123,10 +108,6 @@ FaceAnalyser::FaceAnalyser(vector orientation_bins, double scale, int au_prediction_correction_histogram.resize(head_orientations.size()); dyn_scaling.resize(head_orientations.size()); - // The triangulation used for masking out the non-face parts of aligned image - std::ifstream triangulation_file(tri_location); - LandmarkDetector::ReadMat(triangulation_file, triangulation); - } // Utility for getting the names of returned AUs (presence) @@ -267,12 +248,17 @@ int GetViewId(const vector orientations_all, const cv::Vec3d& orienta } -std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const LandmarkDetector::CLNF& clnf, bool visualise) +std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise) { + + // Extract shape parameters from the detected landmarks + cv::Vec6d params_global; + cv::Mat_ params_local; + pdm.CalcParams(params_global, params_local, detected_landmarks); // First align the face - AlignFaceMask(aligned_face_for_au, frame, clnf, triangulation, true, 0.7, 112, 112); - + AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, 0.7, 112, 112); + // Extract HOG descriptor from the frame and convert it to a useable format cv::Mat_ hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); @@ -280,17 +266,17 @@ std::pair>, std::vectorhead_orientations, curr_orient); - + // Geom descriptor and its median - geom_descriptor_frame = clnf.params_local.t(); + geom_descriptor_frame = params_local.t(); // Stack with the actual feature point locations (without mean) - cv::Mat_ locs = clnf.pdm.princ_comp * geom_descriptor_frame.t(); + cv::Mat_ locs = pdm.princ_comp * geom_descriptor_frame.t(); cv::hconcat(locs.t(), geom_descriptor_frame.clone(), geom_descriptor_frame); - + // First convert the face image to double representation as a row vector, TODO rem //cv::Mat_ aligned_face_cols(1, aligned_face_for_au.cols * aligned_face_for_au.rows * aligned_face_for_au.channels(), aligned_face_for_au.data, 1); //cv::Mat_ aligned_face_cols_double; @@ -320,44 +306,54 @@ std::pair>, std::vector& detected_landmarks, bool success, double timestamp_seconds, bool online, bool visualise) { frames_tracking++; + // Extract shape parameters from the detected landmarks + cv::Vec6d params_global; + cv::Mat_ params_local; + pdm.CalcParams(params_global, params_local, detected_landmarks); + // First align the face if tracking was successfull - if (clnf_model.detection_success) + if(success) { // The aligned face requirement for AUs - AlignFaceMask(aligned_face_for_au, frame, clnf_model, triangulation, true, 0.7, 112, 112); + AlignFaceMask(aligned_face_for_au, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_au, align_width_au, align_height_au); // If the output requirement matches use the already computed one, else compute it again - if (align_scale == 0.7 && align_width == 112 && align_height == 112) + if(align_scale_out == align_scale_au && align_width_out == align_width_au && align_height_out == align_height_au) { aligned_face_for_output = aligned_face_for_au.clone(); } else { - AlignFaceMask(aligned_face_for_output, frame, clnf_model, triangulation, true, align_scale, align_width, align_height); + AlignFaceMask(aligned_face_for_output, frame, detected_landmarks, params_global, pdm, triangulation, true, align_scale_out, align_width_out, align_height_out); } } else { - aligned_face_for_output = cv::Mat(align_height, align_width, CV_8UC3); - aligned_face_for_au = cv::Mat(112, 112, CV_8UC3); + aligned_face_for_output = cv::Mat(align_height_out, align_width_out, CV_8UC3); + aligned_face_for_au = cv::Mat(align_height_au, align_width_au, CV_8UC3); aligned_face_for_output.setTo(0); aligned_face_for_au.setTo(0); } + if (aligned_face_for_output.channels() == 3 && out_grayscale) + { + cvtColor(aligned_face_for_output, aligned_face_for_output, CV_BGR2GRAY); + } + // Extract HOG descriptor from the frame and convert it to a useable format cv::Mat_ hog_descriptor; Extract_FHOG_descriptor(hog_descriptor, aligned_face_for_au, this->num_hog_rows, this->num_hog_cols); - + // Store the descriptor hog_desc_frame = hog_descriptor; - cv::Vec3d curr_orient(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); + cv::Vec3d curr_orient(params_global[1], params_global[2], params_global[3]); int orientation_to_use = GetViewId(this->head_orientations, curr_orient); // Only update the running median if predictions are not high @@ -389,33 +385,33 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // } //} - update_median = update_median & clnf_model.detection_success; + update_median = update_median & success; - if (clnf_model.detection_success) + if (success) frames_tracking_succ++; // A small speedup - if (frames_tracking % 2 == 1) + if(frames_tracking % 2 == 1) { UpdateRunningMedian(this->hog_desc_hist[orientation_to_use], this->hog_hist_sum[orientation_to_use], this->hog_desc_median, hog_descriptor, update_median, this->num_bins_hog, this->min_val_hog, this->max_val_hog); this->hog_desc_median.setTo(0, this->hog_desc_median < 0); - } + } // Geom descriptor and its median - geom_descriptor_frame = clnf_model.params_local.t(); - - if (!clnf_model.detection_success) + geom_descriptor_frame = params_local.t(); + + if(!success) { geom_descriptor_frame.setTo(0); } // Stack with the actual feature point locations (without mean) - cv::Mat_ locs = clnf_model.pdm.princ_comp * geom_descriptor_frame.t(); - + cv::Mat_ locs = pdm.princ_comp * geom_descriptor_frame.t(); + cv::hconcat(locs.t(), geom_descriptor_frame.clone(), geom_descriptor_frame); - + // A small speedup - if (frames_tracking % 2 == 1) + if(frames_tracking % 2 == 1) { UpdateRunningMedian(this->geom_desc_hist, this->geom_hist_sum, this->geom_descriptor_median, geom_descriptor_frame, update_median, this->num_bins_geom, this->min_val_geom, this->max_val_geom); } @@ -424,9 +420,9 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL //cv::Mat_ aligned_face_cols(1, aligned_face.cols * aligned_face.rows * aligned_face.channels(), aligned_face.data, 1); //cv::Mat_ aligned_face_cols_double; //aligned_face_cols.convertTo(aligned_face_cols_double, CV_64F); - + // Visualising the median HOG - if (visualise) + if(visualise) { FaceAnalysis::Visualise_FHOG(hog_descriptor, num_hog_rows, num_hog_cols, hog_descriptor_visualisation); } @@ -435,9 +431,9 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_reg = PredictCurrentAUs(orientation_to_use); std::vector> AU_predictions_reg_corrected; - if (online) + if(online) { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success, true); + AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, success, true); } // Add the reg predictions to the historic data @@ -446,7 +442,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // Find the appropriate AU (if not found add it) // Only add if the detection was successful - if (clnf_model.detection_success) + if(success) { AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second); } @@ -455,7 +451,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(0); } } - + AU_predictions_class = PredictCurrentAUsClass(orientation_to_use); for (size_t au = 0; au < AU_predictions_class.size(); ++au) @@ -463,7 +459,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL // Find the appropriate AU (if not found add it) // Only add if the detection was successful - if (clnf_model.detection_success) + if(success) { AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second); } @@ -472,15 +468,15 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(0); } } + - - if (online) + if(online) { AU_predictions_reg = AU_predictions_reg_corrected; } else { - if (clnf_model.detection_success && frames_tracking_succ - 1 < max_init_frames) + if (success && frames_tracking_succ - 1 < max_init_frames) { hog_desc_frames_init.push_back(hog_descriptor); geom_descriptor_frames_init.push_back(geom_descriptor_frame); @@ -491,10 +487,7 @@ void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const LandmarkDetector::CL this->current_time_seconds = timestamp_seconds; view_used = orientation_to_use; - - bool success = clnf_model.detection_success; - - confidences.push_back(clnf_model.detection_certainty); + valid_preds.push_back(success); timestamps.push_back(timestamp_seconds); @@ -505,79 +498,6 @@ void FaceAnalyser::GetGeomDescriptor(cv::Mat_& geom_desc) geom_desc = this->geom_descriptor_frame.clone(); } -void FaceAnalyser::PredictAUs(const cv::Mat_& hog_features, const cv::Mat_& geom_features, const LandmarkDetector::CLNF& clnf_model, bool online) -{ - // Store the descriptor - hog_desc_frame = hog_features.clone(); - this->geom_descriptor_frame = geom_features.clone(); - - cv::Vec3d curr_orient(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); - int orientation_to_use = GetViewId(this->head_orientations, curr_orient); - - // Perform AU prediction - AU_predictions_reg = PredictCurrentAUs(orientation_to_use); - - std::vector> AU_predictions_reg_corrected; - if(online) - { - AU_predictions_reg_corrected = CorrectOnlineAUs(AU_predictions_reg, orientation_to_use, true, false, clnf_model.detection_success); - } - - // Add the reg predictions to the historic data - for (size_t au = 0; au < AU_predictions_reg.size(); ++au) - { - - // Find the appropriate AU (if not found add it) - // Only add if the detection was successful - if(clnf_model.detection_success) - { - AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(AU_predictions_reg[au].second); - } - else - { - AU_predictions_reg_all_hist[AU_predictions_reg[au].first].push_back(0.0); - } - } - - AU_predictions_class = PredictCurrentAUsClass(orientation_to_use); - - for (size_t au = 0; au < AU_predictions_class.size(); ++au) - { - - // Find the appropriate AU (if not found add it) - // Only add if the detection was successful - if(clnf_model.detection_success) - { - AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(AU_predictions_class[au].second); - } - else - { - AU_predictions_class_all_hist[AU_predictions_class[au].first].push_back(0.0); - } - } - - if(online) - { - AU_predictions_reg = AU_predictions_reg_corrected; - } - - for(size_t i = 0; i < AU_predictions_reg.size(); ++i) - { - AU_predictions_combined.push_back(AU_predictions_reg[i]); - } - for(size_t i = 0; i < AU_predictions_class.size(); ++i) - { - AU_predictions_combined.push_back(AU_predictions_class[i]); - } - - view_used = orientation_to_use; - - bool success = clnf_model.detection_success; - - confidences.push_back(clnf_model.detection_certainty); - valid_preds.push_back(success); -} - // Perform prediction on initial n frames anew as the current neutral face estimate is better now void FaceAnalyser::PostprocessPredictions() { @@ -636,7 +556,6 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector> aus_valid; vector offsets; - confidences = this->confidences; successes = this->valid_preds; vector dyn_au_names = AU_SVR_dynamic_appearance_lin_regressors.GetAUNames(); @@ -683,7 +602,7 @@ void FaceAnalyser::ExtractAllPredictionsOfflineReg(vector au_vals_tmp = au_iter->second; - for (int i = (window_size - 1) / 2; i < (int)au_iter->second.size() - (window_size - 1) / 2; ++i) + for (size_t i = (window_size - 1) / 2; i < au_iter->second.size() - (window_size - 1) / 2; ++i) { double sum = 0; - int count_over = 0; for (int w = -(window_size - 1) / 2; w <= (window_size - 1) / 2; ++w) { sum += au_vals_tmp[i + w]; - count_over++; } - sum = sum / count_over; + sum = sum / window_size; au_iter->second[i] = sum; } @@ -765,16 +682,14 @@ void FaceAnalyser::ExtractAllPredictionsOfflineClass(vector au_vals_tmp = au_vals; - for (int i = (window_size - 1)/2; i < (int)au_vals.size() - (window_size - 1) / 2; ++i) + for (size_t i = (window_size - 1)/2; i < au_vals.size() - (window_size - 1) / 2; ++i) { double sum = 0; - int count_over = 0; for (int w = -(window_size - 1) / 2; w <= (window_size - 1) / 2; ++w) { sum += au_vals_tmp[i + w]; - count_over++; } - sum = sum / count_over; + sum = sum / window_size; if (sum < 0.5) sum = 0; else @@ -787,7 +702,6 @@ void FaceAnalyser::ExtractAllPredictionsOfflineClass(vectorconfidences; successes = this->valid_preds; } @@ -830,7 +744,6 @@ void FaceAnalyser::Reset() timestamps.clear(); AU_predictions_reg_all_hist.clear(); AU_predictions_class_all_hist.clear(); - confidences.clear(); valid_preds.clear(); // Clean up the postprocessing data as well @@ -1086,10 +999,78 @@ vector> FaceAnalyser::GetCurrentAUsCombined() const return AU_predictions_combined; } +void FaceAnalyser::Read(std::string model_loc) +{ + // Reading in the modules for AU recognition + + cout << "Reading the AU analysis module from: " << model_loc << endl; + + ifstream locations(model_loc.c_str(), ios_base::in); + if (!locations.is_open()) + { + cout << "Couldn't open the model file, aborting" << endl; + return; + } + string line; + + // The other module locations should be defined as relative paths from the main model + boost::filesystem::path root = boost::filesystem::path(model_loc).parent_path(); + + // The main file contains the references to other files + while (!locations.eof()) + { + getline(locations, line); + + stringstream lineStream(line); + + string module; + string location; + + // figure out which module is to be read from which file + lineStream >> module; + + lineStream >> location; + + // remove carriage return at the end for compatibility with unix systems + if (location.size() > 0 && location.at(location.size() - 1) == '\r') + { + location = location.substr(0, location.size() - 1); + } + + // append to root + location = (root / location).string(); + if (module.compare("AUPredictor") == 0) + { + // The AU predictors + cout << "Reading the AU predictors from: " << location; + ReadAU(location); + cout << "... Done" << endl; + } + else if (module.compare("PDM") == 0) + { + cout << "Reading the PDM from: " << location; + pdm = PDM::PDM(); + pdm.Read(location); + cout << "... Done" << endl; + } + else if (module.compare("Triangulation") == 0) + { + cout << "Reading the triangulation from:" << location; + // The triangulation used for masking out the non-face parts of aligned image + std::ifstream triangulation_file(location); + ReadMat(triangulation_file, triangulation); + cout << "... Done" << endl; + } + } + +} + // Reading in AU prediction modules void FaceAnalyser::ReadAU(std::string au_model_location) { + + // Open the list of the regressors in the file ifstream locations(au_model_location.c_str(), ios::in); @@ -1121,7 +1102,7 @@ void FaceAnalyser::ReadAU(std::string au_model_location) // Parse comma separated names that this regressor produces name = lineStream.str(); - int index = name.find_first_of(' '); + int index = (int)name.find_first_of(' '); if(index >= 0) { @@ -1155,13 +1136,13 @@ void FaceAnalyser::UpdatePredictionTrack(cv::Mat_& prediction_corr // The median update if(prediction_corr_histogram.empty()) { - prediction_corr_histogram = cv::Mat_(predictions.size(), num_bins, (unsigned int)0); + prediction_corr_histogram = cv::Mat_((int)predictions.size(), num_bins, (unsigned int)0); } for(int i = 0; i < prediction_corr_histogram.rows; ++i) { // Find the bins corresponding to the current descriptor - int index = (predictions[i].second - min_val)*((double)num_bins)/(length); + int index = (int)((predictions[i].second - min_val)*((double)num_bins)/(length)); if(index < 0) { index = 0; @@ -1179,7 +1160,7 @@ void FaceAnalyser::UpdatePredictionTrack(cv::Mat_& prediction_corr if(prediction_correction_count >= min_frames) { // Recompute the correction - int cutoff_point = ratio * prediction_correction_count; + int cutoff_point = (int)(ratio * prediction_correction_count); // For each dimension for(int i = 0; i < prediction_corr_histogram.rows; ++i) @@ -1209,7 +1190,7 @@ void FaceAnalyser::GetSampleHist(cv::Mat_& prediction_corr_histogr sample.resize(prediction_corr_histogram.rows, 0); // Recompute the correction - int cutoff_point = ratio * prediction_correction_count; + int cutoff_point = (int)(ratio * prediction_correction_count); // For each dimension for(int i = 0; i < prediction_corr_histogram.rows; ++i) @@ -1260,7 +1241,7 @@ double FaceAnalyser::GetCurrentTimeSeconds() { } // Allows for post processing of the AU signal -void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) +void FaceAnalyser::PostprocessOutputFile(string output_file) { vector certainties; @@ -1273,8 +1254,8 @@ void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) ExtractAllPredictionsOfflineReg(predictions_reg, certainties, successes, timestamps, dynamic); ExtractAllPredictionsOfflineClass(predictions_class, certainties, successes, timestamps, dynamic); - int num_class = predictions_class.size(); - int num_reg = predictions_reg.size(); + int num_class = (int)predictions_class.size(); + int num_reg = (int)predictions_reg.size(); // Extract the indices of writing out first vector au_reg_names = GetAURegNames(); @@ -1331,7 +1312,7 @@ void FaceAnalyser::PostprocessOutputFile(string output_file, bool dynamic) { if (tokens[i].find("AU") != string::npos && begin_ind == -1) { - begin_ind = i; + begin_ind = (int)i; break; } } diff --git a/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp new file mode 100644 index 00000000..05ef0074 --- /dev/null +++ b/lib/local/FaceAnalyser/src/FaceAnalyserParameters.cpp @@ -0,0 +1,254 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "FaceAnalyserParameters.h" + +// System includes +#include +#include +#include + +#ifndef CONFIG_DIR +#define CONFIG_DIR "~" +#endif + +using namespace std; + +using namespace FaceAnalysis; + +FaceAnalyserParameters::FaceAnalyserParameters():root() +{ + // initialise the default values + init(); +} + +FaceAnalyserParameters::FaceAnalyserParameters(string root_dir) +{ + this->root = root_dir; + init(); + +} +FaceAnalyserParameters::FaceAnalyserParameters(vector &arguments):root() +{ + + // First element is reserved for the executable location (useful for finding relative model locs) + this->root = boost::filesystem::path(arguments[0]).parent_path(); + + // initialise the default values + init(); + + bool* valid = new bool[arguments.size()]; + valid[0] = true; + + bool scale_set = false; + bool size_set = false; + + for (size_t i = 1; i < arguments.size(); ++i) + { + valid[i] = true; + + if (arguments[i].compare("-au_static") == 0) + { + dynamic = false; + valid[i] = false; + } + else if (arguments[i].compare("-g") == 0) + { + grayscale = true; + valid[i] = false; + } + else if (arguments[i].compare("-simscale") == 0) + { + sim_scale_out = stod(arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + scale_set = true; + i++; + } + else if (arguments[i].compare("-simsize") == 0) + { + sim_size_out = stoi(arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + size_set = true; + i++; + } + } + + for (int i = (int)arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + if (dynamic) + { + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + } + else + { + this->model_location = "AU_predictors/main_static_svms.txt"; + } + + // If we set the size but not the scale, adapt the scale to the right size + if (!scale_set && size_set) sim_scale_out = sim_size_out * (0.7 / 112.0); + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root/model_path)) + { + this->model_location = (root/model_path).string(); + } + else if (boost::filesystem::exists(config_path/model_path)) + { + this->model_location = (config_path/model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } +} + +void FaceAnalyserParameters::init() +{ + // Initialize default parameter values + this->dynamic = true; + this->grayscale = false; + this->sim_scale_out = 0.7; + this->sim_size_out = 112; + + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } + + orientation_bins = vector(); + +} + +// Use getters and setters for these as they might need to reload models and make sure the scale and size ratio makes sense +void FaceAnalyserParameters::setAlignedOutput(int output_size, double scale) +{ + this->sim_size_out = output_size; + // If we set the size but not the scale, adapt the scale to the right size + if (scale ==-1) this->sim_scale_out = sim_size_out * (0.7 / 112.0); + else this->sim_scale_out = sim_scale_out; + +} +// This will also change the model location +void FaceAnalyserParameters::OptimizeForVideos() +{ + // Set the post-processing to true and load a dynamic model + dynamic = true; + + this->model_location = "AU_predictors/main_dynamic_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } + +} + +void FaceAnalyserParameters::OptimizeForImages() +{ + // Set the post-processing to true and load a dynamic model + dynamic = false; + + this->model_location = "AU_predictors/main_static_svms.txt"; + + // Make sure model_location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); + boost::filesystem::path model_path = boost::filesystem::path(this->model_location); + if (boost::filesystem::exists(model_path)) + { + this->model_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + this->model_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + this->model_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the AU detection model to load" << std::endl; + } +} + diff --git a/lib/local/FaceAnalyser/src/Face_utils.cpp b/lib/local/FaceAnalyser/src/Face_utils.cpp index b6d4eb58..f9a6fd30 100644 --- a/lib/local/FaceAnalyser/src/Face_utils.cpp +++ b/lib/local/FaceAnalyser/src/Face_utils.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -57,13 +33,16 @@ /////////////////////////////////////////////////////////////////////////////// #include +#include // OpenCV includes #include #include +#include // For FHOG visualisation #include +#include using namespace std; @@ -136,15 +115,15 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, double sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = clnf_model.pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = clnf_model.detected_landmarks.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points @@ -153,7 +132,7 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - cv::Matx22d scale_rot_matrix = LandmarkDetector::AlignShapesWithScale(source_landmarks, destination_landmarks); + cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); cv::Matx23d warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); @@ -161,8 +140,8 @@ namespace FaceAnalysis warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = clnf_model.params_global[4]; - double ty = clnf_model.params_global[5]; + double tx = params_global[4]; + double ty = params_global[5]; cv::Vec2d T(tx, ty); T = scale_rot_matrix * T; @@ -175,15 +154,15 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const LandmarkDetector::CLNF& clnf_model, const cv::Mat_& triangulation, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid, double sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = clnf_model.pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = clnf_model.detected_landmarks.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points @@ -192,7 +171,7 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - cv::Matx22d scale_rot_matrix = LandmarkDetector::AlignShapesWithScale(source_landmarks, destination_landmarks); + cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); cv::Matx23d warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); @@ -200,8 +179,8 @@ namespace FaceAnalysis warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = clnf_model.params_global[4]; - double ty = clnf_model.params_global[5]; + double tx = params_global[4]; + double ty = params_global[5]; cv::Vec2d T(tx, ty); T = scale_rot_matrix * T; @@ -215,7 +194,7 @@ namespace FaceAnalysis // Move the destination landmarks there as well cv::Matx22d warp_matrix_2d(warp_matrix(0,0), warp_matrix(0,1), warp_matrix(1,0), warp_matrix(1,1)); - destination_landmarks = cv::Mat(clnf_model.detected_landmarks.reshape(1, 2).t()) * cv::Mat(warp_matrix_2d).t(); + destination_landmarks = cv::Mat(detected_landmarks.reshape(1, 2).t()) * cv::Mat(warp_matrix_2d).t(); destination_landmarks.col(0) = destination_landmarks.col(0) + warp_matrix(0,2); destination_landmarks.col(1) = destination_landmarks.col(1) + warp_matrix(1,2); @@ -237,7 +216,7 @@ namespace FaceAnalysis destination_landmarks = cv::Mat(destination_landmarks.t()).reshape(1, 1).t(); - LandmarkDetector::PAW paw(destination_landmarks, triangulation, 0, 0, aligned_face.cols-1, aligned_face.rows-1); + PAW paw(destination_landmarks, triangulation, 0, 0, aligned_face.cols-1, aligned_face.rows-1); // Mask each of the channels (a bit of a roundabout way, but OpenCV 3.1 in debug mode doesn't seem to be able to handle a more direct way using split and merge) vector aligned_face_channels(aligned_face.channels()); @@ -379,4 +358,298 @@ namespace FaceAnalysis new_descriptor.copyTo(descriptors.row(row_to_change)); } + //=========================================================================== + // Point set and landmark manipulation functions + //=========================================================================== + // Using Kabsch's algorithm for aligning shapes + //This assumes that align_from and align_to are already mean normalised + cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to) + { + + cv::SVD svd(align_from.t() * align_to); + + // make sure no reflection is there + // corr ensures that we do only rotaitons and not reflections + double d = cv::determinant(svd.vt.t() * svd.u.t()); + + cv::Matx22d corr = cv::Matx22d::eye(); + if (d > 0) + { + corr(1, 1) = 1; + } + else + { + corr(1, 1) = -1; + } + + cv::Matx22d R; + cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); + + return R; + } + + //============================================================================= + // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other + cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) + { + int n = src.rows; + + // First we mean normalise both src and dst + double mean_src_x = cv::mean(src.col(0))[0]; + double mean_src_y = cv::mean(src.col(1))[0]; + + double mean_dst_x = cv::mean(dst.col(0))[0]; + double mean_dst_y = cv::mean(dst.col(1))[0]; + + cv::Mat_ src_mean_normed = src.clone(); + src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; + src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; + + cv::Mat_ dst_mean_normed = dst.clone(); + dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; + dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; + + // Find the scaling factor of each + cv::Mat src_sq; + cv::pow(src_mean_normed, 2, src_sq); + + cv::Mat dst_sq; + cv::pow(dst_mean_normed, 2, dst_sq); + + double s_src = sqrt(cv::sum(src_sq)[0] / n); + double s_dst = sqrt(cv::sum(dst_sq)[0] / n); + + src_mean_normed = src_mean_normed / s_src; + dst_mean_normed = dst_mean_normed / s_dst; + + double s = s_dst / s_src; + + // Get the rotation + cv::Matx22d R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); + + cv::Matx22d A; + cv::Mat(s * R).copyTo(A); + + cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); + cv::Mat_ offset = dst - aligned; + + double t_x = cv::mean(offset.col(0))[0]; + double t_y = cv::mean(offset.col(1))[0]; + + return A; + + } + + + //=========================================================================== + // Visualisation functions + //=========================================================================== + void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) + { + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + double X, Y, Z; + + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + double x; + double y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + + } + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) + { + cv::Matx33d rotation_matrix; + + double s1 = sin(eulerAngles[0]); + double s2 = sin(eulerAngles[1]); + double s3 = sin(eulerAngles[2]); + + double c1 = cos(eulerAngles[0]); + double c2 = cos(eulerAngles[1]); + double c3 = cos(eulerAngles[2]); + + rotation_matrix(0, 0) = c2 * c3; + rotation_matrix(0, 1) = -c2 *s3; + rotation_matrix(0, 2) = s2; + rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; + rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; + rotation_matrix(1, 2) = -c2 * s1; + rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; + rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; + rotation_matrix(2, 2) = c1 * c2; + + return rotation_matrix; + } + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) + { + double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0; + double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0); + double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0); + double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0); + + double t1 = 2.0 * (q0*q2 + q1*q3); + + double yaw = asin(2.0 * (q0*q2 + q1*q3)); + double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); + double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); + + return cv::Vec3d(pitch, yaw, roll); + } + + cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) + { + cv::Matx33d rotMatrix = Euler2RotationMatrix(euler); + cv::Vec3d axis_angle; + cv::Rodrigues(rotMatrix, axis_angle); + return axis_angle; + } + + cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return RotationMatrix2Euler(rotation_matrix); + } + + cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return rotation_matrix; + } + + cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) + { + cv::Vec3d axis_angle; + cv::Rodrigues(rotation_matrix, axis_angle); + return axis_angle; + } + + + //============================================================================ + // Matrix reading functionality + //============================================================================ + + // Reading in a matrix from a stream + void ReadMat(std::ifstream& stream, cv::Mat &output_mat) + { + // Read in the number of rows, columns and the data type + int row, col, type; + + stream >> row >> col >> type; + + output_mat = cv::Mat(row, col, type); + + switch (output_mat.type()) + { + case CV_64FC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_32FC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_32SC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + case CV_8UC1: + { + cv::MatIterator_ begin_it = output_mat.begin(); + cv::MatIterator_ end_it = output_mat.end(); + while (begin_it != end_it) + { + stream >> *begin_it++; + } + } + break; + default: + printf("ERROR(%s,%d) : Unsupported Matrix type %d!\n", __FILE__, __LINE__, output_mat.type()); abort(); + + + } + } + + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) + { + // Read in the number of rows, columns and the data type + int row, col, type; + + stream.read((char*)&row, 4); + stream.read((char*)&col, 4); + stream.read((char*)&type, 4); + + output_mat = cv::Mat(row, col, type); + int size = output_mat.rows * output_mat.cols * output_mat.elemSize(); + stream.read((char *)output_mat.data, size); + + } + + // Skipping lines that start with # (together with empty lines) + void SkipComments(std::ifstream& stream) + { + while (stream.peek() == '#' || stream.peek() == '\n' || stream.peek() == ' ' || stream.peek() == '\r') + { + std::string skipped; + std::getline(stream, skipped); + } + } + + } \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/PAW.cpp b/lib/local/FaceAnalyser/src/PAW.cpp new file mode 100644 index 00000000..4c5e397c --- /dev/null +++ b/lib/local/FaceAnalyser/src/PAW.cpp @@ -0,0 +1,500 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + + +#include "PAW.h" + +// OpenCV includes +#include +#include + +#include "Face_utils.h" + +using namespace FaceAnalysis; + +// Copy constructor +PAW::PAW(const PAW& other) : destination_landmarks(other.destination_landmarks.clone()), source_landmarks(other.source_landmarks.clone()), triangulation(other.triangulation.clone()), +triangle_id(other.triangle_id.clone()), pixel_mask(other.pixel_mask.clone()), coefficients(other.coefficients.clone()), alpha(other.alpha.clone()), beta(other.beta.clone()), map_x(other.map_x.clone()), map_y(other.map_y.clone()) +{ + this->number_of_pixels = other.number_of_pixels; + this->min_x = other.min_x; + this->min_y = other.min_y; +} + +// A constructor from destination shape and triangulation +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) +{ + // Initialise some variables directly + this->destination_landmarks = destination_shape; + this->triangulation = triangulation; + + int num_points = destination_shape.rows/2; + + int num_tris = triangulation.rows; + + // Pre-compute the rest + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + + // Create a vector representation of the control points + std::vector> destination_points; + + for (int tri = 0; tri < num_tris; ++tri) + { + int j = triangulation.at(tri, 0); + int k = triangulation.at(tri, 1); + int l = triangulation.at(tri, 2); + + double c1 = ys.at(l) - ys.at(j); + double c2 = xs.at(l) - xs.at(j); + double c4 = ys.at(k) - ys.at(j); + double c3 = xs.at(k) - xs.at(j); + + double c5 = c3*c1 - c2*c4; + + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; + + // Add points corresponding to triangles as optimisation + std::vector triangle_points(10); + + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + + double min_x, max_x, min_y, max_y; + cv::minMaxIdx(xs_three, &min_x, &max_x); + cv::minMaxIdx(ys_three, &min_y, &max_y); + + triangle_points[6] = max_x; + triangle_points[7] = max_y; + + triangle_points[8] = min_x; + triangle_points[9] = min_y; + + destination_points.push_back(triangle_points); + + } + + double max_x; + double max_y; + + minMaxLoc(xs, &min_x, &max_x); + minMaxLoc(ys, &min_y, &max_y); + + int w = (int)(max_x - min_x + 1.5); + int h = (int)(max_y - min_y + 1.5); + + // Round the min_x and min_y for simplicity? + + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + + int curr_tri = -1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + for(int x = 0; x < pixel_mask.cols; x++) + { + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + // If there is a triangle at this location + if(curr_tri != -1) + { + triangle_id.at(y, x) = curr_tri; + pixel_mask.at(y, x) = 1; + } + } + } + + // Preallocate maps and coefficients + coefficients.create(num_tris, 6); + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + + +} + +// Manually define min and max values +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y) +{ + // Initialise some variables directly + this->destination_landmarks = destination_shape; + this->triangulation = triangulation; + + int num_points = destination_shape.rows/2; + + int num_tris = triangulation.rows; + + // Pre-compute the rest + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + + // Create a vector representation of the control points + std::vector> destination_points; + + for (int tri = 0; tri < num_tris; ++tri) + { + int j = triangulation.at(tri, 0); + int k = triangulation.at(tri, 1); + int l = triangulation.at(tri, 2); + + double c1 = ys.at(l) - ys.at(j); + double c2 = xs.at(l) - xs.at(j); + double c4 = ys.at(k) - ys.at(j); + double c3 = xs.at(k) - xs.at(j); + + double c5 = c3*c1 - c2*c4; + + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; + + // Add points corresponding to triangles as optimisation + std::vector triangle_points(10); + + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + + double min_x, max_x, min_y, max_y; + cv::minMaxIdx(xs_three, &min_x, &max_x); + cv::minMaxIdx(ys_three, &min_y, &max_y); + + triangle_points[6] = max_x; + triangle_points[7] = max_y; + + triangle_points[8] = min_x; + triangle_points[9] = min_y; + + destination_points.push_back(triangle_points); + + } + + double max_x; + double max_y; + + min_x = in_min_x; + min_y = in_min_y; + + max_x = in_max_x; + max_y = in_max_y; + + int w = (int)(max_x - min_x + 1.5); + int h = (int)(max_y - min_y + 1.5); + + // Round the min_x and min_y for simplicity? + + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + + int curr_tri = -1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + for(int x = 0; x < pixel_mask.cols; x++) + { + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + // If there is a triangle at this location + if(curr_tri != -1) + { + triangle_id.at(y, x) = curr_tri; + pixel_mask.at(y, x) = 1; + } + } + } + + // Preallocate maps and coefficients + coefficients.create(num_tris, 6); + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + +} + +//=========================================================================== +void PAW::Read(std::ifstream& stream) +{ + + stream.read ((char*)&number_of_pixels, 4); + stream.read ((char*)&min_x, 8); + stream.read ((char*)&min_y, 8); + + ReadMatBin(stream, destination_landmarks); + + ReadMatBin(stream, triangulation); + + ReadMatBin(stream, triangle_id); + + cv::Mat tmpMask; + ReadMatBin(stream, tmpMask); + tmpMask.convertTo(pixel_mask, CV_8U); + + ReadMatBin(stream, alpha); + + ReadMatBin(stream, beta); + + map_x.create(pixel_mask.rows,pixel_mask.cols); + map_y.create(pixel_mask.rows,pixel_mask.cols); + + coefficients.create(this->NumberOfTriangles(),6); + + source_landmarks = destination_landmarks; +} + +//============================================================================= +// cropping from the source image to the destination image using the shape in s, used to determine if shape fitting converged successfully +void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) +{ + + // set the current shape + source_landmarks = landmarks_to_warp.clone(); + + // prepare the mapping coefficients using the current shape + this->CalcCoeff(); + + // Do the actual mapping computation (where to warp from) + this->WarpRegion(map_x, map_y); + + // Do the actual warp (with bi-linear interpolation) + remap(image_to_warp, destination_image, map_x, map_y, CV_INTER_LINEAR); + +} + + +//============================================================================= +// Calculate the warping coefficients +void PAW::CalcCoeff() +{ + int p = this->NumberOfLandmarks(); + + for(int l = 0; l < this->NumberOfTriangles(); l++) + { + + int i = triangulation.at(l,0); + int j = triangulation.at(l,1); + int k = triangulation.at(l,2); + + double c1 = source_landmarks.at(i , 0); + double c2 = source_landmarks.at(j , 0) - c1; + double c3 = source_landmarks.at(k , 0) - c1; + double c4 = source_landmarks.at(i + p, 0); + double c5 = source_landmarks.at(j + p, 0) - c4; + double c6 = source_landmarks.at(k + p, 0) - c4; + + // Get a pointer to the coefficient we will be precomputing + double *coeff = coefficients.ptr(l); + + // Extract the relevant alphas and betas + double *c_alpha = alpha.ptr(l); + double *c_beta = beta.ptr(l); + + coeff[0] = c1 + c2 * c_alpha[0] + c3 * c_beta[0]; + coeff[1] = c2 * c_alpha[1] + c3 * c_beta[1]; + coeff[2] = c2 * c_alpha[2] + c3 * c_beta[2]; + coeff[3] = c4 + c5 * c_alpha[0] + c6 * c_beta[0]; + coeff[4] = c5 * c_alpha[1] + c6 * c_beta[1]; + coeff[5] = c5 * c_alpha[2] + c6 * c_beta[2]; + } +} + +//====================================================================== +// Compute the mapping coefficients +void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) +{ + + cv::MatIterator_ xp = mapx.begin(); + cv::MatIterator_ yp = mapy.begin(); + cv::MatIterator_ mp = pixel_mask.begin(); + cv::MatIterator_ tp = triangle_id.begin(); + + // The coefficients corresponding to the current triangle + double * a; + + // Current triangle being processed + int k=-1; + + for(int y = 0; y < pixel_mask.rows; y++) + { + double yi = double(y) + min_y; + + for(int x = 0; x < pixel_mask.cols; x++) + { + double xi = double(x) + min_x; + + if(*mp == 0) + { + *xp = -1; + *yp = -1; + } + else + { + // triangle corresponding to the current pixel + int j = *tp; + + // If it is different from the previous triangle point to new coefficients + // This will always be the case in the first iteration, hence a will not point to nothing + if(j != k) + { + // Update the coefficient pointer if a new triangle is being processed + a = coefficients.ptr(j); + k = j; + } + + //ap is now the pointer to the coefficients + double *ap = a; + + //look at the first coefficient (and increment). first coefficient is an x offset + double xo = *ap++; + //second coefficient is an x scale as a function of x + xo += *ap++ * xi; + //third coefficient ap(2) is an x scale as a function of y + *xp = float(xo + *ap++ * yi); + + //then fourth coefficient ap(3) is a y offset + double yo = *ap++; + //fifth coeff adds coeff[4]*x to y + yo += *ap++ * xi; + //final coeff adds coeff[5]*y to y + *yp = float(yo + *ap++ * yi); + + } + mp++; tp++; xp++; yp++; + } + } +} + +// ============================================================ +// Helper functions to determine which point a triangle lies in +// ============================================================ + +// Is the point (x0,y0) on same side as a half-plane defined by (x1,y1), (x2, y2), and (x3, y3) +bool PAW::sameSide(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +{ + + double x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); + double y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); + + return x*y >= 0; + +} + +// if point (x0, y0) is on same side for all three half-planes it is in a triangle +bool PAW::pointInTriangle(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +{ + bool same_1 = sameSide(x0, y0, x1, y1, x2, y2, x3, y3); + bool same_2 = sameSide(x0, y0, x2, y2, x1, y1, x3, y3); + bool same_3 = sameSide(x0, y0, x3, y3, x1, y1, x2, y2); + + return same_1 && same_2 && same_3; + +} + +// Find if a given point lies in the triangles +int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) +{ + + int num_tris = control_points.size(); + + int tri = -1; + + double x0 = point.x; + double y0 = point.y; + + // Allow a guess for speed (so as not to go through all triangles) + if(guess != -1) + { + + bool in_triangle = pointInTriangle(x0, y0, control_points[guess][0], control_points[guess][1], control_points[guess][2], control_points[guess][3], control_points[guess][4], control_points[guess][5]); + if(in_triangle) + { + return guess; + } + } + + + for (int i = 0; i < num_tris; ++i) + { + + double max_x = control_points[i][6]; + double max_y = control_points[i][7]; + + double min_x = control_points[i][8]; + double min_y = control_points[i][9]; + + // Skip the check if the point is outside the bounding box of the triangle + + if( max_x < x0 || min_x > x0 || max_y < y0 || min_y > y0) + { + continue; + } + + bool in_triangle = pointInTriangle(x0, y0, + control_points[i][0], control_points[i][1], + control_points[i][2], control_points[i][3], + control_points[i][4], control_points[i][5]); + + if(in_triangle) + { + tri = i; + break; + } + } + return tri; +} \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/PDM.cpp b/lib/local/FaceAnalyser/src/PDM.cpp new file mode 100644 index 00000000..29945aed --- /dev/null +++ b/lib/local/FaceAnalyser/src/PDM.cpp @@ -0,0 +1,613 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + + +#include + +// OpenCV include +#include +#include + +// Math includes +#define _USE_MATH_DEFINES +#include + +#ifndef M_PI + #define M_PI 3.14159265358979323846 +#endif + +#include + +using namespace FaceAnalysis; +//=========================================================================== + +//============================================================================= +// Orthonormalising the 3x3 rotation matrix +void PDM::Orthonormalise(cv::Matx33d &R) +{ + + cv::SVD svd(R,cv::SVD::MODIFY_A); + + // get the orthogonal matrix from the initial rotation matrix + cv::Mat_ X = svd.u*svd.vt; + + // This makes sure that the handedness is preserved and no reflection happened + // by making sure the determinant is 1 and not -1 + cv::Mat_ W = cv::Mat_::eye(3,3); + double d = determinant(X); + W(2,2) = determinant(X); + cv::Mat Rt = svd.u*W*svd.vt; + + Rt.copyTo(R); + +} + +// A copy constructor +PDM::PDM(const PDM& other) { + + // Make sure the matrices are allocated properly + this->mean_shape = other.mean_shape.clone(); + this->princ_comp = other.princ_comp.clone(); + this->eigen_values = other.eigen_values.clone(); +} + +//=========================================================================== +// Compute the 3D representation of shape (in object space) using the local parameters +void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local) const +{ + out_shape.create(mean_shape.rows, mean_shape.cols); + out_shape = mean_shape + princ_comp*p_local; +} + +//=========================================================================== +// Get the 2D shape (in image space) from global and local parameters +void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const +{ + + int n = this->NumberOfPoints(); + + double s = params_global[0]; // scaling factor + double tx = params_global[4]; // x offset + double ty = params_global[5]; // y offset + + // get the rotation matrix from the euler angles + cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33d currRot = Euler2RotationMatrix(euler); + + // get the 3D shape of the object + cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; + + // create the 2D shape matrix (if it has not been defined yet) + if((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) + { + out_shape.create(2*n,1); + } + // for every vertex + for(int i = 0; i < n; i++) + { + // Transform this using the weak-perspective mapping to 2D from 3D + out_shape.at(i ,0) = s * ( currRot(0,0) * Shape_3D.at(i, 0) + currRot(0,1) * Shape_3D.at(i+n ,0) + currRot(0,2) * Shape_3D.at(i+n*2,0) ) + tx; + out_shape.at(i+n,0) = s * ( currRot(1,0) * Shape_3D.at(i, 0) + currRot(1,1) * Shape_3D.at(i+n ,0) + currRot(1,2) * Shape_3D.at(i+n*2,0) ) + ty; + } +} + +//=========================================================================== +// provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box +// This all assumes that the bounding box describes face from left outline to right outline of the face and chin to eyebrows +void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation) const +{ + + // get the shape instance based on local params + cv::Mat_ current_shape(mean_shape.size()); + + CalcShape3D(current_shape, params_local); + + // rotate the shape + cv::Matx33d rotation_matrix = Euler2RotationMatrix(rotation); + + cv::Mat_ reshaped = current_shape.reshape(1, 3); + + cv::Mat rotated_shape = (cv::Mat(rotation_matrix) * reshaped); + + // Get the width of expected shape + double min_x; + double max_x; + cv::minMaxLoc(rotated_shape.row(0), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(rotated_shape.row(1), &min_y, &max_y); + + double width = abs(min_x - max_x); + double height = abs(min_y - max_y); + + double scaling = ((bounding_box.width / width) + (bounding_box.height / height)) / 2; + + // The estimate of face center also needs some correction + double tx = bounding_box.x + bounding_box.width / 2; + double ty = bounding_box.y + bounding_box.height / 2; + + // Correct it so that the bounding box is just around the minimum and maximum point in the initialised face + tx = tx - scaling * (min_x + max_x)/2; + ty = ty - scaling * (min_y + max_y)/2; + + out_params_global = cv::Vec6d(scaling, rotation[0], rotation[1], rotation[2], tx, ty); +} + +//=========================================================================== +// provided the model parameters, compute the bounding box of a face +// The bounding box describes face from left outline to right outline of the face and chin to eyebrows +void PDM::CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local) const +{ + + // get the shape instance based on local params + cv::Mat_ current_shape; + CalcShape2D(current_shape, params_local, params_global); + + // Get the width of expected shape + double min_x; + double max_x; + cv::minMaxLoc(current_shape(cv::Rect(0, 0, 1, this->NumberOfPoints())), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(current_shape(cv::Rect(0, this->NumberOfPoints(), 1, this->NumberOfPoints())), &min_y, &max_y); + + double width = abs(min_x - max_x); + double height = abs(min_y - max_y); + + out_bounding_box = cv::Rect((int)min_x, (int)min_y, (int)width, (int)height); +} + +//=========================================================================== +// Calculate the PDM's Jacobian over rigid parameters (rotation, translation and scaling), the additional input W represents trust for each of the landmarks and is part of Non-Uniform RLMS +void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& params_global, cv::Mat_ &Jacob, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const +{ + + // number of verts + int n = this->NumberOfPoints(); + + Jacob.create(n * 2, 6); + + float X,Y,Z; + + float s = (float)params_global[0]; + + cv::Mat_ shape_3D_d; + cv::Mat_ p_local_d; + p_local.convertTo(p_local_d, CV_64F); + this->CalcShape3D(shape_3D_d, p_local_d); + + cv::Mat_ shape_3D; + shape_3D_d.convertTo(shape_3D, CV_32F); + + // Get the rotation matrix + cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33d currRot = Euler2RotationMatrix(euler); + + float r11 = (float) currRot(0,0); + float r12 = (float) currRot(0,1); + float r13 = (float) currRot(0,2); + float r21 = (float) currRot(1,0); + float r22 = (float) currRot(1,1); + float r23 = (float) currRot(1,2); + float r31 = (float) currRot(2,0); + float r32 = (float) currRot(2,1); + float r33 = (float) currRot(2,2); + + cv::MatIterator_ Jx = Jacob.begin(); + cv::MatIterator_ Jy = Jx + n * 6; + + for(int i = 0; i < n; i++) + { + + X = shape_3D.at(i,0); + Y = shape_3D.at(i+n,0); + Z = shape_3D.at(i+n*2,0); + + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') + // where R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + // And this is derived using the small angle assumption on the axis angle rotation matrix parametrisation + + // scaling term + *Jx++ = (X * r11 + Y * r12 + Z * r13); + *Jy++ = (X * r21 + Y * r22 + Z * r23); + + // rotation terms + *Jx++ = (s * (Y * r13 - Z * r12) ); + *Jy++ = (s * (Y * r23 - Z * r22) ); + *Jx++ = (-s * (X * r13 - Z * r11)); + *Jy++ = (-s * (X * r23 - Z * r21)); + *Jx++ = (s * (X * r12 - Y * r11) ); + *Jy++ = (s * (X * r22 - Y * r21) ); + + // translation terms + *Jx++ = 1.0f; + *Jy++ = 0.0f; + *Jx++ = 0.0f; + *Jy++ = 1.0f; + + } + + cv::Mat Jacob_w = cv::Mat::zeros(Jacob.rows, Jacob.cols, Jacob.type()); + + Jx = Jacob.begin(); + Jy = Jx + n*6; + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n*6; + + // Iterate over all Jacobian values and multiply them by the weight in diagonal of W + for(int i = 0; i < n; i++) + { + float w_x = W.at(i, i); + float w_y = W.at(i+n, i+n); + + for(int j = 0; j < Jacob.cols; ++j) + { + *Jx_w++ = *Jx++ * w_x; + *Jy_w++ = *Jy++ * w_y; + } + } + + Jacob_t_w = Jacob_w.t(); +} + +//=========================================================================== +// Calculate the PDM's Jacobian over all parameters (rigid and non-rigid), the additional input W represents trust for each of the landmarks and is part of Non-Uniform RLMS +void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& params_global, cv::Mat_ &Jacobian, const cv::Mat_ W, cv::Mat_ &Jacob_t_w) const +{ + + // number of vertices + int n = this->NumberOfPoints(); + + // number of non-rigid parameters + int m = this->NumberOfModes(); + + Jacobian.create(n * 2, 6 + m); + + float X,Y,Z; + + float s = (float) params_global[0]; + + cv::Mat_ shape_3D_d; + cv::Mat_ p_local_d; + params_local.convertTo(p_local_d, CV_64F); + this->CalcShape3D(shape_3D_d, p_local_d); + + cv::Mat_ shape_3D; + shape_3D_d.convertTo(shape_3D, CV_32F); + + cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); + cv::Matx33d currRot = Euler2RotationMatrix(euler); + + float r11 = (float) currRot(0,0); + float r12 = (float) currRot(0,1); + float r13 = (float) currRot(0,2); + float r21 = (float) currRot(1,0); + float r22 = (float) currRot(1,1); + float r23 = (float) currRot(1,2); + float r31 = (float) currRot(2,0); + float r32 = (float) currRot(2,1); + float r33 = (float) currRot(2,2); + + cv::MatIterator_ Jx = Jacobian.begin(); + cv::MatIterator_ Jy = Jx + n * (6 + m); + cv::MatConstIterator_ Vx = this->princ_comp.begin(); + cv::MatConstIterator_ Vy = Vx + n*m; + cv::MatConstIterator_ Vz = Vy + n*m; + + for(int i = 0; i < n; i++) + { + + X = shape_3D.at(i,0); + Y = shape_3D.at(i+n,0); + Z = shape_3D.at(i+n*2,0); + + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') + // where R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + // And this is derived using the small angle assumption on the axis angle rotation matrix parametrisation + + // scaling term + *Jx++ = (X * r11 + Y * r12 + Z * r13); + *Jy++ = (X * r21 + Y * r22 + Z * r23); + + // rotation terms + *Jx++ = (s * (Y * r13 - Z * r12) ); + *Jy++ = (s * (Y * r23 - Z * r22) ); + *Jx++ = (-s * (X * r13 - Z * r11)); + *Jy++ = (-s * (X * r23 - Z * r21)); + *Jx++ = (s * (X * r12 - Y * r11) ); + *Jy++ = (s * (X * r22 - Y * r21) ); + + // translation terms + *Jx++ = 1.0f; + *Jy++ = 0.0f; + *Jx++ = 0.0f; + *Jy++ = 1.0f; + + for(int j = 0; j < m; j++,++Vx,++Vy,++Vz) + { + // How much the change of the non-rigid parameters (when object is rotated) affect 2D motion + *Jx++ = (float) ( s*(r11*(*Vx) + r12*(*Vy) + r13*(*Vz)) ); + *Jy++ = (float) ( s*(r21*(*Vx) + r22*(*Vy) + r23*(*Vz)) ); + } + } + + // Adding the weights here + cv::Mat Jacob_w = Jacobian.clone(); + + if(cv::trace(W)[0] != W.rows) + { + Jx = Jacobian.begin(); + Jy = Jx + n*(6+m); + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n*(6+m); + + // Iterate over all Jacobian values and multiply them by the weight in diagonal of W + for(int i = 0; i < n; i++) + { + float w_x = W.at(i, i); + float w_y = W.at(i+n, i+n); + + for(int j = 0; j < Jacobian.cols; ++j) + { + *Jx_w++ = *Jx++ * w_x; + *Jy_w++ = *Jy++ * w_y; + } + } + } + Jacob_t_w = Jacob_w.t(); + +} + +//=========================================================================== +// Updating the parameters (more details in my thesis) +void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& params_local, cv::Vec6d& params_global) const +{ + + // The scaling and translation parameters can be just added + params_global[0] += (double)delta_p.at(0,0); + params_global[4] += (double)delta_p.at(4,0); + params_global[5] += (double)delta_p.at(5,0); + + // get the original rotation matrix + cv::Vec3d eulerGlobal(params_global[1], params_global[2], params_global[3]); + cv::Matx33d R1 = Euler2RotationMatrix(eulerGlobal); + + // construct R' = [1, -wz, wy + // wz, 1, -wx + // -wy, wx, 1] + cv::Matx33d R2 = cv::Matx33d::eye(); + + R2(1,2) = -1.0*(R2(2,1) = (double)delta_p.at(1,0)); + R2(2,0) = -1.0*(R2(0,2) = (double)delta_p.at(2,0)); + R2(0,1) = -1.0*(R2(1,0) = (double)delta_p.at(3,0)); + + // Make sure it's orthonormal + Orthonormalise(R2); + + // Combine rotations + cv::Matx33d R3 = R1 *R2; + + // Extract euler angle (through axis angle first to make sure it's legal) + cv::Vec3d axis_angle = RotationMatrix2AxisAngle(R3); + cv::Vec3d euler = AxisAngle2Euler(axis_angle); + + params_global[1] = euler[0]; + params_global[2] = euler[1]; + params_global[3] = euler[2]; + + // Local parameter update, just simple addition + if(delta_p.rows > 6) + { + params_local = params_local + delta_p(cv::Rect(0,6,1, this->NumberOfModes())); + } + +} + +void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation) const +{ + + int m = this->NumberOfModes(); + int n = this->NumberOfPoints(); + + // The new number of points + n = this->mean_shape.rows / 3; + + // Compute the initial global parameters + double min_x; + double max_x; + cv::minMaxLoc(landmark_locations(cv::Rect(0, 0, 1, this->NumberOfPoints())), &min_x, &max_x); + + double min_y; + double max_y; + cv::minMaxLoc(landmark_locations(cv::Rect(0, this->NumberOfPoints(), 1, this->NumberOfPoints())), &min_y, &max_y); + + double width = abs(min_x - max_x); + double height = abs(min_y - max_y); + + cv::Rect model_bbox; + CalcBoundingBox(model_bbox, cv::Vec6d(1.0, 0.0, 0.0, 0.0, 0.0, 0.0), cv::Mat_(this->NumberOfModes(), 1, 0.0)); + + cv::Rect bbox((int)min_x, (int)min_y, (int)width, (int)height); + + double scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; + + cv::Vec3d rotation_init = rotation; + cv::Matx33d R = Euler2RotationMatrix(rotation_init); + cv::Vec2d translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); + + cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); + cv::Vec6d glob_params(scaling, rotation_init[0], rotation_init[1], rotation_init[2], translation[0], translation[1]); + + // get the 3D shape of the object + cv::Mat_ loc_params_d; + loc_params.convertTo(loc_params_d, CV_64F); + cv::Mat_ shape_3D = mean_shape + princ_comp * loc_params_d; + + cv::Mat_ curr_shape(2*n, 1); + + // for every vertex + for(int i = 0; i < n; i++) + { + // Transform this using the weak-perspective mapping to 2D from 3D + curr_shape.at(i ,0) = scaling * ( R(0,0) * shape_3D.at(i, 0) + R(0,1) * shape_3D.at(i+n ,0) + R(0,2) * shape_3D.at(i+n*2,0) ) + translation[0]; + curr_shape.at(i+n,0) = scaling * ( R(1,0) * shape_3D.at(i, 0) + R(1,1) * shape_3D.at(i+n ,0) + R(1,2) * shape_3D.at(i+n*2,0) ) + translation[1]; + } + + double currError = cv::norm(curr_shape - landmark_locations); + + cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); + + double reg_factor = 1; + + // Setting the regularisation to the inverse of eigenvalues + cv::Mat(reg_factor / this->eigen_values).copyTo(regularisations(cv::Rect(6, 0, m, 1))); + cv::Mat_ regTerm_d = cv::Mat::diag(regularisations.t()); + regTerm_d.convertTo(regularisations, CV_32F); + + cv::Mat_ WeightMatrix = cv::Mat_::eye(n*2, n*2); + + int not_improved_in = 0; + + for (size_t i = 0; i < 1000; ++i) + { + // get the 3D shape of the object + cv::Mat_ loc_params_d; + loc_params.convertTo(loc_params_d, CV_64F); + shape_3D = mean_shape + princ_comp * loc_params_d; + + shape_3D = shape_3D.reshape(1, 3); + + cv::Matx23d R_2D(R(0,0), R(0,1), R(0,2), R(1,0), R(1,1), R(1,2)); + + cv::Mat_ curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); + curr_shape_2D.col(0) = curr_shape_2D.col(0) + translation(0); + curr_shape_2D.col(1) = curr_shape_2D.col(1) + translation(1); + + curr_shape_2D = cv::Mat(curr_shape_2D.t()).reshape(1, n * 2); + + cv::Mat_ error_resid; + cv::Mat(landmark_locations - curr_shape_2D).convertTo(error_resid, CV_32F); + + cv::Mat_ J, J_w_t; + this->ComputeJacobian(loc_params, glob_params, J, WeightMatrix, J_w_t); + + // projection of the meanshifts onto the jacobians (using the weighted Jacobian, see Baltrusaitis 2013) + cv::Mat_ J_w_t_m = J_w_t * error_resid; + + // Add the regularisation term + J_w_t_m(cv::Rect(0,6,1, m)) = J_w_t_m(cv::Rect(0,6,1, m)) - regularisations(cv::Rect(6,6, m, m)) * loc_params; + + cv::Mat_ Hessian = J_w_t * J; + + // Add the Tikhonov regularisation + Hessian = Hessian + regularisations; + + // Solve for the parameter update (from Baltrusaitis 2013 based on eq (36) Saragih 2011) + cv::Mat_ param_update; + cv::solve(Hessian, J_w_t_m, param_update, CV_CHOLESKY); + + // To not overshoot, have the gradient decent rate a bit smaller + param_update = 0.5 * param_update; + + UpdateModelParameters(param_update, loc_params, glob_params); + + scaling = glob_params[0]; + rotation_init[0] = glob_params[1]; + rotation_init[1] = glob_params[2]; + rotation_init[2] = glob_params[3]; + + translation[0] = glob_params[4]; + translation[1] = glob_params[5]; + + R = Euler2RotationMatrix(rotation_init); + + R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); + R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); + + curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); + curr_shape_2D.col(0) = curr_shape_2D.col(0) + translation(0); + curr_shape_2D.col(1) = curr_shape_2D.col(1) + translation(1); + + curr_shape_2D = cv::Mat(curr_shape_2D.t()).reshape(1, n * 2); + + double error = cv::norm(curr_shape_2D - landmark_locations); + + if(0.999 * currError < error) + { + not_improved_in++; + if (not_improved_in == 5) + { + break; + } + } + + currError = error; + + } + + out_params_global = glob_params; + loc_params.convertTo(out_params_local, CV_64F); + + +} + +void PDM::Read(std::string location) +{ + + std::ifstream pdmLoc(location, std::ios_base::in); + + SkipComments(pdmLoc); + + // Reading mean values + ReadMat(pdmLoc,mean_shape); + + SkipComments(pdmLoc); + + // Reading principal components + ReadMat(pdmLoc,princ_comp); + + SkipComments(pdmLoc); + + // Reading eigenvalues + ReadMat(pdmLoc,eigen_values); + +} diff --git a/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp b/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp index 32334b27..9c224c09 100644 --- a/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp +++ b/lib/local/FaceAnalyser/src/SVM_dynamic_lin.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -55,11 +31,10 @@ // in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// +#include "Face_utils.h" #include "SVM_dynamic_lin.h" -#include "LandmarkCoreIncludes.h" - using namespace FaceAnalysis; void SVM_dynamic_lin::Read(std::ifstream& stream, const std::vector& au_names) @@ -67,20 +42,20 @@ void SVM_dynamic_lin::Read(std::ifstream& stream, const std::vector if(this->means.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVM dynamic classifiers" << endl; + std::cout << "Something went wrong with the SVM dynamic classifiers" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVM_static_lin.cpp b/lib/local/FaceAnalyser/src/SVM_static_lin.cpp index 5c0f5b1f..89b04adf 100644 --- a/lib/local/FaceAnalyser/src/SVM_static_lin.cpp +++ b/lib/local/FaceAnalyser/src/SVM_static_lin.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -55,11 +31,10 @@ // in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. // /////////////////////////////////////////////////////////////////////////////// +#include "Face_utils.h" #include "SVM_static_lin.h" -#include "LandmarkCoreIncludes.h" - using namespace FaceAnalysis; void SVM_static_lin::Read(std::ifstream& stream, const std::vector& au_names) @@ -67,20 +42,20 @@ void SVM_static_lin::Read(std::ifstream& stream, const std::vector& if(this->means.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVM static classifiers" << endl; + std::cout << "Something went wrong with the SVM static classifiers" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp b/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp index 6e02600a..1255b3d6 100644 --- a/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp +++ b/lib/local/FaceAnalyser/src/SVR_dynamic_lin_regressors.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -56,9 +32,9 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "SVR_dynamic_lin_regressors.h" +#include "Face_utils.h" -#include "LandmarkCoreIncludes.h" +#include "SVR_dynamic_lin_regressors.h" using namespace FaceAnalysis; @@ -73,20 +49,20 @@ void SVR_dynamic_lin_regressors::Read(std::ifstream& stream, const std::vectormeans.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVR dynamic regressors" << endl; + std::cout << "Something went wrong with the SVR dynamic regressors" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp b/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp index 9fb984f2..71e26b7d 100644 --- a/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp +++ b/lib/local/FaceAnalyser/src/SVR_static_lin_regressors.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -56,9 +32,9 @@ // /////////////////////////////////////////////////////////////////////////////// -#include "SVR_static_lin_regressors.h" +#include "Face_utils.h" -#include "LandmarkCoreIncludes.h" +#include "SVR_static_lin_regressors.h" using namespace FaceAnalysis; @@ -67,20 +43,20 @@ void SVR_static_lin_regressors::Read(std::ifstream& stream, const std::vectormeans.empty()) { - LandmarkDetector::ReadMatBin(stream, this->means); + ReadMatBin(stream, this->means); } else { cv::Mat_ m_tmp; - LandmarkDetector::ReadMatBin(stream, m_tmp); + ReadMatBin(stream, m_tmp); if(cv::norm(m_tmp - this->means > 0.00001)) { - cout << "Something went wrong with the SVR static regressors" << endl; + std::cout << "Something went wrong with the SVR static regressors" << std::endl; } } cv::Mat_ support_vectors_curr; - LandmarkDetector::ReadMatBin(stream, support_vectors_curr); + ReadMatBin(stream, support_vectors_curr); double bias; stream.read((char *)&bias, 8); diff --git a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj new file mode 100644 index 00000000..9b22cad7 --- /dev/null +++ b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj @@ -0,0 +1,158 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {5F915541-F531-434F-9C81-79F5DB58012B} + GazeAnalyser + 8.1 + + + + StaticLibrary + true + v140 + Unicode + + + StaticLibrary + false + v140 + true + Unicode + + + StaticLibrary + true + v140 + Unicode + + + StaticLibrary + false + v140 + true + Unicode + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(ProjectDir)$(Configuration)\ + + + $(ProjectDir)$(Configuration)\ + + + + Level3 + Disabled + true + ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + StreamingSIMDExtensions2 + + + + + Level3 + Disabled + true + ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + WIN64;_DEBUG;_LIB;EIGEN_MPL2_ONLY;%(PreprocessorDefinitions) + AdvancedVectorExtensions + + + + + Level3 + MaxSpeed + true + true + + + ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + true + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + StreamingSIMDExtensions2 + + + true + true + + + + + Level3 + MaxSpeed + true + true + + + ./include;../LandmarkDetector/include;%(AdditionalIncludeDirectories) + true + WIN64;NDEBUG;_LIB;%(PreprocessorDefinitions) + AdvancedVectorExtensions + + + true + true + + + + + + + + + + + {bdc1d107-de17-4705-8e7b-cdde8bfb2bf8} + + + + + + \ No newline at end of file diff --git a/lib/local/GazeAnalyser/GazeAnalyser.vcxproj.filters b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj.filters new file mode 100644 index 00000000..e15fb020 --- /dev/null +++ b/lib/local/GazeAnalyser/GazeAnalyser.vcxproj.filters @@ -0,0 +1,23 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + + + Header Files + + + + + Source Files + + + \ No newline at end of file diff --git a/lib/local/GazeAnalyser/include/GazeEstimation.h b/lib/local/GazeAnalyser/include/GazeEstimation.h new file mode 100644 index 00000000..2f302828 --- /dev/null +++ b/lib/local/GazeAnalyser/include/GazeEstimation.h @@ -0,0 +1,55 @@ +/////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, +// all rights reserved. +// +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt +// +// * Any publications arising from the use of this software, including but +// not limited to academic journal and conference publications, technical +// reports and manuals, must cite at least one of the following works: +// +// OpenFace: an open source facial behavior analysis toolkit +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency +// in IEEE Winter Conference on Applications of Computer Vision, 2016 +// +// Rendering of Eyes for Eye-Shape Registration and Gaze Estimation +// Erroll Wood, Tadas Baltrušaitis, Xucong Zhang, Yusuke Sugano, Peter Robinson, and Andreas Bulling +// in IEEE International. Conference on Computer Vision (ICCV), 2015 +// +// Cross-dataset learning and person-speci?c normalisation for automatic Action Unit detection +// Tadas Baltrušaitis, Marwa Mahmoud, and Peter Robinson +// in Facial Expression Recognition and Analysis Challenge, +// IEEE International Conference on Automatic Face and Gesture Recognition, 2015 +// +// Constrained Local Neural Fields for robust facial landmark detection in the wild. +// Tadas Baltrušaitis, Peter Robinson, and Louis-Philippe Morency. +// in IEEE Int. Conference on Computer Vision Workshops, 300 Faces in-the-Wild Challenge, 2013. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef __GAZEESTIMATION_h_ +#define __GAZEESTIMATION_h_ + +#include "LandmarkDetectorModel.h" + +#include "opencv2/core/core.hpp" + +namespace GazeAnalysis +{ + + void EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye); + void DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy); + + // Getting the gaze angle in radians with respect to head pose (need to call EstimateGaze first) + cv::Vec2d GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2, cv::Vec6d head_pose); + + // Some utilities + cv::Point3f GetPupilPosition(cv::Mat_ eyeLdmks3d); + +} +#endif \ No newline at end of file diff --git a/lib/local/FaceAnalyser/src/GazeEstimation.cpp b/lib/local/GazeAnalyser/src/GazeEstimation.cpp similarity index 69% rename from lib/local/FaceAnalyser/src/GazeEstimation.cpp rename to lib/local/GazeAnalyser/src/GazeEstimation.cpp index bd3451bc..b0889f60 100644 --- a/lib/local/FaceAnalyser/src/GazeEstimation.cpp +++ b/lib/local/GazeAnalyser/src/GazeEstimation.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -64,9 +40,12 @@ #include "GazeEstimation.h" +#include "LandmarkDetectorUtils.h" +#include "LandmarkDetectorFunc.h" + using namespace std; -using namespace FaceAnalysis; +using namespace GazeAnalysis; // For subpixel accuracy drawing const int gaze_draw_shiftbits = 4; @@ -100,7 +79,7 @@ cv::Point3f RaySphereIntersect(cv::Point3f rayOrigin, cv::Point3f rayDir, cv::Po return rayOrigin + rayDir * t; } -cv::Point3f GetPupilPosition(cv::Mat_ eyeLdmks3d){ +cv::Point3f GazeAnalysis::GetPupilPosition(cv::Mat_ eyeLdmks3d){ eyeLdmks3d = eyeLdmks3d.t(); @@ -110,9 +89,9 @@ cv::Point3f GetPupilPosition(cv::Mat_ eyeLdmks3d){ return p; } -void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye) +void GazeAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Point3f& gaze_absolute, float fx, float fy, float cx, float cy, bool left_eye) { - cv::Vec6d headPose = LandmarkDetector::GetPoseCamera(clnf_model, fx, fy, cx, cy); + cv::Vec6d headPose = LandmarkDetector::GetPose(clnf_model, fx, fy, cx, cy); cv::Vec3d eulerAngles(headPose(3), headPose(4), headPose(5)); cv::Matx33d rotMat = LandmarkDetector::Euler2RotationMatrix(eulerAngles); @@ -141,7 +120,7 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po cv::Mat faceLdmks3d = clnf_model.GetShape(fx, fy, cx, cy); faceLdmks3d = faceLdmks3d.t(); - cv::Mat offset = (cv::Mat_(3, 1) << 0, -3.50, 0); + cv::Mat offset = (cv::Mat_(3, 1) << 0, -3.5, 7.0); int eyeIdx = 1; if (left_eye) { @@ -157,8 +136,23 @@ void FaceAnalysis::EstimateGaze(const LandmarkDetector::CLNF& clnf_model, cv::Po gaze_absolute = gazeVecAxis / norm(gazeVecAxis); } +cv::Vec2d GazeAnalysis::GetGazeAngle(cv::Point3f& gaze_vector_1, cv::Point3f& gaze_vector_2, cv::Vec6d head_pose) +{ -void FaceAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy) + cv::Vec3d eulerAngles(head_pose(3), head_pose(4), head_pose(5)); + cv::Matx33d rotMat = LandmarkDetector::Euler2RotationMatrix(eulerAngles); + cv::Point3f gaze_point = (gaze_vector_1 + gaze_vector_2) / 2; + cv::Vec3d gaze(gaze_point.x, gaze_point.y, gaze_point.z); + + gaze = rotMat * gaze; + + double x_angle = atan2(gaze[0], -gaze[2]); + double y_angle = atan2(gaze[1], -gaze[2]); + + return cv::Vec2d(x_angle, y_angle); + +} +void GazeAnalysis::DrawGaze(cv::Mat img, const LandmarkDetector::CLNF& clnf_model, cv::Point3f gazeVecAxisLeft, cv::Point3f gazeVecAxisRight, float fx, float fy, float cx, float cy) { cv::Mat cameraMat = (cv::Mat_(3, 3) << fx, 0, cx, 0, fy, cy, 0, 0, 0); diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h b/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h index c5833f3a..b588bdc5 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorFunc.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -100,20 +76,12 @@ namespace LandmarkDetector //================================================================ // Helper function for getting head pose from CLNF parameters - // Return the current estimate of the head pose, this can be either in camera or world coordinate space + // Return the current estimate of the head pose in world coordinates with camera at origin (0,0,0) // The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] - cv::Vec6d GetPoseCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy); - cv::Vec6d GetPoseWorld(const CLNF& clnf_model, double fx, double fy, double cx, double cy); - - // Getting a head pose estimate from the currently detected landmarks, with appropriate correction for perspective - // This is because rotation estimate under orthographic assumption is only correct close to the centre of the image - // These methods attempt to correct for that - // The pose returned can be either in camera or world coordinates + cv::Vec6d GetPose(const CLNF& clnf_model, double fx, double fy, double cx, double cy); + + // Return the current estimate of the head pose in world coordinates with camera at origin (0,0,0), but with rotation representing if the head is looking at the camera // The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] - cv::Vec6d GetCorrectedPoseCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy); - cv::Vec6d GetCorrectedPoseWorld(const CLNF& clnf_model, double fx, double fy, double cx, double cy); - - //=========================================================================== - + cv::Vec6d GetPoseWRTCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy); } #endif diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index cdd6ab3a..0f8882bf 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -77,7 +53,7 @@ namespace LandmarkDetector // Helper functions for parsing the inputs //============================================================================================= void get_video_input_output_params(vector &input_video_file, vector &depth_dir, vector &output_files, - vector &output_video_files, bool& world_coordinates_pose, string &output_codec, vector &arguments); + vector &output_video_files, string &output_codec, vector &arguments); void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments); @@ -90,14 +66,14 @@ namespace LandmarkDetector // This is a modified version of openCV code that allows for precomputed dfts of templates and for precomputed dfts of an image // _img is the input img, _img_dft it's dft (optional), _integral_img the images integral image (optional), squared integral image (optional), // templ is the template we are convolving with, templ_dfts it's dfts at varying windows sizes (optional), _result - the output, method the type of convolution - void matchTemplate_m( const cv::Mat_& input_img, cv::Mat_& img_dft, cv::Mat& _integral_img, cv::Mat& _integral_img_sq, const cv::Mat_& templ, map >& templ_dfts, cv::Mat_& result, int method ); + void matchTemplate_m(const cv::Mat_& input_img, cv::Mat_& img_dft, cv::Mat& _integral_img, cv::Mat& _integral_img_sq, const cv::Mat_& templ, map >& templ_dfts, cv::Mat_& result, int method); //=========================================================================== // Point set and landmark manipulation functions //=========================================================================== // Using Kabsch's algorithm for aligning shapes //This assumes that align_from and align_to are already mean normalised - cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to ); + cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); //============================================================================= // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other @@ -113,8 +89,14 @@ namespace LandmarkDetector vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); - vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_& visibilities); - vector CalculateLandmarks(CLNF& clnf_model); + vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); + vector CalculateVisibleLandmarks(const CLNF& clnf_model); + vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); + + vector CalculateAllLandmarks(const cv::Mat_& shape2D); + vector CalculateAllLandmarks(const CLNF& clnf_model); + vector CalculateAllEyeLandmarks(const CLNF& clnf_model); + void DrawLandmarks(cv::Mat img, vector landmarks); void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); @@ -143,16 +125,16 @@ namespace LandmarkDetector //============================================================================ // Face detection using Haar cascade classifier - bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity); - bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier); + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); // The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen - bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, const cv::Point preference = cv::Point(-1,-1)); + bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, const cv::Point preference = cv::Point(-1, -1), double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); // Face detection using HOG-SVM classifier - bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences); - bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, std::vector& confidences); + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, std::vector& confidences, double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); // The preference point allows for disambiguation if multiple faces are present (pick the closest one), if it is not set the biggest face is chosen - bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, double& confidence, const cv::Point preference = cv::Point(-1,-1)); + bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity, dlib::frontal_face_detector& classifier, double& confidence, const cv::Point preference = cv::Point(-1, -1), double min_width = -1, cv::Rect_ roi = cv::Rect_(0.0, 0.0, 1.0, 1.0)); //============================================================================ // Matrix reading functionality diff --git a/lib/local/LandmarkDetector/model/cen_general.txt b/lib/local/LandmarkDetector/model/cen_general.txt index dd40388e..f76e85c9 100644 --- a/lib/local/LandmarkDetector/model/cen_general.txt +++ b/lib/local/LandmarkDetector/model/cen_general.txt @@ -1,4 +1,4 @@ -PDM pdms/In-the-wild_aligned_PDM_68.txt +PDM pdms/pdm_68_aligned_menpo.txt Triangulations tris_68.txt PatchesCEN patch_experts/cen_patches_0.25_menpo.dat PatchesCEN patch_experts/cen_patches_0.35_menpo.dat diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index bc773dcf..a89562ac 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -71,72 +71,20 @@ using namespace LandmarkDetector; -// Getting a head pose estimate from the currently detected landmarks (rotation with respect to point camera) +// Getting a head pose estimate from the currently detected landmarks, with appropriate correction due to the PDM assuming an orthographic camera +// which is only correct close to the centre of the image +// This method returns a corrected pose estimate with respect to world coordinates with camera at origin (0,0,0) // The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] -cv::Vec6d LandmarkDetector::GetPoseCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy) +cv::Vec6d LandmarkDetector::GetPose(const CLNF& clnf_model, double fx, double fy, double cx, double cy) { - if(!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) - { - double Z = fx / clnf_model.params_global[0]; - - double X = ((clnf_model.params_global[4] - cx) * (1.0/fx)) * Z; - double Y = ((clnf_model.params_global[5] - cy) * (1.0/fy)) * Z; - - return cv::Vec6d(X, Y, Z, clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); - } - else - { - return cv::Vec6d(0,0,0,0,0,0); - } -} - -// Getting a head pose estimate from the currently detected landmarks (rotation in world coordinates) -// The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] -cv::Vec6d LandmarkDetector::GetPoseWorld(const CLNF& clnf_model, double fx, double fy, double cx, double cy) -{ - if(!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) - { - double Z = fx / clnf_model.params_global[0]; - - double X = ((clnf_model.params_global[4] - cx) * (1.0/fx)) * Z; - double Y = ((clnf_model.params_global[5] - cy) * (1.0/fy)) * Z; - - // Here we correct for the camera orientation, for this need to determine the angle the camera makes with the head pose - double z_x = cv::sqrt(X * X + Z * Z); - double eul_x = atan2(Y, z_x); - - double z_y = cv::sqrt(Y * Y + Z * Z); - double eul_y = -atan2(X, z_y); - - cv::Matx33d camera_rotation = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(eul_x, eul_y, 0)); - cv::Matx33d head_rotation = LandmarkDetector::AxisAngle2RotationMatrix(cv::Vec3d(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3])); - - cv::Matx33d corrected_rotation = camera_rotation.t() * head_rotation; - - cv::Vec3d euler_corrected = LandmarkDetector::RotationMatrix2Euler(corrected_rotation); - - return cv::Vec6d(X, Y, Z, euler_corrected[0], euler_corrected[1], euler_corrected[2]); - } - else - { - return cv::Vec6d(0,0,0,0,0,0); - } -} - -// Getting a head pose estimate from the currently detected landmarks, with appropriate correction due to orthographic camera issue -// This is because rotation estimate under orthographic assumption is only correct close to the centre of the image -// This method returns a corrected pose estimate with respect to world coordinates (Experimental) -// The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] -cv::Vec6d LandmarkDetector::GetCorrectedPoseWorld(const CLNF& clnf_model, double fx, double fy, double cx, double cy) -{ - if(!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) + if (!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) { // This is used as an initial estimate for the iterative PnP algorithm double Z = fx / clnf_model.params_global[0]; - - double X = ((clnf_model.params_global[4] - cx) * (1.0/fx)) * Z; - double Y = ((clnf_model.params_global[5] - cy) * (1.0/fy)) * Z; - + + double X = ((clnf_model.params_global[4] - cx) * (1.0 / fx)) * Z; + double Y = ((clnf_model.params_global[5] - cy) * (1.0 / fy)) * Z; + // Correction for orientation // 2D points @@ -154,35 +102,35 @@ cv::Vec6d LandmarkDetector::GetCorrectedPoseWorld(const CLNF& clnf_model, double // The camera matrix cv::Matx33d camera_matrix(fx, 0, cx, 0, fy, cy, 0, 0, 1); - + cv::Vec3d vec_trans(X, Y, Z); cv::Vec3d vec_rot(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); - + cv::solvePnP(landmarks_3D, landmarks_2D, camera_matrix, cv::Mat(), vec_rot, vec_trans, true); cv::Vec3d euler = LandmarkDetector::AxisAngle2Euler(vec_rot); - - return cv::Vec6d(vec_trans[0], vec_trans[1], vec_trans[2], vec_rot[0], vec_rot[1], vec_rot[2]); + + return cv::Vec6d(vec_trans[0], vec_trans[1], vec_trans[2], euler[0], euler[1], euler[2]); } else { - return cv::Vec6d(0,0,0,0,0,0); + return cv::Vec6d(0, 0, 0, 0, 0, 0); } } // Getting a head pose estimate from the currently detected landmarks, with appropriate correction due to perspective projection -// This method returns a corrected pose estimate with respect to a point camera (NOTE not the world coordinates) (Experimental) +// This method returns a corrected pose estimate with respect to a point camera (NOTE not the world coordinates), which is useful to find out if the person is looking at a camera // The format returned is [Tx, Ty, Tz, Eul_x, Eul_y, Eul_z] -cv::Vec6d LandmarkDetector::GetCorrectedPoseCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy) +cv::Vec6d LandmarkDetector::GetPoseWRTCamera(const CLNF& clnf_model, double fx, double fy, double cx, double cy) { - if(!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) + if (!clnf_model.detected_landmarks.empty() && clnf_model.params_global[0] != 0) { double Z = fx / clnf_model.params_global[0]; - - double X = ((clnf_model.params_global[4] - cx) * (1.0/fx)) * Z; - double Y = ((clnf_model.params_global[5] - cy) * (1.0/fy)) * Z; - + + double X = ((clnf_model.params_global[4] - cx) * (1.0 / fx)) * Z; + double Y = ((clnf_model.params_global[5] - cy) * (1.0 / fy)) * Z; + // Correction for orientation // 3D points @@ -193,17 +141,17 @@ cv::Vec6d LandmarkDetector::GetCorrectedPoseCamera(const CLNF& clnf_model, doubl // 2D points cv::Mat_ landmarks_2D = clnf_model.detected_landmarks; - + landmarks_2D = landmarks_2D.reshape(1, 2).t(); // Solving the PNP model // The camera matrix cv::Matx33d camera_matrix(fx, 0, cx, 0, fy, cy, 0, 0, 1); - + cv::Vec3d vec_trans(X, Y, Z); cv::Vec3d vec_rot(clnf_model.params_global[1], clnf_model.params_global[2], clnf_model.params_global[3]); - + cv::solvePnP(landmarks_3D, landmarks_2D, camera_matrix, cv::Mat(), vec_rot, vec_trans, true); // Here we correct for the camera orientation, for this need to determine the angle the camera makes with the head pose @@ -219,12 +167,12 @@ cv::Vec6d LandmarkDetector::GetCorrectedPoseCamera(const CLNF& clnf_model, doubl cv::Matx33d corrected_rotation = camera_rotation * head_rotation; cv::Vec3d euler_corrected = LandmarkDetector::RotationMatrix2Euler(corrected_rotation); - + return cv::Vec6d(vec_trans[0], vec_trans[1], vec_trans[2], euler_corrected[0], euler_corrected[1], euler_corrected[2]); } else { - return cv::Vec6d(0,0,0,0,0,0); + return cv::Vec6d(0, 0, 0, 0, 0, 0); } } diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 349c3d1a..c1ef3c9c 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -769,14 +769,16 @@ bool CLNF::Fit(const cv::Mat_& im, const cv::Mat_& depthImg, const if(parameters.refine_parameters == true) { - // Adapt the parameters based on scale (wan't to reduce regularisation as scale increases, but increa sigma and tikhonov) - tmp_parameters.reg_factor = parameters.reg_factor - 15 * log(patch_experts.patch_scaling[scale]/0.25)/log(2); + int scale_max = scale >= 2 ? 2 : scale; + + // Adapt the parameters based on scale (wan't to reduce regularisation as scale increases, but increase sigma and Tikhonov) + tmp_parameters.reg_factor = parameters.reg_factor - 15 * log(patch_experts.patch_scaling[scale_max]/0.25)/log(2); if(tmp_parameters.reg_factor <= 0) tmp_parameters.reg_factor = 0.001; - tmp_parameters.sigma = parameters.sigma + 0.25 * log(patch_experts.patch_scaling[scale]/0.25)/log(2); - tmp_parameters.weight_factor = parameters.weight_factor + 2 * parameters.weight_factor * log(patch_experts.patch_scaling[scale]/0.25)/log(2); + tmp_parameters.sigma = parameters.sigma + 0.25 * log(patch_experts.patch_scaling[scale_max]/0.25)/log(2); + tmp_parameters.weight_factor = parameters.weight_factor + 2 * parameters.weight_factor * log(patch_experts.patch_scaling[scale_max]/0.25)/log(2); } // Get the current landmark locations diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 2d7f77dc..ab451cd9 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -181,7 +181,7 @@ FaceModelParameters::FaceModelParameters(vector &arguments) { // For in the wild fitting these parameters are suitable window_sizes_init = vector(4); - window_sizes_init[0] = 15; window_sizes_init[1] = 13; window_sizes_init[2] = 11; window_sizes_init[3] = 9; + window_sizes_init[0] = 15; window_sizes_init[1] = 13; window_sizes_init[2] = 11; window_sizes_init[3] = 11; sigma = 1.25; reg_factor = 35; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index caf366aa..4701199b 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -1,38 +1,14 @@ /////////////////////////////////////////////////////////////////////////////// -// Copyright (C) 2016, Carnegie Mellon University and University of Cambridge, +// Copyright (C) 2017, Carnegie Mellon University and University of Cambridge, // all rights reserved. // -// THIS SOFTWARE IS PROVIDED “AS IS” FOR ACADEMIC USE ONLY AND ANY EXPRESS -// OR IMPLIED WARRANTIES WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS -// BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY. -// OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -// HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. +// ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY +// +// BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. +// IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. +// +// License can be found in OpenFace-license.txt // -// Notwithstanding the license granted herein, Licensee acknowledges that certain components -// of the Software may be covered by so-called “open source” software licenses (“Open Source -// Components”), which means any software licenses approved as open source licenses by the -// Open Source Initiative or any substantially similar licenses, including without limitation any -// license that, as a condition of distribution of the software licensed under such license, -// requires that the distributor make the software available in source code format. Licensor shall -// provide a list of Open Source Components for a particular version of the Software upon -// Licensee’s request. Licensee will comply with the applicable terms of such licenses and to -// the extent required by the licenses covering Open Source Components, the terms of such -// licenses will apply in lieu of the terms of this Agreement. To the extent the terms of the -// licenses applicable to Open Source Components prohibit any of the restrictions in this -// License Agreement with respect to such Open Source Component, such restrictions will not -// apply to such Open Source Component. To the extent the terms of the licenses applicable to -// Open Source Components require Licensor to make an offer to provide source code or -// related information in connection with the Software, such offer is hereby made. Any request -// for source code or related information should be directed to cl-face-tracker-distribution@lists.cam.ac.uk -// Licensee acknowledges receipt of notices for the Open Source Components for the initial -// delivery of the Software. - // * Any publications arising from the use of this software, including but // not limited to academic journal and conference publications, technical // reports and manuals, must cite at least one of the following works: @@ -76,1464 +52,1548 @@ using namespace std; namespace LandmarkDetector { -// For subpixel accuracy drawing -const int draw_shiftbits = 4; -const int draw_multiplier = 1 << 4; + // For subpixel accuracy drawing + const int draw_shiftbits = 4; + const int draw_multiplier = 1 << 4; -// Useful utility for creating directories for storing the output files -void create_directory_from_file(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = path(path(output_path).parent_path()); - - if(!p.empty() && !boost::filesystem::exists(p)) + // Useful utility for creating directories for storing the output files + void create_directory_from_file(string output_path) { - bool success = boost::filesystem::create_directories(p); - if(!success) + + // Creating the right directory structure + + // First get rid of the file + auto p = path(path(output_path).parent_path()); + + if (!p.empty() && !boost::filesystem::exists(p)) { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// Useful utility for creating directories for storing the output files -void create_directories(string output_path) -{ - - // Creating the right directory structure - - // First get rid of the file - auto p = path(output_path); - - if(!p.empty() && !boost::filesystem::exists(p)) - { - bool success = boost::filesystem::create_directories(p); - if(!success) - { - cout << "Failed to create a directory... " << p.string() << endl; - } - } -} - -// Extracting the following command line arguments -f, -fd, -op, -of, -ov (and possible ordered repetitions) -void get_video_input_output_params(vector &input_video_files, vector &depth_dirs, vector &output_files, - vector &output_video_files, bool& world_coordinates_pose, string& output_codec, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for(size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - } - - // By default use rotation with respect to camera (not world coordinates) - world_coordinates_pose = false; - - // By default use DIVX codec - output_codec = "DIVX"; - - string input_root = ""; - string output_root = ""; - - string separator = string(1, boost::filesystem::path::preferred_separator); - - // First check if there is a root argument (so that videos and outputs could be defined more easilly) - for(size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-root") == 0) - { - input_root = arguments[i + 1] + separator; - output_root = arguments[i + 1] + separator; - - // Add the / or \ to the directory - i++; - } - if (arguments[i].compare("-inroot") == 0) - { - input_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-outroot") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - } - - for(size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-f") == 0) - { - input_video_files.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fd") == 0) - { - depth_dirs.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-of") == 0) - { - output_files.push_back(output_root + arguments[i + 1]); - create_directory_from_file(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-ov") == 0) - { - output_video_files.push_back(output_root + arguments[i + 1]); - create_directory_from_file(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-world_coord") == 0) - { - world_coordinates_pose = true; - } - else if (arguments[i].compare("-oc") == 0) - { - if(arguments[i + 1].length() == 4) - output_codec = arguments[i + 1]; - } - } - - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } - -} - -void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - for(size_t i=0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-fx") == 0) - { - stringstream data(arguments[i+1]); - data >> fx; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fy") == 0) - { - stringstream data(arguments[i+1]); - data >> fy; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-cx") == 0) - { - stringstream data(arguments[i+1]); - data >> cx; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-cy") == 0) - { - stringstream data(arguments[i+1]); - data >> cy; - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-device") == 0) - { - stringstream data(arguments[i+1]); - data >> device; - valid[i] = false; - valid[i+1] = false; - i++; - } - } - - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } -} - -void get_image_input_output_params(vector &input_image_files, vector &input_depth_files, vector &output_feature_files, vector &output_pose_files, vector &output_image_files, - vector> &input_bounding_boxes, vector &arguments) -{ - bool* valid = new bool[arguments.size()]; - - string out_pts_dir, out_pose_dir, out_img_dir; - - string input_root = ""; - string output_root = ""; - - string separator = string(1, boost::filesystem::path::preferred_separator); - - // First check if there is a root argument (so that videos and outputs could be defined more easilly) - for (size_t i = 0; i < arguments.size(); ++i) - { - if (arguments[i].compare("-root") == 0) - { - input_root = arguments[i + 1] + separator; - output_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-inroot") == 0) - { - input_root = arguments[i + 1] + separator; - i++; - } - if (arguments[i].compare("-outroot") == 0) - { - output_root = arguments[i + 1] + separator; - i++; - } - } - - for(size_t i = 0; i < arguments.size(); ++i) - { - valid[i] = true; - if (arguments[i].compare("-f") == 0) - { - input_image_files.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fd") == 0) - { - input_depth_files.push_back(input_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-fdir") == 0) - { - - // parse the -fdir directory by reading in all of the .png and .jpg files in it - path image_directory (arguments[i+1]); - - try + bool success = boost::filesystem::create_directories(p); + if (!success) { - // does the file exist and is it a directory - if (exists(image_directory) && is_directory(image_directory)) + cout << "Failed to create a directory... " << p.string() << endl; + } + } + } + + // Useful utility for creating directories for storing the output files + void create_directories(string output_path) + { + + // Creating the right directory structure + + // First get rid of the file + auto p = path(output_path); + + if (!p.empty() && !boost::filesystem::exists(p)) + { + bool success = boost::filesystem::create_directories(p); + if (!success) + { + cout << "Failed to create a directory... " << p.string() << endl; + } + } + } + + // Extracting the following command line arguments -f, -fd, -op, -of, -ov (and possible ordered repetitions) + void get_video_input_output_params(vector &input_video_files, vector &depth_dirs, vector &output_files, + vector &output_video_files, string& output_codec, vector &arguments) + { + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + } + + // By default use DIVX codec + output_codec = "DIVX"; + + string input_root = ""; + string output_root = ""; + + string separator = string(1, boost::filesystem::path::preferred_separator); + + // First check if there is a root argument (so that videos and outputs could be defined more easilly) + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-root") == 0) + { + input_root = arguments[i + 1] + separator; + output_root = arguments[i + 1] + separator; + + // Add the / or \ to the directory + i++; + } + if (arguments[i].compare("-inroot") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-outroot") == 0) + { + output_root = arguments[i + 1] + separator; + i++; + } + } + + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-f") == 0) + { + input_video_files.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fd") == 0) + { + depth_dirs.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-of") == 0) + { + output_files.push_back(output_root + arguments[i + 1]); + create_directory_from_file(output_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-ov") == 0) + { + output_video_files.push_back(output_root + arguments[i + 1]); + create_directory_from_file(output_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-oc") == 0) + { + if (arguments[i + 1].length() == 4) + output_codec = arguments[i + 1]; + } + } + + for (int i = arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + } + + void get_camera_params(int &device, float &fx, float &fy, float &cx, float &cy, vector &arguments) + { + bool* valid = new bool[arguments.size()]; + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + if (arguments[i].compare("-fx") == 0) + { + stringstream data(arguments[i + 1]); + data >> fx; + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fy") == 0) + { + stringstream data(arguments[i + 1]); + data >> fy; + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-cx") == 0) + { + stringstream data(arguments[i + 1]); + data >> cx; + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-cy") == 0) + { + stringstream data(arguments[i + 1]); + data >> cy; + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-device") == 0) + { + stringstream data(arguments[i + 1]); + data >> device; + valid[i] = false; + valid[i + 1] = false; + i++; + } + } + + for (int i = arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + } + + void get_image_input_output_params(vector &input_image_files, vector &input_depth_files, vector &output_feature_files, vector &output_pose_files, vector &output_image_files, + vector> &input_bounding_boxes, vector &arguments) + { + bool* valid = new bool[arguments.size()]; + + string out_pts_dir, out_pose_dir, out_img_dir; + + string input_root = ""; + string output_root = ""; + + string separator = string(1, boost::filesystem::path::preferred_separator); + + // First check if there is a root argument (so that videos and outputs could be defined more easilly) + for (size_t i = 0; i < arguments.size(); ++i) + { + if (arguments[i].compare("-root") == 0) + { + input_root = arguments[i + 1] + separator; + output_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-inroot") == 0) + { + input_root = arguments[i + 1] + separator; + i++; + } + if (arguments[i].compare("-outroot") == 0) + { + output_root = arguments[i + 1] + separator; + i++; + } + } + + for (size_t i = 0; i < arguments.size(); ++i) + { + valid[i] = true; + if (arguments[i].compare("-f") == 0) + { + input_image_files.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fd") == 0) + { + input_depth_files.push_back(input_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-fdir") == 0) + { + + // parse the -fdir directory by reading in all of the .png and .jpg files in it + path image_directory(arguments[i + 1]); + + try { - - vector file_in_directory; - copy(directory_iterator(image_directory), directory_iterator(), back_inserter(file_in_directory)); - - // Sort the images in the directory first - sort(file_in_directory.begin(), file_in_directory.end()); - - for (vector::const_iterator file_iterator (file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) + // does the file exist and is it a directory + if (exists(image_directory) && is_directory(image_directory)) { - // Possible image extension .jpg and .png - if(file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) - { - - - input_image_files.push_back(file_iterator->string()); - - // If there exists a .txt file corresponding to the image, it is assumed that it contains a bounding box definition for a face - // [minx, miny, maxx, maxy] - path current_file = *file_iterator; - path bbox = current_file.replace_extension("txt"); - // If there is a bounding box file push it to the list of bounding boxes - if(exists(bbox)) + vector file_in_directory; + copy(directory_iterator(image_directory), directory_iterator(), back_inserter(file_in_directory)); + + // Sort the images in the directory first + sort(file_in_directory.begin(), file_in_directory.end()); + + for (vector::const_iterator file_iterator(file_in_directory.begin()); file_iterator != file_in_directory.end(); ++file_iterator) + { + // Possible image extension .jpg and .png + if (file_iterator->extension().string().compare(".jpg") == 0 || file_iterator->extension().string().compare(".png") == 0 || file_iterator->extension().string().compare(".bmp") == 0) { - std::ifstream in_bbox(bbox.string().c_str(), ios_base::in); - double min_x, min_y, max_x, max_y; + input_image_files.push_back(file_iterator->string()); - in_bbox >> min_x >> min_y >> max_x >> max_y; + // If there exists a .txt file corresponding to the image, it is assumed that it contains a bounding box definition for a face + // [minx, miny, maxx, maxy] + path current_file = *file_iterator; + path bbox = current_file.replace_extension("txt"); - in_bbox.close(); + // If there is a bounding box file push it to the list of bounding boxes + if (exists(bbox)) + { - input_bounding_boxes.push_back(cv::Rect_(min_x, min_y, max_x - min_x, max_y - min_y)); + std::ifstream in_bbox(bbox.string().c_str(), ios_base::in); + + double min_x, min_y, max_x, max_y; + + in_bbox >> min_x >> min_y >> max_x >> max_y; + + in_bbox.close(); + + input_bounding_boxes.push_back(cv::Rect_(min_x, min_y, max_x - min_x, max_y - min_y)); + } } } } } + catch (const filesystem_error& ex) + { + cout << ex.what() << '\n'; + } + + valid[i] = false; + valid[i + 1] = false; + i++; } - catch (const filesystem_error& ex) + else if (arguments[i].compare("-ofdir") == 0) { - cout << ex.what() << '\n'; + out_pts_dir = arguments[i + 1]; + create_directories(out_pts_dir); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-opdir") == 0) + { + out_pose_dir = arguments[i + 1]; + create_directories(out_pose_dir); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-oidir") == 0) + { + out_img_dir = arguments[i + 1]; + create_directories(out_img_dir); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-op") == 0) + { + output_pose_files.push_back(output_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-of") == 0) + { + output_feature_files.push_back(output_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + else if (arguments[i].compare("-oi") == 0) + { + output_image_files.push_back(output_root + arguments[i + 1]); + valid[i] = false; + valid[i + 1] = false; + i++; + } + } + + // If any output directories are defined populate them based on image names + if (!out_img_dir.empty()) + { + for (size_t i = 0; i < input_image_files.size(); ++i) + { + path image_loc(input_image_files[i]); + + path fname = image_loc.filename(); + fname = fname.replace_extension("bmp"); + output_image_files.push_back(out_img_dir + "/" + fname.string()); + + } + if (!input_image_files.empty()) + { + create_directory_from_file(output_image_files[0]); + } + } + + if (!out_pts_dir.empty()) + { + for (size_t i = 0; i < input_image_files.size(); ++i) + { + path image_loc(input_image_files[i]); + + path fname = image_loc.filename(); + fname = fname.replace_extension("pts"); + output_feature_files.push_back(out_pts_dir + "/" + fname.string()); + } + create_directory_from_file(output_feature_files[0]); + } + + if (!out_pose_dir.empty()) + { + for (size_t i = 0; i < input_image_files.size(); ++i) + { + path image_loc(input_image_files[i]); + + path fname = image_loc.filename(); + fname = fname.replace_extension("pose"); + output_pose_files.push_back(out_pose_dir + "/" + fname.string()); + } + create_directory_from_file(output_pose_files[0]); + } + + // Make sure the same number of images and bounding boxes is present, if any bounding boxes are defined + if (input_bounding_boxes.size() > 0) + { + assert(input_bounding_boxes.size() == input_image_files.size()); + } + + // Clear up the argument list + for (int i = arguments.size() - 1; i >= 0; --i) + { + if (!valid[i]) + { + arguments.erase(arguments.begin() + i); + } + } + + } + + //=========================================================================== + // Fast patch expert response computation (linear model across a ROI) using normalised cross-correlation + //=========================================================================== + + void crossCorr_m(const cv::Mat_& img, cv::Mat_& img_dft, const cv::Mat_& _templ, map >& _templ_dfts, cv::Mat_& corr) + { + // Our model will always be under min block size so can ignore this + //const double blockScale = 4.5; + //const int minBlockSize = 256; + + int maxDepth = CV_64F; + + cv::Size dftsize; + + dftsize.width = cv::getOptimalDFTSize(corr.cols + _templ.cols - 1); + dftsize.height = cv::getOptimalDFTSize(corr.rows + _templ.rows - 1); + + // Compute block size + cv::Size blocksize; + blocksize.width = dftsize.width - _templ.cols + 1; + blocksize.width = MIN(blocksize.width, corr.cols); + blocksize.height = dftsize.height - _templ.rows + 1; + blocksize.height = MIN(blocksize.height, corr.rows); + + cv::Mat_ dftTempl; + + // if this has not been precomputed, precompute it, otherwise use it + if (_templ_dfts.find(dftsize.width) == _templ_dfts.end()) + { + dftTempl.create(dftsize.height, dftsize.width); + + cv::Mat_ src = _templ; + + cv::Mat_ dst(dftTempl, cv::Rect(0, 0, dftsize.width, dftsize.height)); + + cv::Mat_ dst1(dftTempl, cv::Rect(0, 0, _templ.cols, _templ.rows)); + + if (dst1.data != src.data) + src.convertTo(dst1, dst1.depth()); + + if (dst.cols > _templ.cols) + { + cv::Mat_ part(dst, cv::Range(0, _templ.rows), cv::Range(_templ.cols, dst.cols)); + part.setTo(0); } - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-ofdir") == 0) - { - out_pts_dir = arguments[i + 1]; - create_directories(out_pts_dir); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-opdir") == 0) - { - out_pose_dir = arguments[i + 1]; - create_directories(out_pose_dir); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-oidir") == 0) - { - out_img_dir = arguments[i + 1]; - create_directories(out_img_dir); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-op") == 0) - { - output_pose_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i + 1] = false; - i++; - } - else if (arguments[i].compare("-of") == 0) - { - output_feature_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - else if (arguments[i].compare("-oi") == 0) - { - output_image_files.push_back(output_root + arguments[i + 1]); - valid[i] = false; - valid[i+1] = false; - i++; - } - } + // Perform DFT of the template + dft(dst, dst, 0, _templ.rows); - // If any output directories are defined populate them based on image names - if(!out_img_dir.empty()) - { - for(size_t i=0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); + _templ_dfts[dftsize.width] = dftTempl; - path fname = image_loc.filename(); - fname = fname.replace_extension("bmp"); - output_image_files.push_back(out_img_dir + "/" + fname.string()); - - } - if(!input_image_files.empty()) - { - create_directory_from_file(output_image_files[0]); - } - } - - if(!out_pts_dir.empty()) - { - for(size_t i=0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); - - path fname = image_loc.filename(); - fname = fname.replace_extension("pts"); - output_feature_files.push_back(out_pts_dir + "/" + fname.string()); - } - create_directory_from_file(output_feature_files[0]); - } - - if (!out_pose_dir.empty()) - { - for (size_t i = 0; i < input_image_files.size(); ++i) - { - path image_loc(input_image_files[i]); - - path fname = image_loc.filename(); - fname = fname.replace_extension("pose"); - output_pose_files.push_back(out_pose_dir + "/" + fname.string()); - } - create_directory_from_file(output_pose_files[0]); - } - - // Make sure the same number of images and bounding boxes is present, if any bounding boxes are defined - if(input_bounding_boxes.size() > 0) - { - assert(input_bounding_boxes.size() == input_image_files.size()); - } - - // Clear up the argument list - for(int i=arguments.size()-1; i >= 0; --i) - { - if(!valid[i]) - { - arguments.erase(arguments.begin()+i); - } - } - -} - -//=========================================================================== -// Fast patch expert response computation (linear model across a ROI) using normalised cross-correlation -//=========================================================================== - -void crossCorr_m( const cv::Mat_& img, cv::Mat_& img_dft, const cv::Mat_& _templ, map >& _templ_dfts, cv::Mat_& corr) -{ - // Our model will always be under min block size so can ignore this - //const double blockScale = 4.5; - //const int minBlockSize = 256; - - int maxDepth = CV_64F; - - cv::Size dftsize; - - dftsize.width = cv::getOptimalDFTSize(corr.cols + _templ.cols - 1); - dftsize.height = cv::getOptimalDFTSize(corr.rows + _templ.rows - 1); - - // Compute block size - cv::Size blocksize; - blocksize.width = dftsize.width - _templ.cols + 1; - blocksize.width = MIN( blocksize.width, corr.cols ); - blocksize.height = dftsize.height - _templ.rows + 1; - blocksize.height = MIN( blocksize.height, corr.rows ); - - cv::Mat_ dftTempl; - - // if this has not been precomputed, precompute it, otherwise use it - if(_templ_dfts.find(dftsize.width) == _templ_dfts.end()) - { - dftTempl.create(dftsize.height, dftsize.width); - - cv::Mat_ src = _templ; - - cv::Mat_ dst(dftTempl, cv::Rect(0, 0, dftsize.width, dftsize.height)); - - cv::Mat_ dst1(dftTempl, cv::Rect(0, 0, _templ.cols, _templ.rows)); - - if( dst1.data != src.data ) - src.convertTo(dst1, dst1.depth()); - - if( dst.cols > _templ.cols ) - { - cv::Mat_ part(dst, cv::Range(0, _templ.rows), cv::Range(_templ.cols, dst.cols)); - part.setTo(0); - } - - // Perform DFT of the template - dft(dst, dst, 0, _templ.rows); - - _templ_dfts[dftsize.width] = dftTempl; - - } - else - { - // use the precomputed version - dftTempl = _templ_dfts.find(dftsize.width)->second; - } - - cv::Size bsz(std::min(blocksize.width, corr.cols), std::min(blocksize.height, corr.rows)); - cv::Mat src; - - cv::Mat cdst(corr, cv::Rect(0, 0, bsz.width, bsz.height)); - - cv::Mat_ dftImg; - - if(img_dft.empty()) - { - dftImg.create(dftsize); - dftImg.setTo(0.0); - - cv::Size dsz(bsz.width + _templ.cols - 1, bsz.height + _templ.rows - 1); - - int x2 = std::min(img.cols, dsz.width); - int y2 = std::min(img.rows, dsz.height); - - cv::Mat src0(img, cv::Range(0, y2), cv::Range(0, x2)); - cv::Mat dst(dftImg, cv::Rect(0, 0, dsz.width, dsz.height)); - cv::Mat dst1(dftImg, cv::Rect(0, 0, x2, y2)); - - src = src0; - - if( dst1.data != src.data ) - src.convertTo(dst1, dst1.depth()); - - dft( dftImg, dftImg, 0, dsz.height ); - img_dft = dftImg.clone(); - } - - cv::Mat dftTempl1(dftTempl, cv::Rect(0, 0, dftsize.width, dftsize.height)); - cv::mulSpectrums(img_dft, dftTempl1, dftImg, 0, true); - cv::dft( dftImg, dftImg, cv::DFT_INVERSE + cv::DFT_SCALE, bsz.height ); - - src = dftImg(cv::Rect(0, 0, bsz.width, bsz.height)); - - src.convertTo(cdst, CV_32F); - -} - -//////////////////////////////////////////////////////////////////////////////////////////////////////// - -void matchTemplate_m( const cv::Mat_& input_img, cv::Mat_& img_dft, cv::Mat& _integral_img, cv::Mat& _integral_img_sq, const cv::Mat_& templ, map >& templ_dfts, cv::Mat_& result, int method ) -{ - - int numType = method == CV_TM_CCORR || method == CV_TM_CCORR_NORMED ? 0 : - method == CV_TM_CCOEFF || method == CV_TM_CCOEFF_NORMED ? 1 : 2; - bool isNormed = method == CV_TM_CCORR_NORMED || - method == CV_TM_SQDIFF_NORMED || - method == CV_TM_CCOEFF_NORMED; - - // Assume result is defined properly - if(result.empty()) - { - cv::Size corrSize(input_img.cols - templ.cols + 1, input_img.rows - templ.rows + 1); - result.create(corrSize); - } - LandmarkDetector::crossCorr_m( input_img, img_dft, templ, templ_dfts, result); - - if( method == CV_TM_CCORR ) - return; - - double invArea = 1./((double)templ.rows * templ.cols); - - cv::Mat sum, sqsum; - cv::Scalar templMean, templSdv; - double *q0 = 0, *q1 = 0, *q2 = 0, *q3 = 0; - double templNorm = 0, templSum2 = 0; - - if( method == CV_TM_CCOEFF ) - { - // If it has not been precomputed compute it now - if(_integral_img.empty()) - { - integral(input_img, _integral_img, CV_64F); - } - sum = _integral_img; - - templMean = cv::mean(templ); - } - else - { - // If it has not been precomputed compute it now - if(_integral_img.empty()) - { - integral(input_img, _integral_img, _integral_img_sq, CV_64F); - } - - sum = _integral_img; - sqsum = _integral_img_sq; - - meanStdDev( templ, templMean, templSdv ); - - templNorm = templSdv[0]*templSdv[0] + templSdv[1]*templSdv[1] + templSdv[2]*templSdv[2] + templSdv[3]*templSdv[3]; - - if( templNorm < DBL_EPSILON && method == CV_TM_CCOEFF_NORMED ) - { - result.setTo(1.0); - return; - } - - templSum2 = templNorm + templMean[0]*templMean[0] + templMean[1]*templMean[1] + templMean[2]*templMean[2] + templMean[3]*templMean[3]; - - if( numType != 1 ) - { - templMean = cv::Scalar::all(0); - templNorm = templSum2; - } - - templSum2 /= invArea; - templNorm = std::sqrt(templNorm); - templNorm /= std::sqrt(invArea); // care of accuracy here - - q0 = (double*)sqsum.data; - q1 = q0 + templ.cols; - q2 = (double*)(sqsum.data + templ.rows*sqsum.step); - q3 = q2 + templ.cols; - } - - double* p0 = (double*)sum.data; - double* p1 = p0 + templ.cols; - double* p2 = (double*)(sum.data + templ.rows*sum.step); - double* p3 = p2 + templ.cols; - - int sumstep = sum.data ? (int)(sum.step / sizeof(double)) : 0; - int sqstep = sqsum.data ? (int)(sqsum.step / sizeof(double)) : 0; - - int i, j; - - for( i = 0; i < result.rows; i++ ) - { - float* rrow = result.ptr(i); - int idx = i * sumstep; - int idx2 = i * sqstep; - - for( j = 0; j < result.cols; j++, idx += 1, idx2 += 1 ) - { - double num = rrow[j], t; - double wndMean2 = 0, wndSum2 = 0; - - if( numType == 1 ) - { - - t = p0[idx] - p1[idx] - p2[idx] + p3[idx]; - wndMean2 += t*t; - num -= t*templMean[0]; - - wndMean2 *= invArea; - } - - if( isNormed || numType == 2 ) - { - - t = q0[idx2] - q1[idx2] - q2[idx2] + q3[idx2]; - wndSum2 += t; - - if( numType == 2 ) - { - num = wndSum2 - 2*num + templSum2; - num = MAX(num, 0.); - } - } - - if( isNormed ) - { - t = std::sqrt(MAX(wndSum2 - wndMean2,0))*templNorm; - if( fabs(num) < t ) - num /= t; - else if( fabs(num) < t*1.125 ) - num = num > 0 ? 1 : -1; - else - num = method != CV_TM_SQDIFF_NORMED ? 0 : 1; - } - - rrow[j] = (float)num; - } - } -} - - -//=========================================================================== -// Point set and landmark manipulation functions -//=========================================================================== -// Using Kabsch's algorithm for aligning shapes -//This assumes that align_from and align_to are already mean normalised -cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to ) -{ - - cv::SVD svd(align_from.t() * align_to); - - // make sure no reflection is there - // corr ensures that we do only rotaitons and not reflections - double d = cv::determinant(svd.vt.t() * svd.u.t()); - - cv::Matx22d corr = cv::Matx22d::eye(); - if(d > 0) - { - corr(1,1) = 1; - } - else - { - corr(1,1) = -1; - } - - cv::Matx22d R; - cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); - - return R; -} - -//============================================================================= -// Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other -cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) -{ - int n = src.rows; - - // First we mean normalise both src and dst - double mean_src_x = cv::mean(src.col(0))[0]; - double mean_src_y = cv::mean(src.col(1))[0]; - - double mean_dst_x = cv::mean(dst.col(0))[0]; - double mean_dst_y = cv::mean(dst.col(1))[0]; - - cv::Mat_ src_mean_normed = src.clone(); - src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; - src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; - - cv::Mat_ dst_mean_normed = dst.clone(); - dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; - dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; - - // Find the scaling factor of each - cv::Mat src_sq; - cv::pow(src_mean_normed, 2, src_sq); - - cv::Mat dst_sq; - cv::pow(dst_mean_normed, 2, dst_sq); - - double s_src = sqrt(cv::sum(src_sq)[0]/n); - double s_dst = sqrt(cv::sum(dst_sq)[0]/n); - - src_mean_normed = src_mean_normed / s_src; - dst_mean_normed = dst_mean_normed / s_dst; - - double s = s_dst / s_src; - - // Get the rotation - cv::Matx22d R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); - - cv::Matx22d A; - cv::Mat(s * R).copyTo(A); - - cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); - cv::Mat_ offset = dst - aligned; - - double t_x = cv::mean(offset.col(0))[0]; - double t_y = cv::mean(offset.col(1))[0]; - - return A; - -} - - -//=========================================================================== -// Visualisation functions -//=========================================================================== -void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) -{ - dest = cv::Mat_(mesh.rows,2, 0.0); - - int num_points = mesh.rows; - - double X, Y, Z; - - - cv::Mat_::const_iterator mData = mesh.begin(); - cv::Mat_::iterator projected = dest.begin(); - - for(int i = 0;i < num_points; i++) - { - // Get the points - X = *(mData++); - Y = *(mData++); - Z = *(mData++); - - double x; - double y; - - // if depth is 0 the projection is different - if(Z != 0) - { - x = ((X * fx / Z) + cx); - y = ((Y * fy / Z) + cy); } else { - x = X; - y = Y; + // use the precomputed version + dftTempl = _templ_dfts.find(dftsize.width)->second; } - // Project and store in dest matrix - (*projected++) = x; - (*projected++) = y; - } + cv::Size bsz(std::min(blocksize.width, corr.cols), std::min(blocksize.height, corr.rows)); + cv::Mat src; -} + cv::Mat cdst(corr, cv::Rect(0, 0, bsz.width, bsz.height)); -void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) -{ - double boxVerts[] = {-1, 1, -1, - 1, 1, -1, - 1, 1, 1, - -1, 1, 1, - 1, -1, 1, - 1, -1, -1, - -1, -1, -1, - -1, -1, 1}; + cv::Mat_ dftImg; - vector> edges; - edges.push_back(pair(0,1)); - edges.push_back(pair(1,2)); - edges.push_back(pair(2,3)); - edges.push_back(pair(0,3)); - edges.push_back(pair(2,4)); - edges.push_back(pair(1,5)); - edges.push_back(pair(0,6)); - edges.push_back(pair(3,7)); - edges.push_back(pair(6,5)); - edges.push_back(pair(5,4)); - edges.push_back(pair(4,7)); - edges.push_back(pair(7,6)); - - // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; - - cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); - cv::Mat_ rotBox; - - // Rotate the box - rotBox = cv::Mat(rot) * box.t(); - rotBox = rotBox.t(); - - // Move the bounding box to head position - rotBox.col(0) = rotBox.col(0) + pose[0]; - rotBox.col(1) = rotBox.col(1) + pose[1]; - rotBox.col(2) = rotBox.col(2) + pose[2]; - - // draw the lines - cv::Mat_ rotBoxProj; - Project(rotBoxProj, rotBox, fx, fy, cx, cy); - - cv::Rect image_rect(0,0,image.cols * draw_multiplier, image.rows * draw_multiplier); - - for (size_t i = 0; i < edges.size(); ++i) - { - cv::Mat_ begin; - cv::Mat_ end; - - rotBoxProj.row(edges[i].first).copyTo(begin); - rotBoxProj.row(edges[i].second).copyTo(end); - - - cv::Point p1(cvRound(begin.at(0) * (double)draw_multiplier), cvRound(begin.at(1) * (double)draw_multiplier)); - cv::Point p2(cvRound(end.at(0) * (double)draw_multiplier), cvRound(end.at(1) * (double)draw_multiplier)); - - // Only draw the line if one of the points is inside the image - if(p1.inside(image_rect) || p2.inside(image_rect)) + if (img_dft.empty()) { - cv::line(image, p1, p2, color, thickness, CV_AA, draw_shiftbits); + dftImg.create(dftsize); + dftImg.setTo(0.0); + + cv::Size dsz(bsz.width + _templ.cols - 1, bsz.height + _templ.rows - 1); + + int x2 = std::min(img.cols, dsz.width); + int y2 = std::min(img.rows, dsz.height); + + cv::Mat src0(img, cv::Range(0, y2), cv::Range(0, x2)); + cv::Mat dst(dftImg, cv::Rect(0, 0, dsz.width, dsz.height)); + cv::Mat dst1(dftImg, cv::Rect(0, 0, x2, y2)); + + src = src0; + + if (dst1.data != src.data) + src.convertTo(dst1, dst1.depth()); + + dft(dftImg, dftImg, 0, dsz.height); + img_dft = dftImg.clone(); } - + + cv::Mat dftTempl1(dftTempl, cv::Rect(0, 0, dftsize.width, dftsize.height)); + cv::mulSpectrums(img_dft, dftTempl1, dftImg, 0, true); + cv::dft(dftImg, dftImg, cv::DFT_INVERSE + cv::DFT_SCALE, bsz.height); + + src = dftImg(cv::Rect(0, 0, bsz.width, bsz.height)); + + src.convertTo(cdst, CV_32F); + } -} + //////////////////////////////////////////////////////////////////////////////////////////////////////// -vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) -{ - double boxVerts[] = {-1, 1, -1, - 1, 1, -1, - 1, 1, 1, - -1, 1, 1, - 1, -1, 1, - 1, -1, -1, - -1, -1, -1, - -1, -1, 1}; - - vector> edges; - edges.push_back(pair(0,1)); - edges.push_back(pair(1,2)); - edges.push_back(pair(2,3)); - edges.push_back(pair(0,3)); - edges.push_back(pair(2,4)); - edges.push_back(pair(1,5)); - edges.push_back(pair(0,6)); - edges.push_back(pair(3,7)); - edges.push_back(pair(6,5)); - edges.push_back(pair(5,4)); - edges.push_back(pair(4,7)); - edges.push_back(pair(7,6)); - - // The size of the head is roughly 200mm x 200mm x 200mm - cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; - - cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); - cv::Mat_ rotBox; - - // Rotate the box - rotBox = cv::Mat(rot) * box.t(); - rotBox = rotBox.t(); - - // Move the bounding box to head position - rotBox.col(0) = rotBox.col(0) + pose[0]; - rotBox.col(1) = rotBox.col(1) + pose[1]; - rotBox.col(2) = rotBox.col(2) + pose[2]; - - // draw the lines - cv::Mat_ rotBoxProj; - Project(rotBoxProj, rotBox, fx, fy, cx, cy); - - vector> lines; - - for (size_t i = 0; i < edges.size(); ++i) + void matchTemplate_m(const cv::Mat_& input_img, cv::Mat_& img_dft, cv::Mat& _integral_img, cv::Mat& _integral_img_sq, const cv::Mat_& templ, map >& templ_dfts, cv::Mat_& result, int method) { - cv::Mat_ begin; - cv::Mat_ end; - - rotBoxProj.row(edges[i].first).copyTo(begin); - rotBoxProj.row(edges[i].second).copyTo(end); - cv::Point2d p1(begin.at(0), begin.at(1)); - cv::Point2d p2(end.at(0), end.at(1)); - - lines.push_back(pair(p1,p2)); - - } + int numType = method == CV_TM_CCORR || method == CV_TM_CCORR_NORMED ? 0 : + method == CV_TM_CCOEFF || method == CV_TM_CCOEFF_NORMED ? 1 : 2; + bool isNormed = method == CV_TM_CCORR_NORMED || + method == CV_TM_SQDIFF_NORMED || + method == CV_TM_CCOEFF_NORMED; - return lines; -} - -void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness) -{ - cv::Rect image_rect(0,0,image.cols, image.rows); - - for (size_t i = 0; i < lines.size(); ++i) - { - cv::Point p1 = lines.at(i).first; - cv::Point p2 = lines.at(i).second; - // Only draw the line if one of the points is inside the image - if(p1.inside(image_rect) || p2.inside(image_rect)) + // Assume result is defined properly + if (result.empty()) { - cv::line(image, p1, p2, color, thickness, CV_AA); + cv::Size corrSize(input_img.cols - templ.cols + 1, input_img.rows - templ.rows + 1); + result.create(corrSize); + } + LandmarkDetector::crossCorr_m(input_img, img_dft, templ, templ_dfts, result); + + if (method == CV_TM_CCORR) + return; + + double invArea = 1. / ((double)templ.rows * templ.cols); + + cv::Mat sum, sqsum; + cv::Scalar templMean, templSdv; + double *q0 = 0, *q1 = 0, *q2 = 0, *q3 = 0; + double templNorm = 0, templSum2 = 0; + + if (method == CV_TM_CCOEFF) + { + // If it has not been precomputed compute it now + if (_integral_img.empty()) + { + integral(input_img, _integral_img, CV_64F); + } + sum = _integral_img; + + templMean = cv::mean(templ); + } + else + { + // If it has not been precomputed compute it now + if (_integral_img.empty()) + { + integral(input_img, _integral_img, _integral_img_sq, CV_64F); + } + + sum = _integral_img; + sqsum = _integral_img_sq; + + meanStdDev(templ, templMean, templSdv); + + templNorm = templSdv[0] * templSdv[0] + templSdv[1] * templSdv[1] + templSdv[2] * templSdv[2] + templSdv[3] * templSdv[3]; + + if (templNorm < DBL_EPSILON && method == CV_TM_CCOEFF_NORMED) + { + result.setTo(1.0); + return; + } + + templSum2 = templNorm + templMean[0] * templMean[0] + templMean[1] * templMean[1] + templMean[2] * templMean[2] + templMean[3] * templMean[3]; + + if (numType != 1) + { + templMean = cv::Scalar::all(0); + templNorm = templSum2; + } + + templSum2 /= invArea; + templNorm = std::sqrt(templNorm); + templNorm /= std::sqrt(invArea); // care of accuracy here + + q0 = (double*)sqsum.data; + q1 = q0 + templ.cols; + q2 = (double*)(sqsum.data + templ.rows*sqsum.step); + q3 = q2 + templ.cols; + } + + double* p0 = (double*)sum.data; + double* p1 = p0 + templ.cols; + double* p2 = (double*)(sum.data + templ.rows*sum.step); + double* p3 = p2 + templ.cols; + + int sumstep = sum.data ? (int)(sum.step / sizeof(double)) : 0; + int sqstep = sqsum.data ? (int)(sqsum.step / sizeof(double)) : 0; + + int i, j; + + for (i = 0; i < result.rows; i++) + { + float* rrow = result.ptr(i); + int idx = i * sumstep; + int idx2 = i * sqstep; + + for (j = 0; j < result.cols; j++, idx += 1, idx2 += 1) + { + double num = rrow[j], t; + double wndMean2 = 0, wndSum2 = 0; + + if (numType == 1) + { + + t = p0[idx] - p1[idx] - p2[idx] + p3[idx]; + wndMean2 += t*t; + num -= t*templMean[0]; + + wndMean2 *= invArea; + } + + if (isNormed || numType == 2) + { + + t = q0[idx2] - q1[idx2] - q2[idx2] + q3[idx2]; + wndSum2 += t; + + if (numType == 2) + { + num = wndSum2 - 2 * num + templSum2; + num = MAX(num, 0.); + } + } + + if (isNormed) + { + t = std::sqrt(MAX(wndSum2 - wndMean2, 0))*templNorm; + if (fabs(num) < t) + num /= t; + else if (fabs(num) < t*1.125) + num = num > 0 ? 1 : -1; + else + num = method != CV_TM_SQDIFF_NORMED ? 0 : 1; + } + + rrow[j] = (float)num; + } } - } -} -// Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(const cv::Mat_& shape2D, cv::Mat_& visibilities) -{ - int n = shape2D.rows/2; - vector landmarks; + //=========================================================================== + // Point set and landmark manipulation functions + //=========================================================================== + // Using Kabsch's algorithm for aligning shapes + //This assumes that align_from and align_to are already mean normalised + cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to) + { - for( int i = 0; i < n; ++i) - { - if(visibilities.at(i)) + cv::SVD svd(align_from.t() * align_to); + + // make sure no reflection is there + // corr ensures that we do only rotaitons and not reflections + double d = cv::determinant(svd.vt.t() * svd.u.t()); + + cv::Matx22d corr = cv::Matx22d::eye(); + if (d > 0) { - cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i +n)); + corr(1, 1) = 1; + } + else + { + corr(1, 1) = -1; + } + + cv::Matx22d R; + cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); + + return R; + } + + //============================================================================= + // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other + cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) + { + int n = src.rows; + + // First we mean normalise both src and dst + double mean_src_x = cv::mean(src.col(0))[0]; + double mean_src_y = cv::mean(src.col(1))[0]; + + double mean_dst_x = cv::mean(dst.col(0))[0]; + double mean_dst_y = cv::mean(dst.col(1))[0]; + + cv::Mat_ src_mean_normed = src.clone(); + src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; + src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; + + cv::Mat_ dst_mean_normed = dst.clone(); + dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; + dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; + + // Find the scaling factor of each + cv::Mat src_sq; + cv::pow(src_mean_normed, 2, src_sq); + + cv::Mat dst_sq; + cv::pow(dst_mean_normed, 2, dst_sq); + + double s_src = sqrt(cv::sum(src_sq)[0] / n); + double s_dst = sqrt(cv::sum(dst_sq)[0] / n); + + src_mean_normed = src_mean_normed / s_src; + dst_mean_normed = dst_mean_normed / s_dst; + + double s = s_dst / s_src; + + // Get the rotation + cv::Matx22d R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); + + cv::Matx22d A; + cv::Mat(s * R).copyTo(A); + + cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); + cv::Mat_ offset = dst - aligned; + + double t_x = cv::mean(offset.col(0))[0]; + double t_y = cv::mean(offset.col(1))[0]; + + return A; + + } + + + //=========================================================================== + // Visualisation functions + //=========================================================================== + void Project(cv::Mat_& dest, const cv::Mat_& mesh, double fx, double fy, double cx, double cy) + { + dest = cv::Mat_(mesh.rows, 2, 0.0); + + int num_points = mesh.rows; + + double X, Y, Z; + + + cv::Mat_::const_iterator mData = mesh.begin(); + cv::Mat_::iterator projected = dest.begin(); + + for (int i = 0; i < num_points; i++) + { + // Get the points + X = *(mData++); + Y = *(mData++); + Z = *(mData++); + + double x; + double y; + + // if depth is 0 the projection is different + if (Z != 0) + { + x = ((X * fx / Z) + cx); + y = ((Y * fy / Z) + cy); + } + else + { + x = X; + y = Y; + } + + // Project and store in dest matrix + (*projected++) = x; + (*projected++) = y; + } + + } + + void DrawBox(cv::Mat image, cv::Vec6d pose, cv::Scalar color, int thickness, float fx, float fy, float cx, float cy) + { + double boxVerts[] = { -1, 1, -1, + 1, 1, -1, + 1, 1, 1, + -1, 1, 1, + 1, -1, 1, + 1, -1, -1, + -1, -1, -1, + -1, -1, 1 }; + + vector> edges; + edges.push_back(pair(0, 1)); + edges.push_back(pair(1, 2)); + edges.push_back(pair(2, 3)); + edges.push_back(pair(0, 3)); + edges.push_back(pair(2, 4)); + edges.push_back(pair(1, 5)); + edges.push_back(pair(0, 6)); + edges.push_back(pair(3, 7)); + edges.push_back(pair(6, 5)); + edges.push_back(pair(5, 4)); + edges.push_back(pair(4, 7)); + edges.push_back(pair(7, 6)); + + // The size of the head is roughly 200mm x 200mm x 200mm + cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; + + cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); + cv::Mat_ rotBox; + + // Rotate the box + rotBox = cv::Mat(rot) * box.t(); + rotBox = rotBox.t(); + + // Move the bounding box to head position + rotBox.col(0) = rotBox.col(0) + pose[0]; + rotBox.col(1) = rotBox.col(1) + pose[1]; + rotBox.col(2) = rotBox.col(2) + pose[2]; + + // draw the lines + cv::Mat_ rotBoxProj; + Project(rotBoxProj, rotBox, fx, fy, cx, cy); + + cv::Rect image_rect(0, 0, image.cols * draw_multiplier, image.rows * draw_multiplier); + + for (size_t i = 0; i < edges.size(); ++i) + { + cv::Mat_ begin; + cv::Mat_ end; + + rotBoxProj.row(edges[i].first).copyTo(begin); + rotBoxProj.row(edges[i].second).copyTo(end); + + + cv::Point p1(cvRound(begin.at(0) * (double)draw_multiplier), cvRound(begin.at(1) * (double)draw_multiplier)); + cv::Point p2(cvRound(end.at(0) * (double)draw_multiplier), cvRound(end.at(1) * (double)draw_multiplier)); + + // Only draw the line if one of the points is inside the image + if (p1.inside(image_rect) || p2.inside(image_rect)) + { + cv::line(image, p1, p2, color, thickness, CV_AA, draw_shiftbits); + } + + } + + } + + vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy) + { + double boxVerts[] = { -1, 1, -1, + 1, 1, -1, + 1, 1, 1, + -1, 1, 1, + 1, -1, 1, + 1, -1, -1, + -1, -1, -1, + -1, -1, 1 }; + + vector> edges; + edges.push_back(pair(0, 1)); + edges.push_back(pair(1, 2)); + edges.push_back(pair(2, 3)); + edges.push_back(pair(0, 3)); + edges.push_back(pair(2, 4)); + edges.push_back(pair(1, 5)); + edges.push_back(pair(0, 6)); + edges.push_back(pair(3, 7)); + edges.push_back(pair(6, 5)); + edges.push_back(pair(5, 4)); + edges.push_back(pair(4, 7)); + edges.push_back(pair(7, 6)); + + // The size of the head is roughly 200mm x 200mm x 200mm + cv::Mat_ box = cv::Mat(8, 3, CV_64F, boxVerts).clone() * 100; + + cv::Matx33d rot = LandmarkDetector::Euler2RotationMatrix(cv::Vec3d(pose[3], pose[4], pose[5])); + cv::Mat_ rotBox; + + // Rotate the box + rotBox = cv::Mat(rot) * box.t(); + rotBox = rotBox.t(); + + // Move the bounding box to head position + rotBox.col(0) = rotBox.col(0) + pose[0]; + rotBox.col(1) = rotBox.col(1) + pose[1]; + rotBox.col(2) = rotBox.col(2) + pose[2]; + + // draw the lines + cv::Mat_ rotBoxProj; + Project(rotBoxProj, rotBox, fx, fy, cx, cy); + + vector> lines; + + for (size_t i = 0; i < edges.size(); ++i) + { + cv::Mat_ begin; + cv::Mat_ end; + + rotBoxProj.row(edges[i].first).copyTo(begin); + rotBoxProj.row(edges[i].second).copyTo(end); + + cv::Point2d p1(begin.at(0), begin.at(1)); + cv::Point2d p2(end.at(0), end.at(1)); + + lines.push_back(pair(p1, p2)); + + } + + return lines; + } + + void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness) + { + cv::Rect image_rect(0, 0, image.cols, image.rows); + + for (size_t i = 0; i < lines.size(); ++i) + { + cv::Point p1 = lines.at(i).first; + cv::Point p2 = lines.at(i).second; + // Only draw the line if one of the points is inside the image + if (p1.inside(image_rect) || p2.inside(image_rect)) + { + cv::line(image, p1, p2, color, thickness, CV_AA); + } + + } + + } + + // Computing landmarks (to be drawn later possibly) + vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities) + { + int n = shape2D.rows / 2; + vector landmarks; + + for (int i = 0; i < n; ++i) + { + if (visibilities.at(i)) + { + cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i + n)); + + landmarks.push_back(featurePoint); + } + } + + return landmarks; + } + + // Computing landmarks (to be drawn later possibly) + vector CalculateAllLandmarks(const cv::Mat_& shape2D) + { + + int n; + vector landmarks; + + if (shape2D.cols == 2) + { + n = shape2D.rows; + } + else if (shape2D.cols == 1) + { + n = shape2D.rows / 2; + } + + for (int i = 0; i < n; ++i) + { + cv::Point2d featurePoint; + if (shape2D.cols == 1) + { + featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i + n)); + } + else + { + featurePoint = cv::Point2d(shape2D.at(i, 0), shape2D.at(i, 1)); + } landmarks.push_back(featurePoint); } + + return landmarks; } - return landmarks; -} - -// Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(cv::Mat img, const cv::Mat_& shape2D) -{ - - int n; - vector landmarks; - - if(shape2D.cols == 2) + // Computing landmarks (to be drawn later possibly) + vector CalculateAllLandmarks(const CLNF& clnf_model) { - n = shape2D.rows; - } - else if(shape2D.cols == 1) - { - n = shape2D.rows/2; + return CalculateAllLandmarks(clnf_model.detected_landmarks); } - for( int i = 0; i < n; ++i) - { - cv::Point2d featurePoint; - if(shape2D.cols == 1) + // Computing landmarks (to be drawn later possibly) + vector CalculateVisibleLandmarks(const CLNF& clnf_model) + { + // If the detection was not successful no landmarks are visible + if (clnf_model.detection_success) { - featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i +n)); + int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); + // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation + return CalculateVisibleLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); } else { - featurePoint = cv::Point2d(shape2D.at(i, 0), shape2D.at(i, 1)); + return vector(); } - - landmarks.push_back(featurePoint); } - - return landmarks; -} -// Computing landmarks (to be drawn later possibly) -vector CalculateLandmarks(CLNF& clnf_model) -{ - - int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); - - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation - return CalculateLandmarks(clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); - -} - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) -{ - int n = shape2D.rows/2; - - - // Drawing feature points - if(n >= 66) + // Computing eye landmarks (to be drawn later or in different interfaces) + vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model) { - for( int i = 0; i < n; ++i) - { - if(visibilities.at(i)) + + vector to_return; + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + + auto lmks = CalculateVisibleLandmarks(clnf_model.hierarchical_models[i]); + for (auto lmk : lmks) + { + to_return.push_back(lmk); + } + } + } + return to_return; + } + + // Computing eye landmarks (to be drawn later or in different interfaces) + vector CalculateAllEyeLandmarks(const CLNF& clnf_model) + { + + vector to_return; + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + + if (clnf_model.hierarchical_model_names[i].compare("left_eye_28") == 0 || + clnf_model.hierarchical_model_names[i].compare("right_eye_28") == 0) + { + + auto lmks = CalculateAllLandmarks(clnf_model.hierarchical_models[i]); + for (auto lmk : lmks) + { + to_return.push_back(lmk); + } + } + } + return to_return; + } + + // Drawing landmarks on a face image + void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) + { + int n = shape2D.rows / 2; + + + // Drawing feature points + if (n >= 66) + { + for (int i = 0; i < n; ++i) + { + if (visibilities.at(i)) + { + cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0); + + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + + } + } + } + else if (n == 28) // drawing eyes + { + for (int i = 0; i < n; ++i) { cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); // A rough heuristic for drawn point size - int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.0* ((double)img.cols) / 640.0); + int thickness = 1.0; + int thickness_2 = 1.0; + + int next_point = i + 1; + if (i == 7) + next_point = 0; + if (i == 19) + next_point = 8; + if (i == 27) + next_point = 20; + + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); + if (i < 8 || i > 19) + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + else + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } } - } - else if(n == 28) // drawing eyes - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 7) - next_point = 0; - if(i == 19) - next_point = 8; - if(i == 27) - next_point = 20; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - if( i < 8 || i > 19) - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - else - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(0, 0, 255), thickness_2, CV_AA, draw_shiftbits); - - - } - } - else if(n == 6) - { - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); - - // A rough heuristic for drawn point size - int thickness = 1.0; - int thickness_2 = 1.0; - - int next_point = i + 1; - if(i == 5) - next_point = 0; - - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); - cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); - } - } -} - -// Drawing landmarks on a face image -void Draw(cv::Mat img, const cv::Mat_& shape2D) -{ - - int n; - - if(shape2D.cols == 2) - { - n = shape2D.rows; - } - else if(shape2D.cols == 1) - { - n = shape2D.rows/2; - } - - for( int i = 0; i < n; ++i) - { - cv::Point featurePoint; - if(shape2D.cols == 1) + else if (n == 6) { - featurePoint = cv::Point(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + for (int i = 0; i < n; ++i) + { + cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + + // A rough heuristic for drawn point size + int thickness = 1.0; + int thickness_2 = 1.0; + + int next_point = i + 1; + if (i == 5) + next_point = 0; + + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); + cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + } + } + } + + // Drawing landmarks on a face image + void Draw(cv::Mat img, const cv::Mat_& shape2D) + { + + int n; + + if (shape2D.cols == 2) + { + n = shape2D.rows; + } + else if (shape2D.cols == 1) + { + n = shape2D.rows / 2; + } + + for (int i = 0; i < n; ++i) + { + cv::Point featurePoint; + if (shape2D.cols == 1) + { + featurePoint = cv::Point(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + } + else + { + featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (double)draw_multiplier), cvRound(shape2D.at(i, 1) * (double)draw_multiplier)); + } + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); + + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); + cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); + + } + + } + + // Drawing detected landmarks on a face image + void Draw(cv::Mat img, const CLNF& clnf_model) + { + + int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); + + // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation + Draw(img, clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); + + // If the model has hierarchical updates draw those too + for (size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + { + if (clnf_model.hierarchical_models[i].pdm.NumberOfPoints() != clnf_model.hierarchical_mapping[i].size()) + { + Draw(img, clnf_model.hierarchical_models[i]); + } + } + } + + void DrawLandmarks(cv::Mat img, vector landmarks) + { + for (cv::Point p : landmarks) + { + + // A rough heuristic for drawn point size + int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); + int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); + + cv::circle(img, p, 1, cv::Scalar(0, 0, 255), thickness, CV_AA); + cv::circle(img, p, 1, cv::Scalar(255, 0, 0), thickness_2, CV_AA); + } + + } + + //=========================================================================== + // Angle representation conversion helpers + //=========================================================================== + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) + { + cv::Matx33d rotation_matrix; + + double s1 = sin(eulerAngles[0]); + double s2 = sin(eulerAngles[1]); + double s3 = sin(eulerAngles[2]); + + double c1 = cos(eulerAngles[0]); + double c2 = cos(eulerAngles[1]); + double c3 = cos(eulerAngles[2]); + + rotation_matrix(0, 0) = c2 * c3; + rotation_matrix(0, 1) = -c2 *s3; + rotation_matrix(0, 2) = s2; + rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; + rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; + rotation_matrix(1, 2) = -c2 * s1; + rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; + rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; + rotation_matrix(2, 2) = c1 * c2; + + return rotation_matrix; + } + + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) + { + double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0; + double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0); + double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0); + double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0); + + double t1 = 2.0 * (q0*q2 + q1*q3); + + double yaw = asin(2.0 * (q0*q2 + q1*q3)); + double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); + double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); + + return cv::Vec3d(pitch, yaw, roll); + } + + cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) + { + cv::Matx33d rotMatrix = LandmarkDetector::Euler2RotationMatrix(euler); + cv::Vec3d axis_angle; + cv::Rodrigues(rotMatrix, axis_angle); + return axis_angle; + } + + cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return RotationMatrix2Euler(rotation_matrix); + } + + cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) + { + cv::Matx33d rotation_matrix; + cv::Rodrigues(axis_angle, rotation_matrix); + return rotation_matrix; + } + + cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) + { + cv::Vec3d axis_angle; + cv::Rodrigues(rotation_matrix, axis_angle); + return axis_angle; + } + + //=========================================================================== + + //============================================================================ + // Face detection helpers + //============================================================================ + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, double min_width, cv::Rect_ roi) + { + cv::CascadeClassifier classifier("./classifiers/haarcascade_frontalface_alt.xml"); + if (classifier.empty()) + { + cout << "Couldn't load the Haar cascade classifier" << endl; + return false; } else { - featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (double)draw_multiplier), cvRound(shape2D.at(i, 1) * (double)draw_multiplier)); + return DetectFaces(o_regions, intensity, classifier, min_width, roi); } - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(0, 0, 255), thickness, CV_AA, draw_shiftbits); - cv::circle(img, featurePoint, 1 * draw_multiplier, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } - -} -// Drawing detected landmarks on a face image -void Draw(cv::Mat img, const CLNF& clnf_model) -{ - - int idx = clnf_model.patch_experts.GetViewIdx(clnf_model.params_global, 0); - - // Because we only draw visible points, need to find which points patch experts consider visible at a certain orientation - Draw(img, clnf_model.detected_landmarks, clnf_model.patch_experts.visibilities[0][idx]); - - // If the model has hierarchical updates draw those too - for(size_t i = 0; i < clnf_model.hierarchical_models.size(); ++i) + bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier, double min_width, cv::Rect_ roi) { - if(clnf_model.hierarchical_models[i].pdm.NumberOfPoints() != clnf_model.hierarchical_mapping[i].size()) + + vector face_detections; + if (min_width == -1) { - Draw(img, clnf_model.hierarchical_models[i]); + classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(50, 50)); } - } -} - -void DrawLandmarks(cv::Mat img, vector landmarks) -{ - for(cv::Point p : landmarks) - { - - // A rough heuristic for drawn point size - int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); - int thickness_2 = (int)std::ceil(1.5* ((double)img.cols) / 640.0); - - cv::circle(img, p, 1, cv::Scalar(0,0,255), thickness, CV_AA); - cv::circle(img, p, 1, cv::Scalar(255,0,0), thickness_2, CV_AA); - } - -} - -//=========================================================================== -// Angle representation conversion helpers -//=========================================================================== - -// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign -cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles) -{ - cv::Matx33d rotation_matrix; - - double s1 = sin(eulerAngles[0]); - double s2 = sin(eulerAngles[1]); - double s3 = sin(eulerAngles[2]); - - double c1 = cos(eulerAngles[0]); - double c2 = cos(eulerAngles[1]); - double c3 = cos(eulerAngles[2]); - - rotation_matrix(0,0) = c2 * c3; - rotation_matrix(0,1) = -c2 *s3; - rotation_matrix(0,2) = s2; - rotation_matrix(1,0) = c1 * s3 + c3 * s1 * s2; - rotation_matrix(1,1) = c1 * c3 - s1 * s2 * s3; - rotation_matrix(1,2) = -c2 * s1; - rotation_matrix(2,0) = s1 * s3 - c1 * c3 * s2; - rotation_matrix(2,1) = c3 * s1 + c1 * s2 * s3; - rotation_matrix(2,2) = c1 * c2; - - return rotation_matrix; -} - -// Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign -cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) -{ - double q0 = sqrt( 1 + rotation_matrix(0,0) + rotation_matrix(1,1) + rotation_matrix(2,2) ) / 2.0; - double q1 = (rotation_matrix(2,1) - rotation_matrix(1,2)) / (4.0*q0) ; - double q2 = (rotation_matrix(0,2) - rotation_matrix(2,0)) / (4.0*q0) ; - double q3 = (rotation_matrix(1,0) - rotation_matrix(0,1)) / (4.0*q0) ; - - double t1 = 2.0 * (q0*q2 + q1*q3); - - double yaw = asin(2.0 * (q0*q2 + q1*q3)); - double pitch= atan2(2.0 * (q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3); - double roll = atan2(2.0 * (q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3); - - return cv::Vec3d(pitch, yaw, roll); -} - -cv::Vec3d Euler2AxisAngle(const cv::Vec3d& euler) -{ - cv::Matx33d rotMatrix = LandmarkDetector::Euler2RotationMatrix(euler); - cv::Vec3d axis_angle; - cv::Rodrigues(rotMatrix, axis_angle); - return axis_angle; -} - -cv::Vec3d AxisAngle2Euler(const cv::Vec3d& axis_angle) -{ - cv::Matx33d rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return RotationMatrix2Euler(rotation_matrix); -} - -cv::Matx33d AxisAngle2RotationMatrix(const cv::Vec3d& axis_angle) -{ - cv::Matx33d rotation_matrix; - cv::Rodrigues(axis_angle, rotation_matrix); - return rotation_matrix; -} - -cv::Vec3d RotationMatrix2AxisAngle(const cv::Matx33d& rotation_matrix) -{ - cv::Vec3d axis_angle; - cv::Rodrigues(rotation_matrix, axis_angle); - return axis_angle; -} - -//=========================================================================== - -//============================================================================ -// Face detection helpers -//============================================================================ -bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity) -{ - cv::CascadeClassifier classifier("./classifiers/haarcascade_frontalface_alt.xml"); - if(classifier.empty()) - { - cout << "Couldn't load the Haar cascade classifier" << endl; - return false; - } - else - { - return DetectFaces(o_regions, intensity, classifier); - } - -} - -bool DetectFaces(vector >& o_regions, const cv::Mat_& intensity, cv::CascadeClassifier& classifier) -{ - - vector face_detections; - classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(50, 50)); - - // Convert from int bounding box do a double one with corrections - o_regions.resize(face_detections.size()); - - for( size_t face = 0; face < o_regions.size(); ++face) - { - // OpenCV is overgenerous with face size and y location is off - // CLNF detector expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate - - // The scalings were learned using the Face Detections on LFPW, Helen, AFW and iBUG datasets, using ground truth and detections from openCV - - // Correct for scale - o_regions[face].width = face_detections[face].width * 0.8924; - o_regions[face].height = face_detections[face].height * 0.8676; - - // Move the face slightly to the right (as the width was made smaller) - o_regions[face].x = face_detections[face].x + 0.0578 * face_detections[face].width; - // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested - o_regions[face].y = face_detections[face].y + face_detections[face].height * 0.2166; - - - } - return o_regions.size() > 0; -} - -bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity_image, cv::CascadeClassifier& classifier, cv::Point preference) -{ - // The tracker can return multiple faces - vector > face_detections; - - bool detect_success = LandmarkDetector::DetectFaces(face_detections, intensity_image, classifier); - - if(detect_success) - { - - bool use_preferred = (preference.x != -1) && (preference.y != -1); - - if(face_detections.size() > 1) + else { - // keep the closest one if preference point not set - double best = -1; - int bestIndex = -1; - for( size_t i = 0; i < face_detections.size(); ++i) + classifier.detectMultiScale(intensity, face_detections, 1.2, 2, 0, cv::Size(min_width, min_width)); + } + + // Convert from int bounding box do a double one with corrections + for (size_t face = 0; face < o_regions.size(); ++face) + { + // OpenCV is overgenerous with face size and y location is off + // CLNF detector expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate + + // The scalings were learned using the Face Detections on LFPW, Helen, AFW and iBUG datasets, using ground truth and detections from openCV + cv::Rect_ region; + // Correct for scale + region.width = face_detections[face].width * 0.8924; + region.height = face_detections[face].height * 0.8676; + + // Move the face slightly to the right (as the width was made smaller) + region.x = face_detections[face].x + 0.0578 * face_detections[face].width; + // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested + region.y = face_detections[face].y + face_detections[face].height * 0.2166; + + if (min_width != -1) { + if (region.width < min_width || region.x < ((double)intensity.cols) * roi.x || region.y < ((double)intensity.cols) * roi.y || + region.x + region.width >((double)intensity.cols) * (roi.x + roi.width) || region.y + region.height >((double)intensity.rows) * (roi.y + roi.height)) + continue; + } + + + o_regions.push_back(region); + } + return o_regions.size() > 0; + } + + bool DetectSingleFace(cv::Rect_& o_region, const cv::Mat_& intensity_image, cv::CascadeClassifier& classifier, cv::Point preference, double min_width, cv::Rect_ roi) + { + // The tracker can return multiple faces + vector > face_detections; + + bool detect_success = LandmarkDetector::DetectFaces(face_detections, intensity_image, classifier, min_width, roi); + + if (detect_success) + { + + bool use_preferred = (preference.x != -1) && (preference.y != -1); + + if (face_detections.size() > 1) + { + // keep the closest one if preference point not set + double best = -1; + int bestIndex = -1; + for (size_t i = 0; i < face_detections.size(); ++i) + { + double dist; + bool better; + + if (use_preferred) + { + dist = sqrt((preference.x) * (face_detections[i].width / 2 + face_detections[i].x) + + (preference.y) * (face_detections[i].height / 2 + face_detections[i].y)); + better = dist < best; + } + else + { + dist = face_detections[i].width; + better = face_detections[i].width > best; + } + + // Pick a closest face to preffered point or the biggest face + if (i == 0 || better) + { + bestIndex = i; + best = dist; + } + } + + o_region = face_detections[bestIndex]; + + } + else + { + o_region = face_detections[0]; + } + + } + else + { + // if not detected + o_region = cv::Rect_(0, 0, 0, 0); + } + return detect_success; + } + + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences, double min_width, cv::Rect_ roi) + { + dlib::frontal_face_detector detector = dlib::get_frontal_face_detector(); + + return DetectFacesHOG(o_regions, intensity, detector, confidences, min_width, roi); + + } + + bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& detector, std::vector& o_confidences, double min_width, cv::Rect_ roi) + { + + cv::Mat_ upsampled_intensity; + + double scaling = 1.3; + + cv::resize(intensity, upsampled_intensity, cv::Size((int)(intensity.cols * scaling), (int)(intensity.rows * scaling))); + + dlib::cv_image cv_grayscale(upsampled_intensity); + + std::vector face_detections; + detector(cv_grayscale, face_detections, -0.2); + + // Convert from int bounding box do a double one with corrections + //o_regions.resize(face_detections.size()); + //o_confidences.resize(face_detections.size()); + + for (size_t face = 0; face < face_detections.size(); ++face) + { + // CLNF expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate + + cv::Rect_ region; + // Move the face slightly to the right (as the width was made smaller) + region.x = (face_detections[face].rect.get_rect().tl_corner().x() + 0.0389 * face_detections[face].rect.get_rect().width()) / scaling; + // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested + region.y = (face_detections[face].rect.get_rect().tl_corner().y() + 0.1278 * face_detections[face].rect.get_rect().height()) / scaling; + + // Correct for scale + region.width = (face_detections[face].rect.get_rect().width() * 0.9611) / scaling; + region.height = (face_detections[face].rect.get_rect().height() * 0.9388) / scaling; + + // The scalings were learned using the Face Detections on LFPW and Helen using ground truth and detections from the HOG detector + if (min_width != -1) + { + if (region.width < min_width || region.x < ((double)intensity.cols) * roi.x || region.y < ((double)intensity.cols) * roi.y || + region.x + region.width >((double)intensity.cols) * (roi.x + roi.width) || region.y + region.height >((double)intensity.rows) * (roi.y + roi.height)) + continue; + } + + + o_regions.push_back(region); + o_confidences.push_back(face_detections[face].detection_confidence); + + + } + return o_regions.size() > 0; + } + + bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity_img, dlib::frontal_face_detector& detector, double& confidence, cv::Point preference, double min_width, cv::Rect_ roi) + { + // The tracker can return multiple faces + vector > face_detections; + vector confidences; + bool detect_success = LandmarkDetector::DetectFacesHOG(face_detections, intensity_img, detector, confidences, min_width, roi); + + // In case of multiple faces pick the biggest one + bool use_size = true; + + if (detect_success) + { + + bool use_preferred = (preference.x != -1) && (preference.y != -1); + + // keep the most confident one or the one closest to preference point if set + double best_so_far; + if (use_preferred) + { + best_so_far = sqrt((preference.x - (face_detections[0].width / 2 + face_detections[0].x)) * (preference.x - (face_detections[0].width / 2 + face_detections[0].x)) + + (preference.y - (face_detections[0].height / 2 + face_detections[0].y)) * (preference.y - (face_detections[0].height / 2 + face_detections[0].y))); + } + else if (use_size) + { + best_so_far = (face_detections[0].width + face_detections[0].height) / 2.0; + } + else + { + best_so_far = confidences[0]; + } + int bestIndex = 0; + + for (size_t i = 1; i < face_detections.size(); ++i) + { + double dist; bool better; - if(use_preferred) + if (use_preferred) { - dist = sqrt((preference.x) * (face_detections[i].width/2 + face_detections[i].x) + - (preference.y) * (face_detections[i].height/2 + face_detections[i].y)); - better = dist < best; + dist = sqrt((preference.x - (face_detections[i].width / 2 + face_detections[i].x)) * (preference.x - (face_detections[i].width / 2 + face_detections[i].x)) + + (preference.y - (face_detections[i].height / 2 + face_detections[i].y)) * (preference.y - (face_detections[i].height / 2 + face_detections[i].y))); + + better = dist < best_so_far; + } + else if (use_size) + { + dist = (face_detections[i].width + face_detections[i].height) / 2.0; + better = dist > best_so_far; } else { - dist = face_detections[i].width; - better = face_detections[i].width > best; + dist = confidences[i]; + better = dist > best_so_far; } - // Pick a closest face to preffered point or the biggest face - if(i == 0 || better) + // Pick a closest face + if (better) { - bestIndex = i; - best = dist; - } + best_so_far = dist; + bestIndex = i; + } } o_region = face_detections[bestIndex]; - - } - else - { - o_region = face_detections[0]; - } - - } - else - { - // if not detected - o_region = cv::Rect_(0,0,0,0); - } - return detect_success; -} - -bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, std::vector& confidences) -{ - dlib::frontal_face_detector detector = dlib::get_frontal_face_detector(); - - return DetectFacesHOG(o_regions, intensity, detector, confidences); - -} - -bool DetectFacesHOG(vector >& o_regions, const cv::Mat_& intensity, dlib::frontal_face_detector& detector, std::vector& o_confidences) -{ - - cv::Mat_ upsampled_intensity; - - double scaling = 1.3; - - cv::resize(intensity, upsampled_intensity, cv::Size((int)(intensity.cols * scaling), (int)(intensity.rows * scaling))); - - dlib::cv_image cv_grayscale(upsampled_intensity); - - std::vector face_detections; - detector(cv_grayscale, face_detections, -0.2); - - // Convert from int bounding box do a double one with corrections - o_regions.resize(face_detections.size()); - o_confidences.resize(face_detections.size()); - - for( size_t face = 0; face < o_regions.size(); ++face) - { - // CLNF expects the bounding box to encompass from eyebrow to chin in y, and from cheeck outline to cheeck outline in x, so we need to compensate - - // The scalings were learned using the Face Detections on LFPW and Helen using ground truth and detections from the HOG detector - - // Move the face slightly to the right (as the width was made smaller) - o_regions[face].x = (face_detections[face].rect.get_rect().tl_corner().x() + 0.0389 * face_detections[face].rect.get_rect().width())/scaling; - // Shift face down as OpenCV Haar Cascade detects the forehead as well, and we're not interested - o_regions[face].y = (face_detections[face].rect.get_rect().tl_corner().y() + 0.1278 * face_detections[face].rect.get_rect().height())/scaling; - - // Correct for scale - o_regions[face].width = (face_detections[face].rect.get_rect().width() * 0.9611)/scaling; - o_regions[face].height = (face_detections[face].rect.get_rect().height() * 0.9388)/scaling; - - o_confidences[face] = face_detections[face].detection_confidence; - - - } - return o_regions.size() > 0; -} - -bool DetectSingleFaceHOG(cv::Rect_& o_region, const cv::Mat_& intensity_img, dlib::frontal_face_detector& detector, double& confidence, cv::Point preference) -{ - // The tracker can return multiple faces - vector > face_detections; - vector confidences; - - bool detect_success = LandmarkDetector::DetectFacesHOG(face_detections, intensity_img, detector, confidences); - - if(detect_success) - { - - bool use_preferred = (preference.x != -1) && (preference.y != -1); - - // keep the most confident one or the one closest to preference point if set - double best_so_far; - if(use_preferred) - { - best_so_far = sqrt((preference.x - (face_detections[0].width/2 + face_detections[0].x)) * (preference.x - (face_detections[0].width/2 + face_detections[0].x)) + - (preference.y - (face_detections[0].height/2 + face_detections[0].y)) * (preference.y - (face_detections[0].height/2 + face_detections[0].y))); + confidence = confidences[bestIndex]; } else { - best_so_far = confidences[0]; + // if not detected + o_region = cv::Rect_(0, 0, 0, 0); + // A completely unreliable detection (shouldn't really matter what is returned here) + confidence = -2; } - int bestIndex = 0; + return detect_success; + } - for( size_t i = 1; i < face_detections.size(); ++i) + //============================================================================ + // Matrix reading functionality + //============================================================================ + + // Reading in a matrix from a stream + void ReadMat(std::ifstream& stream, cv::Mat &output_mat) + { + // Read in the number of rows, columns and the data type + int row, col, type; + + stream >> row >> col >> type; + + output_mat = cv::Mat(row, col, type); + + switch (output_mat.type()) { - - double dist; - bool better; - - if(use_preferred) - { - dist = sqrt((preference.x - (face_detections[0].width/2 + face_detections[0].x)) * (preference.x - (face_detections[0].width/2 + face_detections[0].x)) + - (preference.y - (face_detections[0].height/2 + face_detections[0].y)) * (preference.y - (face_detections[0].height/2 + face_detections[0].y))); - better = dist < best_so_far; - } - else - { - dist = confidences[i]; - better = dist > best_so_far; - } - - // Pick a closest face - if(better) - { - best_so_far = dist; - bestIndex = i; - } - } - - o_region = face_detections[bestIndex]; - confidence = confidences[bestIndex]; - } - else - { - // if not detected - o_region = cv::Rect_(0,0,0,0); - // A completely unreliable detection (shouldn't really matter what is returned here) - confidence = -2; - } - return detect_success; -} - -//============================================================================ -// Matrix reading functionality -//============================================================================ - -// Reading in a matrix from a stream -void ReadMat(std::ifstream& stream, cv::Mat &output_mat) -{ - // Read in the number of rows, columns and the data type - int row,col,type; - - stream >> row >> col >> type; - - output_mat = cv::Mat(row, col, type); - - switch(output_mat.type()) - { - case CV_64FC1: + case CV_64FC1: { cv::MatIterator_ begin_it = output_mat.begin(); cv::MatIterator_ end_it = output_mat.end(); - - while(begin_it != end_it) + + while (begin_it != end_it) { stream >> *begin_it++; } @@ -1544,7 +1604,7 @@ void ReadMat(std::ifstream& stream, cv::Mat &output_mat) cv::MatIterator_ begin_it = output_mat.begin(); cv::MatIterator_ end_it = output_mat.end(); - while(begin_it != end_it) + while (begin_it != end_it) { stream >> *begin_it++; } @@ -1554,7 +1614,7 @@ void ReadMat(std::ifstream& stream, cv::Mat &output_mat) { cv::MatIterator_ begin_it = output_mat.begin(); cv::MatIterator_ end_it = output_mat.end(); - while(begin_it != end_it) + while (begin_it != end_it) { stream >> *begin_it++; } @@ -1564,42 +1624,42 @@ void ReadMat(std::ifstream& stream, cv::Mat &output_mat) { cv::MatIterator_ begin_it = output_mat.begin(); cv::MatIterator_ end_it = output_mat.end(); - while(begin_it != end_it) + while (begin_it != end_it) { stream >> *begin_it++; } } break; default: - printf("ERROR(%s,%d) : Unsupported Matrix type %d!\n", __FILE__,__LINE__,output_mat.type()); abort(); + printf("ERROR(%s,%d) : Unsupported Matrix type %d!\n", __FILE__, __LINE__, output_mat.type()); abort(); + } } -} -void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) -{ - // Read in the number of rows, columns and the data type - int row, col, type; - - stream.read ((char*)&row, 4); - stream.read ((char*)&col, 4); - stream.read ((char*)&type, 4); - - output_mat = cv::Mat(row, col, type); - int size = output_mat.rows * output_mat.cols * output_mat.elemSize(); - stream.read((char *)output_mat.data, size); - -} - -// Skipping lines that start with # (together with empty lines) -void SkipComments(std::ifstream& stream) -{ - while(stream.peek() == '#' || stream.peek() == '\n'|| stream.peek() == ' ' || stream.peek() == '\r') + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) { - std::string skipped; - std::getline(stream, skipped); + // Read in the number of rows, columns and the data type + int row, col, type; + + stream.read((char*)&row, 4); + stream.read((char*)&col, 4); + stream.read((char*)&type, 4); + + output_mat = cv::Mat(row, col, type); + int size = output_mat.rows * output_mat.cols * output_mat.elemSize(); + stream.read((char *)output_mat.data, size); + + } + + // Skipping lines that start with # (together with empty lines) + void SkipComments(std::ifstream& stream) + { + while (stream.peek() == '#' || stream.peek() == '\n' || stream.peek() == ' ' || stream.peek() == '\r') + { + std::string skipped; + std::getline(stream, skipped); + } } -} }