From 71174dc32b8d6463a14bef59edb8f1feb54d4f42 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 2 Oct 2017 21:26:56 +0100 Subject: [PATCH] Moving more stuff over to floats (landmark locations and piece-wise affine warps) --- exe/FeatureExtraction/FeatureExtraction.cpp | 2 +- lib/local/FaceAnalyser/include/FaceAnalyser.h | 4 +- lib/local/FaceAnalyser/include/Face_utils.h | 10 +- lib/local/FaceAnalyser/include/PAW.h | 26 +- lib/local/FaceAnalyser/include/PDM.h | 8 +- lib/local/FaceAnalyser/src/FaceAnalyser.cpp | 4 +- lib/local/FaceAnalyser/src/Face_utils.cpp | 128 +++--- lib/local/FaceAnalyser/src/PAW.cpp | 212 +++++----- lib/local/FaceAnalyser/src/PDM.cpp | 291 ++++++------- .../include/LandmarkDetectionValidator.h | 2 +- .../include/LandmarkDetectorModel.h | 16 +- .../include/LandmarkDetectorParameters.h | 10 +- .../include/LandmarkDetectorUtils.h | 10 +- lib/local/LandmarkDetector/include/PAW.h | 22 +- lib/local/LandmarkDetector/include/PDM.h | 4 +- .../src/LandmarkDetectionValidator.cpp | 10 +- .../src/LandmarkDetectorFunc.cpp | 6 +- .../src/LandmarkDetectorModel.cpp | 101 +---- .../src/LandmarkDetectorParameters.cpp | 14 +- .../src/LandmarkDetectorUtils.cpp | 102 ++++- lib/local/LandmarkDetector/src/PAW.cpp | 395 +++++++++--------- lib/local/LandmarkDetector/src/PDM.cpp | 30 +- .../LandmarkDetector/src/Patch_experts.cpp | 25 +- 23 files changed, 733 insertions(+), 699 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index 507319e9..dbb1b4ee 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -796,7 +796,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo { if (face_model.tracking_initialised) { - *output_file << ", " << face_model.detected_landmarks.at(i); + *output_file << ", " << face_model.detected_landmarks.at(i); } else { diff --git a/lib/local/FaceAnalyser/include/FaceAnalyser.h b/lib/local/FaceAnalyser/include/FaceAnalyser.h index 4ce4838a..cae0b224 100644 --- a/lib/local/FaceAnalyser/include/FaceAnalyser.h +++ b/lib/local/FaceAnalyser/include/FaceAnalyser.h @@ -64,7 +64,7 @@ public: // Constructor for FaceAnalyser using the parameters structure FaceAnalyser(const FaceAnalysis::FaceAnalyserParameters& face_analyser_params); - void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false, bool visualise = true); + void AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online = false, bool visualise = true); cv::Mat GetLatestHOGDescriptorVisualisation(); @@ -77,7 +77,7 @@ public: // A standalone call for predicting AUs from a static image, the first element in the pair represents occurence the second intensity // This call is useful for detecting action units in images - std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise = true); + std::pair>, std::vector>> PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise = true); void Reset(); diff --git a/lib/local/FaceAnalyser/include/Face_utils.h b/lib/local/FaceAnalyser/include/Face_utils.h index 51ebf9b5..af3126b7 100644 --- a/lib/local/FaceAnalyser/include/Face_utils.h +++ b/lib/local/FaceAnalyser/include/Face_utils.h @@ -46,8 +46,8 @@ namespace FaceAnalysis // Defining a set of useful utility functions to be used within FaceAnalyser // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid = true, double scale = 0.6, int width = 96, int height = 96); + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid = true, float scale = 0.6, int width = 96, int height = 96); + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid = true, float scale = 0.6, int width = 96, int height = 96); void Extract_FHOG_descriptor(cv::Mat_& descriptor, const cv::Mat& image, int& num_rows, int& num_cols, int cell_size = 8); @@ -62,11 +62,11 @@ namespace FaceAnalysis //=========================================================================== // Using Kabsch's algorithm for aligning shapes //This assumes that align_from and align_to are already mean normalised - cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); + cv::Matx22f AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); //============================================================================= // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other - cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); + cv::Matx22f AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); //=========================================================================== // Visualisation functions @@ -77,6 +77,8 @@ namespace FaceAnalysis // Angle representation conversion helpers //=========================================================================== cv::Matx33d Euler2RotationMatrix(const cv::Vec3d& eulerAngles); + + cv::Matx33f Euler2RotationMatrix_f(const cv::Vec3f& eulerAngles); // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix); diff --git a/lib/local/FaceAnalyser/include/PAW.h b/lib/local/FaceAnalyser/include/PAW.h index 64ee14df..968a2ee3 100644 --- a/lib/local/FaceAnalyser/include/PAW.h +++ b/lib/local/FaceAnalyser/include/PAW.h @@ -56,16 +56,16 @@ public: int number_of_pixels; // Minimum x coordinate in destination - double min_x; + float min_x; // minimum y coordinate in destination - double min_y; + float min_y; // Destination points (landmarks to be warped to) - cv::Mat_ destination_landmarks; + cv::Mat_ destination_landmarks; // Destination points (landmarks to be warped from) - cv::Mat_ source_landmarks; + cv::Mat_ source_landmarks; // Triangulation, each triangle is warped using an affine transform cv::Mat_ triangulation; @@ -81,13 +81,13 @@ public: // affine coefficients for all triangles (see Matthews and Baker 2004) // 6 coefficients for each triangle (are computed from alpha and beta) // This is computed during each warp based on source landmarks - cv::Mat_ coefficients; + cv::Mat_ coefficients; // matrix of (c,x,y) coeffs for alpha - cv::Mat_ alpha; + cv::Mat_ alpha; // matrix of (c,x,y) coeffs for alpha - cv::Mat_ beta; + cv::Mat_ beta; // x-source of warped points cv::Mat_ map_x; @@ -99,10 +99,10 @@ public: PAW(){;} // Construct a warp from a destination shape and triangulation - PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); // The final optional argument allows for optimisation if the triangle indices from previous frame are known (for tracking in video) - PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y); + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y); // Copy constructor PAW(const PAW& other); @@ -110,7 +110,7 @@ public: void Read(std::ifstream &s); // The actual warping - void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); + void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); // Compute coefficients needed for warping void CalcCoeff(); @@ -128,9 +128,9 @@ public: private: // Helper functions for dealing with triangles - static bool sameSide(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3); - static bool pointInTriangle(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3); - static int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1); + static bool sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3); + static bool pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3); + static int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1); }; //=========================================================================== diff --git a/lib/local/FaceAnalyser/include/PDM.h b/lib/local/FaceAnalyser/include/PDM.h index d3bfe333..a76f0ebb 100644 --- a/lib/local/FaceAnalyser/include/PDM.h +++ b/lib/local/FaceAnalyser/include/PDM.h @@ -75,16 +75,16 @@ class PDM{ inline int NumberOfModes() const {return princ_comp.cols;} // Compute shape in object space (3D) - void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; + void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; // Compute shape in image space (2D) - void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; + void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box - void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; + void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3f rotation = cv::Vec3f(0.0f)) const; // Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results) - void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; + void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation = cv::Vec3f(0.0f)) const; // provided the model parameters, compute the bounding box of a face void CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local) const; diff --git a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp index 7b662836..8a39db6b 100644 --- a/lib/local/FaceAnalyser/src/FaceAnalyser.cpp +++ b/lib/local/FaceAnalyser/src/FaceAnalyser.cpp @@ -248,7 +248,7 @@ int GetViewId(const vector orientations_all, const cv::Vec3d& orienta } -std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise) +std::pair>, std::vector>> FaceAnalyser::PredictStaticAUs(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool visualise) { // Extract shape parameters from the detected landmarks @@ -310,7 +310,7 @@ std::pair>, std::vector& detected_landmarks, bool success, double timestamp_seconds, bool online, bool visualise) +void FaceAnalyser::AddNextFrame(const cv::Mat& frame, const cv::Mat_& detected_landmarks, bool success, double timestamp_seconds, bool online, bool visualise) { frames_tracking++; diff --git a/lib/local/FaceAnalyser/src/Face_utils.cpp b/lib/local/FaceAnalyser/src/Face_utils.cpp index f9a6fd30..1fef10c3 100644 --- a/lib/local/FaceAnalyser/src/Face_utils.cpp +++ b/lib/local/FaceAnalyser/src/Face_utils.cpp @@ -50,12 +50,12 @@ namespace FaceAnalysis { // Pick only the more stable/rigid points under changes of expression - void extract_rigid_points(cv::Mat_& source_points, cv::Mat_& destination_points) + void extract_rigid_points(cv::Mat_& source_points, cv::Mat_& destination_points) { if(source_points.rows == 68) { - cv::Mat_ tmp_source = source_points.clone(); - source_points = cv::Mat_(); + cv::Mat_ tmp_source = source_points.clone(); + source_points = cv::Mat_(); // Push back the rigid points (some face outline, eyes, and nose) source_points.push_back(tmp_source.row(1)); @@ -83,8 +83,8 @@ namespace FaceAnalysis source_points.push_back(tmp_source.row(46)); source_points.push_back(tmp_source.row(47)); - cv::Mat_ tmp_dest = destination_points.clone(); - destination_points = cv::Mat_(); + cv::Mat_ tmp_dest = destination_points.clone(); + destination_points = cv::Mat_(); // Push back the rigid points destination_points.push_back(tmp_dest.row(1)); @@ -115,16 +115,16 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFace(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, bool rigid, float sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); - cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); + cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points if(rigid) @@ -154,16 +154,16 @@ namespace FaceAnalysis } // Aligning a face to a common reference frame - void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid, double sim_scale, int out_width, int out_height) + void AlignFaceMask(cv::Mat& aligned_face, const cv::Mat& frame, const cv::Mat_& detected_landmarks, cv::Vec6d params_global, const PDM& pdm, const cv::Mat_& triangulation, bool rigid, float sim_scale, int out_width, int out_height) { // Will warp to scaled mean shape - cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; + cv::Mat_ similarity_normalised_shape = pdm.mean_shape * sim_scale; // Discard the z component similarity_normalised_shape = similarity_normalised_shape(cv::Rect(0, 0, 1, 2*similarity_normalised_shape.rows/3)).clone(); - cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); - cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); + cv::Mat_ source_landmarks = detected_landmarks.reshape(1, 2).t(); + cv::Mat_ destination_landmarks = similarity_normalised_shape.reshape(1, 2).t(); // Aligning only the more rigid points if(rigid) @@ -171,18 +171,18 @@ namespace FaceAnalysis extract_rigid_points(source_landmarks, destination_landmarks); } - cv::Matx22d scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); - cv::Matx23d warp_matrix; + cv::Matx22f scale_rot_matrix = AlignShapesWithScale(source_landmarks, destination_landmarks); + cv::Matx23f warp_matrix; warp_matrix(0,0) = scale_rot_matrix(0,0); warp_matrix(0,1) = scale_rot_matrix(0,1); warp_matrix(1,0) = scale_rot_matrix(1,0); warp_matrix(1,1) = scale_rot_matrix(1,1); - double tx = params_global[4]; - double ty = params_global[5]; + float tx = params_global[4]; + float ty = params_global[5]; - cv::Vec2d T(tx, ty); + cv::Vec2f T(tx, ty); T = scale_rot_matrix * T; // Make sure centering is correct @@ -192,7 +192,7 @@ namespace FaceAnalysis cv::warpAffine(frame, aligned_face, warp_matrix, cv::Size(out_width, out_height), cv::INTER_LINEAR); // Move the destination landmarks there as well - cv::Matx22d warp_matrix_2d(warp_matrix(0,0), warp_matrix(0,1), warp_matrix(1,0), warp_matrix(1,1)); + cv::Matx22f warp_matrix_2d(warp_matrix(0,0), warp_matrix(0,1), warp_matrix(1,0), warp_matrix(1,1)); destination_landmarks = cv::Mat(detected_landmarks.reshape(1, 2).t()) * cv::Mat(warp_matrix_2d).t(); @@ -200,19 +200,19 @@ namespace FaceAnalysis destination_landmarks.col(1) = destination_landmarks.col(1) + warp_matrix(1,2); // Move the eyebrows up to include more of upper face - destination_landmarks.at(0,1) -= (30/0.7)*sim_scale; - destination_landmarks.at(16,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(0,1) -= (30/0.7)*sim_scale; + destination_landmarks.at(16,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(17,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(18,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(19,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(20,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(21,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(22,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(23,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(24,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(25,1) -= (30 / 0.7)*sim_scale; - destination_landmarks.at(26,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(17,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(18,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(19,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(20,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(21,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(22,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(23,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(24,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(25,1) -= (30 / 0.7)*sim_scale; + destination_landmarks.at(26,1) -= (30 / 0.7)*sim_scale; destination_landmarks = cv::Mat(destination_landmarks.t()).reshape(1, 1).t(); @@ -363,16 +363,16 @@ namespace FaceAnalysis //=========================================================================== // Using Kabsch's algorithm for aligning shapes //This assumes that align_from and align_to are already mean normalised - cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to) + cv::Matx22f AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to) { cv::SVD svd(align_from.t() * align_to); // make sure no reflection is there // corr ensures that we do only rotaitons and not reflections - double d = cv::determinant(svd.vt.t() * svd.u.t()); + float d = cv::determinant(svd.vt.t() * svd.u.t()); - cv::Matx22d corr = cv::Matx22d::eye(); + cv::Matx22f corr = cv::Matx22f::eye(); if (d > 0) { corr(1, 1) = 1; @@ -382,7 +382,7 @@ namespace FaceAnalysis corr(1, 1) = -1; } - cv::Matx22d R; + cv::Matx22f R; cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); return R; @@ -390,22 +390,22 @@ namespace FaceAnalysis //============================================================================= // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other - cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) + cv::Matx22f AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) { int n = src.rows; // First we mean normalise both src and dst - double mean_src_x = cv::mean(src.col(0))[0]; - double mean_src_y = cv::mean(src.col(1))[0]; + float mean_src_x = cv::mean(src.col(0))[0]; + float mean_src_y = cv::mean(src.col(1))[0]; - double mean_dst_x = cv::mean(dst.col(0))[0]; - double mean_dst_y = cv::mean(dst.col(1))[0]; + float mean_dst_x = cv::mean(dst.col(0))[0]; + float mean_dst_y = cv::mean(dst.col(1))[0]; - cv::Mat_ src_mean_normed = src.clone(); + cv::Mat_ src_mean_normed = src.clone(); src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; - cv::Mat_ dst_mean_normed = dst.clone(); + cv::Mat_ dst_mean_normed = dst.clone(); dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; @@ -416,25 +416,25 @@ namespace FaceAnalysis cv::Mat dst_sq; cv::pow(dst_mean_normed, 2, dst_sq); - double s_src = sqrt(cv::sum(src_sq)[0] / n); - double s_dst = sqrt(cv::sum(dst_sq)[0] / n); + float s_src = sqrt(cv::sum(src_sq)[0] / n); + float s_dst = sqrt(cv::sum(dst_sq)[0] / n); src_mean_normed = src_mean_normed / s_src; dst_mean_normed = dst_mean_normed / s_dst; - double s = s_dst / s_src; + float s = s_dst / s_src; // Get the rotation - cv::Matx22d R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); + cv::Matx22f R = AlignShapesKabsch2D(src_mean_normed, dst_mean_normed); - cv::Matx22d A; + cv::Matx22f A; cv::Mat(s * R).copyTo(A); - cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); - cv::Mat_ offset = dst - aligned; + cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); + cv::Mat_ offset = dst - aligned; - double t_x = cv::mean(offset.col(0))[0]; - double t_y = cv::mean(offset.col(1))[0]; + float t_x = cv::mean(offset.col(0))[0]; + float t_y = cv::mean(offset.col(1))[0]; return A; @@ -514,6 +514,32 @@ namespace FaceAnalysis return rotation_matrix; } + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign + cv::Matx33f Euler2RotationMatrix_f(const cv::Vec3f& eulerAngles) + { + cv::Matx33f rotation_matrix; + + float s1 = sin(eulerAngles[0]); + float s2 = sin(eulerAngles[1]); + float s3 = sin(eulerAngles[2]); + + float c1 = cos(eulerAngles[0]); + float c2 = cos(eulerAngles[1]); + float c3 = cos(eulerAngles[2]); + + rotation_matrix(0, 0) = c2 * c3; + rotation_matrix(0, 1) = -c2 *s3; + rotation_matrix(0, 2) = s2; + rotation_matrix(1, 0) = c1 * s3 + c3 * s1 * s2; + rotation_matrix(1, 1) = c1 * c3 - s1 * s2 * s3; + rotation_matrix(1, 2) = -c2 * s1; + rotation_matrix(2, 0) = s1 * s3 - c1 * c3 * s2; + rotation_matrix(2, 1) = c3 * s1 + c1 * s2 * s3; + rotation_matrix(2, 2) = c1 * c2; + + return rotation_matrix; + } + // Using the XYZ convention R = Rx * Ry * Rz, left-handed positive sign cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) { diff --git a/lib/local/FaceAnalyser/src/PAW.cpp b/lib/local/FaceAnalyser/src/PAW.cpp index 4c5e397c..542c68eb 100644 --- a/lib/local/FaceAnalyser/src/PAW.cpp +++ b/lib/local/FaceAnalyser/src/PAW.cpp @@ -53,7 +53,7 @@ triangle_id(other.triangle_id.clone()), pixel_mask(other.pixel_mask.clone()), co } // A constructor from destination shape and triangulation -PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) { // Initialise some variables directly this->destination_landmarks = destination_shape; @@ -64,14 +64,14 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu int num_tris = triangulation.rows; // Pre-compute the rest - alpha = cv::Mat_(num_tris, 3); - beta = cv::Mat_(num_tris, 3); + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); - cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); - cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); // Create a vector representation of the control points - std::vector> destination_points; + std::vector> destination_points; for (int tri = 0; tri < num_tris; ++tri) { @@ -79,43 +79,43 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu int k = triangulation.at(tri, 1); int l = triangulation.at(tri, 2); - double c1 = ys.at(l) - ys.at(j); - double c2 = xs.at(l) - xs.at(j); - double c4 = ys.at(k) - ys.at(j); - double c3 = xs.at(k) - xs.at(j); + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); - double c5 = c3*c1 - c2*c4; + float c5 = c3*c1 - c2*c4; - alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; - alpha.at(tri, 1) = c1/c5; - alpha.at(tri, 2) = -c2/c5; + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; - beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; - beta.at(tri, 1) = -c4/c5; - beta.at(tri, 2) = c3/c5; + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; // Add points corresponding to triangles as optimisation - std::vector triangle_points(10); + std::vector triangle_points(10); - triangle_points[0] = xs.at(j); - triangle_points[1] = ys.at(j); - triangle_points[2] = xs.at(k); - triangle_points[3] = ys.at(k); - triangle_points[4] = xs.at(l); - triangle_points[5] = ys.at(l); + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); - cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); - cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); double min_x, max_x, min_y, max_y; cv::minMaxIdx(xs_three, &min_x, &max_x); cv::minMaxIdx(ys_three, &min_y, &max_y); - triangle_points[6] = max_x; - triangle_points[7] = max_y; + triangle_points[6] = (float) max_x; + triangle_points[7] = (float) max_y; - triangle_points[8] = min_x; - triangle_points[9] = min_y; + triangle_points[8] = (float) min_x; + triangle_points[9] = (float) min_y; destination_points.push_back(triangle_points); @@ -123,9 +123,14 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu double max_x; double max_y; + double min_x_d; + double min_y_d; - minMaxLoc(xs, &min_x, &max_x); - minMaxLoc(ys, &min_y, &max_y); + minMaxLoc(xs, &min_x_d, &max_x); + minMaxLoc(ys, &min_y_d, &max_y); + + min_x = min_x_d; + min_y = min_y_d; int w = (int)(max_x - min_x + 1.5); int h = (int)(max_y - min_y + 1.5); @@ -141,7 +146,7 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu { for(int x = 0; x < pixel_mask.cols; x++) { - curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); // If there is a triangle at this location if(curr_tri != -1) { @@ -160,7 +165,7 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu } // Manually define min and max values -PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y) +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y) { // Initialise some variables directly this->destination_landmarks = destination_shape; @@ -171,14 +176,14 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu int num_tris = triangulation.rows; // Pre-compute the rest - alpha = cv::Mat_(num_tris, 3); - beta = cv::Mat_(num_tris, 3); + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); - cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); - cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); // Create a vector representation of the control points - std::vector> destination_points; + std::vector> destination_points; for (int tri = 0; tri < num_tris; ++tri) { @@ -186,50 +191,50 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu int k = triangulation.at(tri, 1); int l = triangulation.at(tri, 2); - double c1 = ys.at(l) - ys.at(j); - double c2 = xs.at(l) - xs.at(j); - double c4 = ys.at(k) - ys.at(j); - double c3 = xs.at(k) - xs.at(j); + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); - double c5 = c3*c1 - c2*c4; + float c5 = c3*c1 - c2*c4; - alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; - alpha.at(tri, 1) = c1/c5; - alpha.at(tri, 2) = -c2/c5; + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1/c5; + alpha.at(tri, 2) = -c2/c5; - beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; - beta.at(tri, 1) = -c4/c5; - beta.at(tri, 2) = c3/c5; + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; + beta.at(tri, 1) = -c4/c5; + beta.at(tri, 2) = c3/c5; // Add points corresponding to triangles as optimisation - std::vector triangle_points(10); + std::vector triangle_points(10); - triangle_points[0] = xs.at(j); - triangle_points[1] = ys.at(j); - triangle_points[2] = xs.at(k); - triangle_points[3] = ys.at(k); - triangle_points[4] = xs.at(l); - triangle_points[5] = ys.at(l); + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); - cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); - cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); double min_x, max_x, min_y, max_y; cv::minMaxIdx(xs_three, &min_x, &max_x); cv::minMaxIdx(ys_three, &min_y, &max_y); - triangle_points[6] = max_x; - triangle_points[7] = max_y; + triangle_points[6] = (float)max_x; + triangle_points[7] = (float)max_y; - triangle_points[8] = min_x; - triangle_points[9] = min_y; + triangle_points[8] = (float)min_x; + triangle_points[9] = (float)min_y; destination_points.push_back(triangle_points); } - double max_x; - double max_y; + float max_x; + float max_y; min_x = in_min_x; min_y = in_min_y; @@ -251,7 +256,7 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu { for(int x = 0; x < pixel_mask.cols; x++) { - curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); // If there is a triangle at this location if(curr_tri != -1) { @@ -273,10 +278,15 @@ void PAW::Read(std::ifstream& stream) { stream.read ((char*)&number_of_pixels, 4); - stream.read ((char*)&min_x, 8); - stream.read ((char*)&min_y, 8); + double min_x_d, min_y_d; + stream.read ((char*)&min_x_d, 8); + stream.read ((char*)&min_y_d, 8); + min_x = (float)min_x_d; + min_y = (float)min_y_d; - ReadMatBin(stream, destination_landmarks); + cv::Mat_ destination_landmarks_d; + ReadMatBin(stream, destination_landmarks_d); + destination_landmarks_d.convertTo(destination_landmarks, CV_32F); ReadMatBin(stream, triangulation); @@ -286,9 +296,13 @@ void PAW::Read(std::ifstream& stream) ReadMatBin(stream, tmpMask); tmpMask.convertTo(pixel_mask, CV_8U); - ReadMatBin(stream, alpha); + cv::Mat_ alpha_d; + ReadMatBin(stream, alpha_d); + alpha_d.convertTo(alpha, CV_32F); - ReadMatBin(stream, beta); + cv::Mat_ beta_d; + ReadMatBin(stream, beta_d); + beta_d.convertTo(beta, CV_32F); map_x.create(pixel_mask.rows,pixel_mask.cols); map_y.create(pixel_mask.rows,pixel_mask.cols); @@ -300,7 +314,7 @@ void PAW::Read(std::ifstream& stream) //============================================================================= // cropping from the source image to the destination image using the shape in s, used to determine if shape fitting converged successfully -void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) +void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) { // set the current shape @@ -331,19 +345,19 @@ void PAW::CalcCoeff() int j = triangulation.at(l,1); int k = triangulation.at(l,2); - double c1 = source_landmarks.at(i , 0); - double c2 = source_landmarks.at(j , 0) - c1; - double c3 = source_landmarks.at(k , 0) - c1; - double c4 = source_landmarks.at(i + p, 0); - double c5 = source_landmarks.at(j + p, 0) - c4; - double c6 = source_landmarks.at(k + p, 0) - c4; + float c1 = source_landmarks.at(i , 0); + float c2 = source_landmarks.at(j , 0) - c1; + float c3 = source_landmarks.at(k , 0) - c1; + float c4 = source_landmarks.at(i + p, 0); + float c5 = source_landmarks.at(j + p, 0) - c4; + float c6 = source_landmarks.at(k + p, 0) - c4; // Get a pointer to the coefficient we will be precomputing - double *coeff = coefficients.ptr(l); + float *coeff = coefficients.ptr(l); // Extract the relevant alphas and betas - double *c_alpha = alpha.ptr(l); - double *c_beta = beta.ptr(l); + float *c_alpha = alpha.ptr(l); + float *c_beta = beta.ptr(l); coeff[0] = c1 + c2 * c_alpha[0] + c3 * c_beta[0]; coeff[1] = c2 * c_alpha[1] + c3 * c_beta[1]; @@ -365,18 +379,18 @@ void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) cv::MatIterator_ tp = triangle_id.begin(); // The coefficients corresponding to the current triangle - double * a; + float * a; // Current triangle being processed int k=-1; for(int y = 0; y < pixel_mask.rows; y++) { - double yi = double(y) + min_y; + float yi = float(y) + min_y; for(int x = 0; x < pixel_mask.cols; x++) { - double xi = double(x) + min_x; + float xi = float(x) + min_x; if(*mp == 0) { @@ -393,22 +407,22 @@ void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) if(j != k) { // Update the coefficient pointer if a new triangle is being processed - a = coefficients.ptr(j); + a = coefficients.ptr(j); k = j; } //ap is now the pointer to the coefficients - double *ap = a; + float *ap = a; //look at the first coefficient (and increment). first coefficient is an x offset - double xo = *ap++; + float xo = *ap++; //second coefficient is an x scale as a function of x xo += *ap++ * xi; //third coefficient ap(2) is an x scale as a function of y *xp = float(xo + *ap++ * yi); //then fourth coefficient ap(3) is a y offset - double yo = *ap++; + float yo = *ap++; //fifth coeff adds coeff[4]*x to y yo += *ap++ * xi; //final coeff adds coeff[5]*y to y @@ -425,18 +439,18 @@ void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) // ============================================================ // Is the point (x0,y0) on same side as a half-plane defined by (x1,y1), (x2, y2), and (x3, y3) -bool PAW::sameSide(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +bool PAW::sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) { - double x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); - double y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); + float x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); + float y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); return x*y >= 0; } // if point (x0, y0) is on same side for all three half-planes it is in a triangle -bool PAW::pointInTriangle(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +bool PAW::pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) { bool same_1 = sameSide(x0, y0, x1, y1, x2, y2, x3, y3); bool same_2 = sameSide(x0, y0, x2, y2, x1, y1, x3, y3); @@ -447,15 +461,15 @@ bool PAW::pointInTriangle(double x0, double y0, double x1, double y1, double x2, } // Find if a given point lies in the triangles -int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) +int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) { int num_tris = control_points.size(); int tri = -1; - double x0 = point.x; - double y0 = point.y; + float x0 = point.x; + float y0 = point.y; // Allow a guess for speed (so as not to go through all triangles) if(guess != -1) @@ -472,11 +486,11 @@ int PAW::findTriangle(const cv::Point_& point, const std::vector& out_shape, const cv::Mat_& p_local) const +void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local) const { out_shape.create(mean_shape.rows, mean_shape.cols); - - cv::Mat_ out_shape_f = mean_shape + princ_comp*p_local; - // TODO move away from doubles - out_shape_f.convertTo(out_shape, CV_64F); + out_shape = mean_shape + princ_comp * p_local; } //=========================================================================== // Get the 2D shape (in image space) from global and local parameters -void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const +void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const { int n = this->NumberOfPoints(); - double s = params_global[0]; // scaling factor - double tx = params_global[4]; // x offset - double ty = params_global[5]; // y offset + // TODO move to float + float s = (float)params_global[0]; // scaling factor + float tx = (float)params_global[4]; // x offset + float ty = (float)params_global[5]; // y offset - // get the rotation matrix from the euler angles - cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); + // get the rotation matrix from the euler angles + cv::Vec3f euler((float)params_global[1], (float)params_global[2], (float)params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix_f(euler); // get the 3D shape of the object - cv::Mat_ Shape_3D_f = mean_shape + princ_comp * params_local; - cv::Mat_ Shape_3D; - Shape_3D_f.convertTo(Shape_3D, CV_64F); + cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; // create the 2D shape matrix (if it has not been defined yet) - if((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) + if ((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) { - out_shape.create(2*n,1); + out_shape.create(2 * n, 1); } // for every vertex - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { // Transform this using the weak-perspective mapping to 2D from 3D - out_shape.at(i ,0) = s * ( currRot(0,0) * Shape_3D.at(i, 0) + currRot(0,1) * Shape_3D.at(i+n ,0) + currRot(0,2) * Shape_3D.at(i+n*2,0) ) + tx; - out_shape.at(i+n,0) = s * ( currRot(1,0) * Shape_3D.at(i, 0) + currRot(1,1) * Shape_3D.at(i+n ,0) + currRot(1,2) * Shape_3D.at(i+n*2,0) ) + ty; + out_shape.at(i, 0) = s * (currRot(0, 0) * Shape_3D.at(i, 0) + currRot(0, 1) * Shape_3D.at(i + n, 0) + currRot(0, 2) * Shape_3D.at(i + n * 2, 0)) + tx; + out_shape.at(i + n, 0) = s * (currRot(1, 0) * Shape_3D.at(i, 0) + currRot(1, 1) * Shape_3D.at(i + n, 0) + currRot(1, 2) * Shape_3D.at(i + n * 2, 0)) + ty; } } //=========================================================================== // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box // This all assumes that the bounding box describes face from left outline to right outline of the face and chin to eyebrows -void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation) const +void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3f rotation) const { // get the shape instance based on local params - cv::Mat_ current_shape(mean_shape.size()); + cv::Mat_ current_shape(mean_shape.size()); CalcShape3D(current_shape, params_local); // rotate the shape - cv::Matx33d rotation_matrix = Euler2RotationMatrix(rotation); + cv::Matx33f rotation_matrix = Euler2RotationMatrix_f(rotation); - cv::Mat_ reshaped = current_shape.reshape(1, 3); + cv::Mat_ reshaped = current_shape.reshape(1, 3); cv::Mat rotated_shape = (cv::Mat(rotation_matrix) * reshaped); // Get the width of expected shape double min_x; double max_x; - cv::minMaxLoc(rotated_shape.row(0), &min_x, &max_x); + cv::minMaxLoc(rotated_shape.row(0), &min_x, &max_x); double min_y; double max_y; @@ -168,8 +164,8 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& boun double ty = bounding_box.y + bounding_box.height / 2; // Correct it so that the bounding box is just around the minimum and maximum point in the initialised face - tx = tx - scaling * (min_x + max_x)/2; - ty = ty - scaling * (min_y + max_y)/2; + tx = tx - scaling * (min_x + max_x) / 2; + ty = ty - scaling * (min_y + max_y) / 2; out_params_global = cv::Vec6d(scaling, rotation[0], rotation[1], rotation[2], tx, ty); } @@ -181,9 +177,9 @@ void PDM::CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_gl { // get the shape instance based on local params - cv::Mat_ current_shape; + cv::Mat_ current_shape; CalcShape2D(current_shape, params_local, params_global); - + // Get the width of expected shape double min_x; double max_x; @@ -206,43 +202,40 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& // number of verts int n = this->NumberOfPoints(); - + Jacob.create(n * 2, 6); - float X,Y,Z; + float X, Y, Z; float s = (float)params_global[0]; - - cv::Mat_ shape_3D_d; - this->CalcShape3D(shape_3D_d, p_local); cv::Mat_ shape_3D; - shape_3D_d.convertTo(shape_3D, CV_32F); + this->CalcShape3D(shape_3D, p_local); - // Get the rotation matrix - cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); - - float r11 = (float) currRot(0,0); - float r12 = (float) currRot(0,1); - float r13 = (float) currRot(0,2); - float r21 = (float) currRot(1,0); - float r22 = (float) currRot(1,1); - float r23 = (float) currRot(1,2); - float r31 = (float) currRot(2,0); - float r32 = (float) currRot(2,1); - float r33 = (float) currRot(2,2); + // Get the rotation matrix (TODO move away from doubles) + cv::Vec3f euler((float)params_global[1], (float)params_global[2], (float)params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix_f(euler); + + float r11 = currRot(0, 0); + float r12 = currRot(0, 1); + float r13 = currRot(0, 2); + float r21 = currRot(1, 0); + float r22 = currRot(1, 1); + float r23 = currRot(1, 2); + float r31 = currRot(2, 0); + float r32 = currRot(2, 1); + float r33 = currRot(2, 2); cv::MatIterator_ Jx = Jacob.begin(); cv::MatIterator_ Jy = Jx + n * 6; - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { - - X = shape_3D.at(i,0); - Y = shape_3D.at(i+n,0); - Z = shape_3D.at(i+n*2,0); - + + X = shape_3D.at(i, 0); + Y = shape_3D.at(i + n, 0); + Z = shape_3D.at(i + n * 2, 0); + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') // where R' = [1, -wz, wy // wz, 1, -wx @@ -250,17 +243,17 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& // And this is derived using the small angle assumption on the axis angle rotation matrix parametrisation // scaling term - *Jx++ = (X * r11 + Y * r12 + Z * r13); - *Jy++ = (X * r21 + Y * r22 + Z * r23); - + *Jx++ = (X * r11 + Y * r12 + Z * r13); + *Jy++ = (X * r21 + Y * r22 + Z * r23); + // rotation terms - *Jx++ = (s * (Y * r13 - Z * r12) ); - *Jy++ = (s * (Y * r23 - Z * r22) ); + *Jx++ = (s * (Y * r13 - Z * r12)); + *Jy++ = (s * (Y * r23 - Z * r22)); *Jx++ = (-s * (X * r13 - Z * r11)); *Jy++ = (-s * (X * r23 - Z * r21)); - *Jx++ = (s * (X * r12 - Y * r11) ); - *Jy++ = (s * (X * r22 - Y * r21) ); - + *Jx++ = (s * (X * r12 - Y * r11)); + *Jy++ = (s * (X * r22 - Y * r21)); + // translation terms *Jx++ = 1.0f; *Jy++ = 0.0f; @@ -270,24 +263,24 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& } cv::Mat Jacob_w = cv::Mat::zeros(Jacob.rows, Jacob.cols, Jacob.type()); - - Jx = Jacob.begin(); - Jy = Jx + n*6; - cv::MatIterator_ Jx_w = Jacob_w.begin(); - cv::MatIterator_ Jy_w = Jx_w + n*6; + Jx = Jacob.begin(); + Jy = Jx + n * 6; + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n * 6; // Iterate over all Jacobian values and multiply them by the weight in diagonal of W - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { float w_x = W.at(i, i); - float w_y = W.at(i+n, i+n); + float w_y = W.at(i + n, i + n); - for(int j = 0; j < Jacob.cols; ++j) + for (int j = 0; j < Jacob.cols; ++j) { *Jx_w++ = *Jx++ * w_x; *Jy_w++ = *Jy++ * w_y; - } + } } Jacob_t_w = Jacob_w.t(); @@ -300,49 +293,45 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& // number of vertices int n = this->NumberOfPoints(); - + // number of non-rigid parameters int m = this->NumberOfModes(); Jacobian.create(n * 2, 6 + m); - - float X,Y,Z; - - float s = (float) params_global[0]; - - // TODO move away from doubles - cv::Mat_ shape_3D_d; - this->CalcShape3D(shape_3D_d, params_local); + + float X, Y, Z; + + float s = (float)params_global[0]; cv::Mat_ shape_3D; - shape_3D_d.convertTo(shape_3D, CV_32F); + this->CalcShape3D(shape_3D, params_local); - cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); - - float r11 = (float) currRot(0,0); - float r12 = (float) currRot(0,1); - float r13 = (float) currRot(0,2); - float r21 = (float) currRot(1,0); - float r22 = (float) currRot(1,1); - float r23 = (float) currRot(1,2); - float r31 = (float) currRot(2,0); - float r32 = (float) currRot(2,1); - float r33 = (float) currRot(2,2); + cv::Vec3f euler((float)params_global[1], (float)params_global[2], (float)params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix_f(euler); - cv::MatIterator_ Jx = Jacobian.begin(); - cv::MatIterator_ Jy = Jx + n * (6 + m); - cv::MatConstIterator_ Vx = this->princ_comp.begin(); - cv::MatConstIterator_ Vy = Vx + n*m; - cv::MatConstIterator_ Vz = Vy + n*m; + float r11 = currRot(0, 0); + float r12 = currRot(0, 1); + float r13 = currRot(0, 2); + float r21 = currRot(1, 0); + float r22 = currRot(1, 1); + float r23 = currRot(1, 2); + float r31 = currRot(2, 0); + float r32 = currRot(2, 1); + float r33 = currRot(2, 2); - for(int i = 0; i < n; i++) + cv::MatIterator_ Jx = Jacobian.begin(); + cv::MatIterator_ Jy = Jx + n * (6 + m); + cv::MatConstIterator_ Vx = this->princ_comp.begin(); + cv::MatConstIterator_ Vy = Vx + n*m; + cv::MatConstIterator_ Vz = Vy + n*m; + + for (int i = 0; i < n; i++) { - - X = shape_3D.at(i,0); - Y = shape_3D.at(i+n,0); - Z = shape_3D.at(i+n*2,0); - + + X = shape_3D.at(i, 0); + Y = shape_3D.at(i + n, 0); + Z = shape_3D.at(i + n * 2, 0); + // The rigid jacobian from the axis angle rotation matrix approximation using small angle assumption (R * R') // where R' = [1, -wz, wy // wz, 1, -wx @@ -352,47 +341,47 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& // scaling term *Jx++ = (X * r11 + Y * r12 + Z * r13); *Jy++ = (X * r21 + Y * r22 + Z * r23); - + // rotation terms - *Jx++ = (s * (Y * r13 - Z * r12) ); - *Jy++ = (s * (Y * r23 - Z * r22) ); + *Jx++ = (s * (Y * r13 - Z * r12)); + *Jy++ = (s * (Y * r23 - Z * r22)); *Jx++ = (-s * (X * r13 - Z * r11)); *Jy++ = (-s * (X * r23 - Z * r21)); - *Jx++ = (s * (X * r12 - Y * r11) ); - *Jy++ = (s * (X * r22 - Y * r21) ); - + *Jx++ = (s * (X * r12 - Y * r11)); + *Jy++ = (s * (X * r22 - Y * r21)); + // translation terms *Jx++ = 1.0f; *Jy++ = 0.0f; *Jx++ = 0.0f; *Jy++ = 1.0f; - for(int j = 0; j < m; j++,++Vx,++Vy,++Vz) + for (int j = 0; j < m; j++, ++Vx, ++Vy, ++Vz) { // How much the change of the non-rigid parameters (when object is rotated) affect 2D motion - *Jx++ = ( s*(r11*(*Vx) + r12*(*Vy) + r13*(*Vz)) ); - *Jy++ = ( s*(r21*(*Vx) + r22*(*Vy) + r23*(*Vz)) ); + *Jx++ = (s*(r11*(*Vx) + r12*(*Vy) + r13*(*Vz))); + *Jy++ = (s*(r21*(*Vx) + r22*(*Vy) + r23*(*Vz))); } - } + } // Adding the weights here cv::Mat Jacob_w = Jacobian.clone(); - - if(cv::trace(W)[0] != W.rows) - { - Jx = Jacobian.begin(); - Jy = Jx + n*(6+m); - cv::MatIterator_ Jx_w = Jacob_w.begin(); - cv::MatIterator_ Jy_w = Jx_w + n*(6+m); + if (cv::trace(W)[0] != W.rows) + { + Jx = Jacobian.begin(); + Jy = Jx + n*(6 + m); + + cv::MatIterator_ Jx_w = Jacob_w.begin(); + cv::MatIterator_ Jy_w = Jx_w + n*(6 + m); // Iterate over all Jacobian values and multiply them by the weight in diagonal of W - for(int i = 0; i < n; i++) + for (int i = 0; i < n; i++) { float w_x = W.at(i, i); - float w_y = W.at(i+n, i+n); + float w_y = W.at(i + n, i + n); - for(int j = 0; j < Jacobian.cols; ++j) + for (int j = 0; j < Jacobian.cols; ++j) { *Jx_w++ = *Jx++ * w_x; *Jy_w++ = *Jy++ * w_y; @@ -449,7 +438,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& } // void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)) const; -void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation) const +void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation) const { int m = this->NumberOfModes(); @@ -467,55 +456,45 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l double max_y; cv::minMaxLoc(landmark_locations(cv::Rect(0, this->NumberOfPoints(), 1, this->NumberOfPoints())), &min_y, &max_y); - double width = abs(min_x - max_x); - double height = abs(min_y - max_y); + float width = (float)abs(min_x - max_x); + float height = (float)abs(min_y - max_y); cv::Rect model_bbox; CalcBoundingBox(model_bbox, cv::Vec6d(1.0, 0.0, 0.0, 0.0, 0.0, 0.0), cv::Mat_(this->NumberOfModes(), 1, 0.0)); cv::Rect bbox((int)min_x, (int)min_y, (int)width, (int)height); - double scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; + float scaling = ((width / model_bbox.width) + (height / model_bbox.height)) / 2; - cv::Vec3d rotation_init = rotation; - cv::Matx33d R = Euler2RotationMatrix(rotation_init); - cv::Vec2d translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); + cv::Vec3f rotation_init(rotation[0], rotation[1], rotation[2]); + cv::Matx33f R = Euler2RotationMatrix_f(rotation_init); + cv::Vec2f translation((min_x + max_x) / 2.0, (min_y + max_y) / 2.0); cv::Mat_ loc_params(this->NumberOfModes(),1, 0.0); cv::Vec6d glob_params(scaling, rotation_init[0], rotation_init[1], rotation_init[2], translation[0], translation[1]); // get the 3D shape of the object - cv::Mat_ loc_params_d; - loc_params.convertTo(loc_params_d, CV_64F); - - // TODO move to doubles - cv::Mat_ mean_shape_d; - cv::Mat_ princ_comp_d; - mean_shape.convertTo(mean_shape_d, CV_64F); - princ_comp.convertTo(princ_comp_d, CV_64F); + cv::Mat_ shape_3D = mean_shape + princ_comp * loc_params; - cv::Mat_ shape_3D = mean_shape_d + princ_comp_d * loc_params_d; - - cv::Mat_ curr_shape(2*n, 1); + cv::Mat_ curr_shape(2*n, 1); // for every vertex for(int i = 0; i < n; i++) { // Transform this using the weak-perspective mapping to 2D from 3D - curr_shape.at(i ,0) = scaling * ( R(0,0) * shape_3D.at(i, 0) + R(0,1) * shape_3D.at(i+n ,0) + R(0,2) * shape_3D.at(i+n*2,0) ) + translation[0]; - curr_shape.at(i+n,0) = scaling * ( R(1,0) * shape_3D.at(i, 0) + R(1,1) * shape_3D.at(i+n ,0) + R(1,2) * shape_3D.at(i+n*2,0) ) + translation[1]; + curr_shape.at(i ,0) = scaling * ( R(0,0) * shape_3D.at(i, 0) + R(0,1) * shape_3D.at(i+n ,0) + R(0,2) * shape_3D.at(i+n*2,0) ) + translation[0]; + curr_shape.at(i+n,0) = scaling * ( R(1,0) * shape_3D.at(i, 0) + R(1,1) * shape_3D.at(i+n ,0) + R(1,2) * shape_3D.at(i+n*2,0) ) + translation[1]; } - double currError = cv::norm(curr_shape - landmark_locations); + float currError = cv::norm(curr_shape - landmark_locations); - cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); + cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); - double reg_factor = 1; + float reg_factor = 1; // Setting the regularisation to the inverse of eigenvalues cv::Mat(reg_factor / this->eigen_values).copyTo(regularisations(cv::Rect(6, 0, m, 1))); - cv::Mat_ regTerm_d = cv::Mat::diag(regularisations.t()); - regTerm_d.convertTo(regularisations, CV_32F); + regularisations = cv::Mat::diag(regularisations.t()); cv::Mat_ WeightMatrix = cv::Mat_::eye(n*2, n*2); @@ -523,16 +502,14 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l for (size_t i = 0; i < 1000; ++i) { - // get the 3D shape of the object, TODO move away from doubles - cv::Mat_ loc_params_d; - loc_params.convertTo(loc_params_d, CV_64F); - shape_3D = mean_shape_d + princ_comp_d * loc_params_d; + // get the 3D shape of the object + shape_3D = mean_shape + princ_comp * loc_params; shape_3D = shape_3D.reshape(1, 3); - cv::Matx23d R_2D(R(0,0), R(0,1), R(0,2), R(1,0), R(1,1), R(1,2)); + cv::Matx23f R_2D(R(0,0), R(0,1), R(0,2), R(1,0), R(1,1), R(1,2)); - cv::Mat_ curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); + cv::Mat_ curr_shape_2D = scaling * shape_3D.t() * cv::Mat(R_2D).t(); curr_shape_2D.col(0) = curr_shape_2D.col(0) + translation(0); curr_shape_2D.col(1) = curr_shape_2D.col(1) + translation(1); @@ -577,7 +554,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l translation[0] = glob_params[4]; translation[1] = glob_params[5]; - R = Euler2RotationMatrix(rotation_init); + R = Euler2RotationMatrix_f(rotation_init); R_2D(0,0) = R(0,0);R_2D(0,1) = R(0,1); R_2D(0,2) = R(0,2); R_2D(1,0) = R(1,0);R_2D(1,1) = R(1,1); R_2D(1,2) = R(1,2); @@ -588,7 +565,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l curr_shape_2D = cv::Mat(curr_shape_2D.t()).reshape(1, n * 2); - double error = cv::norm(curr_shape_2D - landmark_locations); + float error = cv::norm(curr_shape_2D - landmark_locations); if(0.999 * currError < error) { diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h b/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h index 4fcc6640..89a9ef2f 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectionValidator.h @@ -117,7 +117,7 @@ public: DetectionValidator(const DetectionValidator& other); // Given an image, orientation and detected landmarks output the result of the appropriate regressor - double Check(const cv::Vec3d& orientation, const cv::Mat_& intensity_img, cv::Mat_& detected_landmarks); + float Check(const cv::Vec3d& orientation, const cv::Mat_& intensity_img, cv::Mat_& detected_landmarks); // Reading in the model void Read(string location); diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index 40f42690..1704d2d2 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -108,7 +108,7 @@ public: bool tracking_initialised; // The actual output of the regressor (-1 is perfect detection 1 is worst detection) - double detection_certainty; + float detection_certainty; // Indicator if eye model is there for eye detection bool eye_model; @@ -120,7 +120,7 @@ public: // Member variables that retain the state of the tracking (reflecting the state of the lastly tracked (detected) image // Lastly detect 2D model shape [x1,x2,...xn,y1,...yn] - cv::Mat_ detected_landmarks; + cv::Mat_ detected_landmarks; // The landmark detection likelihoods (combined and per patch expert) float model_likelihood; @@ -195,21 +195,11 @@ private: // The actual model optimisation (update step), returns the model likelihood float NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, - const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_idx, bool rigid, int scale, cv::Mat_& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood); + const cv::Mat_& base_shape, const cv::Matx22f& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_idx, bool rigid, int scale, cv::Mat_& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood); // Generating the weight matrix for the Weighted least squares void GetWeightMatrix(cv::Mat_& WeightMatrix, int scale, int view_id, const FaceModelParameters& parameters); - //======================================================= - // Legacy functions that are not used at the moment - //======================================================= - - // Mean shift computation - void NonVectorisedMeanShift(cv::Mat_& out_mean_shifts, const vector >& patch_expert_responses, const cv::Mat_ &dxs, const cv::Mat_ &dys, int resp_size, double a, int scale, int view_id); - - // A vectorised version of mean shift (Not actually used) - void VectorisedMeanShift(cv::Mat_& meanShifts, const vector >& patch_expert_responses, const cv::Mat_ &iis, const cv::Mat_ &jjs, const cv::Mat_ &dxs, const cv::Mat_ &dys, const cv::Size patchSize, double sigma, int scale, int view_id); - }; //=========================================================================== } diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h index 2e259fbd..58c84d87 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h @@ -56,7 +56,7 @@ struct FaceModelParameters bool validate_detections; // Landmark detection validator boundary for correct detection, the regressor output -1 (perfect alignment) 1 (bad alignment), - double validation_boundary; + float validation_boundary; // Used when tracking is going well vector window_sizes_small; @@ -68,17 +68,17 @@ struct FaceModelParameters vector window_sizes_current; // How big is the tracking template that helps with large motions - double face_template_scale; + float face_template_scale; bool use_face_template; // Where to load the model from string model_location; // this is used for the smooting of response maps (KDE sigma) - double sigma; + float sigma; - double reg_factor; // weight put to regularisation - double weight_factor; // factor for weighted least squares + float reg_factor; // weight put to regularisation + float weight_factor; // factor for weighted least squares // should multiple views be considered during reinit bool multi_view; diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h index 3336f42a..1f046843 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorUtils.h @@ -75,10 +75,12 @@ namespace LandmarkDetector // Using Kabsch's algorithm for aligning shapes //This assumes that align_from and align_to are already mean normalised cv::Matx22d AlignShapesKabsch2D(const cv::Mat_& align_from, const cv::Mat_& align_to); + cv::Matx22f AlignShapesKabsch2D_f(const cv::Mat_& align_from, const cv::Mat_& align_to); //============================================================================= // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst); + cv::Matx22f AlignShapesWithScale_f(cv::Mat_& src, cv::Mat_ dst); //=========================================================================== // Visualisation functions @@ -90,18 +92,18 @@ namespace LandmarkDetector vector> CalculateBox(cv::Vec6d pose, float fx, float fy, float cx, float cy); void DrawBox(vector> lines, cv::Mat image, cv::Scalar color, int thickness); - vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); + vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities); vector CalculateVisibleLandmarks(const CLNF& clnf_model); vector CalculateVisibleEyeLandmarks(const CLNF& clnf_model); - vector CalculateAllLandmarks(const cv::Mat_& shape2D); + vector CalculateAllLandmarks(const cv::Mat_& shape2D); vector CalculateAllLandmarks(const CLNF& clnf_model); vector CalculateAllEyeLandmarks(const CLNF& clnf_model); void DrawLandmarks(cv::Mat img, vector landmarks); - void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); - void Draw(cv::Mat img, const cv::Mat_& shape2D); + void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities); + void Draw(cv::Mat img, const cv::Mat_& shape2D); void Draw(cv::Mat img, const CLNF& clnf_model); diff --git a/lib/local/LandmarkDetector/include/PAW.h b/lib/local/LandmarkDetector/include/PAW.h index f9ac33bc..dd0e27f5 100644 --- a/lib/local/LandmarkDetector/include/PAW.h +++ b/lib/local/LandmarkDetector/include/PAW.h @@ -56,16 +56,16 @@ public: int number_of_pixels; // Minimum x coordinate in destination - double min_x; + float min_x; // minimum y coordinate in destination - double min_y; + float min_y; // Destination points (landmarks to be warped to) - cv::Mat_ destination_landmarks; + cv::Mat_ destination_landmarks; // Destination points (landmarks to be warped from) - cv::Mat_ source_landmarks; + cv::Mat_ source_landmarks; // Triangulation, each triangle is warped using an affine transform cv::Mat_ triangulation; @@ -81,13 +81,13 @@ public: // affine coefficients for all triangles (see Matthews and Baker 2004) // 6 coefficients for each triangle (are computed from alpha and beta) // This is computed during each warp based on source landmarks - cv::Mat_ coefficients; + cv::Mat_ coefficients; // matrix of (c,x,y) coeffs for alpha - cv::Mat_ alpha; + cv::Mat_ alpha; // matrix of (c,x,y) coeffs for alpha - cv::Mat_ beta; + cv::Mat_ beta; // x-source of warped points cv::Mat_ map_x; @@ -99,10 +99,10 @@ public: PAW(){;} // Construct a warp from a destination shape and triangulation - PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation); // The final optional argument allows for optimisation if the triangle indices from previous frame are known (for tracking in video) - PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y); + PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y); // Copy constructor PAW(const PAW& other); @@ -110,7 +110,7 @@ public: void Read(std::ifstream &s); // The actual warping - void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); + void Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp); // Compute coefficients needed for warping void CalcCoeff(); @@ -127,7 +127,7 @@ public: private: - int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1) const; + int findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess = -1) const; }; //=========================================================================== diff --git a/lib/local/LandmarkDetector/include/PDM.h b/lib/local/LandmarkDetector/include/PDM.h index 491f799f..f8e08858 100644 --- a/lib/local/LandmarkDetector/include/PDM.h +++ b/lib/local/LandmarkDetector/include/PDM.h @@ -78,13 +78,13 @@ class PDM{ void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; // Compute shape in image space (2D) - void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; + void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3f rotation = cv::Vec3f(0.0f)); // Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results) - void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation = cv::Vec3f(0.0f)); + void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation = cv::Vec3f(0.0f)); // provided the model parameters, compute the bounding box of a face void CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp index 1485f602..8ec54ea4 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectionValidator.cpp @@ -338,7 +338,7 @@ void DetectionValidator::Read(string location) //=========================================================================== // Check if the fitting actually succeeded -double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_& intensity_img, cv::Mat_& detected_landmarks) +float DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_& intensity_img, cv::Mat_& detected_landmarks) { int id = GetViewId(orientation); @@ -347,10 +347,10 @@ double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_ warped; // First only use the ROI of the image of interest - cv::Mat_& detected_landmarks_local = detected_landmarks.clone(); + cv::Mat_& detected_landmarks_local = detected_landmarks.clone(); double min_x_d, max_x_d, min_y_d, max_y_d; - cv::Mat_ xs = detected_landmarks_local(cv::Rect(0, 0, 1, detected_landmarks.rows / 2)); - cv::Mat_ ys = detected_landmarks_local(cv::Rect(0, detected_landmarks.rows / 2, 1, detected_landmarks.rows / 2)); + cv::Mat_ xs = detected_landmarks_local(cv::Rect(0, 0, 1, detected_landmarks.rows / 2)); + cv::Mat_ ys = detected_landmarks_local(cv::Rect(0, detected_landmarks.rows / 2, 1, detected_landmarks.rows / 2)); cv::minMaxLoc(xs, &min_x_d, &max_x_d); cv::minMaxLoc(ys, &min_y_d, &max_y_d); @@ -376,7 +376,7 @@ double DetectionValidator::Check(const cv::Vec3d& orientation, const cv::Mat_& warped_img, int view_id) diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index 02196ff5..97cc585b 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -315,9 +315,9 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat &image, CLNF& clnf_m // Keep track of old model values so that they can be restored if redetection fails cv::Vec6d params_global_init = clnf_model.params_global; - cv::Mat_ params_local_init = clnf_model.params_local.clone(); - double likelihood_init = clnf_model.model_likelihood; - cv::Mat_ detected_landmarks_init = clnf_model.detected_landmarks.clone(); + cv::Mat_ params_local_init = clnf_model.params_local.clone(); + float likelihood_init = clnf_model.model_likelihood; + cv::Mat_ detected_landmarks_init = clnf_model.detected_landmarks.clone(); cv::Mat_ landmark_likelihoods_init = clnf_model.landmark_likelihoods.clone(); // Use the detected bounding box and empty local parameters diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index e06c3b38..5e7f10ec 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -598,13 +598,13 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa vector> mappings = this->hierarchical_mapping[part_model]; - cv::Mat_ part_model_locs(n_part_points * 2, 1, 0.0); + cv::Mat_ part_model_locs(n_part_points * 2, 1, 0.0); // Extract the corresponding landmarks for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) { - part_model_locs.at(mappings[mapping_ind].second) = detected_landmarks.at(mappings[mapping_ind].first); - part_model_locs.at(mappings[mapping_ind].second + n_part_points) = detected_landmarks.at(mappings[mapping_ind].first + this->pdm.NumberOfPoints()); + part_model_locs.at(mappings[mapping_ind].second) = detected_landmarks.at(mappings[mapping_ind].first); + part_model_locs.at(mappings[mapping_ind].second + n_part_points) = detected_landmarks.at(mappings[mapping_ind].first + this->pdm.NumberOfPoints()); } // Fit the part based model PDM @@ -644,8 +644,8 @@ bool CLNF::DetectLandmarks(const cv::Mat_ &image, FaceModelParameters& pa // Reincorporate the models into main tracker for (size_t mapping_ind = 0; mapping_ind < mappings.size(); ++mapping_ind) { - detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); - detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); + detected_landmarks.at(mappings[mapping_ind].first) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second); + detected_landmarks.at(mappings[mapping_ind].first + pdm.NumberOfPoints()) = hierarchical_models[part_model].detected_landmarks.at(mappings[mapping_ind].second + hierarchical_models[part_model].pdm.NumberOfPoints()); } } } @@ -689,7 +689,7 @@ bool CLNF::Fit(const cv::Mat_& im, const std::vector& window_sizes, assert(im.channels() == 1); // Placeholder for the landmarks - cv::Mat_ current_shape(2 * pdm.NumberOfPoints() , 1, 0.0); + cv::Mat_ current_shape(2 * pdm.NumberOfPoints() , 1, 0.0f); int n = pdm.NumberOfPoints(); @@ -858,6 +858,7 @@ void CLNF::NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, // Iterate over the patch responses here cv::MatConstIterator_ p = patch_expert_responses[i].begin(); + // TODO maybe do through MatMuls instead? for(int ii = 0; ii < resp_size; ii++) { for(int jj = 0; jj < resp_size; jj++) @@ -939,16 +940,16 @@ void CLNF::GetWeightMatrix(cv::Mat_& WeightMatrix, int scale, int view_id //============================================================================= float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, - const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_id, bool rigid, int scale, cv::Mat_& landmark_lhoods, + const cv::Mat_& base_shape, const cv::Matx22f& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_id, bool rigid, int scale, cv::Mat_& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood) { int n = pdm.NumberOfPoints(); // Mean, eigenvalues, eigenvectors - cv::Mat_ M = this->pdm.mean_shape; - cv::Mat_ E = this->pdm.eigen_values; - //Mat_ V = this->pdm.princ_comp; + cv::Mat_ M = this->pdm.mean_shape; + cv::Mat_ E = this->pdm.eigen_values; + //Mat_ V = this->pdm.princ_comp; int m = pdm.NumberOfModes(); @@ -956,8 +957,8 @@ float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const cv::Mat_ current_local = initial_local.clone(); - cv::Mat_ current_shape; - cv::Mat_ previous_shape; + cv::Mat_ current_shape; + cv::Mat_ previous_shape; // Pre-calculate the regularisation term cv::Mat_ regTerm; @@ -968,12 +969,11 @@ float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const } else { - cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); + cv::Mat_ regularisations = cv::Mat_::zeros(1, 6 + m); // Setting the regularisation to the inverse of eigenvalues cv::Mat(parameters.reg_factor / E).copyTo(regularisations(cv::Rect(6, 0, m, 1))); - cv::Mat_ regTerm_d = cv::Mat::diag(regularisations.t()); - regTerm_d.convertTo(regTerm, CV_32F); + regTerm = cv::Mat::diag(regularisations.t()); } cv::Mat_ WeightMatrix; @@ -1017,8 +1017,8 @@ float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const // useful for mean shift calculation float a = -0.5/(parameters.sigma * parameters.sigma); - cv::Mat_ current_shape_2D = current_shape.reshape(1, 2).t(); - cv::Mat_ base_shape_2D = base_shape.reshape(1, 2).t(); + cv::Mat_ current_shape_2D = current_shape.reshape(1, 2).t(); + cv::Mat_ base_shape_2D = base_shape.reshape(1, 2).t(); cv::Mat_ offsets; cv::Mat((current_shape_2D - base_shape_2D) * cv::Mat(sim_img_to_ref).t()).convertTo(offsets, CV_32F); @@ -1168,8 +1168,8 @@ cv::Mat_ CLNF::GetShape(float fx, float fy, float cx, float cy) const { float Z = Zavg + shape3d.at(i,2); - float X = Z * ((((float)this->detected_landmarks.at(i)) - cx)/fx); - float Y = Z * ((((float)this->detected_landmarks.at(i + n)) - cy)/fy); + float X = Z * ((this->detected_landmarks.at(i) - cx)/fx); + float Y = Z * ((this->detected_landmarks.at(i + n) - cy)/fy); outShape.at(i,0) = X; outShape.at(i,1) = Y; @@ -1185,8 +1185,8 @@ cv::Mat_ CLNF::GetShape(float fx, float fy, float cx, float cy) const // A utility bounding box function cv::Rect_ CLNF::GetBoundingBox() const { - cv::Mat_ xs = this->detected_landmarks(cv::Rect(0,0,1,this->detected_landmarks.rows/2)); - cv::Mat_ ys = this->detected_landmarks(cv::Rect(0,this->detected_landmarks.rows/2, 1, this->detected_landmarks.rows/2)); + cv::Mat_ xs = this->detected_landmarks(cv::Rect(0,0,1,this->detected_landmarks.rows/2)); + cv::Mat_ ys = this->detected_landmarks(cv::Rect(0,this->detected_landmarks.rows/2, 1, this->detected_landmarks.rows/2)); double min_x, max_x; double min_y, max_y; @@ -1197,62 +1197,3 @@ cv::Rect_ CLNF::GetBoundingBox() const cv::Rect_ model_rect(min_x, min_y, max_x - min_x, max_y - min_y); return model_rect; } - -// Legacy function not used at the moment -void CLNF::NonVectorisedMeanShift(cv::Mat_& out_mean_shifts, const vector >& patch_expert_responses, const cv::Mat_ &dxs, const cv::Mat_ &dys, int resp_size, double a, int scale, int view_id) -{ - - int n = dxs.rows; - - for(int i = 0; i < n; i++) - { - - if(patch_experts.visibilities[scale][view_id].at(i,0) == 0 || sum(patch_expert_responses[i])[0] == 0) - { - out_mean_shifts.at(i,0) = 0; - out_mean_shifts.at(i+n,0) = 0; - continue; - } - - // indices of dx, dy - double dx = dxs.at(i); - double dy = dys.at(i); - - int ii,jj; - double v,vx,vy,mx=0.0,my=0.0,sum=0.0; - - // Iterate over the patch responses here - cv::MatConstIterator_ p = patch_expert_responses[i].begin(); - - for(ii = 0; ii < resp_size; ii++) - { - vx = (dy-ii)*(dy-ii); - for(jj = 0; jj < resp_size; jj++) - { - vy = (dx-jj)*(dx-jj); - - // the probability at the current, xi, yi - v = *p++; - - // the KDE evaluation of that point - double kd = exp(a*(vx+vy)); - v *= kd; - - sum += v; - - // mean shift in x and y - mx += v*jj; - my += v*ii; - - } - } - - // setting the actual mean shift update - double msx = (mx/sum - dx); - double msy = (my/sum - dy); - - out_mean_shifts.at(i, 0) = msx; - out_mean_shifts.at(i + n, 0) = msy; - - } -} diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index 85121769..cb263598 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -213,8 +213,8 @@ FaceModelParameters::FaceModelParameters(vector &arguments) if (model_path.stem().string().compare("main_ceclm_general") == 0) { - sigma = 1.5 * sigma; - reg_factor = 0.9 * reg_factor; + sigma = 1.5f * sigma; + reg_factor = 0.9f * reg_factor; } // Make sure face detector location is valid @@ -288,7 +288,7 @@ void FaceModelParameters::init() window_sizes_init.at(2) = 7; window_sizes_init.at(3) = 5; - face_template_scale = 0.3; + face_template_scale = 0.3f; // Off by default (as it might lead to some slight inaccuracies in slowly moving faces) use_face_template = false; @@ -297,11 +297,11 @@ void FaceModelParameters::init() model_location = "model/main_ceclm_general.txt"; - sigma = 1.5; - reg_factor = 25; - weight_factor = 0; // By default do not use NU-RLMS for videos as it does not work as well for them + sigma = 1.5f; + reg_factor = 25.0f; + weight_factor = 0.0f; // By default do not use NU-RLMS for videos as it does not work as well for them - validation_boundary = -0.45; + validation_boundary = -0.45f; limit_pose = true; multi_view = false; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 8138f3ed..fbab6e39 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -717,6 +717,31 @@ namespace LandmarkDetector return R; } + cv::Matx22f AlignShapesKabsch2D_f(const cv::Mat_& align_from, const cv::Mat_& align_to) + { + + cv::SVD svd(align_from.t() * align_to); + + // make sure no reflection is there + // corr ensures that we do only rotaitons and not reflections + float d = cv::determinant(svd.vt.t() * svd.u.t()); + + cv::Matx22f corr = cv::Matx22f::eye(); + if (d > 0) + { + corr(1, 1) = 1; + } + else + { + corr(1, 1) = -1; + } + + cv::Matx22f R; + cv::Mat(svd.vt.t()*cv::Mat(corr)*svd.u.t()).copyTo(R); + + return R; + } + //============================================================================= // Basically Kabsch's algorithm but also allows the collection of points to be different in scale from each other cv::Matx22d AlignShapesWithScale(cv::Mat_& src, cv::Mat_ dst) @@ -769,6 +794,55 @@ namespace LandmarkDetector } + cv::Matx22f AlignShapesWithScale_f(cv::Mat_& src, cv::Mat_ dst) + { + int n = src.rows; + + // First we mean normalise both src and dst + float mean_src_x = cv::mean(src.col(0))[0]; + float mean_src_y = cv::mean(src.col(1))[0]; + + float mean_dst_x = cv::mean(dst.col(0))[0]; + float mean_dst_y = cv::mean(dst.col(1))[0]; + + cv::Mat_ src_mean_normed = src.clone(); + src_mean_normed.col(0) = src_mean_normed.col(0) - mean_src_x; + src_mean_normed.col(1) = src_mean_normed.col(1) - mean_src_y; + + cv::Mat_ dst_mean_normed = dst.clone(); + dst_mean_normed.col(0) = dst_mean_normed.col(0) - mean_dst_x; + dst_mean_normed.col(1) = dst_mean_normed.col(1) - mean_dst_y; + + // Find the scaling factor of each + cv::Mat src_sq; + cv::pow(src_mean_normed, 2, src_sq); + + cv::Mat dst_sq; + cv::pow(dst_mean_normed, 2, dst_sq); + + float s_src = sqrt(cv::sum(src_sq)[0] / n); + float s_dst = sqrt(cv::sum(dst_sq)[0] / n); + + src_mean_normed = src_mean_normed / s_src; + dst_mean_normed = dst_mean_normed / s_dst; + + float s = s_dst / s_src; + + // Get the rotation + cv::Matx22f R = AlignShapesKabsch2D_f(src_mean_normed, dst_mean_normed); + + cv::Matx22f A; + cv::Mat(s * R).copyTo(A); + + cv::Mat_ aligned = (cv::Mat(cv::Mat(A) * src.t())).t(); + cv::Mat_ offset = dst - aligned; + + float t_x = cv::mean(offset.col(0))[0]; + float t_y = cv::mean(offset.col(1))[0]; + + return A; + + } //=========================================================================== // Visualisation functions @@ -965,7 +1039,7 @@ namespace LandmarkDetector } // Computing landmarks (to be drawn later possibly) - vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities) + vector CalculateVisibleLandmarks(const cv::Mat_& shape2D, const cv::Mat_& visibilities) { int n = shape2D.rows / 2; vector landmarks; @@ -974,7 +1048,7 @@ namespace LandmarkDetector { if (visibilities.at(i)) { - cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i + n)); + cv::Point2d featurePoint(shape2D.at(i), shape2D.at(i + n)); landmarks.push_back(featurePoint); } @@ -984,7 +1058,7 @@ namespace LandmarkDetector } // Computing landmarks (to be drawn later possibly) - vector CalculateAllLandmarks(const cv::Mat_& shape2D) + vector CalculateAllLandmarks(const cv::Mat_& shape2D) { int n; @@ -1004,11 +1078,11 @@ namespace LandmarkDetector cv::Point2d featurePoint; if (shape2D.cols == 1) { - featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i + n)); + featurePoint = cv::Point2d(shape2D.at(i), shape2D.at(i + n)); } else { - featurePoint = cv::Point2d(shape2D.at(i, 0), shape2D.at(i, 1)); + featurePoint = cv::Point2d(shape2D.at(i, 0), shape2D.at(i, 1)); } landmarks.push_back(featurePoint); @@ -1086,7 +1160,7 @@ namespace LandmarkDetector } // Drawing landmarks on a face image - void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) + void Draw(cv::Mat img, const cv::Mat_& shape2D, const cv::Mat_& visibilities) { int n = shape2D.rows / 2; @@ -1098,7 +1172,7 @@ namespace LandmarkDetector { if (visibilities.at(i)) { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(shape2D.at(i) * (float)draw_multiplier), cvRound(shape2D.at(i + n) * (float)draw_multiplier)); // A rough heuristic for drawn point size int thickness = (int)std::ceil(3.0* ((double)img.cols) / 640.0); @@ -1114,7 +1188,7 @@ namespace LandmarkDetector { for (int i = 0; i < n; ++i) { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(shape2D.at(i) * (float)draw_multiplier), cvRound(shape2D.at(i + n) * (float)draw_multiplier)); // A rough heuristic for drawn point size int thickness = 1.0; @@ -1128,7 +1202,7 @@ namespace LandmarkDetector if (i == 27) next_point = 20; - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (float)draw_multiplier), cvRound(shape2D.at(next_point + n) * (float)draw_multiplier)); if (i < 8 || i > 19) cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); else @@ -1141,7 +1215,7 @@ namespace LandmarkDetector { for (int i = 0; i < n; ++i) { - cv::Point featurePoint(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + cv::Point featurePoint(cvRound(shape2D.at(i) * (float)draw_multiplier), cvRound(shape2D.at(i + n) * (float)draw_multiplier)); // A rough heuristic for drawn point size int thickness = 1.0; @@ -1151,14 +1225,14 @@ namespace LandmarkDetector if (i == 5) next_point = 0; - cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (double)draw_multiplier), cvRound(shape2D.at(next_point + n) * (double)draw_multiplier)); + cv::Point nextFeaturePoint(cvRound(shape2D.at(next_point) * (float)draw_multiplier), cvRound(shape2D.at(next_point + n) * (float)draw_multiplier)); cv::line(img, featurePoint, nextFeaturePoint, cv::Scalar(255, 0, 0), thickness_2, CV_AA, draw_shiftbits); } } } // Drawing landmarks on a face image - void Draw(cv::Mat img, const cv::Mat_& shape2D) + void Draw(cv::Mat img, const cv::Mat_& shape2D) { int n; @@ -1177,11 +1251,11 @@ namespace LandmarkDetector cv::Point featurePoint; if (shape2D.cols == 1) { - featurePoint = cv::Point(cvRound(shape2D.at(i) * (double)draw_multiplier), cvRound(shape2D.at(i + n) * (double)draw_multiplier)); + featurePoint = cv::Point(cvRound(shape2D.at(i) * (float)draw_multiplier), cvRound(shape2D.at(i + n) * (float)draw_multiplier)); } else { - featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (double)draw_multiplier), cvRound(shape2D.at(i, 1) * (double)draw_multiplier)); + featurePoint = cv::Point(cvRound(shape2D.at(i, 0) * (float)draw_multiplier), cvRound(shape2D.at(i, 1) * (float)draw_multiplier)); } // A rough heuristic for drawn point size int thickness = (int)std::ceil(5.0* ((double)img.cols) / 640.0); diff --git a/lib/local/LandmarkDetector/src/PAW.cpp b/lib/local/LandmarkDetector/src/PAW.cpp index d07c6fc8..9e5f7321 100644 --- a/lib/local/LandmarkDetector/src/PAW.cpp +++ b/lib/local/LandmarkDetector/src/PAW.cpp @@ -54,69 +54,69 @@ triangle_id(other.triangle_id.clone()), pixel_mask(other.pixel_mask.clone()), co } // A constructor from destination shape and triangulation -PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation) { // Initialise some variables directly this->destination_landmarks = destination_shape; this->triangulation = triangulation; - int num_points = destination_shape.rows/2; + int num_points = destination_shape.rows / 2; int num_tris = triangulation.rows; - + // Pre-compute the rest - alpha = cv::Mat_(num_tris, 3); - beta = cv::Mat_(num_tris, 3); - - cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); - cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); - + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + // Create a vector representation of the control points - vector> destination_points; + std::vector> destination_points; for (int tri = 0; tri < num_tris; ++tri) - { + { int j = triangulation.at(tri, 0); int k = triangulation.at(tri, 1); int l = triangulation.at(tri, 2); - double c1 = ys.at(l) - ys.at(j); - double c2 = xs.at(l) - xs.at(j); - double c4 = ys.at(k) - ys.at(j); - double c3 = xs.at(k) - xs.at(j); - - double c5 = c3*c1 - c2*c4; + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); - alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; - alpha.at(tri, 1) = c1/c5; - alpha.at(tri, 2) = -c2/c5; + float c5 = c3*c1 - c2*c4; - beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; - beta.at(tri, 1) = -c4/c5; - beta.at(tri, 2) = c3/c5; + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1 / c5; + alpha.at(tri, 2) = -c2 / c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3) / c5; + beta.at(tri, 1) = -c4 / c5; + beta.at(tri, 2) = c3 / c5; // Add points corresponding to triangles as optimisation - vector triangle_points(10); + std::vector triangle_points(10); - triangle_points[0] = xs.at(j); - triangle_points[1] = ys.at(j); - triangle_points[2] = xs.at(k); - triangle_points[3] = ys.at(k); - triangle_points[4] = xs.at(l); - triangle_points[5] = ys.at(l); - - cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); - cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); double min_x, max_x, min_y, max_y; cv::minMaxIdx(xs_three, &min_x, &max_x); cv::minMaxIdx(ys_three, &min_y, &max_y); - triangle_points[6] = max_x; - triangle_points[7] = max_y; + triangle_points[6] = (float)max_x; + triangle_points[7] = (float)max_y; - triangle_points[8] = min_x; - triangle_points[9] = min_y; + triangle_points[8] = (float)min_x; + triangle_points[9] = (float)min_y; destination_points.push_back(triangle_points); @@ -124,113 +124,119 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu double max_x; double max_y; + double min_x_d; + double min_y_d; - minMaxLoc(xs, &min_x, &max_x); - minMaxLoc(ys, &min_y, &max_y); + minMaxLoc(xs, &min_x_d, &max_x); + minMaxLoc(ys, &min_y_d, &max_y); + + min_x = min_x_d; + min_y = min_y_d; int w = (int)(max_x - min_x + 1.5); - int h = (int)(max_y - min_y + 1.5); - + int h = (int)(max_y - min_y + 1.5); + // Round the min_x and min_y for simplicity? - pixel_mask = cv::Mat_(h, w, (uchar)0); - triangle_id = cv::Mat_(h, w, -1); - + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + int curr_tri = -1; - for(int y = 0; y < pixel_mask.rows; y++) + for (int y = 0; y < pixel_mask.rows; y++) { - for(int x = 0; x < pixel_mask.cols; x++) + for (int x = 0; x < pixel_mask.cols; x++) { - curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); // If there is a triangle at this location - if(curr_tri != -1) + if (curr_tri != -1) { triangle_id.at(y, x) = curr_tri; - pixel_mask.at(y, x) = 1; - } + pixel_mask.at(y, x) = 1; + } } } - + // Preallocate maps and coefficients coefficients.create(num_tris, 6); - map_x.create(pixel_mask.rows,pixel_mask.cols); - map_y.create(pixel_mask.rows,pixel_mask.cols); + map_x.create(pixel_mask.rows, pixel_mask.cols); + map_y.create(pixel_mask.rows, pixel_mask.cols); } + // Manually define min and max values -PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, double in_min_x, double in_min_y, double in_max_x, double in_max_y) +PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangulation, float in_min_x, float in_min_y, float in_max_x, float in_max_y) { // Initialise some variables directly this->destination_landmarks = destination_shape; this->triangulation = triangulation; - int num_points = destination_shape.rows/2; + int num_points = destination_shape.rows / 2; int num_tris = triangulation.rows; - + // Pre-compute the rest - alpha = cv::Mat_(num_tris, 3); - beta = cv::Mat_(num_tris, 3); - - cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); - cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); + alpha = cv::Mat_(num_tris, 3); + beta = cv::Mat_(num_tris, 3); + + cv::Mat_ xs = destination_shape(cv::Rect(0, 0, 1, num_points)); + cv::Mat_ ys = destination_shape(cv::Rect(0, num_points, 1, num_points)); // Create a vector representation of the control points - vector> destination_points; - + std::vector> destination_points; + for (int tri = 0; tri < num_tris; ++tri) - { + { int j = triangulation.at(tri, 0); int k = triangulation.at(tri, 1); int l = triangulation.at(tri, 2); - double c1 = ys.at(l) - ys.at(j); - double c2 = xs.at(l) - xs.at(j); - double c4 = ys.at(k) - ys.at(j); - double c3 = xs.at(k) - xs.at(j); - - double c5 = c3*c1 - c2*c4; + float c1 = ys.at(l) - ys.at(j); + float c2 = xs.at(l) - xs.at(j); + float c4 = ys.at(k) - ys.at(j); + float c3 = xs.at(k) - xs.at(j); - alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; - alpha.at(tri, 1) = c1/c5; - alpha.at(tri, 2) = -c2/c5; + float c5 = c3*c1 - c2*c4; - beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3)/c5; - beta.at(tri, 1) = -c4/c5; - beta.at(tri, 2) = c3/c5; + alpha.at(tri, 0) = (ys.at(j) * c2 - xs.at(j) * c1) / c5; + alpha.at(tri, 1) = c1 / c5; + alpha.at(tri, 2) = -c2 / c5; + + beta.at(tri, 0) = (xs.at(j) * c4 - ys.at(j) * c3) / c5; + beta.at(tri, 1) = -c4 / c5; + beta.at(tri, 2) = c3 / c5; // Add points corresponding to triangles as optimisation - vector triangle_points(10); + std::vector triangle_points(10); - triangle_points[0] = xs.at(j); - triangle_points[1] = ys.at(j); - triangle_points[2] = xs.at(k); - triangle_points[3] = ys.at(k); - triangle_points[4] = xs.at(l); - triangle_points[5] = ys.at(l); - - cv::Vec3d xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); - cv::Vec3d ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); + triangle_points[0] = xs.at(j); + triangle_points[1] = ys.at(j); + triangle_points[2] = xs.at(k); + triangle_points[3] = ys.at(k); + triangle_points[4] = xs.at(l); + triangle_points[5] = ys.at(l); + + cv::Vec3f xs_three(triangle_points[0], triangle_points[2], triangle_points[4]); + cv::Vec3f ys_three(triangle_points[1], triangle_points[3], triangle_points[5]); double min_x, max_x, min_y, max_y; cv::minMaxIdx(xs_three, &min_x, &max_x); cv::minMaxIdx(ys_three, &min_y, &max_y); - triangle_points[6] = max_x; - triangle_points[7] = max_y; + triangle_points[6] = (float)max_x; + triangle_points[7] = (float)max_y; - triangle_points[8] = min_x; - triangle_points[9] = min_y; + triangle_points[8] = (float)min_x; + triangle_points[9] = (float)min_y; destination_points.push_back(triangle_points); - + } - double max_x; - double max_y; + float max_x; + float max_y; min_x = in_min_x; min_y = in_min_y; @@ -239,33 +245,33 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu max_y = in_max_y; int w = (int)(max_x - min_x + 1.5); - int h = (int)(max_y - min_y + 1.5); - + int h = (int)(max_y - min_y + 1.5); + // Round the min_x and min_y for simplicity? - pixel_mask = cv::Mat_(h, w, (uchar)0); - triangle_id = cv::Mat_(h, w, -1); - + pixel_mask = cv::Mat_(h, w, (uchar)0); + triangle_id = cv::Mat_(h, w, -1); + int curr_tri = -1; - for(int y = 0; y < pixel_mask.rows; y++) + for (int y = 0; y < pixel_mask.rows; y++) { - for(int x = 0; x < pixel_mask.cols; x++) + for (int x = 0; x < pixel_mask.cols; x++) { - curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); + curr_tri = findTriangle(cv::Point_(x + min_x, y + min_y), destination_points, curr_tri); // If there is a triangle at this location - if(curr_tri != -1) + if (curr_tri != -1) { triangle_id.at(y, x) = curr_tri; - pixel_mask.at(y, x) = 1; - } + pixel_mask.at(y, x) = 1; + } } - } + } // Preallocate maps and coefficients coefficients.create(num_tris, 6); - map_x.create(pixel_mask.rows,pixel_mask.cols); - map_y.create(pixel_mask.rows,pixel_mask.cols); + map_x.create(pixel_mask.rows, pixel_mask.cols); + map_y.create(pixel_mask.rows, pixel_mask.cols); } @@ -273,37 +279,46 @@ PAW::PAW(const cv::Mat_& destination_shape, const cv::Mat_& triangu void PAW::Read(std::ifstream& stream) { - stream.read ((char*)&number_of_pixels, 4); - stream.read ((char*)&min_x, 8); - stream.read ((char*)&min_y, 8); + stream.read((char*)&number_of_pixels, 4); + double min_x_d, min_y_d; + stream.read((char*)&min_x_d, 8); + stream.read((char*)&min_y_d, 8); + min_x = (float)min_x_d; + min_y = (float)min_y_d; - LandmarkDetector::ReadMatBin(stream, destination_landmarks); + cv::Mat_ destination_landmarks_d; + LandmarkDetector::ReadMatBin(stream, destination_landmarks_d); + destination_landmarks_d.convertTo(destination_landmarks, CV_32F); LandmarkDetector::ReadMatBin(stream, triangulation); LandmarkDetector::ReadMatBin(stream, triangle_id); - - cv::Mat tmpMask; - LandmarkDetector::ReadMatBin(stream, tmpMask); - tmpMask.convertTo(pixel_mask, CV_8U); - - LandmarkDetector::ReadMatBin(stream, alpha); - LandmarkDetector::ReadMatBin(stream, beta); + cv::Mat tmpMask; + LandmarkDetector::ReadMatBin(stream, tmpMask); + tmpMask.convertTo(pixel_mask, CV_8U); - map_x.create(pixel_mask.rows,pixel_mask.cols); - map_y.create(pixel_mask.rows,pixel_mask.cols); + cv::Mat_ alpha_d; + LandmarkDetector::ReadMatBin(stream, alpha_d); + alpha_d.convertTo(alpha, CV_32F); + + cv::Mat_ beta_d; + LandmarkDetector::ReadMatBin(stream, beta_d); + beta_d.convertTo(beta, CV_32F); + + map_x.create(pixel_mask.rows, pixel_mask.cols); + map_y.create(pixel_mask.rows, pixel_mask.cols); + + coefficients.create(this->NumberOfTriangles(), 6); - coefficients.create(this->NumberOfTriangles(),6); - source_landmarks = destination_landmarks; } //============================================================================= // cropping from the source image to the destination image using the shape in s, used to determine if shape fitting converged successfully -void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) +void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const cv::Mat_& landmarks_to_warp) { - + // set the current shape source_landmarks = landmarks_to_warp.clone(); @@ -312,10 +327,10 @@ void PAW::Warp(const cv::Mat& image_to_warp, cv::Mat& destination_image, const c // Do the actual mapping computation (where to warp from) this->WarpRegion(map_x, map_y); - + // Do the actual warp (with bi-linear interpolation) remap(image_to_warp, destination_image, map_x, map_y, CV_INTER_LINEAR); - + } @@ -325,33 +340,33 @@ void PAW::CalcCoeff() { int p = this->NumberOfLandmarks(); - for(int l = 0; l < this->NumberOfTriangles(); l++) + for (int l = 0; l < this->NumberOfTriangles(); l++) { - - int i = triangulation.at(l,0); - int j = triangulation.at(l,1); - int k = triangulation.at(l,2); - double c1 = source_landmarks.at(i , 0); - double c2 = source_landmarks.at(j , 0) - c1; - double c3 = source_landmarks.at(k , 0) - c1; - double c4 = source_landmarks.at(i + p, 0); - double c5 = source_landmarks.at(j + p, 0) - c4; - double c6 = source_landmarks.at(k + p, 0) - c4; + int i = triangulation.at(l, 0); + int j = triangulation.at(l, 1); + int k = triangulation.at(l, 2); + + float c1 = source_landmarks.at(i, 0); + float c2 = source_landmarks.at(j, 0) - c1; + float c3 = source_landmarks.at(k, 0) - c1; + float c4 = source_landmarks.at(i + p, 0); + float c5 = source_landmarks.at(j + p, 0) - c4; + float c6 = source_landmarks.at(k + p, 0) - c4; // Get a pointer to the coefficient we will be precomputing - double *coeff = coefficients.ptr(l); + float *coeff = coefficients.ptr(l); // Extract the relevant alphas and betas - double *c_alpha = alpha.ptr(l); - double *c_beta = beta.ptr(l); + float *c_alpha = alpha.ptr(l); + float *c_beta = beta.ptr(l); coeff[0] = c1 + c2 * c_alpha[0] + c3 * c_beta[0]; - coeff[1] = c2 * c_alpha[1] + c3 * c_beta[1]; - coeff[2] = c2 * c_alpha[2] + c3 * c_beta[2]; + coeff[1] = c2 * c_alpha[1] + c3 * c_beta[1]; + coeff[2] = c2 * c_alpha[2] + c3 * c_beta[2]; coeff[3] = c4 + c5 * c_alpha[0] + c6 * c_beta[0]; - coeff[4] = c5 * c_alpha[1] + c6 * c_beta[1]; - coeff[5] = c5 * c_alpha[2] + c6 * c_beta[2]; + coeff[4] = c5 * c_alpha[1] + c6 * c_beta[1]; + coeff[5] = c5 * c_alpha[2] + c6 * c_beta[2]; } } @@ -359,27 +374,27 @@ void PAW::CalcCoeff() // Compute the mapping coefficients void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) { - + cv::MatIterator_ xp = mapx.begin(); cv::MatIterator_ yp = mapy.begin(); cv::MatIterator_ mp = pixel_mask.begin(); cv::MatIterator_ tp = triangle_id.begin(); - + // The coefficients corresponding to the current triangle - double * a; + float * a; // Current triangle being processed - int k=-1; + int k = -1; - for(int y = 0; y < pixel_mask.rows; y++) + for (int y = 0; y < pixel_mask.rows; y++) { - double yi = double(y) + min_y; - - for(int x = 0; x < pixel_mask.cols; x++) - { - double xi = double(x) + min_x; + float yi = float(y) + min_y; - if(*mp == 0) + for (int x = 0; x < pixel_mask.cols; x++) + { + float xi = float(x) + min_x; + + if (*mp == 0) { *xp = -1; *yp = -1; @@ -391,32 +406,32 @@ void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) // If it is different from the previous triangle point to new coefficients // This will always be the case in the first iteration, hence a will not point to nothing - if(j != k) + if (j != k) { // Update the coefficient pointer if a new triangle is being processed - a = coefficients.ptr(j); + a = coefficients.ptr(j); k = j; - } + } //ap is now the pointer to the coefficients - double *ap = a; + float *ap = a; //look at the first coefficient (and increment). first coefficient is an x offset - double xo = *ap++; + float xo = *ap++; //second coefficient is an x scale as a function of x - xo += *ap++ * xi; + xo += *ap++ * xi; //third coefficient ap(2) is an x scale as a function of y - *xp = float(xo + *ap++ * yi); + *xp = float(xo + *ap++ * yi); //then fourth coefficient ap(3) is a y offset - double yo = *ap++; + float yo = *ap++; //fifth coeff adds coeff[4]*x to y - yo += *ap++ * xi; + yo += *ap++ * xi; //final coeff adds coeff[5]*y to y - *yp = float(yo + *ap++ * yi); + *yp = float(yo + *ap++ * yi); } - mp++; tp++; xp++; yp++; + mp++; tp++; xp++; yp++; } } } @@ -426,18 +441,18 @@ void PAW::WarpRegion(cv::Mat_& mapx, cv::Mat_& mapy) // ============================================================ // Is the point (x0,y0) on same side as a half-plane defined by (x1,y1), (x2, y2), and (x3, y3) -bool sameSide(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +bool sameSide(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) { - double x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); - double y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); + float x = (x3-x2)*(y0-y2) - (x0-x2)*(y3-y2); + float y = (x3-x2)*(y1-y2) - (x1-x2)*(y3-y2); return x*y >= 0; } // if point (x0, y0) is on same side for all three half-planes it is in a triangle -bool pointInTriangle(double x0, double y0, double x1, double y1, double x2, double y2, double x3, double y3) +bool pointInTriangle(float x0, float y0, float x1, float y1, float x2, float y2, float x3, float y3) { bool same_1 = sameSide(x0, y0, x1, y1, x2, y2, x3, y3); bool same_2 = sameSide(x0, y0, x2, y2, x1, y1, x3, y3); @@ -448,54 +463,54 @@ bool pointInTriangle(double x0, double y0, double x1, double y1, double x2, doub } // Find if a given point lies in the triangles -int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) const +int PAW::findTriangle(const cv::Point_& point, const std::vector>& control_points, int guess) const { - + int num_tris = control_points.size(); - + int tri = -1; - - double x0 = point.x; - double y0 = point.y; + + float x0 = point.x; + float y0 = point.y; // Allow a guess for speed (so as not to go through all triangles) - if(guess != -1) + if (guess != -1) { - + bool in_triangle = pointInTriangle(x0, y0, control_points[guess][0], control_points[guess][1], control_points[guess][2], control_points[guess][3], control_points[guess][4], control_points[guess][5]); - if(in_triangle) + if (in_triangle) { return guess; } } - for (int i = 0; i < num_tris; ++i) + for (int i = 0; i < num_tris; ++i) { - double max_x = control_points[i][6]; - double max_y = control_points[i][7]; + float max_x = control_points[i][6]; + float max_y = control_points[i][7]; - double min_x = control_points[i][8]; - double min_y = control_points[i][9]; + float min_x = control_points[i][8]; + float min_y = control_points[i][9]; // Skip the check if the point is outside the bounding box of the triangle - if( max_x < x0 || min_x > x0 || max_y < y0 || min_y > y0) + if (max_x < x0 || min_x > x0 || max_y < y0 || min_y > y0) { continue; } - bool in_triangle = pointInTriangle(x0, y0, + bool in_triangle = pointInTriangle(x0, y0, control_points[i][0], control_points[i][1], control_points[i][2], control_points[i][3], control_points[i][4], control_points[i][5]); - if(in_triangle) + if (in_triangle) { - tri = i; - break; - } + tri = i; + break; + } } return tri; } \ No newline at end of file diff --git a/lib/local/LandmarkDetector/src/PDM.cpp b/lib/local/LandmarkDetector/src/PDM.cpp index a0a8a4c8..fb9260e1 100644 --- a/lib/local/LandmarkDetector/src/PDM.cpp +++ b/lib/local/LandmarkDetector/src/PDM.cpp @@ -147,23 +147,22 @@ void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local //=========================================================================== // Get the 2D shape (in image space) from global and local parameters -void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const +void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const { int n = this->NumberOfPoints(); - double s = params_global[0]; // scaling factor - double tx = params_global[4]; // x offset - double ty = params_global[5]; // y offset + // TODO move to float + float s = (float)params_global[0]; // scaling factor + float tx = (float)params_global[4]; // x offset + float ty = (float)params_global[5]; // y offset // get the rotation matrix from the euler angles - cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); - cv::Matx33d currRot = Euler2RotationMatrix(euler); + cv::Vec3f euler((float)params_global[1], (float)params_global[2], (float)params_global[3]); + cv::Matx33f currRot = Euler2RotationMatrix_f(euler); // get the 3D shape of the object - cv::Mat_ Shape_3D_f = mean_shape + princ_comp * params_local; - cv::Mat_ Shape_3D; - Shape_3D_f.convertTo(Shape_3D, CV_64F); + cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; // create the 2D shape matrix (if it has not been defined yet) if((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) @@ -174,8 +173,8 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params for(int i = 0; i < n; i++) { // Transform this using the weak-perspective mapping to 2D from 3D - out_shape.at(i ,0) = s * ( currRot(0,0) * Shape_3D.at(i, 0) + currRot(0,1) * Shape_3D.at(i+n ,0) + currRot(0,2) * Shape_3D.at(i+n*2,0) ) + tx; - out_shape.at(i+n,0) = s * ( currRot(1,0) * Shape_3D.at(i, 0) + currRot(1,1) * Shape_3D.at(i+n ,0) + currRot(1,2) * Shape_3D.at(i+n*2,0) ) + ty; + out_shape.at(i ,0) = s * ( currRot(0,0) * Shape_3D.at(i, 0) + currRot(0,1) * Shape_3D.at(i+n ,0) + currRot(0,2) * Shape_3D.at(i+n*2,0) ) + tx; + out_shape.at(i+n,0) = s * ( currRot(1,0) * Shape_3D.at(i, 0) + currRot(1,1) * Shape_3D.at(i+n ,0) + currRot(1,2) * Shape_3D.at(i+n*2,0) ) + ty; } } @@ -229,7 +228,7 @@ void PDM::CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_gl { // get the shape instance based on local params - cv::Mat_ current_shape; + cv::Mat_ current_shape; CalcShape2D(current_shape, params_local, params_global); // Get the width of expected shape @@ -489,7 +488,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& } -void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3f rotation) +void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_ & landmark_locations, const cv::Vec3f rotation) { int m = this->NumberOfModes(); @@ -503,7 +502,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l for(int i = 0; i < n; ++i) { // If the landmark is invisible indicate this - if(landmark_locations.at(i) == 0) + if(landmark_locations.at(i) == 0) { visi_ind_2D.at(i) = 0; visi_ind_2D.at(i+n) = 0; @@ -545,8 +544,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_l { if(visi_ind_2D.at(i) == 1) { - // TODO move away from doubles - landmark_locs_vis.at(k) = (float)landmark_locations.at(i); + landmark_locs_vis.at(k) = landmark_locations.at(i); k++; } } diff --git a/lib/local/LandmarkDetector/src/Patch_experts.cpp b/lib/local/LandmarkDetector/src/Patch_experts.cpp index 766243e3..760329ab 100644 --- a/lib/local/LandmarkDetector/src/Patch_experts.cpp +++ b/lib/local/LandmarkDetector/src/Patch_experts.cpp @@ -142,11 +142,11 @@ void Patch_experts::Response(vector >& patch_expert_responses, c int n = pdm.NumberOfPoints(); // Compute the current landmark locations (around which responses will be computed) - cv::Mat_ landmark_locations; + cv::Mat_ landmark_locations; pdm.CalcShape2D(landmark_locations, params_local, params_global); - cv::Mat_ reference_shape; + cv::Mat_ reference_shape; // Initialise the reference shape on which we'll be warping cv::Vec6d global_ref(patch_scaling[scale], 0, 0, 0, 0, 0); @@ -155,19 +155,14 @@ void Patch_experts::Response(vector >& patch_expert_responses, c pdm.CalcShape2D(reference_shape, params_local, global_ref); // similarity and inverse similarity transform to and from image and reference shape - cv::Mat_ reference_shape_2D = (reference_shape.reshape(1, 2).t()); - cv::Mat_ image_shape_2D = landmark_locations.reshape(1, 2).t(); + cv::Mat_ reference_shape_2D = (reference_shape.reshape(1, 2).t()); + cv::Mat_ image_shape_2D = landmark_locations.reshape(1, 2).t(); - sim_img_to_ref = AlignShapesWithScale(image_shape_2D, reference_shape_2D); - cv::Matx22d sim_ref_to_img_d = sim_img_to_ref.inv(cv::DECOMP_LU); + sim_img_to_ref = AlignShapesWithScale_f(image_shape_2D, reference_shape_2D); + sim_ref_to_img = sim_img_to_ref.inv(cv::DECOMP_LU); - double a1 = sim_ref_to_img_d(0,0); - double b1 = -sim_ref_to_img_d(0,1); - - sim_ref_to_img(0,0) = (float)sim_ref_to_img_d(0,0); - sim_ref_to_img(0,1) = (float)sim_ref_to_img_d(0,1); - sim_ref_to_img(1,0) = (float)sim_ref_to_img_d(1,0); - sim_ref_to_img(1,1) = (float)sim_ref_to_img_d(1,1); + float a1 = sim_ref_to_img(0,0); + float b1 = -sim_ref_to_img(0,1); bool use_ccnf = !this->ccnf_expert_intensity.empty(); bool use_cen = !this->cen_expert_intensity.empty(); @@ -248,7 +243,7 @@ void Patch_experts::Response(vector >& patch_expert_responses, c } // scale and rotate to mean shape to reference frame - cv::Mat sim = (cv::Mat_(2, 3) << a1, -b1, landmark_locations.at(ind, 0), b1, a1, landmark_locations.at(ind + n, 0)); + cv::Mat sim = (cv::Mat_(2, 3) << a1, -b1, landmark_locations.at(ind, 0), b1, a1, landmark_locations.at(ind + n, 0)); // Extract the region of interest around the current landmark location cv::Mat_ area_of_interest(area_of_interest_height, area_of_interest_width); @@ -280,7 +275,7 @@ void Patch_experts::Response(vector >& patch_expert_responses, c { // Grab mirrored area of interest - cv::Mat sim_r = (cv::Mat_(2, 3) << a1, -b1, landmark_locations.at(mirror_id, 0), b1, a1, landmark_locations.at(mirror_id + n, 0)); + cv::Mat sim_r = (cv::Mat_(2, 3) << a1, -b1, landmark_locations.at(mirror_id, 0), b1, a1, landmark_locations.at(mirror_id + n, 0)); // Extract the region of interest around the current landmark location cv::Mat_ area_of_interest_r(area_of_interest_height, area_of_interest_width);