From 156cf2e2dd8acedf541a68b54f30e4154e58ca52 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Sun, 1 Oct 2017 09:21:39 +0100 Subject: [PATCH] Moving params_local to floats, preparing more things to move to floats. --- exe/FeatureExtraction/FeatureExtraction.cpp | 2 +- .../include/LandmarkDetectorModel.h | 6 ++-- lib/local/LandmarkDetector/include/PDM.h | 10 +++--- .../src/LandmarkDetectorModel.cpp | 5 ++- lib/local/LandmarkDetector/src/PDM.cpp | 33 +++++++++++-------- 5 files changed, 30 insertions(+), 26 deletions(-) diff --git a/exe/FeatureExtraction/FeatureExtraction.cpp b/exe/FeatureExtraction/FeatureExtraction.cpp index d0d8e0d4..c80ff0ae 100644 --- a/exe/FeatureExtraction/FeatureExtraction.cpp +++ b/exe/FeatureExtraction/FeatureExtraction.cpp @@ -839,7 +839,7 @@ void outputAllFeatures(std::ofstream* output_file, bool output_2D_landmarks, boo { if (face_model.tracking_initialised) { - *output_file << ", " << face_model.params_local.at(i, 0); + *output_file << ", " << face_model.params_local.at(i, 0); } else { diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index 33647309..75ef4aaf 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -73,7 +73,7 @@ public: // The local and global parameters describing the current model instance (current landmark detections) // Local parameters describing the non-rigid shape - cv::Mat_ params_local; + cv::Mat_ params_local; // Global parameters describing the rigid shape [scale, euler_x, euler_y, euler_z, tx, ty] cv::Vec6d params_global; @@ -124,7 +124,7 @@ public: // The landmark detection likelihoods (combined and per patch expert) float model_likelihood; - cv::Mat_ landmark_likelihoods; + cv::Mat_ landmark_likelihoods; // Keeping track of how many frames the tracker has failed in so far when tracking in videos // This is useful for knowing when to initialise and reinitialise tracking @@ -194,7 +194,7 @@ private: void NonVectorisedMeanShift_precalc_kde(cv::Mat_& out_mean_shifts, const vector >& patch_expert_responses, const cv::Mat_ &dxs, const cv::Mat_ &dys, int resp_size, float a, int scale, int view_id, map >& mean_shifts); // The actual model optimisation (update step), returns the model likelihood - float NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, + float NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_idx, bool rigid, int scale, cv::Mat_& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood); // Generating the weight matrix for the Weighted least squares diff --git a/lib/local/LandmarkDetector/include/PDM.h b/lib/local/LandmarkDetector/include/PDM.h index 46087871..46db23f3 100644 --- a/lib/local/LandmarkDetector/include/PDM.h +++ b/lib/local/LandmarkDetector/include/PDM.h @@ -75,19 +75,19 @@ class PDM{ void Clamp(cv::Mat_& params_local, cv::Vec6d& params_global, const FaceModelParameters& params); // Compute shape in object space (3D) - void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; + void CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& params_local) const; // Compute shape in image space (2D) - void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; + void CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const; // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box - void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation = cv::Vec3d(0.0)); + void CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation = cv::Vec3d(0.0)); // Provided the landmark location compute global and local parameters best fitting it (can provide optional rotation for potentially better results) - void CalcParams(cv::Vec6d& out_params_global, const cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)); + void CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation = cv::Vec3d(0.0)); // provided the model parameters, compute the bounding box of a face - void CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local); + void CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local); // Helpers for computing Jacobians, and Jacobians with the weight matrix void ComputeRigidJacobian(const cv::Mat_& params_local, const cv::Vec6d& params_global, cv::Mat_ &Jacob, const cv::Mat_ W, cv::Mat_ &Jacob_t_w); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index f1385da9..99d994f8 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -938,7 +938,7 @@ void CLNF::GetWeightMatrix(cv::Mat_& WeightMatrix, int scale, int view_id } //============================================================================= -float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, +float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, const vector >& patch_expert_responses, const cv::Vec6d& initial_global, const cv::Mat_& initial_local, const cv::Mat_& base_shape, const cv::Matx22d& sim_img_to_ref, const cv::Matx22f& sim_ref_to_img, int resp_size, int view_id, bool rigid, int scale, cv::Mat_& landmark_lhoods, const FaceModelParameters& parameters, bool compute_lhood) { @@ -954,8 +954,7 @@ float CLNF::NU_RLMS(cv::Vec6d& final_global, cv::Mat_& final_local, cons cv::Vec6d current_global(initial_global); - cv::Mat_ current_local; - initial_local.convertTo(current_local, CV_32F); + cv::Mat_ current_local = initial_local.clone(); cv::Mat_ current_shape; cv::Mat_ previous_shape; diff --git a/lib/local/LandmarkDetector/src/PDM.cpp b/lib/local/LandmarkDetector/src/PDM.cpp index a0ea88e6..cae45f13 100644 --- a/lib/local/LandmarkDetector/src/PDM.cpp +++ b/lib/local/LandmarkDetector/src/PDM.cpp @@ -139,15 +139,20 @@ void PDM::Clamp(cv::Mat_& local_params, cv::Vec6d& params_global, const F } //=========================================================================== // Compute the 3D representation of shape (in object space) using the local parameters -void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local) const +void PDM::CalcShape3D(cv::Mat_& out_shape, const cv::Mat_& p_local) const { out_shape.create(mean_shape.rows, mean_shape.cols); - out_shape = mean_shape + princ_comp*p_local; + + // TODO change it to all floats internally + cv::Mat_ p_local_d; + p_local.convertTo(p_local_d, CV_64F); + + out_shape = mean_shape + princ_comp*p_local_d; } //=========================================================================== // Get the 2D shape (in image space) from global and local parameters -void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const +void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& params_local, const cv::Vec6d& params_global) const { int n = this->NumberOfPoints(); @@ -160,8 +165,12 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& param cv::Vec3d euler(params_global[1], params_global[2], params_global[3]); cv::Matx33d currRot = Euler2RotationMatrix(euler); + // TODO change it to all floats internally + cv::Mat_ p_local_d; + params_local.convertTo(p_local_d, CV_64F); + // get the 3D shape of the object - cv::Mat_ Shape_3D = mean_shape + princ_comp * params_local; + cv::Mat_ Shape_3D = mean_shape + princ_comp * p_local_d; // create the 2D shape matrix (if it has not been defined yet) if((out_shape.rows != mean_shape.rows) || (out_shape.cols != 1)) @@ -180,7 +189,7 @@ void PDM::CalcShape2D(cv::Mat_& out_shape, const cv::Mat_& param //=========================================================================== // provided the bounding box of a face and the local parameters (with optional rotation), generates the global parameters that can generate the face with the provided bounding box // This all assumes that the bounding box describes face from left outline to right outline of the face and chin to eyebrows -void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation) +void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& bounding_box, const cv::Mat_& params_local, const cv::Vec3d rotation) { // get the shape instance based on local params @@ -223,7 +232,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Rect_& boun //=========================================================================== // provided the model parameters, compute the bounding box of a face // The bounding box describes face from left outline to right outline of the face and chin to eyebrows -void PDM::CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local) +void PDM::CalcBoundingBox(cv::Rect& out_bounding_box, const cv::Vec6d& params_global, const cv::Mat_& params_local) { // get the shape instance based on local params @@ -260,9 +269,7 @@ void PDM::ComputeRigidJacobian(const cv::Mat_& p_local, const cv::Vec6d& float s = (float)params_global[0]; cv::Mat_ shape_3D_d; - cv::Mat_ p_local_d; - p_local.convertTo(p_local_d, CV_64F); - this->CalcShape3D(shape_3D_d, p_local_d); + this->CalcShape3D(shape_3D_d, p_local); cv::Mat_ shape_3D; shape_3D_d.convertTo(shape_3D, CV_32F); @@ -359,9 +366,7 @@ void PDM::ComputeJacobian(const cv::Mat_& params_local, const cv::Vec6d& float s = (float) params_global[0]; cv::Mat_ shape_3D_d; - cv::Mat_ p_local_d; - params_local.convertTo(p_local_d, CV_64F); - this->CalcShape3D(shape_3D_d, p_local_d); + this->CalcShape3D(shape_3D_d, params_local); cv::Mat_ shape_3D; shape_3D_d.convertTo(shape_3D, CV_32F); @@ -497,7 +502,7 @@ void PDM::UpdateModelParameters(const cv::Mat_& delta_p, cv::Mat_& } -void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation) +void PDM::CalcParams(cv::Vec6d& out_params_global, cv::Mat_& out_params_local, const cv::Mat_& landmark_locations, const cv::Vec3d rotation) { int m = this->NumberOfModes(); @@ -697,7 +702,7 @@ void PDM::CalcParams(cv::Vec6d& out_params_global, const cv::Mat_& out_p } out_params_global = glob_params; - loc_params.convertTo(out_params_local, CV_64F); + out_params_local = loc_params; this->mean_shape = m_old; this->princ_comp = v_old;