From 596593cca37a48aba8649d8a34bd83d18ca993ee Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 19 Jun 2017 16:35:11 -0400 Subject: [PATCH] Moving towards an optimized C++ version of CEN. --- .../include/LandmarkDetectorModel.h | 3 + .../LandmarkDetector/include/Patch_experts.h | 11 +- .../LandmarkDetector/model/cen_general.txt | 1 + .../LandmarkDetector/model/early_term_cen.txt | 1 + .../src/LandmarkDetectorFunc.cpp | 239 +++++++++++++++--- .../src/LandmarkDetectorModel.cpp | 10 +- .../src/LandmarkDetectorParameters.cpp | 7 + .../LandmarkDetector/src/Patch_experts.cpp | 34 ++- .../Run_OF_on_images.m | 2 +- ...OpenFace_feature_point_tests_300W_ceclm.m} | 24 +- ...m => run_head_pose_tests_OpenFace_CECLM.m} | 31 +-- .../experiments_JANUS/results/Janus-full.pdf | Bin 7224 -> 7224 bytes .../experiments_JANUS/results/Janus-full.png | Bin 175087 -> 175087 bytes .../results/Janus-no-outline.pdf | Bin 8500 -> 8500 bytes .../results/Janus-no-outline.png | Bin 198243 -> 198243 bytes matlab_version/fitting/Fitting_from_bb.m | 2 +- ...{PatchResponseDNN.m => PatchResponseCEN.m} | 2 +- .../learn_error_mapping/early_term_cen.txt | 1 + .../learn_error_mapping/output_corrections.m | 18 ++ matlab_version/learn_error_mapping/readme.txt | 4 +- 20 files changed, 327 insertions(+), 63 deletions(-) create mode 100644 lib/local/LandmarkDetector/model/early_term_cen.txt rename matlab_runners/Feature Point Experiments/{run_OpenFace_feature_point_tests_300W.m => run_OpenFace_feature_point_tests_300W_ceclm.m} (86%) rename matlab_runners/Head Pose Experiments/{run_head_pose_tests_OpenFace_DCLM.m => run_head_pose_tests_OpenFace_CECLM.m} (57%) rename matlab_version/fitting/{PatchResponseDNN.m => PatchResponseCEN.m} (97%) create mode 100644 matlab_version/learn_error_mapping/early_term_cen.txt create mode 100644 matlab_version/learn_error_mapping/output_corrections.m diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index b34d3f94..041f0f00 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -155,6 +155,9 @@ public: // Useful when resetting or initialising the model closer to a specific location (when multiple faces are present) cv::Point_ preference_det; + // Tracking which view was used last + int view_used; + // A default constructor CLNF(); diff --git a/lib/local/LandmarkDetector/include/Patch_experts.h b/lib/local/LandmarkDetector/include/Patch_experts.h index 9b532bf4..e4aaf17a 100644 --- a/lib/local/LandmarkDetector/include/Patch_experts.h +++ b/lib/local/LandmarkDetector/include/Patch_experts.h @@ -103,6 +103,12 @@ public: // Landmark visibilities for each scale and view vector > > visibilities; + // Early termination calibration values, useful for CE-CLM model to speed up the multi-hypothesis setup + vector early_term_weights; + vector early_term_biases; + vector early_term_cutoffs; + + // A default constructor Patch_experts(){;} @@ -123,9 +129,8 @@ public: inline int nViews(size_t scale = 0) const { return (int)centers[scale].size(); }; // Reading in all of the patch experts - void Read(vector intensity_svr_expert_locations, vector depth_svr_expert_locations, vector intensity_ccnf_expert_locations, vector intensity_cen_expert_locations); - - + void Read(vector intensity_svr_expert_locations, vector depth_svr_expert_locations, vector intensity_ccnf_expert_locations, vector intensity_cen_expert_locations, + string early_term_loc = ""); private: diff --git a/lib/local/LandmarkDetector/model/cen_general.txt b/lib/local/LandmarkDetector/model/cen_general.txt index adecf281..dd40388e 100644 --- a/lib/local/LandmarkDetector/model/cen_general.txt +++ b/lib/local/LandmarkDetector/model/cen_general.txt @@ -4,3 +4,4 @@ PatchesCEN patch_experts/cen_patches_0.25_menpo.dat PatchesCEN patch_experts/cen_patches_0.35_menpo.dat PatchesCEN patch_experts/cen_patches_0.50_menpo.dat PatchesCEN patch_experts/cen_patches_1.00_menpo.dat +EarlyTermination early_term_cen.txt \ No newline at end of file diff --git a/lib/local/LandmarkDetector/model/early_term_cen.txt b/lib/local/LandmarkDetector/model/early_term_cen.txt new file mode 100644 index 00000000..8e0765ba --- /dev/null +++ b/lib/local/LandmarkDetector/model/early_term_cen.txt @@ -0,0 +1 @@ +1.000 0.987 0.768 0.899 0.899 0.768 0.987 0.000 0.033 -0.332 -0.434 -0.434 -0.332 0.033 -0.240 -0.260 0.050 0.210 0.210 0.050 -0.260 \ No newline at end of file diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index 7cedb3b8..b66320fb 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -67,6 +67,7 @@ // System includes #include +#include using namespace LandmarkDetector; @@ -455,37 +456,12 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // Optionally can provide a bounding box in which detection is performed (this is useful if multiple faces are to be detected in images) //================================================================================================================ -// This is the one where the actual work gets done, other DetectLandmarksInImage calls lead to this one -bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_image, const cv::Mat_ depth_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) +bool DetectLandmarksInImageMultiHypBasic(const cv::Mat_ &grayscale_image, const cv::Mat_ depth_image, vector rotation_hypotheses, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) { - // Can have multiple hypotheses - vector rotation_hypotheses; - - if(params.multi_view) - { - // Try out different orientation initialisations - // It is possible to add other orientation hypotheses easilly by just pushing to this vector - rotation_hypotheses.push_back(cv::Vec3d(0,0,0)); - rotation_hypotheses.push_back(cv::Vec3d(0,0.5236,0)); - rotation_hypotheses.push_back(cv::Vec3d(0,-0.5236,0)); - - // If not using CEN patch experts add additional hypotheses - if(clnf_model.patch_experts.cen_expert_intensity.size() == 0) - { - rotation_hypotheses.push_back(cv::Vec3d(0.5236,0,0)); - rotation_hypotheses.push_back(cv::Vec3d(-0.5236,0,0)); - } - } - else - { - // Assume the face is close to frontal - rotation_hypotheses.push_back(cv::Vec3d(0,0,0)); - } - // Use the initialisation size for the landmark detection params.window_sizes_current = params.window_sizes_init; - + // Store the current best estimate double best_likelihood; cv::Vec6d best_global_parameters; @@ -501,7 +477,7 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i vector> best_detected_landmarks_h(clnf_model.hierarchical_models.size()); vector> best_landmark_likelihoods_h(clnf_model.hierarchical_models.size()); - for(size_t hypothesis = 0; hypothesis < rotation_hypotheses.size(); ++hypothesis) + for (size_t hypothesis = 0; hypothesis < rotation_hypotheses.size(); ++hypothesis) { // Reset the potentially set clnf_model parameters clnf_model.params_local.setTo(0.0); @@ -513,10 +489,10 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i // calculate the local and global parameters from the generated 2D shape (mapping from the 2D to 3D because camera params are unknown) clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local, rotation_hypotheses[hypothesis]); - - bool success = clnf_model.DetectLandmarks(grayscale_image, depth_image, params); - if(hypothesis == 0 || best_likelihood < clnf_model.model_likelihood) + bool success = clnf_model.DetectLandmarks(grayscale_image, depth_image, params); + + if (hypothesis == 0 || best_likelihood < clnf_model.model_likelihood) { best_likelihood = clnf_model.model_likelihood; best_global_parameters = clnf_model.params_global; @@ -557,6 +533,207 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i } return best_success; + + +} + +// Helper index sorting function +template std::vector sort_indexes(const vector &v) { + + // initialize original index locations + vector idx(v.size()); + std::iota(idx.begin(), idx.end(), 0); + + // sort indexes based on comparing values in v + sort(idx.begin(), idx.end(), + [&v](size_t i1, size_t i2) {return v[i1] < v[i2]; }); + + return idx; +} + +bool DetectLandmarksInImageMultiHypEarlyTerm(const cv::Mat_ &grayscale_image, const cv::Mat_ depth_image, vector rotation_hypotheses, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) +{ + FaceModelParameters old_params(params); + + // Use the initialisation size for the landmark detection + params.window_sizes_current = params.window_sizes_init; + + bool early_term = false; + + // Setup the parameters accordingly + // Only do the first iteration + for (int i = 1; i < params.window_sizes_current.size(); ++i) + { + params.window_sizes_current[i] = 0; + } + params.refine_hierarchical = false; + params.validate_detections = false; + + bool success = false; + + // Keeping track of converges + vector likelihoods; + vector global_parameters; + vector> local_parameters; + + for (size_t hypothesis = 0; hypothesis < rotation_hypotheses.size(); ++hypothesis) + { + // Reset the potentially set clnf_model parameters + clnf_model.params_local.setTo(0.0); + + for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) + { + clnf_model.hierarchical_models[part].params_local.setTo(0.0); + } + + // calculate the local and global parameters from the generated 2D shape (mapping from the 2D to 3D because camera params are unknown) + clnf_model.pdm.CalcParams(clnf_model.params_global, bounding_box, clnf_model.params_local, rotation_hypotheses[hypothesis]); + + // Perform landmark detection in first scale + clnf_model.DetectLandmarks(grayscale_image, depth_image, params); + + double lhood = clnf_model.model_likelihood * clnf_model.patch_experts.early_term_weights[clnf_model.view_used] + clnf_model.patch_experts.early_term_biases[clnf_model.view_used]; + + // If likelihood higher than cutoff continue on this model + if (lhood > clnf_model.patch_experts.early_term_cutoffs[clnf_model.view_used]) + { + params.refine_hierarchical = old_params.refine_hierarchical; + params.window_sizes_current = params.window_sizes_init; + params.window_sizes_current[0] = 0; + params.validate_detections = old_params.validate_detections; + success = clnf_model.DetectLandmarks(grayscale_image, depth_image, params); + break; + } + else + { + likelihoods.push_back(lhood); + global_parameters.push_back(clnf_model.params_global); + local_parameters.push_back(clnf_model.params_local); + } + } + + params.refine_hierarchical = old_params.refine_hierarchical; + params.window_sizes_current = params.window_sizes_init; + params.window_sizes_current[0] = 0; + params.validate_detections = old_params.validate_detections; + + if (!early_term) + { + + // Store the current best estimate + double best_likelihood; + cv::Vec6d best_global_parameters; + cv::Mat_ best_local_parameters; + cv::Mat_ best_detected_landmarks; + cv::Mat_ best_landmark_likelihoods; + bool best_success; + + // The hierarchical model parameters + vector best_likelihood_h(clnf_model.hierarchical_models.size()); + vector best_global_parameters_h(clnf_model.hierarchical_models.size()); + vector> best_local_parameters_h(clnf_model.hierarchical_models.size()); + vector> best_detected_landmarks_h(clnf_model.hierarchical_models.size()); + vector> best_landmark_likelihoods_h(clnf_model.hierarchical_models.size()); + + // Sort the likelihoods and pick the best top 3 models + vector indices = sort_indexes(likelihoods); + + for (size_t i = 0; i < 3; ++i) + { + // Reset the potentially set clnf_model parameters + clnf_model.params_local = local_parameters[indices[i]]; + clnf_model.params_global = global_parameters[indices[i]]; + for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) + { + clnf_model.hierarchical_models[part].params_local.setTo(0.0); + } + + // Perform landmark detection in first scale + success = clnf_model.DetectLandmarks(grayscale_image, depth_image, params); + + if (i == 0 || best_likelihood < clnf_model.model_likelihood) + { + best_likelihood = clnf_model.model_likelihood; + best_global_parameters = clnf_model.params_global; + best_local_parameters = clnf_model.params_local.clone(); + best_detected_landmarks = clnf_model.detected_landmarks.clone(); + best_landmark_likelihoods = clnf_model.landmark_likelihoods.clone(); + best_success = success; + } + + for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) + { + if (i == 0 || best_likelihood < clnf_model.hierarchical_models[part].model_likelihood) + { + best_likelihood_h[part] = clnf_model.hierarchical_models[part].model_likelihood; + best_global_parameters_h[part] = clnf_model.hierarchical_models[part].params_global; + best_local_parameters_h[part] = clnf_model.hierarchical_models[part].params_local.clone(); + best_detected_landmarks_h[part] = clnf_model.hierarchical_models[part].detected_landmarks.clone(); + best_landmark_likelihoods_h[part] = clnf_model.hierarchical_models[part].landmark_likelihoods.clone(); + } + } + } + + // Store the best estimates in the clnf_model + clnf_model.model_likelihood = best_likelihood; + clnf_model.params_global = best_global_parameters; + clnf_model.params_local = best_local_parameters.clone(); + clnf_model.detected_landmarks = best_detected_landmarks.clone(); + clnf_model.detection_success = best_success; + clnf_model.landmark_likelihoods = best_landmark_likelihoods.clone(); + + for (size_t part = 0; part < clnf_model.hierarchical_models.size(); ++part) + { + clnf_model.hierarchical_models[part].params_global = best_global_parameters_h[part]; + clnf_model.hierarchical_models[part].params_local = best_local_parameters_h[part].clone(); + clnf_model.hierarchical_models[part].detected_landmarks = best_detected_landmarks_h[part].clone(); + clnf_model.hierarchical_models[part].landmark_likelihoods = best_landmark_likelihoods_h[part].clone(); + } + + } + + params = old_params; + + return success; + +} + + +// This is the one where the actual work gets done, other DetectLandmarksInImage calls lead to this one +bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_image, const cv::Mat_ depth_image, const cv::Rect_ bounding_box, CLNF& clnf_model, FaceModelParameters& params) +{ + + // Can have multiple hypotheses + vector rotation_hypotheses; + + if(params.multi_view) + { + // Try out different orientation initialisations + // It is possible to add other orientation hypotheses easilly by just pushing to this vector + rotation_hypotheses.push_back(cv::Vec3d(0,0,0)); + rotation_hypotheses.push_back(cv::Vec3d(0,0.5236,0)); + rotation_hypotheses.push_back(cv::Vec3d(0,-0.5236,0)); + rotation_hypotheses.push_back(cv::Vec3d(0,0, 0.5236)); + rotation_hypotheses.push_back(cv::Vec3d(0,0, 0.5236)); + } + else + { + // Assume the face is close to frontal + rotation_hypotheses.push_back(cv::Vec3d(0,0,0)); + } + + bool success; + + // Either use basic multi-hypothesis testing or clever testing if early termination parameters are present + if(clnf_model.patch_experts.early_term_biases.size() == 0) + { + success = DetectLandmarksInImageMultiHypBasic(grayscale_image, depth_image, rotation_hypotheses, bounding_box, clnf_model, params); + } + else + { + success = DetectLandmarksInImageMultiHypEarlyTerm(grayscale_image, depth_image, rotation_hypotheses, bounding_box, clnf_model, params); + } + return success; } bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_image, const cv::Mat_ depth_image, CLNF& clnf_model, FaceModelParameters& params) diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index 96f5d442..349c3d1a 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -270,6 +270,7 @@ void CLNF::Read_CLNF(string clnf_location) vector depth_expert_locations; vector ccnf_expert_locations; vector cen_expert_locations; + string early_term_loc; // The other module locations should be defined as relative paths from the main model boost::filesystem::path root = boost::filesystem::path(clnf_location).parent_path(); @@ -345,10 +346,14 @@ void CLNF::Read_CLNF(string clnf_location) { cen_expert_locations.push_back(location); } + else if (module.compare("EarlyTermination") == 0) + { + early_term_loc = location; + } } // Initialise the patch experts - patch_experts.Read(intensity_expert_locations, depth_expert_locations, ccnf_expert_locations, cen_expert_locations); + patch_experts.Read(intensity_expert_locations, depth_expert_locations, ccnf_expert_locations, cen_expert_locations, early_term_loc); // Read in a face detector face_detector_HOG = dlib::get_frontal_face_detector(); @@ -358,7 +363,7 @@ void CLNF::Read_CLNF(string clnf_location) void CLNF::Read(string main_location) { - cout << "Reading the CLNF landmark detector/tracker from: " << main_location << endl; + cout << "Reading the landmark detector/tracker from: " << main_location << endl; ifstream locations(main_location.c_str(), ios_base::in); if(!locations.is_open()) @@ -779,6 +784,7 @@ bool CLNF::Fit(const cv::Mat_& im, const cv::Mat_& depthImg, const // Get the view used by patch experts int view_id = patch_experts.GetViewIdx(params_global, scale); + this->view_used = view_id; // the actual optimisation step this->NU_RLMS(params_global, params_local, patch_expert_responses, cv::Vec6d(params_global), params_local.clone(), current_shape, sim_img_to_ref, sim_ref_to_img, window_size, view_id, true, scale, this->landmark_likelihoods, tmp_parameters); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index feb8845a..2d7f77dc 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -204,6 +204,7 @@ FaceModelParameters::FaceModelParameters(vector &arguments) } } + // Make sure model_location is valid // First check working directory, then the executable's directory, then the config path set by the build process. boost::filesystem::path config_path = boost::filesystem::path(CONFIG_DIR); @@ -224,6 +225,12 @@ FaceModelParameters::FaceModelParameters(vector &arguments) { std::cout << "Could not find the landmark detection model to load" << std::endl; } + + if (model_path.stem().string().compare("main_ceclm_general") == 0) + { + sigma = 1.5 * sigma; + reg_factor = 0.9 * reg_factor; + } } void FaceModelParameters::init() diff --git a/lib/local/LandmarkDetector/src/Patch_experts.cpp b/lib/local/LandmarkDetector/src/Patch_experts.cpp index 89f62e56..aac8dff6 100644 --- a/lib/local/LandmarkDetector/src/Patch_experts.cpp +++ b/lib/local/LandmarkDetector/src/Patch_experts.cpp @@ -80,7 +80,9 @@ using namespace LandmarkDetector; // A copy constructor -Patch_experts::Patch_experts(const Patch_experts& other) : patch_scaling(other.patch_scaling), centers(other.centers), svr_expert_intensity(other.svr_expert_intensity), svr_expert_depth(other.svr_expert_depth), ccnf_expert_intensity(other.ccnf_expert_intensity), cen_expert_intensity(other.cen_expert_intensity) +Patch_experts::Patch_experts(const Patch_experts& other) : patch_scaling(other.patch_scaling), centers(other.centers), svr_expert_intensity(other.svr_expert_intensity), + svr_expert_depth(other.svr_expert_depth), ccnf_expert_intensity(other.ccnf_expert_intensity), cen_expert_intensity(other.cen_expert_intensity), + early_term_weights(other.early_term_weights), early_term_biases(other.early_term_biases), early_term_cutoffs(other.early_term_cutoffs) { // Make sure the matrices are allocated properly @@ -338,7 +340,7 @@ int Patch_experts::GetViewIdx(const cv::Vec6d& params_global, int scale) const //=========================================================================== -void Patch_experts::Read(vector intensity_svr_expert_locations, vector depth_svr_expert_locations, vector intensity_ccnf_expert_locations, vector intensity_cen_expert_locations) +void Patch_experts::Read(vector intensity_svr_expert_locations, vector depth_svr_expert_locations, vector intensity_ccnf_expert_locations, vector intensity_cen_expert_locations, string early_term_loc) { // initialise the SVR intensity patch expert parameters @@ -449,6 +451,34 @@ void Patch_experts::Read(vector intensity_svr_expert_locations, vector> weight; + early_term_weights.push_back(weight); + } + + for (int i = 0; i < centers[0].size(); ++i) + { + double bias; + earlyTermFile >> bias; + early_term_biases.push_back(bias); + } + + for (int i = 0; i < centers[0].size(); ++i) + { + double cutoff; + earlyTermFile >> cutoff; + early_term_cutoffs.push_back(cutoff); + } + } + } //======================= Reading the SVR patch experts =========================================// void Patch_experts::Read_SVR_patch_experts(string expert_location, std::vector& centers, std::vector >& visibility, std::vector >& patches, double& scale) diff --git a/matlab_runners/Feature Point Experiments/Run_OF_on_images.m b/matlab_runners/Feature Point Experiments/Run_OF_on_images.m index 3cca6f50..aa09e026 100644 --- a/matlab_runners/Feature Point Experiments/Run_OF_on_images.m +++ b/matlab_runners/Feature Point Experiments/Run_OF_on_images.m @@ -48,7 +48,7 @@ command = cat(2, command, [' -mloc ' model ' ']); command = cat(2, command, [' -multi_view ' num2str(multi_view) ' ']); tic -parfor i=1:numel(dataset_dirs) +for i=1:numel(dataset_dirs) input_loc = ['-fdir "', dataset_dirs{i}, '" ']; command_c = cat(2, command, input_loc); diff --git a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W_ceclm.m similarity index 86% rename from matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m rename to matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W_ceclm.m index ae819473..eb75af5b 100644 --- a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m +++ b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W_ceclm.m @@ -11,6 +11,14 @@ else database_root = '/multicomp/datasets/300-W/'; end +%% Run using CE-CLM model +out_clnf = [curr_dir '/out_ceclm/']; +if(~exist(out_clnf, 'file')) + mkdir(out_clnf); +end + +[err_ceclm, err_no_out_ceclm] = Run_OF_on_images(out_clnf, database_root, 'use_afw', 'use_lfpw', 'use_ibug', 'use_helen', 'verbose', 'model', 'model/main_ceclm_general.txt', 'multi_view', 1); + %% Run using CLNF in the wild model out_clnf = [curr_dir '/out_wild_clnf_wild/']; if(~exist(out_clnf, 'file')) @@ -48,6 +56,8 @@ save('results/landmark_detections.mat'); f = fopen('results/landmark_detections.txt', 'w'); fprintf(f, 'Type, mean, median\n'); +fprintf(f, 'err ce-clm: %f, %f\n', mean(err_ceclm), median(err_ceclm)); + fprintf(f, 'err clnf: %f, %f\n', mean(err_clnf), median(err_clnf)); fprintf(f, 'err clnf wild: %f, %f\n', mean(err_clnf_wild), median(err_clnf_wild)); @@ -107,6 +117,18 @@ intraface_error = compute_error( labels - 0.5, shapes); plot(error_x, error_y, 'g--','DisplayName', 'SDM', 'LineWidth',line_width); hold on; +% load ce-clm errors +load('out_ceclm/res.mat'); +labels = labels([1:60,62:64,66:end],:, detected_cpp); +shapes = shapes([1:60,62:64,66:end],:, detected_cpp); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +ceclm_error_cpp = compute_error( labels, shapes); +[error_x, error_y] = cummErrorCurve(ceclm_error_cpp); +plot(error_x, error_y, 'r','DisplayName', 'CECLM', 'LineWidth',line_width); +hold on; + % load clnf errors load('out_wild_clnf_wild/res.mat'); labels = labels([1:60,62:64,66:end],:, detected_cpp); @@ -116,7 +138,7 @@ shapes = shapes(18:end,:,:); clnf_error_cpp = compute_error( labels, shapes); [error_x, error_y] = cummErrorCurve(clnf_error_cpp); -plot(error_x, error_y, 'r','DisplayName', 'CLM+CLNF', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'CLM+CLNF', 'LineWidth',line_width); hold on; % load svr errors diff --git a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_DCLM.m b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CECLM.m similarity index 57% rename from matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_DCLM.m rename to matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CECLM.m index 9dd7fbe0..7d09c65a 100644 --- a/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_DCLM.m +++ b/matlab_runners/Head Pose Experiments/run_head_pose_tests_OpenFace_CECLM.m @@ -1,54 +1,45 @@ clear; -% fitting parameters more suitable for clnf - %% % Run the BU test with clnf if exist('D:/Datasets/HeadPose', 'file') database_root = 'D:/Datasets/HeadPose/'; elseif(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; -elseif(exist('C:\Users\tbaltrus\Documents\DATA\HeadPose', 'file')) - database_root = 'C:\Users\tbaltrus\Documents\DATA\HeadPose/'; -else +elseif(exist([getenv('USERPROFILE') 'F:/Dropbox/Dropbox/AAM/test data/'], 'file')) database_root = 'F:/Dropbox/Dropbox/AAM/test data/'; +else + database_root = '/multicomp/datasets/head_pose_dbs/'; end buDir = [database_root, '/bu/uniform-light/']; -% The fast and accurate clnf %% -v = 3; -[fps_bu_OF, resFolderBU_OF] = run_bu_experiment(buDir, false, v, 'model', 'model/main_dclm_general.txt'); +[resFolderBU_OF] = run_bu_experiment(buDir, false, 'model', 'model/main_ceclm_general.txt'); [bu_error_OF, pred_hp_bu, gt_hp_bu, all_errors_bu_OF, rels_bu] = calcBUerror(resFolderBU_OF, buDir); %% % Run the Biwi test biwi_dir = '/biwi pose/'; -biwi_results_root = '/biwi pose results/'; -% Intensity -v = 4; -[fps_biwi_OF, res_folder_OF] = run_biwi_experiment(database_root, biwi_dir, biwi_results_root, false, false, v, 'model', 'model/main_dclm_general.txt'); +[res_folder_biwi_OF] = run_biwi_experiment(database_root, biwi_dir, false, false, 'model', 'model/main_ceclm_general.txt'); % Calculate the resulting errors -[biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError([database_root res_folder_OF], [database_root biwi_dir]); +[biwi_error_OF, pred_hp_biwi, gt_hp_biwi, ~, all_errors_biwi_OF, rels_biwi] = calcBiwiError(res_folder_biwi_OF, [database_root biwi_dir]); %% Run the ICT test -ict_dir = ['ict/']; -ict_results_root = ['ict results/']; +ict_dir = ['/ict/']; -v = 4; % Intensity -[fps_ict_OF, res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, ict_results_root, false, false, v, 'model', 'model/main_dclm_general.txt'); +[res_folder_ict_OF] = run_ict_experiment(database_root, ict_dir, false, false, 'model', 'model/main_ceclm_general.txt'); % Calculate the resulting errors -[ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError([database_root res_folder_ict_OF], [database_root ict_dir]); +[ict_error_OF, pred_hp_ict, gt_hp_ict, ~, all_errors_ict_OF, rel_ict] = calcIctError(res_folder_ict_OF, [database_root ict_dir]); %% Save the results -filename = 'results/Pose_OF_dclm'; +filename = 'results/Pose_OF_CECLM'; save(filename); % Also save them in a reasonable .txt format for easy comparison -f = fopen('results/Pose_OF_dclm.txt', 'w'); +f = fopen('results/Pose_OF_CECLM.txt', 'w'); fprintf(f, 'Dataset and model, pitch, yaw, roll, mean, median\n'); fprintf(f, 'biwi error: %.3f, %.3f, %.3f, %.3f, %.3f\n', biwi_error_OF, mean(all_errors_biwi_OF(:)), median(all_errors_biwi_OF(:))); fprintf(f, 'bu error: %.3f, %.3f, %.3f, %.3f, %.3f\n', bu_error_OF, mean(all_errors_bu_OF(:)), median(all_errors_bu_OF(:))); diff --git a/matlab_version/experiments_JANUS/results/Janus-full.pdf b/matlab_version/experiments_JANUS/results/Janus-full.pdf index a724d0bb67696b7cd1b59c66d632faba26d17719..46d4a33b4dc8b269deeadc59eb14271b04846651 100644 GIT binary patch delta 91 zcmdmCvBP3QHix05p|OdDk@?2r7-?q%Q)5>rLpNt*Lt`K|H#T-Pv2ZgrF?TaJGIcVx PaI;geA*5uokc=7tMYI+w delta 91 zcmdmCvBP3QHiv diff --git a/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf b/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf index 45bd660b5c7a5063f4fe630eaee7db994b548108..28c4461313f1ae6cb71113352863ba3a50c14d4c 100644 GIT binary patch delta 92 zcmdnuw8d#c7N?n^rJ=Ejg`xSz;s^z2BSUjjLvu?PS5pH+Qxh`-Cueg*S7$Q|CpSw| SV^ZKz=@vj;%2(OiIuuW28QMeh89*vhE}E)7oX>xX