diff --git a/matlab_runners/Feature Point Experiments/results/fps_yt.txt b/matlab_runners/Feature Point Experiments/results/fps_yt.txt index 6fa35031..872405f4 100644 --- a/matlab_runners/Feature Point Experiments/results/fps_yt.txt +++ b/matlab_runners/Feature Point Experiments/results/fps_yt.txt @@ -1,4 +1,4 @@ Model, mean, median OpenFace (CE-CLM): 0.0499, 0.0474 OpenFace (CLNF): 0.0553, 0.0514 -CLM: 0.0632, 0.0588 +CLM: 0.0634, 0.0587 diff --git a/matlab_runners/Feature Point Experiments/results/landmark_detections.txt b/matlab_runners/Feature Point Experiments/results/landmark_detections.txt index f4606422..3e225a9f 100644 --- a/matlab_runners/Feature Point Experiments/results/landmark_detections.txt +++ b/matlab_runners/Feature Point Experiments/results/landmark_detections.txt @@ -1,4 +1,4 @@ Type, mean, median err ce-clm: 0.042496, 0.034719 err clnf: 0.055028, 0.038369 -err svr: 0.069754, 0.049870 +err svr: 0.069575, 0.049742 diff --git a/matlab_version/demo/face_image_demo_eyes.m b/matlab_version/demo/face_image_demo_eyes.m index 29cfd626..11b4a64f 100644 --- a/matlab_version/demo/face_image_demo_eyes.m +++ b/matlab_version/demo/face_image_demo_eyes.m @@ -12,6 +12,7 @@ views = [0,0,0]; % Loading eye PDM and patch experts [clmParams_eye, pdm_right_eye, pdm_left_eye] = Load_CLM_params_eye_28(); + [patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye); [patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye); clmParams_eye.multi_modal_types = patches_right_eye(1).multi_modal_types; @@ -78,34 +79,11 @@ for img=1:numel(images) % shape correction for matlab format shape = shape + 1; - - [shape_2, shape_r_eye_2] = Fitting_from_bb_hierarch(image, pdm, pdm_right_eye, patches_right_eye, clmParams_eye, shape, right_eye_inds, right_eye_inds_synth); - [shape_2, shape_l_eye_2] = Fitting_from_bb_hierarch(image, pdm, pdm_left_eye, patches_left_eye, clmParams_eye, shape_2, left_eye_inds, left_eye_inds_synth); - + % Perform eye fitting now - shape_r_eye = zeros(numel(pdm_right_eye.M)/3, 2); - shape_r_eye(right_eye_inds_synth,:) = shape(right_eye_inds, :); - - [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_right_eye.M, pdm_right_eye.E, pdm_right_eye.V, shape_r_eye); - - bbox = [min(shape_r_eye(:,1)), min(shape_r_eye(:,2)), max(shape_r_eye(:,1)), max(shape_r_eye(:,2))]; - - g_param = [a; Rot2Euler(R)'; T]; - - [shape_r_eye] = Fitting_from_bb(image, [], bbox, pdm_right_eye, patches_right_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params); - - % Perform eye fitting now - shape_l_eye = zeros(numel(pdm_right_eye.M)/3, 2); - shape_l_eye(left_eye_inds_synth,:) = shape(left_eye_inds, :); - - [ a, R, T, ~, l_params] = fit_PDM_ortho_proj_to_2D(pdm_left_eye.M, pdm_left_eye.E, pdm_left_eye.V, shape_l_eye); - - bbox = [min(shape_l_eye(:,1)), min(shape_l_eye(:,2)), max(shape_l_eye(:,1)), max(shape_l_eye(:,2))]; - - g_param = [a; Rot2Euler(R)'; T]; - - [shape_l_eye] = Fitting_from_bb(image, [], bbox, pdm_left_eye, patches_left_eye, clmParams_eye, 'gparam', g_param, 'lparam', l_params); - + [shape, shape_r_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_right_eye, patches_right_eye, clmParams_eye, shape, right_eye_inds, right_eye_inds_synth); + [shape, shape_l_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_left_eye, patches_left_eye, clmParams_eye, shape, left_eye_inds, left_eye_inds_synth); + plot(shape_l_eye(9:20,1), shape_l_eye(9:20,2), '.g', 'MarkerSize',7); plot(shape_l_eye(1:8,1), shape_l_eye(1:8,2), '.b', 'MarkerSize',7); diff --git a/matlab_version/experiments_300W/Script_CECLM_general.m b/matlab_version/experiments_300W/Script_CECLM_general.m index 4be8b7b6..a46bbf7a 100644 --- a/matlab_version/experiments_300W/Script_CECLM_general.m +++ b/matlab_version/experiments_300W/Script_CECLM_general.m @@ -6,7 +6,7 @@ addpath(genpath('../')); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'F:\Dropbox\AAM\test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); diff --git a/matlab_version/experiments_300W/Script_CECLM_menpo.m b/matlab_version/experiments_300W/Script_CECLM_menpo.m index afd69ddf..f27efc82 100644 --- a/matlab_version/experiments_300W/Script_CECLM_menpo.m +++ b/matlab_version/experiments_300W/Script_CECLM_menpo.m @@ -6,7 +6,7 @@ addpath(genpath('../')); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'F:\Dropbox\AAM\test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); diff --git a/matlab_version/experiments_300W/Script_CLM.m b/matlab_version/experiments_300W/Script_CLM.m index 86f94a09..5f2505c9 100644 --- a/matlab_version/experiments_300W/Script_CLM.m +++ b/matlab_version/experiments_300W/Script_CLM.m @@ -6,7 +6,7 @@ addpath(genpath('../')); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'F:\Dropbox\AAM\test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); diff --git a/matlab_version/experiments_300W/Script_CLNF_general.m b/matlab_version/experiments_300W/Script_CLNF_general.m index e34a92a8..99a57995 100644 --- a/matlab_version/experiments_300W/Script_CLNF_general.m +++ b/matlab_version/experiments_300W/Script_CLNF_general.m @@ -6,7 +6,7 @@ addpath(genpath('../')); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'F:\Dropbox\AAM\test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); @@ -16,6 +16,7 @@ end views = [0,0,0; 0,-30,0; 0,30,0; 0,0,30; 0,0,-30;]; views = views * pi/180; +[ patches_51, pdm_51, clmParams_51, inds_full, inds_inner ] = Load_CLNF_inner(); %% Change if you want to visualize the outputs verbose = false; @@ -53,6 +54,9 @@ for i=1:numel(images) bbox = detections(i,:); [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views); + + % Perform inner landmark fitting now + [shape, shape_inner] = Fitting_from_bb_hierarch(image, pdm, pdm_51, patches_51, clmParams_51, shape, inds_full, inds_inner); shapes_all(:,:,i) = shape; diff --git a/matlab_version/experiments_300W/Script_CLNF_wild.m b/matlab_version/experiments_300W/Script_CLNF_wild.m index acc11095..437fff0f 100644 --- a/matlab_version/experiments_300W/Script_CLNF_wild.m +++ b/matlab_version/experiments_300W/Script_CLNF_wild.m @@ -6,7 +6,7 @@ addpath(genpath('../')); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'F:\Dropbox\AAM\test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); diff --git a/matlab_version/experiments_300W/results/ceclm-300W-68.pdf b/matlab_version/experiments_300W/results/ceclm-300W-68.pdf index ddbf874d..2b023000 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-68.pdf and b/matlab_version/experiments_300W/results/ceclm-300W-68.pdf differ diff --git a/matlab_version/experiments_300W/results/ceclm-300W-68.png b/matlab_version/experiments_300W/results/ceclm-300W-68.png index 6d4e3ea2..82b1d584 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-68.png and b/matlab_version/experiments_300W/results/ceclm-300W-68.png differ diff --git a/matlab_version/experiments_300W/results/ceclm-300W-comm-68.pdf b/matlab_version/experiments_300W/results/ceclm-300W-comm-68.pdf index 312c3f32..208bb86c 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-comm-68.pdf and b/matlab_version/experiments_300W/results/ceclm-300W-comm-68.pdf differ diff --git a/matlab_version/experiments_300W/results/ceclm-300W-comm-68.png b/matlab_version/experiments_300W/results/ceclm-300W-comm-68.png index c35cd6db..44b9a58e 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-comm-68.png and b/matlab_version/experiments_300W/results/ceclm-300W-comm-68.png differ diff --git a/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.pdf b/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.pdf index 14b18ccc..7369a2c7 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.pdf and b/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.pdf differ diff --git a/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.png b/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.png index beca0b9b..d86a3d23 100644 Binary files a/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.png and b/matlab_version/experiments_300W/results/ceclm-300W-ibug-68.png differ diff --git a/matlab_version/experiments_300W/results/results_ceclm_general.mat b/matlab_version/experiments_300W/results/results_ceclm_general.mat index fd4c8b8d..394a4031 100644 Binary files a/matlab_version/experiments_300W/results/results_ceclm_general.mat and b/matlab_version/experiments_300W/results/results_ceclm_general.mat differ diff --git a/matlab_version/experiments_300W/results/results_ceclm_menpo.mat b/matlab_version/experiments_300W/results/results_ceclm_menpo.mat index ed166813..cac64ce5 100644 Binary files a/matlab_version/experiments_300W/results/results_ceclm_menpo.mat and b/matlab_version/experiments_300W/results/results_ceclm_menpo.mat differ diff --git a/matlab_version/experiments_300W/results/results_clm.mat b/matlab_version/experiments_300W/results/results_clm.mat index f581164c..519ab563 100644 Binary files a/matlab_version/experiments_300W/results/results_clm.mat and b/matlab_version/experiments_300W/results/results_clm.mat differ diff --git a/matlab_version/experiments_300W/results/results_clnf_general.mat b/matlab_version/experiments_300W/results/results_clnf_general.mat index 54014de7..a5dbed3e 100644 Binary files a/matlab_version/experiments_300W/results/results_clnf_general.mat and b/matlab_version/experiments_300W/results/results_clnf_general.mat differ diff --git a/matlab_version/experiments_300W/results/results_clnf_wild.mat b/matlab_version/experiments_300W/results/results_clnf_wild.mat index 34a79eb0..9601e467 100644 Binary files a/matlab_version/experiments_300W/results/results_clnf_wild.mat and b/matlab_version/experiments_300W/results/results_clnf_wild.mat differ diff --git a/matlab_version/fitting/Fitting_from_bb_hierarch.m b/matlab_version/fitting/Fitting_from_bb_hierarch.m index 21f776e8..71674908 100644 --- a/matlab_version/fitting/Fitting_from_bb_hierarch.m +++ b/matlab_version/fitting/Fitting_from_bb_hierarch.m @@ -8,7 +8,7 @@ function [ shape2D_full, shape_hierarch, global_params, local_params, final_lhoo % Get the hierarchical model parameters based on current parent % landmark locations - [ a, R, T, ~, l_params, ~] = fit_PDM_ortho_proj_to_2D_no_reg(pdm_hierarch.M, pdm_hierarch.E, pdm_hierarch.V, shape_hierarch); + [ a, R, T, ~, l_params, ~] = fit_PDM_ortho_proj_to_2D(pdm_hierarch.M, pdm_hierarch.E, pdm_hierarch.V, shape_hierarch); g_param = [a; Rot2Euler(R)'; T]; @@ -21,7 +21,7 @@ function [ shape2D_full, shape_hierarch, global_params, local_params, final_lhoo if(a > 0.9) shape2D_full(inds_full, :) = shape_hierarch(inds_hierarch,:); % Make sure the combined version can be expressed by the PDM - [ ~, ~, ~, ~, ~, ~, shape2D_full] = fit_PDM_ortho_proj_to_2D_no_reg(pdm_full.M, pdm_full.E, pdm_full.V, shape2D_full); + [ ~, ~, ~, ~, ~, ~, shape2D_full] = fit_PDM_ortho_proj_to_2D(pdm_full.M, pdm_full.E, pdm_full.V, shape2D_full); end diff --git a/matlab_version/fitting/Fitting_from_bb_multi_hyp.m b/matlab_version/fitting/Fitting_from_bb_multi_hyp.m index 1edc0eaa..8e018773 100644 --- a/matlab_version/fitting/Fitting_from_bb_multi_hyp.m +++ b/matlab_version/fitting/Fitting_from_bb_multi_hyp.m @@ -93,27 +93,4 @@ function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, v end - % Check if a hierarchical model is defined, and if it is apply it here - if(isfield(clmParams, 'hierarch')) - % Perform hierarchical model fitting - %, TODO this should be a loop - shape_hierarch = shape2D(clmParams.hierarch.inds,:); - - [ a, R, T, ~, l_params, err] = fit_PDM_ortho_proj_to_2D_no_reg(clmParams.hierarch.pdm.M, clmParams.hierarch.pdm.E, clmParams.hierarch.pdm.V, shape_hierarch); - if(a > 0.9) - g_param = [a; Rot2Euler(R)'; T]; - - bbox_2 = [min(shape_hierarch(:,1)), min(shape_hierarch(:,2)), max(shape_hierarch(:,1)), max(shape_hierarch(:,2))]; - - [shape_hierarch] = Fitting_from_bb(Image, [], bbox_2, clmParams.hierarch.pdm, clmParams.hierarch.patches, clmParams.hierarch.clmParams, 'gparam', g_param, 'lparam', l_params); - - % Now after detections incorporate the subset of the landmarks into the main model - shape2D(clmParams.hierarch.inds, :) = shape_hierarch; - - [ ~, ~, ~, ~, ~, ~, shape2D] = fit_PDM_ortho_proj_to_2D_no_reg(PDM.M, PDM.E, PDM.V, shape2D); - - end - - end - end \ No newline at end of file diff --git a/matlab_version/models/Load_CLM_params_inner.m b/matlab_version/models/Load_CLM_params_inner.m deleted file mode 100644 index 6f99a857..00000000 --- a/matlab_version/models/Load_CLM_params_inner.m +++ /dev/null @@ -1,28 +0,0 @@ -function [ clmParams, pdm_mouth] = Load_CLM_params_inner() -%LOAD_CLM_PARAMS_WILD Summary of this function goes here -% Detailed explanation goes here - clmParams.window_size = [19,19;]; - clmParams.numPatchIters = size(clmParams.window_size,1); - - % the PDM created from in the wild data - pdmLoc = ['../models/hierarch_pdm/pdm_51_inner.mat']; - - load(pdmLoc); - - pdm_mouth = struct; - pdm_mouth.M = double(M); - pdm_mouth.E = double(E); - pdm_mouth.V = double(V); - - % the default model parameters to use - clmParams.regFactor = 2.5; - clmParams.sigmaMeanShift = 1.75; - clmParams.tikhonov_factor = 2.5; - - clmParams.startScale = 1; - clmParams.num_RLMS_iter = 5; - clmParams.fTol = 0.01; - clmParams.useMultiScale = true; - clmParams.use_multi_modal = 1; - -end \ No newline at end of file diff --git a/matlab_version/models/Load_CLNF_general.m b/matlab_version/models/Load_CLNF_general.m index 4bee872b..d7ea1a23 100644 --- a/matlab_version/models/Load_CLNF_general.m +++ b/matlab_version/models/Load_CLNF_general.m @@ -29,18 +29,5 @@ function [ patches, pdm, clmParams ] = Load_CLNF_general() clmParams.use_multi_modal = 1; clmParams.multi_modal_types = patches(1).multi_modal_types; - % Loading the hierarchical model that will be applied after the main - % model - [clmParams_inner, pdm_inner] = Load_CLM_params_inner(); - clmParams_inner.window_size = [17,17;19,19;21,21;23,23]; - inds_inner = 18:68; - [patches_inner] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams_inner); - clmParams_inner.multi_modal_types = patches_inner(1).multi_modal_types; - - clmParams.hierarch = struct; - clmParams.hierarch(1).clmParams = clmParams_inner; - clmParams.hierarch(1).inds = inds_inner; - clmParams.hierarch(1).patches = patches_inner; - clmParams.hierarch(1).pdm = pdm_inner; end