From 762965dd5385dd12599c7e37be64f3fd04fcbfbd Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Mon, 16 Apr 2018 07:31:30 +0100 Subject: [PATCH] Converting the demos to the new interface. --- matlab_version/demo/face_image_demo.m | 38 +++++++---------- matlab_version/demo/face_image_demo_eyes.m | 35 +++++++++------- matlab_version/models/Load_CLNF_inner.m | 40 ++++++++++++++++++ matlab_version/models/Load_eye_models.m | 49 ++++++++++++++++++++++ 4 files changed, 124 insertions(+), 38 deletions(-) create mode 100644 matlab_version/models/Load_CLNF_inner.m create mode 100644 matlab_version/models/Load_eye_models.m diff --git a/matlab_version/demo/face_image_demo.m b/matlab_version/demo/face_image_demo.m index 4ec384e6..ac8134de 100644 --- a/matlab_version/demo/face_image_demo.m +++ b/matlab_version/demo/face_image_demo.m @@ -6,16 +6,19 @@ addpath(genpath('../face_detection')); addpath('../CCNF/'); %% loading the patch experts - -%[patches, pdm, clmParams, early_term_params] = Load_CECLM_general(); -% An accurate CCNF (or CLNF) model -[clmParams, pdm] = Load_CLM_params_wild(); -[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); -% A simpler (but less accurate SVR) -% [patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams); +% Default OpenFace landmark model, using CE-CLM patch experts +[patches, pdm, clmParams, early_term_params] = Load_CECLM_general(); -clmParams.multi_modal_types = patches(1).multi_modal_types; +% faster but less accurate +%[patches, pdm, clmParams] = Load_CLNF_general(); + +% even faster but even less accurate +%[patches, pdm, clmParams] = Load_CLM_general(); + +% Using a multi-view approach +views = [0,0,0; 0,-30,0; 0,30,0; 0,0,30; 0,0,-30;]; +views = views * pi/180; % Dependencies for face detection (MatConvNet), remove if not present setup_mconvnet; @@ -70,21 +73,12 @@ for img=1:numel(images) bbox = bboxs(i,:); - % have a multi-view version - if(multi_view) - - views = [0,0,0; 0,-30,0; 0,30,0; 0,-55,0; 0,55,0; 0,0,30; 0,0,-30; 0,-90,0; 0,90,0; 0,-70,40; 0,70,-40]; - views = views * pi/180; - - if(exist('early_term_params', 'var')) - [shape,~,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); - else - [shape,~,~,lhood,lmark_lhood,view_used] =... - Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views); - end + if(exist('early_term_params', 'var')) + [shape,~,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views, early_term_params); else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + [shape,~,~,lhood,lmark_lhood,view_used] =... + Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views); end % shape correction for matlab format diff --git a/matlab_version/demo/face_image_demo_eyes.m b/matlab_version/demo/face_image_demo_eyes.m index 11b4a64f..3de1bfa9 100644 --- a/matlab_version/demo/face_image_demo_eyes.m +++ b/matlab_version/demo/face_image_demo_eyes.m @@ -7,22 +7,24 @@ addpath('../CCNF/'); %% loading the patch experts +% Default OpenFace landmark model, using CE-CLM patch experts [patches, pdm, clmParams, early_term_params] = Load_CECLM_general(); -views = [0,0,0]; -% Loading eye PDM and patch experts -[clmParams_eye, pdm_right_eye, pdm_left_eye] = Load_CLM_params_eye_28(); +% faster but less accurate +%[patches, pdm, clmParams] = Load_CLNF_general(); -[patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye); -[patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye); -clmParams_eye.multi_modal_types = patches_right_eye(1).multi_modal_types; -right_eye_inds = [43,44,45,46,47,48]; -left_eye_inds = [37,38,39,40,41,42]; +% even faster but even less accurate +%[patches, pdm, clmParams] = Load_CLM_general(); -right_eye_inds_synth = [9 11 13 15 17 19]; -left_eye_inds_synth = [9 11 13 15 17 19]; +% Using a multi-view approach +views = [0,0,0; 0,-30,0; 0,30,0; 0,0,30; 0,0,-30;]; +views = views * pi/180; -clmParams.multi_modal_types = patches(1).multi_modal_types; +% Load the eye landmark models that will be used +[ clmParams_eye, pdm_right_eye, pdm_left_eye, ... + patches_left_eye, patches_right_eye,... + left_eye_inds_in_68, right_eye_inds_in_68,... + left_eye_inds_in_28, right_eye_inds_in_28] = Load_eye_models(); %% % root_dir = 'C:\Users\Tadas\Dropbox\AAM\test data\gaze_original\p00/'; @@ -77,12 +79,13 @@ for img=1:numel(images) [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb_multi_hyp(image, [], bbox, pdm, patches, clmParams, views); - % shape correction for matlab format - shape = shape + 1; - % Perform eye fitting now - [shape, shape_r_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_right_eye, patches_right_eye, clmParams_eye, shape, right_eye_inds, right_eye_inds_synth); - [shape, shape_l_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_left_eye, patches_left_eye, clmParams_eye, shape, left_eye_inds, left_eye_inds_synth); + [shape, shape_r_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_right_eye, patches_right_eye, clmParams_eye, shape, right_eye_inds_in_68, right_eye_inds_in_28); + [shape, shape_l_eye] = Fitting_from_bb_hierarch(image, pdm, pdm_left_eye, patches_left_eye, clmParams_eye, shape, left_eye_inds_in_68, left_eye_inds_in_28); + + % Convert it to matlab convention + shape_r_eye = shape_r_eye + 1; + shape_l_eye = shape_l_eye + 1; plot(shape_l_eye(9:20,1), shape_l_eye(9:20,2), '.g', 'MarkerSize',7); plot(shape_l_eye(1:8,1), shape_l_eye(1:8,2), '.b', 'MarkerSize',7); diff --git a/matlab_version/models/Load_CLNF_inner.m b/matlab_version/models/Load_CLNF_inner.m new file mode 100644 index 00000000..4ae8f4e2 --- /dev/null +++ b/matlab_version/models/Load_CLNF_inner.m @@ -0,0 +1,40 @@ +function [ patches, pdm, clmParams, inds_full, inds_inner ] = Load_CLNF_inner() +%LOAD_CLNF_Inner Summary of this function goes here +% Detailed explanation goes here + + % 51 landmark model of a face (a subset of 68 landmark model without + % the face outline) + clmParams = struct; + clmParams.window_size = [19,19]; + clmParams.numPatchIters = size(clmParams.window_size,1); + + % the PDM created from in the wild data + pdmLoc = ['../models/hierarch_pdm/pdm_51_inner.mat']; + + load(pdmLoc); + + pdm = struct; + pdm.M = double(M); + pdm.E = double(E); + pdm.V = double(V); + + % the default model parameters to use + clmParams.regFactor = 2.5; + clmParams.sigmaMeanShift = 1.75; + clmParams.tikhonov_factor = 2.5; + + clmParams.startScale = 1; + clmParams.num_RLMS_iter = 5; + clmParams.fTol = 0.01; + clmParams.useMultiScale = true; + clmParams.use_multi_modal = 1; + + [patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams); + clmParams.multi_modal_types = patches(1).multi_modal_types; + + % Corresponding indices between the 68 point and the 51 point version + inds_full = 18:68; + inds_inner = 1:51; + +end + diff --git a/matlab_version/models/Load_eye_models.m b/matlab_version/models/Load_eye_models.m new file mode 100644 index 00000000..8cef04d8 --- /dev/null +++ b/matlab_version/models/Load_eye_models.m @@ -0,0 +1,49 @@ +function [ clmParams_eye, pdm_right_eye, pdm_left_eye, ... + patches_left_eye, patches_right_eye,... + left_eye_inds_in_68, right_eye_inds_in_68,... + left_eye_inds_in_28, right_eye_inds_in_28] = Load_eye_models() + + clmParams_eye.window_size = [17,17; 15,15; 13,13;]; + clmParams_eye.numPatchIters = size(clmParams_eye.window_size,1); + + % the PDM created from in the wild data + pdmLoc = ['../models/hierarch_pdm/pdm_28_r_eye.mat']; + + load(pdmLoc); + + pdm_right_eye = struct; + pdm_right_eye.M = double(M); + pdm_right_eye.E = double(E); + pdm_right_eye.V = double(V); + + pdmLoc = ['../models/hierarch_pdm/pdm_28_l_eye.mat']; + + load(pdmLoc); + + pdm_left_eye = struct; + pdm_left_eye.M = double(M); + pdm_left_eye.E = double(E); + pdm_left_eye.V = double(V); + + % the default model parameters to use + clmParams_eye.regFactor = 2.0; + clmParams_eye.sigmaMeanShift = 1.5; + clmParams_eye.tikhonov_factor = 0; + + clmParams_eye.startScale = 1; + clmParams_eye.num_RLMS_iter = 10; + clmParams_eye.fTol = 0.01; + clmParams_eye.useMultiScale = true; + clmParams_eye.use_multi_modal = 0; + clmParams_eye.tikhonov_factor = 0; + + [patches_right_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_right_eye.mat', [], [], clmParams_eye); + [patches_left_eye] = Load_Patch_Experts( '../models/hierarch/', 'ccnf_patches_*_synth_left_eye.mat', [], [], clmParams_eye); + clmParams_eye.multi_modal_types = patches_right_eye(1).multi_modal_types; + right_eye_inds_in_68 = [43,44,45,46,47,48]; + left_eye_inds_in_68 = [37,38,39,40,41,42]; + + right_eye_inds_in_28 = [9 11 13 15 17 19]; + left_eye_inds_in_28 = [9 11 13 15 17 19]; + +end \ No newline at end of file