Cleaning up Menpo results using CE-CLM

.gitignore

@@ -70,3 +70,6 @@ matlab_version/experiments_menpo/out_profile/
 matlab_version/experiments_menpo/out_semifrontal/
 matlab_version/pdm_generation/menpo_pdm/old_models/
 matlab_version/pdm_generation/menpo_pdm/
+matlab_version/experiments_menpo/menpo_challenge_helpers/out_profile/
+matlab_version/experiments_menpo/menpo_challenge_helpers/out_semifrontal/
+matlab_version/experiments_menpo/menpo_challenge_helpers/out_semifrontal.zip

matlab_version/experiments_menpo/Collect_menpo_imgs.m

@@ -0,0 +1,49 @@
+function [images, detections, labels] = Collect_menpo_imgs(root_dir)
+
+    load('face_detections/menpo_train_dets.mat');
+
+    bb = bboxes;
+    load('face_detections/menpo_valid_dets.mat');
+    bboxes = cat(2, bb, bboxes);
+
+    % Have three bounding box locations (frontal tuned, profile tuned)
+    detections = zeros(numel(bboxes), 4);
+    labels = cell(numel(bboxes),1);
+    
+    for i=1:numel(bboxes)
+        images(i).img = [root_dir,  bboxes(i).name];
+        
+        % If face detected
+        if(~isempty(bboxes(i).bbox))
+            bbox = bboxes(i).bbox(1:4);
+            % Correct the MTCNN bounding box
+            width = bbox(3) - bbox(1);
+            height = bbox(4) - bbox(2);
+            tx = bbox(1);
+            ty = bbox(2);
+
+            % Frontal faces
+            new_width = width * 1.0323;
+            new_height = height * 0.7751;
+            new_tx = width * -0.0075 + tx;
+            new_ty = height * 0.2459 + ty;
+
+            detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
+            
+        else % If face not detected, use the mean location of the face in training data
+            img_size = size(imread([root_dir,  bboxes(i).name]));
+            img_width = img_size(2);
+            img_height = img_size(1);
+            
+            width = img_width * 0.4421;
+            height = img_height * 0.445;
+            
+            tx = img_width * 0.5048 - 0.5 * width;
+            ty = img_height * 0.5166 - 0.5 * height;
+            
+            detections(i,:) = [tx, ty, tx + width, ty + height];
+        end
+        labels{i} = bboxes(i).gt_landmarks;
+    end
+    
+end

matlab_version/experiments_menpo/Collect_valid_imgs.m

@@ -1,6 +1,6 @@
 function [images, detections, labels] = Collect_valid_imgs(root_dir)
 
-    load('menpo_valid_dets.mat');
+    load('face_detections/menpo_valid_dets.mat');
 
     % Have three bounding box locations (frontal tuned, profile tuned)
     detections = zeros(numel(bboxes), 4);

matlab_version/experiments_menpo/Display_menpo_results_full_outline_cross_data.m

@@ -0,0 +1,156 @@
+%% 
+clear;
+
+load('results/results_clnf_cross-data.mat');
+load('results/menpo_labels.mat');
+[clnf_error, frontal_ids] = compute_error_menpo_1( labels,  experiments.shapes);
+clnf_error_frontal = clnf_error(frontal_ids);
+clnf_error_profile = clnf_error(~frontal_ids);
+
+load('results/results_ceclm_cross-data.mat');
+
+[dclm_error, frontal_ids] = compute_error_menpo_1( labels,  experiments.shapes);
+dclm_error_frontal = dclm_error(frontal_ids);
+dclm_error_profile = dclm_error(~frontal_ids);
+
+load('results/tcdcn_menpo.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}+0.5;
+end
+
+[tcdcn_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+tcdcn_error_frontal = tcdcn_error(frontal_ids);
+tcdcn_error_profile = tcdcn_error(~frontal_ids);
+
+load('results/CFAN_menpo_train.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[cfan_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+cfan_error_frontal = cfan_error(frontal_ids);
+cfan_error_profile = cfan_error(~frontal_ids);
+
+load('results/menpo_train_3DDFA.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[error_3ddfa, frontal_ids] = compute_error_menpo_1(labels, shapes);
+error_3ddfa_frontal = error_3ddfa(frontal_ids);
+error_3ddfa_profile = error_3ddfa(~frontal_ids);
+
+
+load('results/Menpo-CFSS_train.mat');
+shapes = cell(size(estimatedPoseFull,1),1);
+
+for i = 1:numel(shapes)
+    shape = cat(2, estimatedPoseFull(i,1:68)', estimatedPoseFull(i,69:end)');
+    shapes{i} = shape-0.5;
+end
+
+[cfss_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+cfss_error_frontal = cfss_error(frontal_ids);
+cfss_error_profile = cfss_error(~frontal_ids);
+
+%%
+
+scrsz = get(0,'ScreenSize');
+figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
+
+set(figure1,'Units','Inches');
+pos = get(figure1,'Position');
+set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
+
+% Create axes
+axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
+
+line_width = 6;
+hold on;
+
+[error_x, error_y] = cummErrorCurve(dclm_error_frontal);
+plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(clnf_error_frontal);
+plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfan_error_frontal);
+plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(error_3ddfa_frontal);
+plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfss_error_frontal);
+plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
+
+
+[error_x, error_y] = cummErrorCurve(tcdcn_error_frontal);
+plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
+
+
+set(gca,'xtick',[0:0.01:0.07])
+xlim([0.01,0.07]);
+xlabel('Size normalised MAE','FontName','Helvetica');
+ylabel('Proportion of images','FontName','Helvetica');
+grid on
+% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
+
+
+leg = legend('show', 'Location', 'SouthEast');
+set(leg,'FontSize',50)
+
+[error_x, error_y] = cummErrorCurve(dclm_error_frontal);
+plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
+
+print -dpdf results/menpo-frontal_full.pdf
+print -dpng results/menpo-frontal_full.png
+%%
+
+scrsz = get(0,'ScreenSize');
+figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
+
+set(figure1,'Units','Inches');
+pos = get(figure1,'Position');
+set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
+
+% Create axes
+axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
+
+line_width = 6;
+hold on;
+
+[error_x, error_y] = cummErrorCurve(dclm_error_profile);
+plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(clnf_error_profile);
+plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfan_error_profile);
+plot(error_x, error_y, 'DisplayName', 'CFAN', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(error_3ddfa_profile);
+plot(error_x, error_y, 'DisplayName', '3DDFA', 'LineWidth',line_width);
+
+[error_x, error_y] = cummErrorCurve(cfss_error_profile);
+plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width);
+
+
+[error_x, error_y] = cummErrorCurve(tcdcn_error_profile);
+plot(error_x, error_y, 'DisplayName', 'TCDCN', 'LineWidth',line_width);
+
+
+set(gca,'xtick',[0.01:0.02:0.11])
+xlim([0.03,0.11]);
+xlabel('Size normalised MAE','FontName','Helvetica');
+ylabel('Proportion of images','FontName','Helvetica');
+grid on
+% title('Fitting on Menpo frontal images','FontSize',60,'FontName','Helvetica');
+
+leg = legend('show', 'Location', 'SouthEast');
+set(leg,'FontSize',50)
+
+[error_x, error_y] = cummErrorCurve(dclm_error_profile);
+plot(error_x, error_y, 'r', 'DisplayName', 'CE-CLM', 'LineWidth',line_width);
+
+print -dpdf results/menpo-profile_full.pdf
+print -dpng results/menpo-profile_full.png

matlab_version/experiments_menpo/Extract_menpo_labels_corrected.m

@@ -0,0 +1,19 @@
+root_dir = 'D:\Datasets\menpo/';
+load('menpo_train_dets.mat');
+
+bb = bboxes;
+load('menpo_valid_dets.mat');
+bboxes = cat(2, bb, bboxes);
+
+labels = cell(numel(bboxes),1);
+    
+for i=1:numel(bboxes)
+    pts_name = [root_dir,  bboxes(i).name(1:end-3), 'pts'];
+    lmarks = importdata(pts_name, ' ', 3);
+    labels{i} = lmarks.data;
+    
+    img = imread([root_dir,  bboxes(i).name]);
+    imshow(img); hold on;plot(labels{i}(:,1), labels{i}(:,2), '.r');hold off;
+    drawnow expose;
+end
+save('results/menpo_labels', 'labels');

matlab_version/experiments_menpo/Extract_table_results_49.m

@@ -0,0 +1,69 @@
+clear
+
+load('results/results_clnf_cross-data.mat');
+load('results/menpo_labels.mat');
+[clnf_error, frontal_ids] = compute_error_menpo_small( labels,  experiments.shapes);
+clnf_error_frontal = clnf_error(frontal_ids);
+clnf_error_profile = clnf_error(~frontal_ids);
+
+load('results/menpo_train_chehra.mat');
+[drmf_error, frontal_ids] = compute_error_menpo_small( labels,  shapes);
+drmf_error_frontal = drmf_error(frontal_ids);
+drmf_error_profile = drmf_error(~frontal_ids);
+
+load('results/menpo_train_sdm.mat');
+
+[sdm_error, frontal_ids] = compute_error_menpo_small( labels,  shapes);
+sdm_error_frontal = sdm_error(frontal_ids);
+sdm_error_profile = sdm_error(~frontal_ids);
+
+load('results/Menpo_train_pocr.mat');
+
+[pocr_error, frontal_ids] = compute_error_menpo_small( labels,  experiments.shapes);
+pocr_error_frontal = pocr_error(frontal_ids);
+pocr_error_profile = pocr_error(~frontal_ids);
+load('results/results_ceclm_cross-data.mat');
+
+[ceclm_error, frontal_ids] = compute_error_menpo_small( labels,  experiments.shapes);
+ceclm_error_frontal = ceclm_error(frontal_ids);
+ceclm_error_profile = ceclm_error(~frontal_ids);
+
+
+load('results/tcdcn_menpo.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}+0.5;
+end
+
+[tcdcn_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
+tcdcn_error_frontal = tcdcn_error(frontal_ids);
+tcdcn_error_profile = tcdcn_error(~frontal_ids);
+
+load('results/CFAN_menpo_train.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[cfan_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
+cfan_error_frontal = cfan_error(frontal_ids);
+cfan_error_profile = cfan_error(~frontal_ids);
+
+load('results/menpo_train_3DDFA.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[error_3ddfa, frontal_ids] = compute_error_menpo_small(labels, shapes);
+error_3ddfa_frontal = error_3ddfa(frontal_ids);
+error_3ddfa_profile = error_3ddfa(~frontal_ids);
+
+load('results/Menpo-CFSS_train.mat');
+shapes = cell(size(estimatedPoseFull,1),1);
+
+for i = 1:numel(shapes)
+    shape = cat(2, estimatedPoseFull(i,1:68)', estimatedPoseFull(i,69:end)');
+    shapes{i} = shape-0.5;
+end
+
+[cfss_error, frontal_ids] = compute_error_menpo_small(labels, shapes);
+cfss_error_frontal = cfss_error(frontal_ids);
+cfss_error_profile = cfss_error(~frontal_ids);

matlab_version/experiments_menpo/Extract_table_results_68.m

@@ -0,0 +1,53 @@
+clear
+
+load('results/results_clnf_cross-data.mat');
+load('results/menpo_labels.mat');
+[clnf_error, frontal_ids] = compute_error_menpo_1( labels,  experiments.shapes);
+clnf_error_frontal = clnf_error(frontal_ids);
+clnf_error_profile = clnf_error(~frontal_ids);
+
+load('results/results_ceclm_cross-data.mat');
+
+[ceclm_error, frontal_ids] = compute_error_menpo_1( labels,  experiments.shapes);
+ceclm_error_frontal = ceclm_error(frontal_ids);
+ceclm_error_profile = ceclm_error(~frontal_ids);
+
+load('results/CFAN_menpo_train.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[cfan_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+cfan_error_frontal = cfan_error(frontal_ids);
+cfan_error_profile = cfan_error(~frontal_ids);
+
+load('results/tcdcn_menpo.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}+0.5;
+end
+
+[tcdcn_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+tcdcn_error_frontal = tcdcn_error(frontal_ids);
+tcdcn_error_profile = tcdcn_error(~frontal_ids);
+
+load('results/menpo_train_3DDFA.mat');
+for i = 1:numel(shapes)
+    shapes{i} = shapes{i}-0.5;
+end
+
+[error_3ddfa, frontal_ids] = compute_error_menpo_1(labels, shapes);
+error_3ddfa_frontal = error_3ddfa(frontal_ids);
+error_3ddfa_profile = error_3ddfa(~frontal_ids);
+
+
+load('results/Menpo-CFSS_train.mat');
+shapes = cell(size(estimatedPoseFull,1),1);
+
+for i = 1:numel(shapes)
+    shape = cat(2, estimatedPoseFull(i,1:68)', estimatedPoseFull(i,69:end)');
+    shapes{i} = shape-0.5;
+end
+
+[cfss_error, frontal_ids] = compute_error_menpo_1(labels, shapes);
+cfss_error_frontal = cfss_error(frontal_ids);
+cfss_error_profile = cfss_error(~frontal_ids);

matlab_version/experiments_menpo/Script_CECLM_menpo_cross_data.m

@@ -1,4 +1,4 @@
-function Script_DCLM_menpo_more_hyp()
+function Script_CECLM_menpo_cross_data()
 
 addpath('../PDM_helpers/');
 addpath('../fitting/normxcorr2_mex_ALL');
@@ -6,7 +6,7 @@ addpath('../fitting/');
 addpath('../CCNF/');
 addpath('../models/');
 
-[images, detections, labels] = Collect_valid_imgs('C:\Users\tbaltrus\Documents\menpo_data_orig/');
+[images, detections, labels] = Collect_menpo_imgs('D:\Datasets\menpo/');
 
 %% loading the patch experts
    
@@ -15,11 +15,11 @@ clmParams = struct;
 clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
 clmParams.numPatchIters = size(clmParams.window_size,1);
 
-[patches] = Load_DCLM_Patch_Experts( '../models/dpn/', 'dpn_patches_*_general.mat', [], [], clmParams);
+[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat', [], [], clmParams);
 
 %% Fitting the model to the provided image
 
-output_root = './menpo_fit_dclm_more_hyp/';
+output_root = './menpo_fit_dclm_more_epochs/';
 
 % the default PDM to use
 pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
@@ -57,8 +57,12 @@ all_views_used = zeros(numel(images),1);
 % Use the multi-hypothesis model, as bounding box tells nothing about
 % orientation
 multi_view = true;
-verbose = true;
+verbose = false;
 tic
+if(verbose)
+    f = figure;
+end
+
 for i=1:numel(images)
 
     image = imread(images(i).img);
@@ -73,7 +77,7 @@ for i=1:numel(images)
     % have a multi-view version
     if(multi_view)
 
-        views = [0,0,0; 0,-30,0; 0,-70,0; 0,30,0; 0,70,0; 0,0,30; 0,0,-30;];
+        views = [0,0,0; 0,-70,40; 0,70,-40; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
         views = views * pi/180;                                                                                     
 
         shapes = zeros(num_points, 2, size(views,1));
@@ -112,7 +116,7 @@ for i=1:numel(images)
 
         actualShape = labels{i};
         %         f = figure('visible','off');
-        f = figure;
+
         try
         if(max(image_orig(:)) > 1)
             imshow(double(image_orig)/255, 'Border', 'tight');
@@ -129,7 +133,6 @@ for i=1:numel(images)
 %                                         close(f);
         hold off;
         drawnow expose
-        close(f);
         catch warn
 
         end
@@ -149,11 +152,9 @@ if(~exist('experiments', 'var'))
 else
     experiments = cat(1, experiments, experiment);
 end
-% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
-%     numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
 
 %%
-output_results = 'results/results_valid_dclm_menpo_more_hyp_2.mat';
+output_results = 'results/results_ceclm_cross-data.mat';
 save(output_results, 'experiments');
     
 end

matlab_version/experiments_menpo/Script_CECLM_menpo_test_frontal.m

@@ -0,0 +1,162 @@
+function Script_CECLM_menpo_test_frontal()
+
+addpath('../PDM_helpers/');
+addpath('../fitting/normxcorr2_mex_ALL');
+addpath('../fitting/');
+addpath('../CCNF/');
+addpath('../models/');
+
+[images, detections] = Collect_menpo_test_frontal('D:\Datasets\menpo\testset\semifrontal/');
+
+%% loading the patch experts
+   
+clmParams = struct;
+
+clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+clmParams.numPatchIters = size(clmParams.window_size,1);
+
+[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
+
+%% Fitting the model to the provided image
+
+output_root = './menpo_fit_ceclm_test_semifrontal/';
+out_pts = './out_semifrontal/';
+mkdir(out_pts);
+% the default PDM to use
+pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
+
+load(pdmLoc);
+
+pdm = struct;
+pdm.M = double(M);
+pdm.E = double(E);
+pdm.V = double(V);
+
+clmParams.regFactor = 0.9 * [35, 27, 20, 20];
+clmParams.sigmaMeanShift = 1.5 * [1.25, 1.375, 1.5, 1.5]; 
+clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+clmParams.startScale = 1;
+clmParams.num_RLMS_iter = 10;
+clmParams.fTol = 0.01;
+clmParams.useMultiScale = true;
+clmParams.use_multi_modal = 1;
+clmParams.multi_modal_types  = patches(1).multi_modal_types;
+clmParams.numPatchIters = 4;
+
+% for recording purposes
+experiment.params = clmParams;
+
+num_points = numel(M)/3;
+
+shapes_all = cell(numel(images), 1);
+labels_all = cell(numel(images), 1);
+lhoods = zeros(numel(images),1);
+all_lmark_lhoods = zeros(num_points, numel(images));
+all_views_used = zeros(numel(images),1);
+
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+multi_view = true;
+verbose = true;
+tic
+if(verbose)
+    f = figure;
+end
+
+% load('../pdm_generation/menpo_pdm/conversion.mat');
+
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = squeeze(detections(i,:));                  
+    
+    % have a multi-view version
+    if(multi_view)
+
+        views = [0,0,0; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
+        views = views * pi/180;                                                                                     
+
+        shapes = zeros(num_points, 2, size(views,1));
+        ls = zeros(size(views,1),1);
+        lmark_lhoods = zeros(num_points,size(views,1));
+        views_used = zeros(size(views,1),1);
+        g_params = zeros(size(views, 1), 6);
+        % Find the best orientation
+        for v = 1:size(views,1)      
+            [shapes(:,:,v),g_params(v,:),~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
+        end
+
+        [lhood, v_ind] = max(ls);
+        lmark_lhood = lmark_lhoods(:,v_ind);
+
+        shape = shapes(:,:,v_ind);
+        view_used = views_used(v_ind);
+        g_params = g_params(v_ind,:);
+    else
+        [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
+    end
+
+    shape = shape + 0.5;
+    [~, name_org, ~] = fileparts(images(i).img);
+    name = [out_pts, name_org, '.pts'];
+    
+    shape = write_menpo_frontal(shape, name);
+    
+    all_lmark_lhoods(:,i) = lmark_lhood;
+    all_views_used(i) = view_used;
+
+    shapes_all{i} = shape;
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(verbose && mod(i,20) == 0)
+        %         f = figure('visible','off');
+
+        try
+        if(max(image_orig(:)) > 1)
+            imshow(double(image_orig)/255, 'Border', 'tight');
+        else
+            imshow(double(image_orig), 'Border', 'tight');
+        end
+        axis equal;
+        hold on;
+        plot(shape(:,1)-0.5, shape(:,2)-0.5,'.r','MarkerSize',20);
+        plot(shape(:,1)-0.5, shape(:,2)-0.5,'.b','MarkerSize',10);
+        rectangle('Position', [bbox(1), bbox(2), bbox(3) - bbox(1), bbox(4) - bbox(2)], 'EdgeColor', 'r', 'LineWidth', 2);
+%                                         print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
+        print(f, '-djpeg', sprintf('%s/%s.jpg', output_root, name_org));
+%                                         close(f);
+        hold off;
+        drawnow expose
+        catch warn
+
+        end
+    end
+
+end
+toc
+
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.all_lmark_lhoods = all_lmark_lhoods;
+experiment.all_views_used = all_views_used;
+
+% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
+%     numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_test_menpo_semifrontal.mat';
+save(output_results, 'experiments');
+    
+end

matlab_version/experiments_menpo/Script_CECLM_menpo_test_profile.m

@@ -0,0 +1,165 @@
+function Script_CECLM_menpo_test_profile()
+
+addpath('../PDM_helpers/');
+addpath('../fitting/normxcorr2_mex_ALL');
+addpath('../fitting/');
+addpath('../CCNF/');
+addpath('../models/');
+
+[images, detections] = Collect_menpo_test_profile('D:\Datasets\menpo\testset\profile/');
+
+%% loading the patch experts
+   
+clmParams = struct;
+
+clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+clmParams.numPatchIters = size(clmParams.window_size,1);
+
+[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
+
+%% Fitting the model to the provided image
+
+output_root = './menpo_fit_ceclm_test_profile/';
+out_pts = './out_profile/';
+mkdir(out_pts);
+% the default PDM to use
+pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
+
+load(pdmLoc);
+
+pdm = struct;
+pdm.M = double(M);
+pdm.E = double(E);
+pdm.V = double(V);
+
+clmParams.regFactor = 0.9 * [35, 27, 20, 20];
+clmParams.sigmaMeanShift = 1.5 * [1.25, 1.375, 1.5, 1.5]; 
+clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+clmParams.startScale = 1;
+clmParams.num_RLMS_iter = 10;
+clmParams.fTol = 0.01;
+clmParams.useMultiScale = true;
+clmParams.use_multi_modal = 1;
+clmParams.multi_modal_types  = patches(1).multi_modal_types;
+clmParams.numPatchIters = 4;
+
+% for recording purposes
+experiment.params = clmParams;
+
+num_points = numel(M)/3;
+
+shapes_all = cell(numel(images), 1);
+labels_all = cell(numel(images), 1);
+lhoods = zeros(numel(images),1);
+all_lmark_lhoods = zeros(num_points, numel(images));
+all_views_used = zeros(numel(images),1);
+
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+multi_view = true;
+verbose = false;
+tic
+if(verbose)
+    f = figure;
+end
+
+load('../pdm_generation/menpo_pdm/conversion.mat');
+
+for i=1:numel(images)
+
+    image = imread(images(i).img);
+    image_orig = image;
+    
+    if(size(image,3) == 3)
+        image = rgb2gray(image);
+    end              
+
+    bbox = squeeze(detections(i,:));                  
+    
+    % have a multi-view version
+    if(multi_view)
+
+        views = [0,0,0; 0,-70,40; 0,70,-40; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
+        views = views * pi/180;                                                                                     
+
+        shapes = zeros(num_points, 2, size(views,1));
+        ls = zeros(size(views,1),1);
+        lmark_lhoods = zeros(num_points,size(views,1));
+        views_used = zeros(size(views,1),1);
+        g_params = zeros(size(views, 1), 6);
+        % Find the best orientation
+        for v = 1:size(views,1)      
+            [shapes(:,:,v),g_params(v,:),~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
+        end
+
+        [lhood, v_ind] = max(ls);
+        lmark_lhood = lmark_lhoods(:,v_ind);
+
+        shape = shapes(:,:,v_ind);
+        view_used = views_used(v_ind);
+        g_params = g_params(v_ind,:);
+    else
+        [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
+    end
+
+    shape = shape + 0.5;
+    [~, name_org, ~] = fileparts(images(i).img);
+    name = [out_pts, name_org, '.pts'];
+    if(g_params(3) > 0)
+        shape = write_menpo_profile(shape, name, a_left, vis_pts_left);
+    else
+        shape = write_menpo_profile(shape, name, a_right, vis_pts_right);        
+    end
+    
+    all_lmark_lhoods(:,i) = lmark_lhood;
+    all_views_used(i) = view_used;
+
+    shapes_all{i} = shape;
+
+    if(mod(i, 200)==0)
+        fprintf('%d done\n', i );
+    end
+
+    lhoods(i) = lhood;
+
+    if(verbose)
+        %         f = figure('visible','off');
+
+        try
+        if(max(image_orig(:)) > 1)
+            imshow(double(image_orig)/255, 'Border', 'tight');
+        else
+            imshow(double(image_orig), 'Border', 'tight');
+        end
+        axis equal;
+        hold on;
+        plot(shape(:,1)-0.5, shape(:,2)-0.5,'.r','MarkerSize',20);
+        plot(shape(:,1)-0.5, shape(:,2)-0.5,'.b','MarkerSize',10);
+        rectangle('Position', [bbox(1), bbox(2), bbox(3) - bbox(1), bbox(4) - bbox(2)], 'EdgeColor', 'r', 'LineWidth', 2);
+%                                         print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
+        print(f, '-djpeg', sprintf('%s/%s.jpg', output_root, name_org));
+%                                         close(f);
+        hold off;
+        drawnow expose
+        catch warn
+
+        end
+    end
+
+end
+toc
+
+experiment.lhoods = lhoods;
+experiment.shapes = shapes_all;
+experiment.all_lmark_lhoods = all_lmark_lhoods;
+experiment.all_views_used = all_views_used;
+
+% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
+%     numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
+
+%%
+output_results = 'results/results_test_menpo_profile.mat';
+save(output_results, 'experiments');
+    
+end

matlab_version/experiments_menpo/Script_CECLM_menpo_valid.m

@@ -1,4 +1,4 @@
-function Script_DCLM_menpo()
+function Script_CECLM_menpo_valid()
 
 addpath('../PDM_helpers/');
 addpath('../fitting/normxcorr2_mex_ALL');
@@ -6,7 +6,7 @@ addpath('../fitting/');
 addpath('../CCNF/');
 addpath('../models/');
 
-[images, detections, labels] = Collect_valid_imgs('C:\Users\tbaltrus\Documents\menpo_data_orig/');
+[images, detections, labels] = Collect_valid_imgs('D:\Datasets\menpo/');
 
 %% loading the patch experts
    
@@ -15,14 +15,14 @@ clmParams = struct;
 clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
 clmParams.numPatchIters = size(clmParams.window_size,1);
 
-[patches] = Load_DCLM_Patch_Experts( '../models/dpn/', 'dpn_patches_*_general.mat', [], [], clmParams);
+[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
 
 %% Fitting the model to the provided image
 
-output_root = './menpo_fit_dclm/';
+output_root = './menpo_fit_ceclm/';
 
 % the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
 
 load(pdmLoc);
 
@@ -31,8 +31,8 @@ pdm.M = double(M);
 pdm.E = double(E);
 pdm.V = double(V);
 
-clmParams.regFactor = [35, 27, 20, 20];
-clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; 
+clmParams.regFactor = 0.9 * [35, 27, 20, 20];
+clmParams.sigmaMeanShift = 1.5 * [1.25, 1.375, 1.5, 1.5];  
 clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
 
 clmParams.startScale = 1;
@@ -57,9 +57,9 @@ all_views_used = zeros(numel(images),1);
 % Use the multi-hypothesis model, as bounding box tells nothing about
 % orientation
 multi_view = true;
-verbose = true;
+verbose = false;
 tic
-for i=1:numel(images)
+for i=29:numel(images)
 
     image = imread(images(i).img);
     image_orig = image;
@@ -73,22 +73,22 @@ for i=1:numel(images)
     % have a multi-view version
     if(multi_view)
 
-        views = [0,0,0; 0,-30,0; 0,-70,0; 0,30,0; 0,70,0;];
+        views = [0,0,0; 0,-70,40; 0,70,-40; 0,-30,0; 0,-60,0; 0,-90,0; 0,30,0; 0,60,0; 0,90,0; 0,0,30; 0,0,-30;];
         views = views * pi/180;                                                                                     
 
         shapes = zeros(num_points, 2, size(views,1));
         ls = zeros(size(views,1),1);
         lmark_lhoods = zeros(num_points,size(views,1));
         views_used = zeros(size(views,1),1);
-
+        globals = zeros(size(views,1),6);
         % Find the best orientation
-        for v = 1:size(views,1)
-            [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
+        for v = 1:size(views,1)            
+            [shapes(:,:,v),globals(v,:),~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
         end
 
         [lhood, v_ind] = max(ls);
         lmark_lhood = lmark_lhoods(:,v_ind);
-
+        glob = globals(v_ind,:);
         shape = shapes(:,:,v_ind);
         view_used = views_used(v_ind);
 
@@ -111,8 +111,8 @@ for i=1:numel(images)
     if(verbose)
 
         actualShape = labels{i};
-        %         f = figure('visible','off');
-        f = figure;
+                f = figure('visible','off');
+%         f = figure;
         try
         if(max(image_orig(:)) > 1)
             imshow(double(image_orig)/255, 'Border', 'tight');
@@ -149,11 +149,8 @@ if(~exist('experiments', 'var'))
 else
     experiments = cat(1, experiments, experiment);
 end
-% fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
-%     numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
 
-%%
-output_results = 'results/results_valid_dclm_menpo.mat';
+output_results = 'results/results_ceclm_valid.mat';
 save(output_results, 'experiments');
     
 end

matlab_version/experiments_menpo/Script_CLM_menpo_valid.m

@@ -1,188 +0,0 @@
-function Script_CLNF_wild_iccv()
-
-addpath('../PDM_helpers/');
-addpath('../fitting/normxcorr2_mex_ALL');
-addpath('../fitting/');
-addpath('../CCNF/');
-addpath('../models/');
-
-% Replace this with the location of in 300 faces in the wild data
-if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
-    root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];    
-else
-    root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/';
-end
-
-% load the images to detect landmarks of
-[images, detections, labels] = Collect_wild_imgs(root_test_data);
-
-%% loading the patch experts
-   
-clmParams = struct;
-
-clmParams.window_size = [25,25; 23,23; 21,21; 19,19;];
-
-clmParams.numPatchIters = size(clmParams.window_size,1);
-
-[patches] = Load_Patch_Experts( '../models/wild/', 'ccnf_patches_*_wild.mat', [], [], clmParams);
-
-%% Fitting the model to the provided image
-
-verbose = false; % set to true to visualise the fitting
-output_root = './wild_fit_clnf/';
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-% the default model parameters to use
-clmParams.regFactor = [35, 27, 20, 5];
-clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.75]; 
-clmParams.tikhonov_factor = [2.5, 5, 7.5, 12.5];
-
-clmParams.startScale = 1;
-clmParams.num_RLMS_iter = 10;
-clmParams.fTol = 0.01;
-clmParams.useMultiScale = true;
-clmParams.use_multi_modal = 1;
-clmParams.multi_modal_types  = patches(1).multi_modal_types;
-   
-% for recording purposes
-experiment.params = clmParams;
-
-num_points = numel(M)/3;
-
-shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
-lhoods = zeros(numel(images),1);
-all_lmark_lhoods = zeros(num_points, numel(images));
-all_views_used = zeros(numel(images),1);
-
-% Use the multi-hypothesis model, as bounding box tells nothing about
-% orientation
-multi_view = true;
-
-tic
-for i=1:numel(images)
-
-    image = imread(images(i).img);
-    image_orig = image;
-    
-    if(size(image,3) == 3)
-        image = rgb2gray(image);
-    end              
-
-    bbox = detections(i,:);
-    
-    % have a multi-view version
-    if(multi_view)
-
-        views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0];
-        views = views * pi/180;                                                                                     
-
-        shapes = zeros(num_points, 2, size(views,1));
-        ls = zeros(size(views,1),1);
-        lmark_lhoods = zeros(num_points,size(views,1));
-        views_used = zeros(num_points,size(views,1));
-
-        % Find the best orientation
-        for v = 1:size(views,1)
-            [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
-        end
-
-        [lhood, v_ind] = max(ls);
-        lmark_lhood = lmark_lhoods(:,v_ind);
-
-        shape = shapes(:,:,v_ind);
-        view_used = views_used(v);
-
-    else
-        [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
-    end
-
-    all_lmark_lhoods(:,i) = lmark_lhood;
-    all_views_used(i) = view_used;
-
-    % shape correction for matlab format
-    shapes_all(:,:,i) = shape;
-    labels_all(:,:,i) = labels(i,:,:);
-
-    if(mod(i, 200)==0)
-        fprintf('%d done\n', i );
-    end
-    
-    lhoods(i) = lhood;
-    if(verbose)
-        % Center the pixel
-        actualShape = squeeze(labels(i,:,:)) - 0.5;
-
-        [height_img, width_img,~] = size(image_orig);
-        width = max(actualShape(:,1)) - min(actualShape(:,1));
-        height = max(actualShape(:,2)) - min(actualShape(:,2));
-
-        img_min_x = max(int32(min(actualShape(:,1))) - width/3,1);
-        img_max_x = min(int32(max(actualShape(:,1))) + width/3,width_img);
-
-        img_min_y = max(int32(min(actualShape(:,2))) - height/3,1);
-        img_max_y = min(int32(max(actualShape(:,2))) + height/3,height_img);
-
-        shape(:,1) = shape(:,1) - double(img_min_x);
-        shape(:,2) = shape(:,2) - double(img_min_y);
-
-        image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :);    
-
-        % valid points to draw (not to draw
-        % occluded ones)
-        v_points = sum(squeeze(labels(i,:,:)),2) > 0;
-
-        f = figure('visible','off');
-        %f = figure;
-        try
-        if(max(image_orig(:)) > 1)
-            imshow(double(image_orig)/255, 'Border', 'tight');
-        else
-            imshow(double(image_orig), 'Border', 'tight');
-        end
-        axis equal;
-        hold on;
-        
-        plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20);
-        plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10);
-%                                         print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
-        print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i));
-%                                         close(f);
-        hold off;
-        close(f);
-        catch warn
-
-        end
-    end
-
-end
-toc
-experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all);
-experiment.lhoods = lhoods;
-experiment.shapes = shapes_all;
-experiment.labels = labels_all;
-experiment.all_lmark_lhoods = all_lmark_lhoods;
-experiment.all_views_used = all_views_used;
-% save the experiment
-if(~exist('experiments', 'var'))
-    experiments = experiment;
-else
-    experiments = cat(1, experiments, experiment);
-end
-fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
-    numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
-
-%%
-output_results = 'results/results_wild_clnf.mat';
-save(output_results, 'experiments');
-    
-end

matlab_version/experiments_menpo/Script_CLNF_menpo.m

@@ -6,7 +6,7 @@ addpath('../fitting/');
 addpath('../CCNF/');
 addpath('../models/');
 
-[images, detections, labels] = Collect_valid_imgs('C:\Users\tbaltrus\Documents\menpo_data_orig/');
+[images, detections, labels] = Collect_menpo_imgs('C:\Users\tbaltrus\Documents\menpo_data_orig/');
 
 %% loading the patch experts
    
@@ -57,7 +57,7 @@ all_views_used = zeros(numel(images),1);
 % Use the multi-hypothesis model, as bounding box tells nothing about
 % orientation
 multi_view = true;
-verbose = true;
+verbose = false;
 tic
 for i=1:numel(images)
 
@@ -68,12 +68,12 @@ for i=1:numel(images)
         image = rgb2gray(image);
     end              
 
-    bbox = squeeze(detections(i,1,:));                  
+    bbox = squeeze(detections(i,:));                  
     
     % have a multi-view version
     if(multi_view)
 
-        views = [0,0,0; 0,-30,0; 0,-70,0; 0,30,0; 0,70,0];
+        views = [0,0,0; 0,-30,0; 0,-70,0; 0,30,0; 0,70,0; 0,0,30; 0,0,-30;];
         views = views * pi/180;                                                                                     
 
         shapes = zeros(num_points, 2, size(views,1));
@@ -153,7 +153,7 @@ end
 %     numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
 
 %%
-output_results = 'results/results_valid_clnf_menpo.mat';
+output_results = 'results/results_clnf_cross-data.mat';
 save(output_results, 'experiments');
     
 end

matlab_version/experiments_menpo/Script_DCLM_menpo_validate_params.m

@@ -1,173 +0,0 @@
-function Script_DCLM_menpo_validate_params()
-
-addpath('../PDM_helpers/');
-addpath('../fitting/normxcorr2_mex_ALL');
-addpath('../fitting/');
-addpath('../CCNF/');
-addpath('../models/');
-
-[images, detections, labels] = Collect_valid_imgs('C:\Users\tbaltrus\Documents\menpo_data_orig/');
-
-%% loading the patch experts
-   
-clmParams = struct;
-
-clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
-clmParams.numPatchIters = size(clmParams.window_size,1);
-
-[patches] = Load_DCLM_Patch_Experts( '../models/dpn/', 'dpn_patches_*_general.mat', [], [], clmParams);
-
-%% Fitting the model to the provided image
-
-output_root = './menpo_fit_dclm_more_hyp/';
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-clmParams.regFactor = [35, 27, 20, 20];
-clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; 
-clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
-
-clmParams.startScale = 1;
-clmParams.num_RLMS_iter = 10;
-clmParams.fTol = 0.01;
-clmParams.useMultiScale = true;
-clmParams.use_multi_modal = 1;
-clmParams.multi_modal_types  = patches(1).multi_modal_types;
-clmParams.numPatchIters = 4;
-
-% for recording purposes
-experiment.params = clmParams;
-
-num_points = numel(M)/3;
-
-shapes_all = cell(numel(images), 1);
-labels_all = cell(numel(images), 1);
-lhoods = zeros(numel(images),1);
-all_lmark_lhoods = zeros(num_points, numel(images));
-all_views_used = zeros(numel(images),1);
-
-% Use the multi-hypothesis model, as bounding box tells nothing about
-% orientation
-multi_view = true;
-verbose = false;
-regs_valid = [0.9];
-sigmas_valid = [1.5];
-t_valid = [0.8,1.0,1.2];
-
-for r=regs_valid
-    for s=sigmas_valid
-        clmParams.regFactor = r*[35, 27, 20, 20];
-        clmParams.sigmaMeanShift = s*[1.25, 1.375, 1.5, 1.5]; 
-
-        tic
-        for i=1:numel(images)
-
-            image = imread(images(i).img);
-            image_orig = image;
-
-            if(size(image,3) == 3)
-                image = rgb2gray(image);
-            end              
-
-            bbox = squeeze(detections(i,:));                  
-
-            % have a multi-view version
-            if(multi_view)
-
-                views = [0,0,0; 0,-30,0; 0,-70,0; 0,30,0; 0,70,0; 0,0,30; 0,0,-30;];
-                views = views * pi/180;                                                                                     
-
-                shapes = zeros(num_points, 2, size(views,1));
-                ls = zeros(size(views,1),1);
-                lmark_lhoods = zeros(num_points,size(views,1));
-                views_used = zeros(size(views,1),1);
-
-                % Find the best orientation
-                for v = 1:size(views,1)
-                    [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));                                            
-                end
-
-                [lhood, v_ind] = max(ls);
-                lmark_lhood = lmark_lhoods(:,v_ind);
-
-                shape = shapes(:,:,v_ind);
-                view_used = views_used(v_ind);
-
-            else
-                [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
-            end
-
-            all_lmark_lhoods(:,i) = lmark_lhood;
-            all_views_used(i) = view_used;
-
-            shapes_all{i} = shape;
-            labels_all{i} = labels{i};
-
-            if(mod(i, 200)==0)
-                fprintf('%d done\n', i );
-            end
-
-            lhoods(i) = lhood;
-
-            if(verbose)
-
-                actualShape = labels{i};
-                %         f = figure('visible','off');
-                f = figure;
-                try
-                if(max(image_orig(:)) > 1)
-                    imshow(double(image_orig)/255, 'Border', 'tight');
-                else
-                    imshow(double(image_orig), 'Border', 'tight');
-                end
-                axis equal;
-                hold on;
-                v_points = logical(patches(1).visibilities(view_used,:))';
-                plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20);
-                plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10);
-        %                                         print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
-                print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i));
-        %                                         close(f);
-                hold off;
-                drawnow expose
-                close(f);
-                catch warn
-
-                end
-            end
-
-        end
-        toc
-
-        experiment.lhoods = lhoods;
-        experiment.shapes = shapes_all;
-        experiment.labels = labels_all;
-        experiment.all_lmark_lhoods = all_lmark_lhoods;
-        experiment.all_views_used = all_views_used;
-        [errs, frontal_ids] = compute_error_menpo_1(labels_all, shapes_all);
-        experiment.errors_frontal = errs(frontal_ids);
-        experiment.errors_profile = errs(~frontal_ids);
-        experiment.params = clmParams;
-        % save the experiment
-        if(~exist('experiments', 'var'))
-            experiments = experiment;
-        else
-            experiments = cat(1, experiments, experiment);
-        end
-        fprintf('experiment %d done: error frontal %.3f error profile %.4f\n', ...
-            numel(experiments), median(experiment.errors_frontal), median(experiment.errors_profile));
-
-        %%
-        output_results = 'results/results_valid_dclm_validate_3.mat';
-        save(output_results, 'experiments');
-    end
-end
-end

matlab_version/experiments_menpo/comp_error_pts.m

@@ -1,45 +0,0 @@
-%gts
-
-dirs = {'../../test data/AFW/';
-    '../../test data/ibug/';
-    '../../test data/helen/testset/';
-    '../../test data/lfpw/testset/';};
-
-landmark_dets = dir('out_clnf_cpp\*.pts');
-
-landmark_det_dir = 'out_clnf_cpp\';
-
-num_imgs = size(landmark_dets,1);
-
-labels = zeros(68,2,num_imgs);
-shapes = zeros(68,2,num_imgs);
-
-landmark_gt = dir(['../../test data/AFW/*.pts']);
-
-curr = 0;
-
-for i=1:numel(dirs)
-    
-    curr = curr+1;
-    
-    gt_labels = dir([dirs{i}, '*.pts']);
-    
-    for g=1:numel(gt_labels)
-           gt_landmarks = importdata([dirs{i}, gt_labels(g).name], ' ', 3);
-           gt_landmarks = gt_landmarks.data;
- 
-           % find the corresponding detection
-           
-           landmark_det = importdata([landmark_det_dir, gt_labels(g).name], ' ', 3);
-           landmark_det = landmark_det.data;
-           
-           labels(:,:,curr) = gt_landmarks;
-           shapes(:,:,curr) = landmark_det;
-           
-    end
-    
-end
-
-
-
-

matlab_version/experiments_menpo/compute_error.m

@@ -1,34 +0,0 @@
-function [ error_per_image ] = compute_error( ground_truth_all, detected_points_all )
-%compute_error
-%   compute the average point-to-point Euclidean error normalized by the
-%   inter-ocular distance (measured as the Euclidean distance between the
-%   outer corners of the eyes)
-%
-%   Inputs:
-%          grounth_truth_all, size: num_of_points x 2 x num_of_images
-%          detected_points_all, size: num_of_points x 2 x num_of_images
-%   Output:
-%          error_per_image, size: num_of_images x 1
-
-
-num_of_images = size(ground_truth_all,3);
-num_of_points = size(ground_truth_all,1);
-
-error_per_image = zeros(num_of_images,1);
-
-for i =1:num_of_images
-    detected_points      = detected_points_all(:,:,i);
-    ground_truth_points  = ground_truth_all(:,:,i);
-    if(num_of_points == 66 || num_of_points == 68)
-        interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:));
-    else
-        interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:));
-    end
-    sum=0;
-    for j=1:num_of_points
-        sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
-    end
-    error_per_image(i) = sum/(num_of_points*interocular_distance);
-end
-
-end

matlab_version/experiments_menpo/compute_error_menpo_1.m

@@ -19,10 +19,12 @@ for i =1:num_of_images
     detected_points      = detected_points_all{i} + 0.5;
     ground_truth_points  = ground_truth_all{i};
     num_of_points = size(ground_truth_points,1);
-    
-    if(num_of_points == 68)
-        interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:));
-    else
+
+   normalization = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
+       (max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
+   normalization = normalization / 2;    
+        
+    if(num_of_points==39)
         frontal_ids(i) = false;
         landmark_labels = zeros(68,2);
         % Need to map to the profile points, and normalize based on size
@@ -65,13 +67,11 @@ for i =1:num_of_images
             landmark_labels(right_to_frontal_map(:,2),:) = ground_truth_points(right_to_frontal_map(:,1),:);                
         end
                            
-       interocular_distance = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
-           (max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
-       interocular_distance = interocular_distance / 2;
        detected_points = detected_points(landmark_labels(:,1)~=0,:);
        ground_truth_points = landmark_labels(landmark_labels(:,1)~=0,:);
        
     end
+
     
     num_of_points = size(ground_truth_points,1);
         
@@ -79,7 +79,7 @@ for i =1:num_of_images
     for j=1:num_of_points
         sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
     end
-    error_per_image(i) = sum/(num_of_points*interocular_distance);
+    error_per_image(i) = sum/(num_of_points*normalization);
 end
 
 end

matlab_version/experiments_menpo/compute_error_menpo_small.m

@@ -0,0 +1,97 @@
+function [ error_per_image, frontal_ids ] = compute_error_menpo_small( ground_truth_all, detected_points_all )
+%compute_error
+%   compute the average point-to-point Euclidean error normalized by the
+%   inter-ocular distance (measured as the Euclidean distance between the
+%   outer corners of the eyes)
+%
+%   Inputs:
+%          grounth_truth_all, size: num_of_points x 2 x num_of_images
+%          detected_points_all, size: num_of_points x 2 x num_of_images
+%   Output:
+%          error_per_image, size: num_of_images x 1
+
+% This script uses the 49 point convention instead of the 68 point one
+num_of_images = numel(ground_truth_all);
+error_per_image = zeros(num_of_images,1);
+frontal_ids = true(num_of_images,1);
+
+for i =1:num_of_images
+    detected_points      = detected_points_all{i} + 0.5;
+    ground_truth_points  = ground_truth_all{i};
+    num_of_points = size(ground_truth_points,1);
+
+    normalization = (max(ground_truth_points(:,1))-min(ground_truth_points(:,1)))+...
+       (max(ground_truth_points(:,2))-min(ground_truth_points(:,2)));
+    normalization = normalization / 2;    
+    landmark_labels = ground_truth_points;
+    if(num_of_points==39)
+        frontal_ids(i) = false;
+        landmark_labels = zeros(68,2);
+        % Need to map to the profile points, and normalize based on size
+        % instead
+        left_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
+                               21,34; 22,32; 23,39; 24,38; 25,37; 26,42; 27,41;
+                               28,52; 29,51; 30,50; 31,49; 32,60; 33,59; 34,58;
+                               35,63; 36,62; 37,61; 38,68; 39,67];
+
+        right_to_frontal_map = [17,28; 18,29; 19,30; 20,31;
+                               21,34; 22,36; 23,44; 24,45; 25,46; 26,47; 27,48;
+                               28,52; 29,53; 30,54; 31,55; 32,56; 33,57; 34,58;
+                               35,63; 36,64; 37,65; 38,66; 39,67];
+                   
+        % Determine if the points are clock-wise or counter clock-wise
+        % Clock-wise points are facing left, counter-clock-wise right
+        sum = 0;
+        for k=1:11
+            step = (ground_truth_points(k+1,1) - ground_truth_points(k,1)) * (ground_truth_points(k+1,2) + ground_truth_points(k,2));
+            sum = sum + step;
+        end
+
+        if(sum > 0)
+            % First need to resample the face outline as there are 9
+            % points in the near-frontal and 10 points in profile for
+            % the outline of the face
+
+            outline = iterate_piece_wise(ground_truth_points(1:10,:), 9);
+            brow = iterate_piece_wise(ground_truth_points(13:16,:), 5);
+            landmark_labels(1:9,:) = outline;
+            landmark_labels(18:22,:) = brow;
+            landmark_labels(left_to_frontal_map(:,2),:) = ground_truth_points(left_to_frontal_map(:,1),:);
+        else
+            outline = iterate_piece_wise(ground_truth_points(10:-1:1,:), 9);
+            brow = iterate_piece_wise(ground_truth_points(16:-1:13,:), 5);
+
+            landmark_labels(9:17,:) = outline;
+            landmark_labels(23:27,:) = brow;
+
+            landmark_labels(right_to_frontal_map(:,2),:) = ground_truth_points(right_to_frontal_map(:,1),:);                
+        end
+                                  
+    end
+
+
+    % First get rid of inner mouth points and then the face outline
+    landmark_labels = landmark_labels([1:60,62:64,66:end],:,:);
+    landmark_labels = landmark_labels(18:end,:);
+
+    % Do the same for 68 markup version
+    if(size(detected_points,1) == 68)
+        detected_points = detected_points([1:60,62:64,66:end],:,:);
+        detected_points = detected_points(18:end,:);
+    end
+
+    % Remove the invisible points in profile
+    ground_truth_points = landmark_labels(landmark_labels(:,1)~=0,:);
+
+    detected_points = detected_points(landmark_labels(:,1)~=0,:);
+
+    num_of_points = size(ground_truth_points,1);
+        
+    sum=0;
+    for j=1:num_of_points
+        sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:));
+    end
+    error_per_image(i) = sum/(num_of_points*normalization);
+end
+
+end

matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_names.m

@@ -0,0 +1,18 @@
+function [images, labels] = Collect_menpo_names(root_dir)
+
+    load('menpo_train_dets.mat');
+
+    bb = bboxes;
+    load('menpo_valid_dets.mat');
+    bboxes = cat(2, bb, bboxes);
+
+    % Have three bounding box locations (frontal tuned, profile tuned)
+    labels = cell(numel(bboxes),1);
+    
+    for i=1:numel(bboxes)
+        images(i).img = [bboxes(i).name];
+               
+        labels{i} = bboxes(i).gt_landmarks;
+    end
+    
+end

matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_test_frontal.m

@@ -0,0 +1,43 @@
+function [images, detections] = Collect_menpo_test_frontal(root_dir)
+
+    load('menpo_test_frontal.mat');
+
+    % Have three bounding box locations (frontal tuned, profile tuned)
+    detections = zeros(numel(bboxes), 4);
+    
+    for i=1:numel(bboxes)
+        images(i).img = [root_dir,  bboxes(i).name];
+        
+        % If face detected
+        if(~isempty(bboxes(i).bbox))
+            bbox = bboxes(i).bbox(1:4);
+            % Correct the MTCNN bounding box
+            width = bbox(3) - bbox(1);
+            height = bbox(4) - bbox(2);
+            tx = bbox(1);
+            ty = bbox(2);
+
+            % Frontal faces
+            new_width = width * 1.0323;
+            new_height = height * 0.7751;
+            new_tx = width * -0.0075 + tx;
+            new_ty = height * 0.2459 + ty;
+
+            detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
+            
+        else % If face not detected, use the mean location of the face in training data
+            img_size = size(imread([root_dir,  bboxes(i).name]));
+            img_width = img_size(2);
+            img_height = img_size(1);
+            
+            width = img_width * 0.4421;
+            height = img_height * 0.445;
+            
+            tx = img_width * 0.5048 - 0.5 * width;
+            ty = img_height * 0.5166 - 0.5 * height;
+            
+            detections(i,:) = [tx, ty, tx + width, ty + height];
+        end
+    end
+    
+end

matlab_version/experiments_menpo/menpo_challenge_helpers/Collect_menpo_test_profile.m

@@ -0,0 +1,43 @@
+function [images, detections] = Collect_menpo_test_profile(root_dir)
+
+    load('menpo_test_profile.mat');
+
+    % Have three bounding box locations (frontal tuned, profile tuned)
+    detections = zeros(numel(bboxes), 4);
+    
+    for i=1:numel(bboxes)
+        images(i).img = [root_dir,  bboxes(i).name];
+        
+        % If face detected
+        if(~isempty(bboxes(i).bbox))
+            bbox = bboxes(i).bbox(1:4);
+            % Correct the MTCNN bounding box
+            width = bbox(3) - bbox(1);
+            height = bbox(4) - bbox(2);
+            tx = bbox(1);
+            ty = bbox(2);
+
+            % Frontal faces
+            new_width = width * 1.0323;
+            new_height = height * 0.7751;
+            new_tx = width * -0.0075 + tx;
+            new_ty = height * 0.2459 + ty;
+
+            detections(i,:) = [new_tx, new_ty, new_tx + new_width, new_ty + new_height];
+            
+        else % If face not detected, use the mean location of the face in training data
+            img_size = size(imread([root_dir,  bboxes(i).name]));
+            img_width = img_size(2);
+            img_height = img_size(1);
+            
+            width = img_width * 0.4421;
+            height = img_height * 0.445;
+            
+            tx = img_width * 0.5048 - 0.5 * width;
+            ty = img_height * 0.5166 - 0.5 * height;
+            
+            detections(i,:) = [tx, ty, tx + width, ty + height];
+        end
+    end
+    
+end

matlab_version/experiments_menpo/menpo_challenge_helpers/Test_menpo_formats.m

@@ -0,0 +1,45 @@
+clear;
+load('results/results_valid_dclm_menpo.mat');
+
+% First compute the error
+[errors, frontal_ids] = compute_error_menpo_1(experiments.labels, experiments.shapes);
+labels_f = experiments.labels(frontal_ids);
+shapes_f = experiments.shapes(frontal_ids);
+errors_f = errors(frontal_ids);
+for i=1:10
+   
+    write_menpo_frontal(shapes_f{i} + 0.5, sprintf('tmp/%d.pts', i));
+    
+    preds = importdata(sprintf('tmp/%d.pts', i), ' ', 3);
+    landmarks = preds.data;
+    
+    [err] = compute_error_menpo_unb(labels_f(i), {landmarks});
+    
+end
+
+%%
+labels_p = experiments.labels(~frontal_ids);
+shapes_p = experiments.shapes(~frontal_ids);
+errors_p = errors(~frontal_ids);
+v_used = experiments.all_views_used(~frontal_ids);
+load('../pdm_generation/menpo_pdm/conversion.mat');
+load('../pdm_generation/menpo_pdm/menpo_68_pts_valid_profile.mat');
+ind_left = 1;
+ind_right = 1;
+for i=1:10
+       
+    if(v_used(i) == 2 || v_used(i) == 3 || v_used(i) == 6)
+        write_menpo_profile(shapes_p{i} + 0.5, sprintf('tmp/%d.pts', i), a_left, vis_pts_left);
+        labs = all_pts_orig_left(:,:,ind_left);
+        ind_left = ind_left + 1;
+    else
+        write_menpo_profile(shapes_p{i} + 0.5, sprintf('tmp/%d.pts', i), a_right, vis_pts_right);
+        labs = all_pts_orig_right(:,:,ind_right);
+        ind_right = ind_right + 1;
+    end
+
+    preds = importdata(sprintf('tmp/%d.pts', i), ' ', 3);
+    landmarks = preds.data;
+    [err] = compute_error_prof_unb({labs}, {landmarks});
+    
+end

matlab_version/experiments_menpo/menpo_challenge_helpers/write_menpo_frontal.m

@@ -0,0 +1,19 @@
+function [ out_dets ] = write_menpo_frontal( detections, name )
+%WRITE_MEPO_PROFILE Summary of this function goes here
+%   Detailed explanation goes here
+    f = fopen(name, 'w');
+    fprintf(f, 'version: 1\n');
+    fprintf(f, 'n_points: 68\n');
+    fprintf(f, '{\n');
+    xs = detections(:,1);
+    ys = detections(:,2);
+   
+    for i=1:size(xs,1)
+        fprintf(f, '%.3f %.3f\n', xs(i), ys(i));
+    end
+    
+    fprintf(f, '}\n');    
+    fclose(f);
+    out_dets = cat(2, xs, ys);
+end
+

matlab_version/experiments_menpo/menpo_challenge_helpers/write_menpo_profile.m

@@ -0,0 +1,20 @@
+function [ out_dets ] = write_menpo_profile( detections, name, conversion, visibilities )
+%WRITE_MEPO_PROFILE Summary of this function goes here
+%   Detailed explanation goes here
+    f = fopen(name, 'w');
+    fprintf(f, 'version: 1\n');
+    fprintf(f, 'n_points: 39\n');
+    fprintf(f, '{\n');
+    dets = conversion * cat(1,detections(visibilities,1), detections(visibilities,2));
+    xs = dets(1:end/2,:);
+    ys = dets(end/2+1:end,:);
+   
+    for i=1:size(xs,1)
+        fprintf(f, '%.3f %.3f\n', xs(i), ys(i));
+    end
+    
+    fprintf(f, '}\n');    
+    fclose(f);
+    out_dets = cat(2, xs, ys);
+end
+