More simplification and cleanup of JANUS code

matlab_version/experiments_JANUS/Script_CECLM_general.m

@@ -6,45 +6,16 @@ addpath(genpath('../'));
 root_test_data = 'D:/Datasets/janus_labeled';
 [images, detections, labels] = Collect_JANUS_imgs(root_test_data);
 
-%% loading the patch experts
+%% loading the CE-CLM model and parameters   
-   
+[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
-clmParams = struct;
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+multi_view = true;
 
-clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+%% Setup recording
-clmParams.numPatchIters = size(clmParams.window_size,1);
-
-[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat', [], [], clmParams);
-
-%% Fitting the model to the provided image
-
-output_root = './fit_ceclm_general/';
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-clmParams.regFactor = 0.9*[35, 27, 20, 20];
-clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
-clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
-
-clmParams.startScale = 1;
-clmParams.num_RLMS_iter = 10;
-clmParams.fTol = 0.01;
-clmParams.useMultiScale = true;
-clmParams.use_multi_modal = 1;
-clmParams.multi_modal_types  = patches(1).multi_modal_types;
-clmParams.numPatchIters = 4;
-
-% for recording purposes
 experiment.params = clmParams;
 
-num_points = numel(M)/3;
+num_points = numel(pdm.M)/3;
 
 shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
 labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
@@ -52,9 +23,7 @@ lhoods = zeros(numel(images),1);
 all_lmark_lhoods = zeros(num_points, numel(images));
 all_views_used = zeros(numel(images),1);
 
-% Use the multi-hypothesis model, as bounding box tells nothing about
+% Change if you want to visualize the outputs
-% orientation
-multi_view = true;
 verbose = false;
 output_img = false;
 
@@ -67,8 +36,9 @@ end
 if(verbose)
     f = figure;
 end
-% As the orientations are not equally reliable reweigh them
+
-load('../learn_error_mapping/cen_general_mapping.mat');
+
+%% Fitting the model to the provided images
 
 tic
 for i=1:numel(images)
@@ -118,7 +88,7 @@ for i=1:numel(images)
 end
 toc
 
-experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
 experiment.lhoods = lhoods;
 experiment.shapes = shapes_all;
 experiment.labels = labels_all;

matlab_version/experiments_JANUS/Script_CECLM_menpo.m

@@ -6,43 +6,16 @@ addpath(genpath('../'));
 root_test_data = 'D:/Datasets/janus_labeled';
 [images, detections, labels] = Collect_JANUS_imgs(root_test_data);
 
-%% loading the patch experts
+%% loading the CE-CLM model and parameters   
-   
+[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
-clmParams = struct;
+% Use the multi-hypothesis model, as bounding box tells nothing about
+% orientation
+multi_view = true;
 
-clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+%% Setup recording
-clmParams.numPatchIters = size(clmParams.window_size,1);
-
-[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
-
-%% Fitting the model to the provided image
-
-% the default PDM to use
-pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
-
-load(pdmLoc);
-
-pdm = struct;
-pdm.M = double(M);
-pdm.E = double(E);
-pdm.V = double(V);
-
-clmParams.regFactor = 0.9*[35, 27, 20, 20];
-clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
-clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
-
-clmParams.startScale = 1;
-clmParams.num_RLMS_iter = 10;
-clmParams.fTol = 0.01;
-clmParams.useMultiScale = true;
-clmParams.use_multi_modal = 1;
-clmParams.multi_modal_types  = patches(1).multi_modal_types;
-clmParams.numPatchIters = 4;
-
-% for recording purposes
 experiment.params = clmParams;
 
-num_points = numel(M)/3;
+num_points = numel(pdm.M)/3;
 
 shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
 labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
@@ -50,14 +23,12 @@ lhoods = zeros(numel(images),1);
 all_lmark_lhoods = zeros(num_points, numel(images));
 all_views_used = zeros(numel(images),1);
 
-% Use the multi-hypothesis model, as bounding box tells nothing about
+% Change if you want to visualize the outputs
-% orientation
-multi_view = true;
 verbose = false;
 output_img = false;
 
 if(output_img)
-    output_root = './ceclm_menpo_out/';
+    output_root = './ceclm_gen_out/';
     if(~exist(output_root, 'dir'))
         mkdir(output_root);
     end
@@ -65,8 +36,9 @@ end
 if(verbose)
     f = figure;
 end
-% As the orientations are not equally reliable reweigh them
+
-load('../learn_error_mapping/cen_menpo_mapping.mat');
+
+%% Fitting the model to the provided images
 
 tic
 for i=1:numel(images)
@@ -116,7 +88,7 @@ for i=1:numel(images)
 end
 toc
 
-experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
 experiment.lhoods = lhoods;
 experiment.shapes = shapes_all;
 experiment.labels = labels_all;

matlab_version/experiments_JANUS/Script_CLNF_general.m

@@ -140,7 +140,7 @@ for i=1:numel(images)
 end
 toc
 
-experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
+experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
 experiment.lhoods = lhoods;
 experiment.shapes = shapes_all;
 experiment.labels = labels_all;

matlab_version/fitting/Fitting_from_bb.m

@@ -136,7 +136,7 @@ function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, v
             responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:));
         elseif(strcmp(patchExperts(scale).type, 'CCNF'))                        
             responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
-        elseif(strcmp(patchExperts(scale).type, 'DNN'))                        
+        elseif(strcmp(patchExperts(scale).type, 'CEN'))                        
             responses = PatchResponseDNN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
         end
         

matlab_version/models/Load_CECLM_Patch_Experts.m

@@ -1,4 +1,4 @@
-function [patches] = Load_DCLM_Patch_Experts( col_patch_dir, col_patch_file, depth_patch_dir, depth_patch_file, clmParams)
+function [patches] = Load_CECLM_Patch_Experts( col_patch_dir, col_patch_file)
 %LOAD_PATCH_EXPERTS Summary of this function goes here
 %   Detailed explanation goes here
    
@@ -19,7 +19,7 @@ function [patches] = Load_DCLM_Patch_Experts( col_patch_dir, col_patch_file, dep
         patch.modalities = patch_experts.types;
         patch.multi_modal_types = patch_experts.types;
 
-        patch.type = 'DNN';
+        patch.type = 'CEN';
 
         % Knowing what normalisation was performed during training is
         % important for fitting

matlab_version/models/Load_CECLM_general.m

@@ -0,0 +1,38 @@
+function [patches, pdm, clmParams, early_term_params] = Load_CECLM_general()
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    % Load the patch experts/local detectors
+    [patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat');
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    clmParams.regFactor = 0.9*[35, 27, 20, 20];
+    clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+    clmParams.numPatchIters = 4;
+    
+    % As the orientations are not equally reliable reweigh them
+    load('../learn_error_mapping/cen_general_mapping.mat');    
+end
+

matlab_version/models/Load_CECLM_menpo.m

@@ -0,0 +1,38 @@
+function [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo()
+%LOAD_PATCH_EXPERTS Summary of this function goes here
+%   Detailed explanation goes here
+   
+    % Load the patch experts/local detectors
+    [patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat');
+
+    % the default PDM to use
+    pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
+
+    load(pdmLoc);
+
+    pdm = struct;
+    pdm.M = double(M);
+    pdm.E = double(E);
+    pdm.V = double(V);
+
+    clmParams = struct;
+
+    clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
+    clmParams.numPatchIters = size(clmParams.window_size,1);
+
+    clmParams.regFactor = 0.9*[35, 27, 20, 20];
+    clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5]; 
+    clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
+
+    clmParams.startScale = 1;
+    clmParams.num_RLMS_iter = 10;
+    clmParams.fTol = 0.01;
+    clmParams.useMultiScale = true;
+    clmParams.use_multi_modal = 1;
+    clmParams.multi_modal_types  = patches(1).multi_modal_types;
+    clmParams.numPatchIters = 4;
+    
+    % As the orientations are not equally reliable reweigh them
+    load('../learn_error_mapping/cen_menpo_mapping.mat');    
+end
+

matlab_version/models/cen/create_cen_experts_gen.m

@@ -1,5 +1,5 @@
 clear;
-load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
 
 mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
   32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -26,6 +26,8 @@ patch_experts.patch_experts = cell(n_views, n_landmarks);
 
 scales = {'0.25', '0.35', '0.50', '1.00'};
 
+root = 'D:/deep_experts/2017-02-02/rmses/';
+
 for s=scales
 
     for c=1:n_views
@@ -38,9 +40,9 @@ for s=scales
                     mirror = false;
                     % Find the relevant file
                     if(c == frontalView)
-                        rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, i);
+                        rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, i);
                     else
-                        rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, i);                    
+                        rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_profile%d_%d_512.mat'], s{1}, c-1, i);                    
                     end
                     if(exist(rel_file, 'file'))
                        load(rel_file); 
@@ -53,7 +55,7 @@ for s=scales
                            continue;
                        end
                        if(c == frontalView)
-                           rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, rel_id);
+                           rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, rel_id);
 
                            mirror = true;
                            load(rel_file);
@@ -106,6 +108,6 @@ for s=scales
         end
     end
     trainingScale = str2num(s{1});
-    save(['dpn_patches_', s{1} '_general_v2.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+    save(['cen_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
-%     write_patch_expert_bin(['dpn_patches_', s{1} '_general_v2.dat'], trainingScale, centers, visiIndex, patch_experts);
+%     write_patch_expert_bin(['cen_patches_', s{1} '_general.dat'], trainingScale, centers, visiIndex, patch_experts);
 end

matlab_version/models/cen/create_cen_experts_menpo.m

@@ -1,5 +1,5 @@
 clear;
-load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
 
 mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
   32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -26,6 +26,10 @@ patch_experts.patch_experts = cell(n_views, n_landmarks);
 
 scales = {'0.25', '0.35', '0.50', '1.00'};
 
+visiIndex = zeros(7, 68);
+
+root = 'D:/deep_experts/menpo/rmses/';
+
 for s=scales
 
     for c=1:n_views
@@ -34,45 +38,39 @@ for s=scales
 
             for i=1:n_landmarks
 
-                if(visiIndex(c,i))
+                mirror = false;
-                    mirror = false;
+                % Find the relevant file
-                    % Find the relevant file
+                if(c == frontalView)
-                    if(c == frontalView)
+                    rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, i);
-                        rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, i);
+                else                    
-                    else
+                    rel_file = sprintf([root, '/%s_profile%d_%d_512.mat'], s{1}, c-1, i);                    
-                        rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, i);                    
-                    end
-                    if(exist(rel_file, 'file'))
-                       load(rel_file); 
-                    else
-                       rel_id = mirrorInds(mirrorInds(:,2)==i,1);
-                       if(isempty(rel_id))
-                           rel_id = mirrorInds(mirrorInds(:,1)==i,2);
-                       end
-                       if(~visiIndex(c, rel_id))
-                           break;
-                       end
-                       if(c == frontalView)
-                           rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, rel_id);
-                       else
-                           rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, rel_id);                    
-                       end
-                       mirror = true;
-                       load(rel_file);
-                    end
-                    patch_experts.correlations(c, i) = correlation_2;
-                    patch_experts.rms_errors(c, i) = rmse;
-
-                    if(~mirror)
-                        patch_experts.patch_experts{c, i} = weights;
-                    else
-                        flips = fliplr(reshape([1:121]', 11, 11));
-                        weights_flipped = weights;
-                        weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
-                        patch_experts.patch_experts{c,i} = weights_flipped;
-                    end
                 end
+                if(exist(rel_file, 'file'))
+                    visiIndex(c,i) = 1;
+                    load(rel_file); 
+                else
+                   rel_id = mirrorInds(mirrorInds(:,2)==i,1);
+                   if(isempty(rel_id))
+                       rel_id = mirrorInds(mirrorInds(:,1)==i,2);
+                   end
+                   if(c == frontalView)                       
+                       rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, rel_id);
+                       mirror = true;
+                       visiIndex(c,i) = 1;
+                       load(rel_file);
+                   end
+                end
+                patch_experts.correlations(c, i) = correlation_2;
+                patch_experts.rms_errors(c, i) = rmse;
 
+                if(~mirror)
+                    patch_experts.patch_experts{c, i} = weights;
+                else
+                    flips = fliplr(reshape([1:121]', 11, 11));
+                    weights_flipped = weights;
+                    weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
+                    patch_experts.patch_experts{c,i} = weights_flipped;
+                end
             end
         else
 
@@ -83,6 +81,11 @@ for s=scales
             corr_T  = swap(corr_T, mirrorInds(:,1), mirrorInds(:,2));
             patch_experts.correlations(c,:) = corr_T;
 
+            vis_T = visiIndex(swap_id,:);
+            % Appending a mirror view instead, based on the profile view               
+            vis_T  = swap(vis_T, mirrorInds(:,1), mirrorInds(:,2));
+            visiIndex(c,:) = vis_T;
+            
             rmsT = patch_experts.rms_errors(swap_id,:);
             rmsT = swap(rmsT, mirrorInds(:,1), mirrorInds(:,2));
             patch_experts.rms_errors(c,:) = rmsT;
@@ -107,6 +110,6 @@ for s=scales
         end
     end
     trainingScale = str2num(s{1});
-    save(['dpn_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+    save(['cen_patches_', s{1} '_menpo.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
-    write_patch_expert_bin(['dpn_patches_', s{1} '_general.dat'], trainingScale, centers, visiIndex, patch_experts);
+    write_patch_expert_bin(['cen_patches_', s{1} '_menpo.dat'], trainingScale, centers, visiIndex, patch_experts);
 end

matlab_version/models/cen/create_cen_experts_sparse.m

@@ -1,5 +1,5 @@
 clear;
-load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
+load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
 
 mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
   32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -135,6 +135,6 @@ for s=scales
     end 
     
     trainingScale = str2num(s{1});
-    save(['dpn_patches_', s{1} '_general_sparse.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
+    save(['cen_patches_', s{1} '_general_sparse.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
-    write_patch_expert_bin(['dpn_patches_', s{1} '_general_sparse.dat'], trainingScale, centers, visiIndex, patch_experts);
+    write_patch_expert_bin(['cen_patches_', s{1} '_general_sparse.dat'], trainingScale, centers, visiIndex, patch_experts);
 end

matlab_version/models/cen/readme.txt

@@ -0,0 +1,4 @@
+Scripts for creating CEN patch experts from already trained models.
+
+To create one from 300W + MultiPIE - create_cen_experts_gen.m
+To create one from 300W + MultiPIE + Menpo - create_cen_experts_menpo.m