ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2025-12-30 04:52:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

More simplification and cleanup of JANUS code

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2017-05-08 13:09:29 -04:00
parent 427c4183cb
commit 23868d634a
21 changed files with 163 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
matlab_version/experiments_JANUS/Script_CECLM_general.m
View File

@@ -6,45 +6,16 @@ addpath(genpath('../'));
root_test_data = 'D:/Datasets/janus_labeled';
[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
%% loading the patch experts
clmParams = struct;
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_general();
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat', [], [], clmParams);
%% Fitting the model to the provided image
output_root = './fit_ceclm_general/';
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = 0.9*[35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
%% Setup recording
experiment.params = clmParams;
num_points = numel(M)/3;
num_points = numel(pdm.M)/3;
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
@@ -52,9 +23,7 @@ lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
@@ -67,8 +36,9 @@ end
if(verbose)
f = figure;
end
% As the orientations are not equally reliable reweigh them
load('../learn_error_mapping/cen_general_mapping.mat');
%% Fitting the model to the provided images
tic
for i=1:numel(images)
@@ -118,7 +88,7 @@ for i=1:numel(images)
end
toc
experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;

54
matlab_version/experiments_JANUS/Script_CECLM_menpo.m
View File

@@ -6,43 +6,16 @@ addpath(genpath('../'));
root_test_data = 'D:/Datasets/janus_labeled';
[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
%% loading the patch experts
clmParams = struct;
%% loading the CE-CLM model and parameters
[patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo();
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat', [], [], clmParams);
%% Fitting the model to the provided image
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = 0.9*[35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
%% Setup recording
experiment.params = clmParams;
num_points = numel(M)/3;
num_points = numel(pdm.M)/3;
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
@@ -50,14 +23,12 @@ lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
% Change if you want to visualize the outputs
verbose = false;
output_img = false;
if(output_img)
output_root = './ceclm_menpo_out/';
output_root = './ceclm_gen_out/';
if(~exist(output_root, 'dir'))
mkdir(output_root);
end
@@ -65,8 +36,9 @@ end
if(verbose)
f = figure;
end
% As the orientations are not equally reliable reweigh them
load('../learn_error_mapping/cen_menpo_mapping.mat');
%% Fitting the model to the provided images
tic
for i=1:numel(images)
@@ -116,7 +88,7 @@ for i=1:numel(images)
end
toc
experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;

2
matlab_version/experiments_JANUS/Script_CLNF_general.m
View File

@@ -140,7 +140,7 @@ for i=1:numel(images)
end
toc
experiment.errors_normed = compute_error(labels_all, shapes_all - 0.5);
experiment.errors_normed = compute_error(labels_all, shapes_all - 1.0);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;

BIN
matlab_version/experiments_JANUS/results/Janus-full.pdf
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/Janus-full.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 171 KiB

After

Width:  |  Height:  |  Size: 171 KiB

BIN
matlab_version/experiments_JANUS/results/Janus-no-outline.pdf
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/Janus-no-outline.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 193 KiB

After

Width:  |  Height:  |  Size: 193 KiB

BIN
matlab_version/experiments_JANUS/results/results_ceclm_general.mat
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/results_ceclm_menpo.mat
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/results_wild_clnf_general.mat
View File

Binary file not shown.

2
matlab_version/fitting/Fitting_from_bb.m
View File

@@ -136,7 +136,7 @@ function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, v
responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'CCNF'))
responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
elseif(strcmp(patchExperts(scale).type, 'DNN'))
elseif(strcmp(patchExperts(scale).type, 'CEN'))
responses = PatchResponseDNN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:));
end

4
matlab_version/models/Load_DCLM_Patch_Experts.m → matlab_version/models/Load_CECLM_Patch_Experts.m
View File

@@ -1,4 +1,4 @@
function [patches] = Load_DCLM_Patch_Experts( col_patch_dir, col_patch_file, depth_patch_dir, depth_patch_file, clmParams)
function [patches] = Load_CECLM_Patch_Experts( col_patch_dir, col_patch_file)
%LOAD_PATCH_EXPERTS Summary of this function goes here
% Detailed explanation goes here
@@ -19,7 +19,7 @@ function [patches] = Load_DCLM_Patch_Experts( col_patch_dir, col_patch_file, dep
patch.modalities = patch_experts.types;
patch.multi_modal_types = patch_experts.types;
patch.type = 'DNN';
patch.type = 'CEN';
% Knowing what normalisation was performed during training is
% important for fitting

38
matlab_version/models/Load_CECLM_general.m Normal file
View File

@@ -0,0 +1,38 @@
function [patches, pdm, clmParams, early_term_params] = Load_CECLM_general()
%LOAD_PATCH_EXPERTS Summary of this function goes here
% Detailed explanation goes here
% Load the patch experts/local detectors
[patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_general.mat');
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
clmParams.regFactor = 0.9*[35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% As the orientations are not equally reliable reweigh them
load('../learn_error_mapping/cen_general_mapping.mat');
end

38
matlab_version/models/Load_CECLM_menpo.m Normal file
View File

@@ -0,0 +1,38 @@
function [patches, pdm, clmParams, early_term_params] = Load_CECLM_menpo()
%LOAD_PATCH_EXPERTS Summary of this function goes here
% Detailed explanation goes here
% Load the patch experts/local detectors
[patches] = Load_CECLM_Patch_Experts( '../models/cen/', 'cen_patches_*_menpo.mat');
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_menpo.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
clmParams.regFactor = 0.9*[35, 27, 20, 20];
clmParams.sigmaMeanShift = 1.5*[1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% As the orientations are not equally reliable reweigh them
load('../learn_error_mapping/cen_menpo_mapping.mat');
end

14
matlab_version/models/dpn/create_dpn_experts_2017_02_02.m → matlab_version/models/cen/create_cen_experts_gen.m
View File

@@ -1,5 +1,5 @@
clear;
load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -26,6 +26,8 @@ patch_experts.patch_experts = cell(n_views, n_landmarks);
scales = {'0.25', '0.35', '0.50', '1.00'};
root = 'D:/deep_experts/2017-02-02/rmses/';
for s=scales
for c=1:n_views
@@ -38,9 +40,9 @@ for s=scales
mirror = false;
% Find the relevant file
if(c == frontalView)
rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, i);
rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, i);
else
rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, i);
rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_profile%d_%d_512.mat'], s{1}, c-1, i);
end
if(exist(rel_file, 'file'))
load(rel_file);
@@ -53,7 +55,7 @@ for s=scales
continue;
end
if(c == frontalView)
rel_file = sprintf('D:/deep_experts/2017-02-02/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, rel_id);
rel_file = sprintf([root, 'MultiGeneral_arch4general_%s_frontal_%d_512.mat'], s{1}, rel_id);
mirror = true;
load(rel_file);
@@ -106,6 +108,6 @@ for s=scales
end
end
trainingScale = str2num(s{1});
save(['dpn_patches_', s{1} '_general_v2.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
% write_patch_expert_bin(['dpn_patches_', s{1} '_general_v2.dat'], trainingScale, centers, visiIndex, patch_experts);
save(['cen_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
% write_patch_expert_bin(['cen_patches_', s{1} '_general.dat'], trainingScale, centers, visiIndex, patch_experts);
end

83
matlab_version/models/dpn/create_dpn_experts.m → matlab_version/models/cen/create_cen_experts_menpo.m
View File

@@ -1,5 +1,5 @@
clear;
load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -26,6 +26,10 @@ patch_experts.patch_experts = cell(n_views, n_landmarks);
scales = {'0.25', '0.35', '0.50', '1.00'};
visiIndex = zeros(7, 68);
root = 'D:/deep_experts/menpo/rmses/';
for s=scales
for c=1:n_views
@@ -34,45 +38,39 @@ for s=scales
for i=1:n_landmarks
if(visiIndex(c,i))
mirror = false;
% Find the relevant file
if(c == frontalView)
rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, i);
else
rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, i);
end
if(exist(rel_file, 'file'))
load(rel_file);
else
rel_id = mirrorInds(mirrorInds(:,2)==i,1);
if(isempty(rel_id))
rel_id = mirrorInds(mirrorInds(:,1)==i,2);
end
if(~visiIndex(c, rel_id))
break;
end
if(c == frontalView)
rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_frontal_%d_512.mat', s{1}, rel_id);
else
rel_file = sprintf('D:/deep_experts/rmses/MultiGeneral_arch4general_%s_profile%d_%d_512.mat', s{1}, c-1, rel_id);
end
mirror = true;
load(rel_file);
end
patch_experts.correlations(c, i) = correlation_2;
patch_experts.rms_errors(c, i) = rmse;
if(~mirror)
patch_experts.patch_experts{c, i} = weights;
else
flips = fliplr(reshape([1:121]', 11, 11));
weights_flipped = weights;
weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
patch_experts.patch_experts{c,i} = weights_flipped;
end
mirror = false;
% Find the relevant file
if(c == frontalView)
rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, i);
else
rel_file = sprintf([root, '/%s_profile%d_%d_512.mat'], s{1}, c-1, i);
end
if(exist(rel_file, 'file'))
visiIndex(c,i) = 1;
load(rel_file);
else
rel_id = mirrorInds(mirrorInds(:,2)==i,1);
if(isempty(rel_id))
rel_id = mirrorInds(mirrorInds(:,1)==i,2);
end
if(c == frontalView)
rel_file = sprintf([root, '/%s_frontal_%d_512.mat'], s{1}, rel_id);
mirror = true;
visiIndex(c,i) = 1;
load(rel_file);
end
end
patch_experts.correlations(c, i) = correlation_2;
patch_experts.rms_errors(c, i) = rmse;
if(~mirror)
patch_experts.patch_experts{c, i} = weights;
else
flips = fliplr(reshape([1:121]', 11, 11));
weights_flipped = weights;
weights_flipped{1}(2:end,:) = weights{1}(flips+1,:);
patch_experts.patch_experts{c,i} = weights_flipped;
end
end
else
@@ -83,6 +81,11 @@ for s=scales
corr_T = swap(corr_T, mirrorInds(:,1), mirrorInds(:,2));
patch_experts.correlations(c,:) = corr_T;
vis_T = visiIndex(swap_id,:);
% Appending a mirror view instead, based on the profile view
vis_T = swap(vis_T, mirrorInds(:,1), mirrorInds(:,2));
visiIndex(c,:) = vis_T;
rmsT = patch_experts.rms_errors(swap_id,:);
rmsT = swap(rmsT, mirrorInds(:,1), mirrorInds(:,2));
patch_experts.rms_errors(c,:) = rmsT;
@@ -107,6 +110,6 @@ for s=scales
end
end
trainingScale = str2num(s{1});
save(['dpn_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
write_patch_expert_bin(['dpn_patches_', s{1} '_general.dat'], trainingScale, centers, visiIndex, patch_experts);
save(['cen_patches_', s{1} '_menpo.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
write_patch_expert_bin(['cen_patches_', s{1} '_menpo.dat'], trainingScale, centers, visiIndex, patch_experts);
end

6
matlab_version/models/dpn/create_dpn_experts_sparse.m → matlab_version/models/cen/create_cen_experts_sparse.m
View File

@@ -1,5 +1,5 @@
clear;
load('C:\Users\tbaltrus\Documents\OpenFace\matlab_version\models\general\ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
load('../general/ccnf_patches_0.25_general.mat', 'centers', 'visiIndex', 'normalisationOptions');
mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
@@ -135,6 +135,6 @@ for s=scales
end
trainingScale = str2num(s{1});
save(['dpn_patches_', s{1} '_general_sparse.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
write_patch_expert_bin(['dpn_patches_', s{1} '_general_sparse.dat'], trainingScale, centers, visiIndex, patch_experts);
save(['cen_patches_', s{1} '_general_sparse.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions');
write_patch_expert_bin(['cen_patches_', s{1} '_general_sparse.dat'], trainingScale, centers, visiIndex, patch_experts);
end

4
matlab_version/models/cen/readme.txt Normal file
View File

@@ -0,0 +1,4 @@
Scripts for creating CEN patch experts from already trained models.
To create one from 300W + MultiPIE - create_cen_experts_gen.m
To create one from 300W + MultiPIE + Menpo - create_cen_experts_menpo.m

0
matlab_version/models/dpn/swap.m → matlab_version/models/cen/swap.m
View File

0
matlab_version/models/dpn/writeMatrixBin.m → matlab_version/models/cen/writeMatrixBin.m
View File

0
matlab_version/models/dpn/write_patch_expert_bin.m → matlab_version/models/cen/write_patch_expert_bin.m
View File