ZF/OpenFace
1
0
Fork 0
mirror of https://gitcode.com/gh_mirrors/ope/OpenFace.git synced 2025-12-30 13:02:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

DCLM more results - JANUS and 300W.

Browse Source
This commit is contained in:
Tadas Baltrusaitis
2016-10-23 14:06:54 -04:00
parent 93c54f3b16
commit af795fc99d
31 changed files with 1295 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -36,3 +36,5 @@ matlab_runners/Demos/output_features_seq/
matlab_runners/Demos/output_features_vid/
matlab_runners/Feature Point Experiments/300VW_features/
matlab_runners/Feature Point Experiments/300VW_features_2/
matlab_version/experiments_JANUS/wild_fit_clnf/
matlab_version/experiments_JANUS/wild_fit_dclm/

33
matlab_version/bounding_box_mapping/learn_error_dets_300w.m Normal file
View File

@@ -0,0 +1,33 @@
% load('ocv.mat')
% Move into Matlab space
% bboxes = bboxes + 1;
clear
% Replace this with the location of in 300 faces in the wild data
if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file'))
root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/'];
else
root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/';
end
[images, detections, labels] = Collect_wild_imgs(root_test_data, true, true, true, true);
%% some visualisations
% Find the width and height mappings
widths_gt = (max(labels(:,:,1)') - min(labels(:,:,1)'))';
heights_gt = (max(labels(:,:,2)') - min(labels(:,:,2)'))';
widths_det = detections(:,3) - detections(:,1);
heights_det = detections(:,4) - detections(:,2);
s_width = sqrt(std(widths_det ./ widths_gt));
s_height = sqrt(std(heights_det ./ heights_gt));
tx_gt = min(labels(:,:,1)')';
ty_gt = min(labels(:,:,2)')';
tx_det = detections(:,1);
ty_det = detections(:,2);
s_tx = std((tx_gt - tx_det) ./ widths_det);
s_ty = std((ty_gt - ty_det) ./ heights_det);

78
matlab_version/experiments_JANUS/Collect_JANUS_imgs.m Normal file
View File

@@ -0,0 +1,78 @@
function [images, detections, labels] = Collect_JANUS_imgs(root_test_data)
dataset_loc = [root_test_data, '/'];
landmarkLabels = dir([dataset_loc '\*.pts']);
num_samples = 5;
num_imgs = size(landmarkLabels,1);
images = struct;
labels = zeros(num_imgs * num_samples, 68, 2);
num_landmarks = 68;
rng(0);
detections = [];
ind = 1;
for imgs = 1:num_imgs
[~,name,~] = fileparts(landmarkLabels(imgs).name);
landmarks = dlmread([dataset_loc, landmarkLabels(imgs).name], ' ', [3,0,num_landmarks+2,1]);
tmp = landmarks(:,1);
landmarks(:,1) = landmarks(:,2);
landmarks(:,2) = tmp;
landmarks(landmarks == - 1) = 0;
% Swap around some points
eyes = landmarks(18:29,:);
brow_l = landmarks(30:34,:);
nose = landmarks(35:43,:);
brow_r = landmarks(44:48,:);
landmarks(18:22,:) = brow_l;
landmarks(23:27,:) = brow_r;
landmarks(28:36,:) = nose;
landmarks(37:48,:) = eyes;
non_occluded = landmarks(:,1) ~= 0;
for s=1:num_samples
% Create detections based on 300W noise level - just shifting in x, y
% and adding some scaling
scale_x = 1 + randn(1,1) * 0.01;
scale_y = 1 + randn(1,1) * 0.015;
tx = 0.06 * randn(1,1);
ty = 0.05 * randn(1,1);
width_gt = (max(landmarks(non_occluded,1)) - min(landmarks(non_occluded,1)));
height_gt = (max(landmarks(non_occluded,2)) - min(landmarks(non_occluded,2)));
width = width_gt * scale_x;
height = height_gt * scale_y;
x = min(landmarks(non_occluded,1)) + width * tx;
y = min(landmarks(non_occluded,2)) + height * ty;
% Add like 5 more
detections = cat(1, detections, [x, y, x+ width, y+height]);
images(ind).img = [dataset_loc, name];
labels(ind,:,:) = landmarks;
% imshow(imread(images(ind).img));
% hold on;
% rectangle('position', [x, y, width, height]);
% hold off;
ind = ind + 1;
end
end
end

213
matlab_version/experiments_JANUS/Display_dclm_results.m Normal file
View File

@@ -0,0 +1,213 @@
clear
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
load('results/results_wild_clnf_general_final_inner.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
shapes = experiments.shapes([1:60,62:64,66:end],:,:);
labels = labels(18:end,:,:);
% center the pixel
shapes = shapes(18:end,:,:);
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
plot(error_x, error_y, 'b','DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width);
hold on;
load('results/results_wild_dclm_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
shapes = experiments.shapes([1:60,62:64,66:end],:,:);
labels = labels(18:end,:,:);
% center the pixel
shapes = shapes(18:end,:,:);
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
plot(error_x, error_y, 'r','DisplayName', 'DCLM', 'LineWidth',line_width);
hold on;
load('results/JANUS_pocr.mat');
% center the pixel
shapes_all = experiments.shapes;
pocr_error = compute_error(labels, shapes_all);
[error_x, error_y] = cummErrorCurve(pocr_error);
plot(error_x, error_y, '.-g','DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width);
hold on;
% load('results/intraface_wild_resize.mat');
% labels_all = labels_all(18:end,:,detected);
% % center the pixel
% shapes_all = shapes_all(18:end,:,detected) + 0.5;
%
% intraface_wild_error = compute_error(labels_all, shapes_all);
%
% [error_x, error_y] = cummErrorCurve(intraface_wild_error);
%
% plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width);
% hold on;
% load('results/GNDPM_300W.mat');
% % center the pixel
% shapes_all = shapes_all(:,:,detected) + 1;
% labels_all = labels_all(:,:,detected);
%
% gndpm_wild_error = compute_error(labels_all, shapes_all);
%
% [error_x, error_y] = cummErrorCurve(gndpm_wild_error);
%
% plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width);
% hold on;
% load('results/zhu_wild.mat');
% labels_all = labels_all(18:end,:,detected);
% shapes_all = shapes_all(18:end,:,detected);
%
% zhu_wild_error = compute_error(labels_all, shapes_all);
%
% [error_x, error_y] = cummErrorCurve(zhu_wild_error);
%
% plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width);
% load('results/results_wild_clm.mat');
% labels = experiments.labels([1:60,62:64,66:end],:,detected);
% shapes = experiments.shapes([1:60,62:64,66:end],:,detected);
% labels = labels(18:end,:,:);
% % center the pixel
% shapes = shapes(18:end,:,:) + 0.5;
%
% clm_error = compute_error( labels, shapes);
%
% [error_x, error_y] = cummErrorCurve(clm_error);
%
% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width);
% load('results/drmf_wild.mat');
% labels_all = labels_all(18:end,:,detected);
% shapes_all = shapes_all(18:end,:,detected);
%
% drmf_error = compute_error(labels_all, shapes_all);
%
% [error_x, error_y] = cummErrorCurve(drmf_error);
%
% plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width);
set(gca,'xtick',[0:0.05:0.15])
xlim([0,0.15]);
xlabel('Size normalised shape RMS error','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',30)
print -dpdf results/Janus-no-outline.pdf
print -dpng results/Janus-no-outline.png
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
load('results/results_wild_clnf_general_final_inner.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
% center the pixel
shapes = experiments.shapes([1:60,62:64,66:end],:,:);
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
hold on;
plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width);
load('results/results_wild_dclm_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
% center the pixel
shapes = experiments.shapes([1:60,62:64,66:end],:,:);
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
hold on;
plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width);
% load('results/zhu_wild.mat');
%
% zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:));
%
% [error_x, error_y] = cummErrorCurve(zhu_wild_error);
%
% plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width);
%
% load('results/yu_wild.mat');
%
% yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:));
% yu_wild_error(isnan(yu_wild_error)) = 1;
% yu_wild_error(isinf(yu_wild_error)) = 1;
%
% [error_x, error_y] = cummErrorCurve(yu_wild_error);
%
% plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width);
%
% load('results/results_wild_clm.mat');
% experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:);
% % center the pixel
% experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5;
%
% clm_error = compute_error( experiments(1).labels, experiments(1).shapes);
%
% [error_x, error_y] = cummErrorCurve(clm_error);
%
% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width);
%
% load('results/drmf_wild.mat');
%
% drmf_error = compute_error(labels_all, shapes_all);
%
% [error_x, error_y] = cummErrorCurve(drmf_error);
%
% plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width);
set(gca,'xtick',[0:0.05:0.15])
xlim([0,0.15]);
xlabel('Size normalised shape RMS error','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
%title('Fitting in the wild','FontSize',60,'FontName','Helvetica');
legend('show', 'Location', 'SouthEast');
print -dpdf results/Janus-full.pdf
print -dpng results/Janus-full.png

216
matlab_version/experiments_JANUS/Script_CLNF_general_no_out.m Normal file
View File

@@ -0,0 +1,216 @@
function Script_CLNF_general_no_out()
addpath('../PDM_helpers/');
addpath('../fitting/normxcorr2_mex_ALL');
addpath('../fitting/');
addpath('../CCNF/');
addpath('../models/');
% Replace this with the location of in 300 faces in the wild data
root_test_data = 'D:/Datasets/janus_labeled';
[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
%% loading the patch experts
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams);
%% Fitting the model to the provided image
verbose = true; % set to true to visualise the fitting
output_root = './wild_fit_clnf/';
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = [35, 27, 20];
clmParams.sigmaMeanShift = [1.25, 1.375, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
% Loading the final scale
[clmParams_inner, pdm_inner] = Load_CLM_params_inner();
clmParams_inner.window_size = [17,17;19,19;21,21;23,23];
inds_inner = 18:68;
[patches_inner] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams_inner);
clmParams_inner.multi_modal_types = patches_inner(1).multi_modal_types;
% load('results/results_wild_clnf_general.mat');
% clear 'experiments';
% for recording purposes
experiment.params = clmParams;
num_points = numel(M)/3;
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = detections(i,:);
% have a multi-view version
if(multi_view)
views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0];
views = views * pi/180;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(num_points,size(views,1));
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
end
[lhood, v_ind] = max(ls);
lmark_lhood = lmark_lhoods(:,v_ind);
shape = shapes(:,:,v_ind);
view_used = views_used(v);
else
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
end
% Perform inner face fitting
shape_inner = shape(inds_inner,:);
[ a, R, T, ~, l_params, err] = fit_PDM_ortho_proj_to_2D_no_reg(pdm_inner.M, pdm_inner.E, pdm_inner.V, shape_inner);
if(a > 0.9)
g_param = [a; Rot2Euler(R)'; T];
bbox = [min(shape_inner(:,1)), min(shape_inner(:,2)), max(shape_inner(:,1)), max(shape_inner(:,2))];
[shape_inner] = Fitting_from_bb(image, [], bbox, pdm_inner, patches_inner, clmParams_inner, 'gparam', g_param, 'lparam', l_params);
% Now after detections incorporate the eyes back
% into the face model
shape(inds_inner, :) = shape_inner;
[ ~, ~, ~, ~, ~, ~, shape_fit] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, shape);
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all(:,:,i) = shape_fit;
else
shapes_all(:,:,i) = shape;
end
labels_all(:,:,i) = labels(i,:,:);
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose)
actualShape = squeeze(labels(i,:,:));
[height_img, width_img,~] = size(image_orig);
width = max(actualShape(:,1)) - min(actualShape(:,1));
height = max(actualShape(:,2)) - min(actualShape(:,2));
v_points = sum(squeeze(labels(i,:,:)),2) > 0;
img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1);
img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img);
img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1);
img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img);
shape(:,1) = shape(:,1) - double(img_min_x);
shape(:,2) = shape(:,2) - double(img_min_y);
image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :);
% valid points to draw (not to draw
% occluded ones)
% f = figure('visible','off');
f = figure;
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
plot(shape(:,1), shape(:,2),'.r','MarkerSize',20);
plot(shape(:,1), shape(:,2),'.b','MarkerSize',10);
% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i));
% close(f);
hold off;
% drawnow expose
close(f);
catch warn
end
end
end
toc
experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
% save the experiment
if(~exist('experiments', 'var'))
experiments = experiment;
else
experiments = cat(1, experiments, experiment);
end
fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
%%
output_results = 'results/results_wild_clnf_general_final_inner.mat';
save(output_results, 'experiments');
end

184
matlab_version/experiments_JANUS/Script_DCLM_general.m Normal file
View File

@@ -0,0 +1,184 @@
function Script_DCLM_general()
addpath('../PDM_helpers/');
addpath('../fitting/normxcorr2_mex_ALL');
addpath('../fitting/');
addpath('../CCNF/');
addpath('../models/');
% Replace this with the location of in 300 faces in the wild data
root_test_data = 'D:/Datasets/janus_labeled';
[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
%% loading the patch experts
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams);
%% Fitting the model to the provided image
output_root = './wild_fit_dclm/';
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = [35, 27, 20, 20];
clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
experiment.params = clmParams;
num_points = numel(M)/3;
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
verbose = true;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = detections(i,:);
% have a multi-view version
if(multi_view)
views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0];
views = views * pi/180;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(num_points,size(views,1));
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
end
[lhood, v_ind] = max(ls);
lmark_lhood = lmark_lhoods(:,v_ind);
shape = shapes(:,:,v_ind);
view_used = views_used(v);
else
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
end
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all(:,:,i) = shape;
labels_all(:,:,i) = labels(i,:,:);
if(mod(i, 200)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose)
actualShape = squeeze(labels(i,:,:));
[height_img, width_img,~] = size(image_orig);
width = max(actualShape(:,1)) - min(actualShape(:,1));
height = max(actualShape(:,2)) - min(actualShape(:,2));
v_points = sum(squeeze(labels(i,:,:)),2) > 0;
img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1);
img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img);
img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1);
img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img);
shape(:,1) = shape(:,1) - double(img_min_x);
shape(:,2) = shape(:,2) - double(img_min_y);
image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :);
% valid points to draw (not to draw
% occluded ones)
% f = figure('visible','off');
f = figure;
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
plot(shape(:,1), shape(:,2),'.r','MarkerSize',20);
plot(shape(:,1), shape(:,2),'.b','MarkerSize',10);
% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i));
% close(f);
hold off;
% drawnow expose
close(f);
catch warn
end
end
end
toc
experiment.errors_normed = compute_error(labels_all, shapes_all + 0.5);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
% save the experiment
if(~exist('experiments', 'var'))
experiments = experiment;
else
experiments = cat(1, experiments, experiment);
end
fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
%%
output_results = 'results/results_wild_dclm_general.mat';
save(output_results, 'experiments');
end

184
matlab_version/experiments_JANUS/Script_DCLM_general_large.m Normal file
View File

@@ -0,0 +1,184 @@
function Script_DCLM_general_large()
addpath('../PDM_helpers/');
addpath('../fitting/normxcorr2_mex_ALL');
addpath('../fitting/');
addpath('../CCNF/');
addpath('../models/');
% Replace this with the location of in 300 faces in the wild data
root_test_data = 'D:/Datasets/janus_labeled';
[images, detections, labels] = Collect_JANUS_imgs(root_test_data);
%% loading the patch experts
clmParams = struct;
clmParams.window_size = [29,29; 27,27; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams);
%% Fitting the model to the provided image
output_root = './wild_fit_dclm/';
% the default PDM to use
pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat'];
load(pdmLoc);
pdm = struct;
pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = [35, 27, 20, 20];
clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 4;
% for recording purposes
experiment.params = clmParams;
num_points = numel(M)/3;
shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1));
labels_all = zeros(size(labels,2),size(labels,3), size(labels,1));
lhoods = zeros(numel(images),1);
all_lmark_lhoods = zeros(num_points, numel(images));
all_views_used = zeros(numel(images),1);
% Use the multi-hypothesis model, as bounding box tells nothing about
% orientation
multi_view = true;
verbose = false;
tic
for i=1:numel(images)
image = imread(images(i).img);
image_orig = image;
if(size(image,3) == 3)
image = rgb2gray(image);
end
bbox = detections(i,:);
% have a multi-view version
if(multi_view)
views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0];
views = views * pi/180;
shapes = zeros(num_points, 2, size(views,1));
ls = zeros(size(views,1),1);
lmark_lhoods = zeros(num_points,size(views,1));
views_used = zeros(num_points,size(views,1));
% Find the best orientation
for v = 1:size(views,1)
[shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:));
end
[lhood, v_ind] = max(ls);
lmark_lhood = lmark_lhoods(:,v_ind);
shape = shapes(:,:,v_ind);
view_used = views_used(v);
else
[shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams);
end
all_lmark_lhoods(:,i) = lmark_lhood;
all_views_used(i) = view_used;
shapes_all(:,:,i) = shape;
labels_all(:,:,i) = labels(i,:,:);
if(mod(i, 50)==0)
fprintf('%d done\n', i );
end
lhoods(i) = lhood;
if(verbose)
actualShape = squeeze(labels(i,:,:));
[height_img, width_img,~] = size(image_orig);
width = max(actualShape(:,1)) - min(actualShape(:,1));
height = max(actualShape(:,2)) - min(actualShape(:,2));
v_points = sum(squeeze(labels(i,:,:)),2) > 0;
img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1);
img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img);
img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1);
img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img);
shape(:,1) = shape(:,1) - double(img_min_x);
shape(:,2) = shape(:,2) - double(img_min_y);
image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :);
% valid points to draw (not to draw
% occluded ones)
% f = figure('visible','off');
f = figure;
try
if(max(image_orig(:)) > 1)
imshow(double(image_orig)/255, 'Border', 'tight');
else
imshow(double(image_orig), 'Border', 'tight');
end
axis equal;
hold on;
plot(shape(:,1), shape(:,2),'.r','MarkerSize',20);
plot(shape(:,1), shape(:,2),'.b','MarkerSize',10);
% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i));
% print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i));
% close(f);
hold off;
% drawnow expose
close(f);
catch warn
end
end
end
toc
experiment.errors_normed = compute_error(labels_all, shapes_all + 0.5);
experiment.lhoods = lhoods;
experiment.shapes = shapes_all;
experiment.labels = labels_all;
experiment.all_lmark_lhoods = all_lmark_lhoods;
experiment.all_views_used = all_views_used;
% save the experiment
if(~exist('experiments', 'var'))
experiments = experiment;
else
experiments = cat(1, experiments, experiment);
end
fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ...
numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed));
%%
output_results = 'results/results_wild_dclm_general_large.mat';
save(output_results, 'experiments');
end

43
matlab_version/experiments_JANUS/compute_error.m Normal file
View File

@@ -0,0 +1,43 @@
function [ error_per_image, err_pp, err_pp_dim ] = compute_error( ground_truth_all, detected_points_all )
%compute_error
% compute the average point-to-point Euclidean error normalized by the
% inter-ocular distance (measured as the Euclidean distance between the
% outer corners of the eyes)
%
% Inputs:
% grounth_truth_all, size: num_of_points x 2 x num_of_images
% detected_points_all, size: num_of_points x 2 x num_of_images
% Output:
% error_per_image, size: num_of_images x 1
num_of_images = size(ground_truth_all,3);
num_of_points = size(ground_truth_all,1);
error_per_image = zeros(num_of_images,1);
err_pp = zeros(num_of_images, num_of_points);
err_pp_dim = zeros(num_of_images, num_of_points, 2);
for i =1:num_of_images
detected_points = detected_points_all(:,:,i);
ground_truth_points = ground_truth_all(:,:,i);
visible = ground_truth_points(:,1) > 0;
normalization_x = max(ground_truth_points(visible,1)) - min(ground_truth_points(visible,1));
normalization_y = max(ground_truth_points(visible,2)) - min(ground_truth_points(visible,2));
normalization = (normalization_x + normalization_y)/2;
sum_c=0;
for j=1:num_of_points
if(visible(j))
sum_c = sum_c+norm(detected_points(j,:)-ground_truth_points(j,:));
err_pp(i,j) = norm(detected_points(j,:)-ground_truth_points(j,:));
err_pp_dim(i,j,1) = detected_points(j,1)-ground_truth_points(j,1);
err_pp_dim(i,j,2) = detected_points(j,2)-ground_truth_points(j,2);
end
end
error_per_image(i) = sum_c/(sum(visible)*normalization);
err_pp(i,:) = err_pp(i,:) ./ normalization;
err_pp_dim(i,:) = err_pp_dim(i,:) ./ normalization;
end
end

19
matlab_version/experiments_JANUS/cummErrorCurve.m Normal file
View File

@@ -0,0 +1,19 @@
function [x, y] = cummErrorCurve( errorVec )
%CUMMERRORCURVE Summary of this function goes here
% Detailed explanation goes here
spacing = 0.001;
sampling = [0:spacing:max(errorVec)];
x = sampling;
y = zeros(numel(sampling,1));
for i=1:numel(sampling)
y(i) = sum(errorVec < sampling(i)) / numel(errorVec);
end
end

BIN
matlab_version/experiments_JANUS/results/JANUS_pocr.mat Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/Janus-full.pdf Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/Janus-full.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 112 KiB

BIN
matlab_version/experiments_JANUS/results/Janus-no-outline.pdf Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/Janus-no-outline.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 121 KiB

BIN
matlab_version/experiments_JANUS/results/results_wild_clnf_general_final_inner.mat Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/results_wild_dclm_general.mat Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_JANUS/results/results_wild_dclm_general_large.mat Normal file
View File

Binary file not shown.

233
matlab_version/experiments_iccv_300w/Display_dclm_results.m Normal file
View File

@@ -0,0 +1,233 @@
clear
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
load('results/intraface_wild_resize.mat');
labels_all = labels_all(18:end,:,:);
shapes_all = shapes_all(18:end,:,:);
intraface_wild_error = compute_error(labels_all, shapes_all);
% removing faces that were not detected by intraface for fairness
detected = intraface_wild_error < 1;
% load('results/results_wild_clnf.mat');
% labels = experiments.labels([1:60,62:64,66:end],:,detected);
% shapes = experiments.shapes([1:60,62:64,66:end],:,detected);
% labels = labels(18:end,:,:);
% % center the pixel
% shapes = shapes(18:end,:,:) + 0.5;
%
% clnf_error = compute_error( labels, shapes);
%
% [error_x, error_y] = cummErrorCurve(clnf_error);
%
% plot(error_x, error_y, 'DisplayName', 'CLNF wild', 'LineWidth',line_width);
% hold on;
load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat');
labels = experiments.labels([1:60,62:64,66:end],:,detected);
shapes = experiments.shapes([1:60,62:64,66:end],:,detected);
labels = labels(18:end,:,:);
% center the pixel
shapes = shapes(18:end,:,:) + 0.5;
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width);
hold on;
load('../experiments_in_the_wild/results/results_wild_dclm_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,detected);
shapes = experiments.shapes([1:60,62:64,66:end],:,detected);
labels = labels(18:end,:,:) - 1.0;
shapes = shapes(18:end,:,:);
dclm_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(dclm_error);
plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width);
load('results/300W_pocr.mat');
% center the pixel
shapes_all = experiments.shapes;
pocr_error = compute_error(labels, shapes_all(:,:,detected));
[error_x, error_y] = cummErrorCurve(pocr_error);
plot(error_x, error_y, 'DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width);
hold on;
load('results/intraface_wild_resize.mat');
labels_all = labels_all(18:end,:,detected);
% center the pixel
shapes_all = shapes_all(18:end,:,detected) + 0.5;
intraface_wild_error = compute_error(labels_all, shapes_all);
[error_x, error_y] = cummErrorCurve(intraface_wild_error);
plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width);
hold on;
load('results/GNDPM_300W.mat');
% center the pixel
shapes_all = shapes_all(:,:,detected) + 1;
labels_all = labels_all(:,:,detected);
gndpm_wild_error = compute_error(labels_all, shapes_all);
[error_x, error_y] = cummErrorCurve(gndpm_wild_error);
plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width);
hold on;
load('results/zhu_wild.mat');
labels_all = labels_all(18:end,:,detected);
shapes_all = shapes_all(18:end,:,detected);
zhu_wild_error = compute_error(labels_all, shapes_all);
[error_x, error_y] = cummErrorCurve(zhu_wild_error);
plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width);
load('results/results_wild_clm.mat');
labels = experiments.labels([1:60,62:64,66:end],:,detected);
shapes = experiments.shapes([1:60,62:64,66:end],:,detected);
labels = labels(18:end,:,:);
% center the pixel
shapes = shapes(18:end,:,:) + 0.5;
clm_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clm_error);
plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width);
load('results/drmf_wild.mat');
labels_all = labels_all(18:end,:,detected);
shapes_all = shapes_all(18:end,:,detected);
drmf_error = compute_error(labels_all, shapes_all);
[error_x, error_y] = cummErrorCurve(drmf_error);
plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width);
set(gca,'xtick',[0:0.05:0.15])
xlim([0,0.08]);
xlabel('Size normalised shape RMS error','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica');
leg = legend('show', 'Location', 'SouthEast');
set(leg,'FontSize',30)
print -dpdf results/dclm-in-the-wild-clnf-no-outline.pdf
print -dpng results/dclm-300W-no-outline.png
%%
scrsz = get(0,'ScreenSize');
figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]);
set(figure1,'Units','Inches');
pos = get(figure1,'Position');
set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)])
% Create axes
axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica');
line_width = 6;
hold on;
load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
% center the pixel
shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 1;
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
hold on;
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
load('../experiments_in_the_wild/results/results_wild_dclm_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
% center the pixel
shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 0.5;
dclm_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(dclm_error);
plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width);
load('results/zhu_wild.mat');
zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:));
[error_x, error_y] = cummErrorCurve(zhu_wild_error);
plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width);
load('results/yu_wild.mat');
yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:));
yu_wild_error(isnan(yu_wild_error)) = 1;
yu_wild_error(isinf(yu_wild_error)) = 1;
[error_x, error_y] = cummErrorCurve(yu_wild_error);
plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width);
load('results/results_wild_clm.mat');
experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:);
% center the pixel
experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5;
clm_error = compute_error( experiments(1).labels, experiments(1).shapes);
[error_x, error_y] = cummErrorCurve(clm_error);
plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width);
load('results/drmf_wild.mat');
drmf_error = compute_error(labels_all, shapes_all);
[error_x, error_y] = cummErrorCurve(drmf_error);
plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width);
set(gca,'xtick',[0:0.05:0.15])
xlim([0,0.15]);
xlabel('Size normalised shape RMS error','FontName','Helvetica');
ylabel('Proportion of images','FontName','Helvetica');
grid on
%title('Fitting in the wild','FontSize',60,'FontName','Helvetica');
legend('show', 'Location', 'SouthEast');
print -dpdf results/dclm-in-the-wild-comparison.pdf
print -dpng results/dclm-300W-outline.png

BIN
matlab_version/experiments_iccv_300w/results/300W_pocr.mat Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 216 KiB

BIN
matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 166 KiB

BIN
matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf Normal file
View File

Binary file not shown.

BIN
matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline.pdf
View File

Binary file not shown.

BIN
matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf
View File

Binary file not shown.

18
matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m
View File

@@ -25,21 +25,7 @@ clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
hold on;
plot(error_x, error_y, 'DisplayName', 'CLNF - 4 scales', 'LineWidth',line_width);
load('results/results_wild_clnf_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
shapes = experiments.shapes([1:60,62:64,66:end],:,:);
labels = labels(18:end,:,:) - 0.5;
shapes = shapes(18:end,:,:);
clnf_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clnf_error);
hold on;
plot(error_x, error_y, 'DisplayName', 'CLNF - 1 scale', 'LineWidth',line_width);
plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width);
load('results/results_wild_dclm_general.mat');
labels = experiments.labels([1:60,62:64,66:end],:,:);
@@ -51,7 +37,7 @@ clm_error = compute_error( labels, shapes);
[error_x, error_y] = cummErrorCurve(clm_error);
plot(error_x, error_y, 'DisplayName', 'DCLM - 1 scale', 'LineWidth',line_width);
plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width);
set(gca,'xtick',[0:0.01:0.08])
xlim([0,0.08]);

10
matlab_version/experiments_in_the_wild/Script_DCLM_general.m
View File

@@ -19,7 +19,7 @@ end
clmParams = struct;
clmParams.window_size = [25,25; 23,23; 21,21;];
clmParams.window_size = [25,25; 23,23; 21,21; 21,21];
clmParams.numPatchIters = size(clmParams.window_size,1);
[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams);
@@ -38,9 +38,9 @@ pdm.M = double(M);
pdm.E = double(E);
pdm.V = double(V);
clmParams.regFactor = [35, 27, 20];
clmParams.sigmaMeanShift = [1.25, 1.375, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5];
clmParams.regFactor = [35, 27, 20, 20];
clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5];
clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5];
clmParams.startScale = 1;
clmParams.num_RLMS_iter = 10;
@@ -48,7 +48,7 @@ clmParams.fTol = 0.01;
clmParams.useMultiScale = true;
clmParams.use_multi_modal = 1;
clmParams.multi_modal_types = patches(1).multi_modal_types;
clmParams.numPatchIters = 3;
clmParams.numPatchIters = 4;
% for recording purposes
experiment.params = clmParams;

BIN
matlab_version/experiments_in_the_wild/results/dclm_results.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 164 KiB

After

Width:  |  Height:  |  Size: 134 KiB

BIN
matlab_version/experiments_in_the_wild/results/results_wild_dclm_general.mat
View File

Binary file not shown.

5
matlab_version/fitting/PatchResponseDNN.m
View File

@@ -17,7 +17,7 @@ function [ responses ] = PatchResponseDNN(patches, patch_experts_class, visibili
smallRegionVec = patches(i,:);
smallRegion = reshape(smallRegionVec, window_size(1), window_size(2));
patch = im2col(smallRegion, patchSize, 'sliding')';
patch = im2col_mine(smallRegion, patchSize)';
% Normalize
if(col_norm)
@@ -43,7 +43,8 @@ function [ responses ] = PatchResponseDNN(patches, patch_experts_class, visibili
patch_normed = weights{(w-1)*2+1}' * patch_normed + repmat(weights{(w-1)*2+2}', 1, size(patch_normed,2));
if w < 3
patch_normed(patch_normed < 0) = 0;
% patch_normed(patch_normed < 0) = 0;
patch_normed = max(0, patch_normed);
else
patch_normed = 1./(1+exp(-patch_normed));
end

80
matlab_version/fitting/im2col_mine.m Normal file
View File

@@ -0,0 +1,80 @@
function b=im2col_mine(a, block)
%IM2COL Rearrange image blocks into columns.
% B = IM2COL(A,[M N],'distinct') rearranges each distinct
% M-by-N block in the image A into a column of B. IM2COL pads A
% with zeros, if necessary, so its size is an integer multiple
% of M-by-N. If A = [A11 A12; A21 A22], where each Aij is
% M-by-N, then B = [A11(:) A21(:) A12(:) A22(:)].
%
% B = IM2COL(A,[M N],'sliding') converts each sliding M-by-N
% block of A into a column of B, with no zero padding. B has
% M*N rows and will contain as many columns as there are M-by-N
% neighborhoods in A. If the size of A is [MM NN], then the
% size of B is (M*N)-by-((MM-M+1)*(NN-N+1). Each column of B
% contains the neighborhoods of A reshaped as NHOOD(:), where
% NHOOD is a matrix containing an M-by-N neighborhood of
% A. IM2COL orders the columns of B so that they can be
% reshaped to form a matrix in the normal way. For example,
% suppose you use a function, such as SUM(B), that returns a
% scalar for each column of B. You can directly store the
% result in a matrix of size (MM-M+1)-by-(NN-N+1) using these
% calls:
%
% B = im2col(A,[M N],'sliding');
% C = reshape(sum(B),MM-M+1,NN-N+1);
%
% B = IM2COL(A,[M N]) uses the default block type of
% 'sliding'.
%
% B = IM2COL(A,'indexed',...) processes A as an indexed image,
% padding with zeros if the class of A is uint8 or uint16, or
% ones if the class of A is double.
%
% Class Support
% -------------
% The input image A can be numeric or logical. The output matrix
% B is of the same class as the input image.
%
% Example
% -------
% Calculate the local mean using a [2 2] neighborhood with zero padding.
%
% A = reshape(linspace(0,1,16),[4 4])'
% B = im2col(A,[2 2])
% M = mean(B)
% newA = col2im(M,[1 1],[3 3])
%
% See also BLOCKPROC, COL2IM, COLFILT, NLFILTER.
% Copyright 1993-2012 The MathWorks, Inc.
[ma,na] = size(a);
m = block(1); n = block(2);
if any([ma na] < [m n]) % if neighborhood is larger than image
b = zeros(m*n,0);
return
end
% Create Hankel-like indexing sub matrix.
mc = block(1); nc = ma-m+1; nn = na-n+1;
cidx = (0:mc-1)'; ridx = 1:nc;
t = cidx(:,ones(nc,1)) + ridx(ones(mc,1),:); % Hankel Subscripts
tt = zeros(mc*n,nc);
rows = 1:mc;
for i=0:n-1,
tt(i*mc+rows,:) = t+ma*i;
end
ttt = zeros(mc*n,nc*nn);
cols = 1:nc;
for j=0:nn-1,
ttt(:,j*nc+cols) = tt+ma*j;
end
% If a is a row vector, change it to a column vector. This change is
% necessary when A is a row vector and [M N] = size(A).
if ndims(a) == 2 && na > 1 && ma == 1
a = a(:);
end
b = a(ttt);