diff --git a/.gitignore b/.gitignore index b53fb9b7..e5462ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ matlab_runners/Demos/output_features_seq/ matlab_runners/Demos/output_features_vid/ matlab_runners/Feature Point Experiments/300VW_features/ matlab_runners/Feature Point Experiments/300VW_features_2/ +matlab_version/experiments_JANUS/wild_fit_clnf/ +matlab_version/experiments_JANUS/wild_fit_dclm/ diff --git a/matlab_version/bounding_box_mapping/learn_error_dets_300w.m b/matlab_version/bounding_box_mapping/learn_error_dets_300w.m new file mode 100644 index 00000000..2dde3529 --- /dev/null +++ b/matlab_version/bounding_box_mapping/learn_error_dets_300w.m @@ -0,0 +1,33 @@ +% load('ocv.mat') +% Move into Matlab space +% bboxes = bboxes + 1; +clear +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data, true, true, true, true); + +%% some visualisations + +% Find the width and height mappings +widths_gt = (max(labels(:,:,1)') - min(labels(:,:,1)'))'; +heights_gt = (max(labels(:,:,2)') - min(labels(:,:,2)'))'; + +widths_det = detections(:,3) - detections(:,1); +heights_det = detections(:,4) - detections(:,2); + +s_width = sqrt(std(widths_det ./ widths_gt)); +s_height = sqrt(std(heights_det ./ heights_gt)); + +tx_gt = min(labels(:,:,1)')'; +ty_gt = min(labels(:,:,2)')'; + +tx_det = detections(:,1); +ty_det = detections(:,2); + +s_tx = std((tx_gt - tx_det) ./ widths_det); +s_ty = std((ty_gt - ty_det) ./ heights_det); \ No newline at end of file diff --git a/matlab_version/experiments_JANUS/Collect_JANUS_imgs.m b/matlab_version/experiments_JANUS/Collect_JANUS_imgs.m new file mode 100644 index 00000000..f6682ed9 --- /dev/null +++ b/matlab_version/experiments_JANUS/Collect_JANUS_imgs.m @@ -0,0 +1,78 @@ +function [images, detections, labels] = Collect_JANUS_imgs(root_test_data) + + dataset_loc = [root_test_data, '/']; + + landmarkLabels = dir([dataset_loc '\*.pts']); + + num_samples = 5; + + num_imgs = size(landmarkLabels,1); + + images = struct; + labels = zeros(num_imgs * num_samples, 68, 2); + + num_landmarks = 68; + + rng(0); + detections = []; + + ind = 1; + + for imgs = 1:num_imgs + + [~,name,~] = fileparts(landmarkLabels(imgs).name); + + landmarks = dlmread([dataset_loc, landmarkLabels(imgs).name], ' ', [3,0,num_landmarks+2,1]); + tmp = landmarks(:,1); + landmarks(:,1) = landmarks(:,2); + landmarks(:,2) = tmp; + + landmarks(landmarks == - 1) = 0; + + % Swap around some points + eyes = landmarks(18:29,:); + brow_l = landmarks(30:34,:); + nose = landmarks(35:43,:); + brow_r = landmarks(44:48,:); + + landmarks(18:22,:) = brow_l; + landmarks(23:27,:) = brow_r; + landmarks(28:36,:) = nose; + landmarks(37:48,:) = eyes; + + non_occluded = landmarks(:,1) ~= 0; + for s=1:num_samples + % Create detections based on 300W noise level - just shifting in x, y + % and adding some scaling + scale_x = 1 + randn(1,1) * 0.01; + scale_y = 1 + randn(1,1) * 0.015; + + tx = 0.06 * randn(1,1); + ty = 0.05 * randn(1,1); + + width_gt = (max(landmarks(non_occluded,1)) - min(landmarks(non_occluded,1))); + height_gt = (max(landmarks(non_occluded,2)) - min(landmarks(non_occluded,2))); + + width = width_gt * scale_x; + height = height_gt * scale_y; + + x = min(landmarks(non_occluded,1)) + width * tx; + y = min(landmarks(non_occluded,2)) + height * ty; + + % Add like 5 more + detections = cat(1, detections, [x, y, x+ width, y+height]); + + images(ind).img = [dataset_loc, name]; + labels(ind,:,:) = landmarks; + +% imshow(imread(images(ind).img)); +% hold on; +% rectangle('position', [x, y, width, height]); +% hold off; + + ind = ind + 1; + end + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_JANUS/Display_dclm_results.m b/matlab_version/experiments_JANUS/Display_dclm_results.m new file mode 100644 index 00000000..9564f707 --- /dev/null +++ b/matlab_version/experiments_JANUS/Display_dclm_results.m @@ -0,0 +1,213 @@ +clear + +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'b','DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); +hold on; + +load('results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'r','DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +load('results/JANUS_pocr.mat'); + +% center the pixel +shapes_all = experiments.shapes; + +pocr_error = compute_error(labels, shapes_all); + +[error_x, error_y] = cummErrorCurve(pocr_error); + +plot(error_x, error_y, '.-g','DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width); +hold on; + +% load('results/intraface_wild_resize.mat'); +% labels_all = labels_all(18:end,:,detected); +% % center the pixel +% shapes_all = shapes_all(18:end,:,detected) + 0.5; +% +% intraface_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(intraface_wild_error); +% +% plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width); +% hold on; + +% load('results/GNDPM_300W.mat'); +% % center the pixel +% shapes_all = shapes_all(:,:,detected) + 1; +% labels_all = labels_all(:,:,detected); +% +% gndpm_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(gndpm_wild_error); +% +% plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width); +% hold on; + + +% load('results/zhu_wild.mat'); +% labels_all = labels_all(18:end,:,detected); +% shapes_all = shapes_all(18:end,:,detected); +% +% zhu_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(zhu_wild_error); +% +% plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width); + +% load('results/results_wild_clm.mat'); +% labels = experiments.labels([1:60,62:64,66:end],:,detected); +% shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +% labels = labels(18:end,:,:); +% % center the pixel +% shapes = shapes(18:end,:,:) + 0.5; +% +% clm_error = compute_error( labels, shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); + +% load('results/drmf_wild.mat'); +% labels_all = labels_all(18:end,:,detected); +% shapes_all = shapes_all(18:end,:,detected); +% +% drmf_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(drmf_error); +% +% plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +leg = legend('show', 'Location', 'SouthEast'); +set(leg,'FontSize',30) + +print -dpdf results/Janus-no-outline.pdf +print -dpng results/Janus-no-outline.png + +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); + +load('results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width); + +% load('results/zhu_wild.mat'); +% +% zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:)); +% +% [error_x, error_y] = cummErrorCurve(zhu_wild_error); +% +% plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width); +% +% load('results/yu_wild.mat'); +% +% yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); +% yu_wild_error(isnan(yu_wild_error)) = 1; +% yu_wild_error(isinf(yu_wild_error)) = 1; +% +% [error_x, error_y] = cummErrorCurve(yu_wild_error); +% +% plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); +% +% load('results/results_wild_clm.mat'); +% experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); +% % center the pixel +% experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; +% +% clm_error = compute_error( experiments(1).labels, experiments(1).shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); +% +% load('results/drmf_wild.mat'); +% +% drmf_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(drmf_error); +% +% plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +%title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/Janus-full.pdf +print -dpng results/Janus-full.png diff --git a/matlab_version/experiments_JANUS/Script_CLNF_general_no_out.m b/matlab_version/experiments_JANUS/Script_CLNF_general_no_out.m new file mode 100644 index 00000000..5a9e93f1 --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_CLNF_general_no_out.m @@ -0,0 +1,216 @@ +function Script_CLNF_general_no_out() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +verbose = true; % set to true to visualise the fitting +output_root = './wild_fit_clnf/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; + +% Loading the final scale +[clmParams_inner, pdm_inner] = Load_CLM_params_inner(); +clmParams_inner.window_size = [17,17;19,19;21,21;23,23]; +inds_inner = 18:68; +[patches_inner] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*general_no_out.mat', [], [], clmParams_inner); +clmParams_inner.multi_modal_types = patches_inner(1).multi_modal_types; + +% load('results/results_wild_clnf_general.mat'); +% clear 'experiments'; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = true; + +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + % Perform inner face fitting + shape_inner = shape(inds_inner,:); + + [ a, R, T, ~, l_params, err] = fit_PDM_ortho_proj_to_2D_no_reg(pdm_inner.M, pdm_inner.E, pdm_inner.V, shape_inner); + if(a > 0.9) + g_param = [a; Rot2Euler(R)'; T]; + + bbox = [min(shape_inner(:,1)), min(shape_inner(:,2)), max(shape_inner(:,1)), max(shape_inner(:,2))]; + + [shape_inner] = Fitting_from_bb(image, [], bbox, pdm_inner, patches_inner, clmParams_inner, 'gparam', g_param, 'lparam', l_params); + + % Now after detections incorporate the eyes back + % into the face model + + shape(inds_inner, :) = shape_inner; + + [ ~, ~, ~, ~, ~, ~, shape_fit] = fit_PDM_ortho_proj_to_2D_no_reg(pdm.M, pdm.E, pdm.V, shape); + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape_fit; + else + shapes_all(:,:,i) = shape; + end + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 200)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(:,1), shape(:,2),'.r','MarkerSize',20); + plot(shape(:,1), shape(:,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); + print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_clnf_general_final_inner.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_JANUS/Script_DCLM_general.m b/matlab_version/experiments_JANUS/Script_DCLM_general.m new file mode 100644 index 00000000..fce922cf --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_DCLM_general.m @@ -0,0 +1,184 @@ +function Script_DCLM_general() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +output_root = './wild_fit_dclm/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = true; +verbose = true; +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape; + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 200)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(:,1), shape(:,2),'.r','MarkerSize',20); + plot(shape(:,1), shape(:,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); + print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all, shapes_all + 0.5); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_dclm_general.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_JANUS/Script_DCLM_general_large.m b/matlab_version/experiments_JANUS/Script_DCLM_general_large.m new file mode 100644 index 00000000..8de962ea --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_DCLM_general_large.m @@ -0,0 +1,184 @@ +function Script_DCLM_general_large() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [29,29; 27,27; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +output_root = './wild_fit_dclm/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = true; +verbose = false; +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape; + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 50)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(:,1), shape(:,2),'.r','MarkerSize',20); + plot(shape(:,1), shape(:,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all, shapes_all + 0.5); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_dclm_general_large.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_JANUS/compute_error.m b/matlab_version/experiments_JANUS/compute_error.m new file mode 100644 index 00000000..5f98e2d3 --- /dev/null +++ b/matlab_version/experiments_JANUS/compute_error.m @@ -0,0 +1,43 @@ +function [ error_per_image, err_pp, err_pp_dim ] = compute_error( ground_truth_all, detected_points_all ) +%compute_error +% compute the average point-to-point Euclidean error normalized by the +% inter-ocular distance (measured as the Euclidean distance between the +% outer corners of the eyes) +% +% Inputs: +% grounth_truth_all, size: num_of_points x 2 x num_of_images +% detected_points_all, size: num_of_points x 2 x num_of_images +% Output: +% error_per_image, size: num_of_images x 1 + + +num_of_images = size(ground_truth_all,3); +num_of_points = size(ground_truth_all,1); + +error_per_image = zeros(num_of_images,1); +err_pp = zeros(num_of_images, num_of_points); +err_pp_dim = zeros(num_of_images, num_of_points, 2); + +for i =1:num_of_images + detected_points = detected_points_all(:,:,i); + ground_truth_points = ground_truth_all(:,:,i); + visible = ground_truth_points(:,1) > 0; + normalization_x = max(ground_truth_points(visible,1)) - min(ground_truth_points(visible,1)); + normalization_y = max(ground_truth_points(visible,2)) - min(ground_truth_points(visible,2)); + normalization = (normalization_x + normalization_y)/2; + + sum_c=0; + for j=1:num_of_points + if(visible(j)) + sum_c = sum_c+norm(detected_points(j,:)-ground_truth_points(j,:)); + err_pp(i,j) = norm(detected_points(j,:)-ground_truth_points(j,:)); + err_pp_dim(i,j,1) = detected_points(j,1)-ground_truth_points(j,1); + err_pp_dim(i,j,2) = detected_points(j,2)-ground_truth_points(j,2); + end + end + error_per_image(i) = sum_c/(sum(visible)*normalization); + err_pp(i,:) = err_pp(i,:) ./ normalization; + err_pp_dim(i,:) = err_pp_dim(i,:) ./ normalization; +end + +end diff --git a/matlab_version/experiments_JANUS/cummErrorCurve.m b/matlab_version/experiments_JANUS/cummErrorCurve.m new file mode 100644 index 00000000..e18a1ada --- /dev/null +++ b/matlab_version/experiments_JANUS/cummErrorCurve.m @@ -0,0 +1,19 @@ +function [x, y] = cummErrorCurve( errorVec ) +%CUMMERRORCURVE Summary of this function goes here +% Detailed explanation goes here + + + spacing = 0.001; + + sampling = [0:spacing:max(errorVec)]; + + x = sampling; + + y = zeros(numel(sampling,1)); + + for i=1:numel(sampling) + + y(i) = sum(errorVec < sampling(i)) / numel(errorVec); + end +end + diff --git a/matlab_version/experiments_JANUS/results/JANUS_pocr.mat b/matlab_version/experiments_JANUS/results/JANUS_pocr.mat new file mode 100644 index 00000000..77f36cad Binary files /dev/null and b/matlab_version/experiments_JANUS/results/JANUS_pocr.mat differ diff --git a/matlab_version/experiments_JANUS/results/Janus-full.pdf b/matlab_version/experiments_JANUS/results/Janus-full.pdf new file mode 100644 index 00000000..38962545 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/Janus-full.pdf differ diff --git a/matlab_version/experiments_JANUS/results/Janus-full.png b/matlab_version/experiments_JANUS/results/Janus-full.png new file mode 100644 index 00000000..23c0dc7f Binary files /dev/null and b/matlab_version/experiments_JANUS/results/Janus-full.png differ diff --git a/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf b/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf new file mode 100644 index 00000000..1f9575aa Binary files /dev/null and b/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf differ diff --git a/matlab_version/experiments_JANUS/results/Janus-no-outline.png b/matlab_version/experiments_JANUS/results/Janus-no-outline.png new file mode 100644 index 00000000..85969f3a Binary files /dev/null and b/matlab_version/experiments_JANUS/results/Janus-no-outline.png differ diff --git a/matlab_version/experiments_JANUS/results/results_wild_clnf_general_final_inner.mat b/matlab_version/experiments_JANUS/results/results_wild_clnf_general_final_inner.mat new file mode 100644 index 00000000..86452079 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/results_wild_clnf_general_final_inner.mat differ diff --git a/matlab_version/experiments_JANUS/results/results_wild_dclm_general.mat b/matlab_version/experiments_JANUS/results/results_wild_dclm_general.mat new file mode 100644 index 00000000..8811d866 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/results_wild_dclm_general.mat differ diff --git a/matlab_version/experiments_JANUS/results/results_wild_dclm_general_large.mat b/matlab_version/experiments_JANUS/results/results_wild_dclm_general_large.mat new file mode 100644 index 00000000..69a91218 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/results_wild_dclm_general_large.mat differ diff --git a/matlab_version/experiments_iccv_300w/Display_dclm_results.m b/matlab_version/experiments_iccv_300w/Display_dclm_results.m new file mode 100644 index 00000000..4cdf6836 --- /dev/null +++ b/matlab_version/experiments_iccv_300w/Display_dclm_results.m @@ -0,0 +1,233 @@ +clear + +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/intraface_wild_resize.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +intraface_wild_error = compute_error(labels_all, shapes_all); + +% removing faces that were not detected by intraface for fairness +detected = intraface_wild_error < 1; + +% load('results/results_wild_clnf.mat'); +% labels = experiments.labels([1:60,62:64,66:end],:,detected); +% shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +% labels = labels(18:end,:,:); +% % center the pixel +% shapes = shapes(18:end,:,:) + 0.5; +% +% clnf_error = compute_error( labels, shapes); +% +% [error_x, error_y] = cummErrorCurve(clnf_error); +% +% plot(error_x, error_y, 'DisplayName', 'CLNF wild', 'LineWidth',line_width); +% hold on; + +load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,detected); +shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:) + 0.5; + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); +hold on; + +load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,detected); +shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +labels = labels(18:end,:,:) - 1.0; +shapes = shapes(18:end,:,:); + +dclm_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(dclm_error); + +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width); + + +load('results/300W_pocr.mat'); + +% center the pixel +shapes_all = experiments.shapes; + +pocr_error = compute_error(labels, shapes_all(:,:,detected)); + +[error_x, error_y] = cummErrorCurve(pocr_error); + +plot(error_x, error_y, 'DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width); +hold on; + +load('results/intraface_wild_resize.mat'); +labels_all = labels_all(18:end,:,detected); +% center the pixel +shapes_all = shapes_all(18:end,:,detected) + 0.5; + +intraface_wild_error = compute_error(labels_all, shapes_all); + +[error_x, error_y] = cummErrorCurve(intraface_wild_error); + +plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width); +hold on; + +load('results/GNDPM_300W.mat'); +% center the pixel +shapes_all = shapes_all(:,:,detected) + 1; +labels_all = labels_all(:,:,detected); + +gndpm_wild_error = compute_error(labels_all, shapes_all); + +[error_x, error_y] = cummErrorCurve(gndpm_wild_error); + +plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width); +hold on; + + +load('results/zhu_wild.mat'); +labels_all = labels_all(18:end,:,detected); +shapes_all = shapes_all(18:end,:,detected); + +zhu_wild_error = compute_error(labels_all, shapes_all); + +[error_x, error_y] = cummErrorCurve(zhu_wild_error); + +plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width); + +load('results/results_wild_clm.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,detected); +shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:) + 0.5; + +clm_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); + +load('results/drmf_wild.mat'); +labels_all = labels_all(18:end,:,detected); +shapes_all = shapes_all(18:end,:,detected); + +drmf_error = compute_error(labels_all, shapes_all); + +[error_x, error_y] = cummErrorCurve(drmf_error); + +plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.08]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +leg = legend('show', 'Location', 'SouthEast'); +set(leg,'FontSize',30) + +print -dpdf results/dclm-in-the-wild-clnf-no-outline.pdf +print -dpng results/dclm-300W-no-outline.png +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 1; + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); + +load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 0.5; + +dclm_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(dclm_error); + +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width); + +load('results/zhu_wild.mat'); + +zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:)); + +[error_x, error_y] = cummErrorCurve(zhu_wild_error); + +plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width); + +load('results/yu_wild.mat'); + +yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); +yu_wild_error(isnan(yu_wild_error)) = 1; +yu_wild_error(isinf(yu_wild_error)) = 1; + +[error_x, error_y] = cummErrorCurve(yu_wild_error); + +plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); + +load('results/results_wild_clm.mat'); +experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); +% center the pixel +experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; + +clm_error = compute_error( experiments(1).labels, experiments(1).shapes); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); + +load('results/drmf_wild.mat'); + +drmf_error = compute_error(labels_all, shapes_all); + +[error_x, error_y] = cummErrorCurve(drmf_error); + +plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +%title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/dclm-in-the-wild-comparison.pdf +print -dpng results/dclm-300W-outline.png \ No newline at end of file diff --git a/matlab_version/experiments_iccv_300w/results/300W_pocr.mat b/matlab_version/experiments_iccv_300w/results/300W_pocr.mat new file mode 100644 index 00000000..0c19af0c Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/300W_pocr.mat differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png b/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png new file mode 100644 index 00000000..6abfef55 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png b/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png new file mode 100644 index 00000000..0200c003 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf new file mode 100644 index 00000000..fbe7c1f2 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf new file mode 100644 index 00000000..0d3c372c Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline.pdf b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline.pdf index 4ce5972b..ecb2602d 100644 Binary files a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline.pdf and b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf b/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf index 2f152c71..e60e5d2d 100644 Binary files a/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf and b/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf differ diff --git a/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m b/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m index 90e3b013..c0598216 100644 --- a/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m +++ b/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m @@ -25,21 +25,7 @@ clnf_error = compute_error( labels, shapes); [error_x, error_y] = cummErrorCurve(clnf_error); hold on; -plot(error_x, error_y, 'DisplayName', 'CLNF - 4 scales', 'LineWidth',line_width); - -load('results/results_wild_clnf_general.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,:); -shapes = experiments.shapes([1:60,62:64,66:end],:,:); -labels = labels(18:end,:,:) - 0.5; -shapes = shapes(18:end,:,:); - -clnf_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clnf_error); -hold on; - -plot(error_x, error_y, 'DisplayName', 'CLNF - 1 scale', 'LineWidth',line_width); - +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); load('results/results_wild_dclm_general.mat'); labels = experiments.labels([1:60,62:64,66:end],:,:); @@ -51,7 +37,7 @@ clm_error = compute_error( labels, shapes); [error_x, error_y] = cummErrorCurve(clm_error); -plot(error_x, error_y, 'DisplayName', 'DCLM - 1 scale', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width); set(gca,'xtick',[0:0.01:0.08]) xlim([0,0.08]); diff --git a/matlab_version/experiments_in_the_wild/Script_DCLM_general.m b/matlab_version/experiments_in_the_wild/Script_DCLM_general.m index 7175f7d4..a36550f6 100644 --- a/matlab_version/experiments_in_the_wild/Script_DCLM_general.m +++ b/matlab_version/experiments_in_the_wild/Script_DCLM_general.m @@ -19,7 +19,7 @@ end clmParams = struct; -clmParams.window_size = [25,25; 23,23; 21,21;]; +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; clmParams.numPatchIters = size(clmParams.window_size,1); [patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams); @@ -38,9 +38,9 @@ pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); -clmParams.regFactor = [35, 27, 20]; -clmParams.sigmaMeanShift = [1.25, 1.375, 1.5]; -clmParams.tikhonov_factor = [2.5, 5, 7.5]; +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; clmParams.startScale = 1; clmParams.num_RLMS_iter = 10; @@ -48,7 +48,7 @@ clmParams.fTol = 0.01; clmParams.useMultiScale = true; clmParams.use_multi_modal = 1; clmParams.multi_modal_types = patches(1).multi_modal_types; -clmParams.numPatchIters = 3; +clmParams.numPatchIters = 4; % for recording purposes experiment.params = clmParams; diff --git a/matlab_version/experiments_in_the_wild/results/dclm_results.png b/matlab_version/experiments_in_the_wild/results/dclm_results.png index b5f0da16..c04fdfc1 100644 Binary files a/matlab_version/experiments_in_the_wild/results/dclm_results.png and b/matlab_version/experiments_in_the_wild/results/dclm_results.png differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general.mat b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general.mat index 4996d9f5..3151a0b2 100644 Binary files a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general.mat and b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general.mat differ diff --git a/matlab_version/fitting/PatchResponseDNN.m b/matlab_version/fitting/PatchResponseDNN.m index ac6884a1..2e2db241 100644 --- a/matlab_version/fitting/PatchResponseDNN.m +++ b/matlab_version/fitting/PatchResponseDNN.m @@ -17,7 +17,7 @@ function [ responses ] = PatchResponseDNN(patches, patch_experts_class, visibili smallRegionVec = patches(i,:); smallRegion = reshape(smallRegionVec, window_size(1), window_size(2)); - patch = im2col(smallRegion, patchSize, 'sliding')'; + patch = im2col_mine(smallRegion, patchSize)'; % Normalize if(col_norm) @@ -43,7 +43,8 @@ function [ responses ] = PatchResponseDNN(patches, patch_experts_class, visibili patch_normed = weights{(w-1)*2+1}' * patch_normed + repmat(weights{(w-1)*2+2}', 1, size(patch_normed,2)); if w < 3 - patch_normed(patch_normed < 0) = 0; +% patch_normed(patch_normed < 0) = 0; + patch_normed = max(0, patch_normed); else patch_normed = 1./(1+exp(-patch_normed)); end diff --git a/matlab_version/fitting/im2col_mine.m b/matlab_version/fitting/im2col_mine.m new file mode 100644 index 00000000..fd90c8fd --- /dev/null +++ b/matlab_version/fitting/im2col_mine.m @@ -0,0 +1,80 @@ +function b=im2col_mine(a, block) +%IM2COL Rearrange image blocks into columns. +% B = IM2COL(A,[M N],'distinct') rearranges each distinct +% M-by-N block in the image A into a column of B. IM2COL pads A +% with zeros, if necessary, so its size is an integer multiple +% of M-by-N. If A = [A11 A12; A21 A22], where each Aij is +% M-by-N, then B = [A11(:) A21(:) A12(:) A22(:)]. +% +% B = IM2COL(A,[M N],'sliding') converts each sliding M-by-N +% block of A into a column of B, with no zero padding. B has +% M*N rows and will contain as many columns as there are M-by-N +% neighborhoods in A. If the size of A is [MM NN], then the +% size of B is (M*N)-by-((MM-M+1)*(NN-N+1). Each column of B +% contains the neighborhoods of A reshaped as NHOOD(:), where +% NHOOD is a matrix containing an M-by-N neighborhood of +% A. IM2COL orders the columns of B so that they can be +% reshaped to form a matrix in the normal way. For example, +% suppose you use a function, such as SUM(B), that returns a +% scalar for each column of B. You can directly store the +% result in a matrix of size (MM-M+1)-by-(NN-N+1) using these +% calls: +% +% B = im2col(A,[M N],'sliding'); +% C = reshape(sum(B),MM-M+1,NN-N+1); +% +% B = IM2COL(A,[M N]) uses the default block type of +% 'sliding'. +% +% B = IM2COL(A,'indexed',...) processes A as an indexed image, +% padding with zeros if the class of A is uint8 or uint16, or +% ones if the class of A is double. +% +% Class Support +% ------------- +% The input image A can be numeric or logical. The output matrix +% B is of the same class as the input image. +% +% Example +% ------- +% Calculate the local mean using a [2 2] neighborhood with zero padding. +% +% A = reshape(linspace(0,1,16),[4 4])' +% B = im2col(A,[2 2]) +% M = mean(B) +% newA = col2im(M,[1 1],[3 3]) +% +% See also BLOCKPROC, COL2IM, COLFILT, NLFILTER. + +% Copyright 1993-2012 The MathWorks, Inc. + +[ma,na] = size(a); +m = block(1); n = block(2); + +if any([ma na] < [m n]) % if neighborhood is larger than image + b = zeros(m*n,0); + return +end + +% Create Hankel-like indexing sub matrix. +mc = block(1); nc = ma-m+1; nn = na-n+1; +cidx = (0:mc-1)'; ridx = 1:nc; +t = cidx(:,ones(nc,1)) + ridx(ones(mc,1),:); % Hankel Subscripts +tt = zeros(mc*n,nc); +rows = 1:mc; +for i=0:n-1, + tt(i*mc+rows,:) = t+ma*i; +end +ttt = zeros(mc*n,nc*nn); +cols = 1:nc; +for j=0:nn-1, + ttt(:,j*nc+cols) = tt+ma*j; +end + +% If a is a row vector, change it to a column vector. This change is +% necessary when A is a row vector and [M N] = size(A). +if ndims(a) == 2 && na > 1 && ma == 1 + a = a(:); +end +b = a(ttt); +