diff --git a/.gitignore b/.gitignore index 967a668e..1cb99f71 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,14 @@ matlab_version/experiments_JANUS/wild_fit_clnf/ matlab_version/experiments_JANUS/wild_fit_dclm/ exe/FeatureExtraction/300VW_features_experiments/ matlab_runners/Feature Point Experiments/300VW_features_experiments/ +matlab_version/experiments_300VW/CLM_res/ +matlab_version/experiments_300VW/CLNF_res/ +matlab_version/experiments_300VW/DCLM_res/ +matlab_version/experiments_in_the_wild/patch_expert_responses/ +matlab_version/experiments_in_the_wild/wild_fit_clnf/ +matlab_version/experiments_in_the_wild/wild_fit_dclm/ +matlab_version/experiments_iccv_300w/wild_fit_cfss/ +matlab_version/experiments_iccv_300w/wild_fit_clnf/ +matlab_version/experiments_iccv_300w/wild_fit_dclm/ +matlab_version/experiments_JANUS/all_fit_cvpr/ +matlab_version/experiments_JANUS/cfss_fit/ diff --git a/matlab_version/experiments_300VW/CLM_300VW.m b/matlab_version/experiments_300VW/CLM_300VW.m new file mode 100644 index 00000000..1f6d38d3 --- /dev/null +++ b/matlab_version/experiments_300VW/CLM_300VW.m @@ -0,0 +1,157 @@ +clear; +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +output_dir = './CLM_res/'; + +%% select database and load bb initializations +db_root = 'D:\Datasets\300VW_Dataset_2015_12_14\300VW_Dataset_2015_12_14/'; +bb_root = '../..//matlab_runners/Feature Point Experiments/300VW_face_dets/'; +extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; + +in_dirs = dir(db_root); +in_dirs = in_dirs(3:end); +in_dirs = in_dirs(1:end-2); +preds_all = []; +gts_all = []; + +%% loading the patch experts and the PDM +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 19,19; 17,17;]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams); + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); +num_points = numel(pdm.M) / 3; + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [0,0,0, 0]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +multi_view = true; +verbose = true; + +%% Select video +for i=1:numel(in_dirs) + + in_file_name = [db_root '/', in_dirs(i).name, '/vid.avi']; + + vid = VideoReader(in_file_name); + + bounding_boxes = dlmread([bb_root, in_dirs(i).name, '_dets.txt'], ','); + preds = []; + n_frames = size(bounding_boxes,1); + for f=1:n_frames + input_image = readFrame(vid); + + reset = true; + + %% Initialize from detected bounding boxes every 30 frames + if (mod(f-1, 30) == 0) + ind = min(f, size(bounding_boxes,1)); + bb = bounding_boxes(ind, :); + % If no face detected use the closest detected BB + if(bb(3) == 0) + ind_next = ind + find(bounding_boxes(ind+1:end,3), 1); + if(isempty(bounding_boxes(ind_next)) || bounding_boxes(ind_next,3)==0) + ind_next = find(bounding_boxes(1:ind,3), 1, 'last'); + end + bb = bounding_boxes(ind_next, :); + end + bb(3) = bb(1) + bb(3); + bb(4) = bb(2) + bb(4); + reset = true; + else + reset = false; + end + + % have a multi-view version for initialization, otherwise use + % previous shape + if(reset && multi_view) + clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; + clmParams.numPatchIters = 4; + clmParams.startScale = 1; + + views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),g_param,l_param,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + clmParams.window_size = [23,23; 21,21; 19,19; 17,17]; + clmParams.numPatchIters = 3; + clmParams.startScale = 2; + [shape,g_param,l_param,lhood,lmark_lhood,view_used] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'gparam', g_param, 'lparam', l_param); + end + + preds = cat(3, preds, shape); + + %% plot the result + imshow(input_image); + hold on; + plot(shape(:,1), shape(:,2), '.r'); +% rectangle('Position', [bb(2), bb(1), bb(4), bb(3)]); + hold off; + drawnow expose + + end + + %% Grab the ground truth + fps_all = dir([db_root, '/', in_dirs(i).name, '/annot/*.pts']); + gt_landmarks = zeros([68, 2, size(fps_all)]); + for k = 1:size(fps_all) + + gt_landmarks_frame = dlmread([db_root, '/', in_dirs(i).name, '/annot/', fps_all(k).name], ' ', 'A4..B71'); + gt_landmarks(:,:,k) = gt_landmarks_frame; + end + + if(size(gt_landmarks,3) ~= size(preds,3)) + fprintf('something went wrong with vid %d\n', i); + end + + % Remove unreliable frames + if(exist([extra_dir, '/', in_dirs(i).name, '.mat'], 'file')) + load([extra_dir, '/', in_dirs(i).name, '.mat']); + gt_landmarks(:,:,int32(error)) = []; + preds(:,:,int32(error))=[]; + end + + vid_name = in_dirs(i).name; + save([output_dir, '/', vid_name], 'preds', 'gt_landmarks'); +end +% [pocr_error, err_pp_pocr] = compute_error( gts_all, preds_all); + + + diff --git a/matlab_version/experiments_300VW/CLNF_300VW.m b/matlab_version/experiments_300VW/CLNF_300VW.m new file mode 100644 index 00000000..e2febb3c --- /dev/null +++ b/matlab_version/experiments_300VW/CLNF_300VW.m @@ -0,0 +1,157 @@ +clear; +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +output_dir = './CLNF_res/'; + +%% select database and load bb initializations +db_root = 'D:\Datasets\300VW_Dataset_2015_12_14\300VW_Dataset_2015_12_14/'; +bb_root = '../..//matlab_runners/Feature Point Experiments/300VW_face_dets/'; +extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; + +in_dirs = dir(db_root); +in_dirs = in_dirs(3:end); +in_dirs = in_dirs(1:end-2); +preds_all = []; +gts_all = []; + +%% loading the patch experts and the PDM +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 19,19; 17,17;]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); +num_points = numel(pdm.M) / 3; + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [0,0,0, 0]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +multi_view = true; +verbose = true; + +%% Select video +for i=1:numel(in_dirs) + + in_file_name = [db_root '/', in_dirs(i).name, '/vid.avi']; + + vid = VideoReader(in_file_name); + + bounding_boxes = dlmread([bb_root, in_dirs(i).name, '_dets.txt'], ','); + preds = []; + n_frames = size(bounding_boxes,1); + for f=1:n_frames + input_image = readFrame(vid); + + reset = true; + + %% Initialize from detected bounding boxes every 30 frames + if (mod(f-1, 30) == 0) + ind = min(f, size(bounding_boxes,1)); + bb = bounding_boxes(ind, :); + % If no face detected use the closest detected BB + if(bb(3) == 0) + ind_next = ind + find(bounding_boxes(ind+1:end,3), 1); + if(isempty(bounding_boxes(ind_next)) || bounding_boxes(ind_next,3)==0) + ind_next = find(bounding_boxes(1:ind,3), 1, 'last'); + end + bb = bounding_boxes(ind_next, :); + end + bb(3) = bb(1) + bb(3); + bb(4) = bb(2) + bb(4); + reset = true; + else + reset = false; + end + + % have a multi-view version for initialization, otherwise use + % previous shape + if(reset && multi_view) + clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; + clmParams.numPatchIters = 4; + clmParams.startScale = 1; + + views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),g_param,l_param,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + clmParams.window_size = [23,23; 21,21; 19,19; 17,17]; + clmParams.numPatchIters = 3; + clmParams.startScale = 2; + [shape,g_param,l_param,lhood,lmark_lhood,view_used] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'gparam', g_param, 'lparam', l_param); + end + + preds = cat(3, preds, shape); + + %% plot the result + imshow(input_image); + hold on; + plot(shape(:,1), shape(:,2), '.r'); +% rectangle('Position', [bb(2), bb(1), bb(4), bb(3)]); + hold off; + drawnow expose + + end + + %% Grab the ground truth + fps_all = dir([db_root, '/', in_dirs(i).name, '/annot/*.pts']); + gt_landmarks = zeros([68, 2, size(fps_all)]); + for k = 1:size(fps_all) + + gt_landmarks_frame = dlmread([db_root, '/', in_dirs(i).name, '/annot/', fps_all(k).name], ' ', 'A4..B71'); + gt_landmarks(:,:,k) = gt_landmarks_frame; + end + + if(size(gt_landmarks,3) ~= size(preds,3)) + fprintf('something went wrong with vid %d\n', i); + end + + % Remove unreliable frames + if(exist([extra_dir, '/', in_dirs(i).name, '.mat'], 'file')) + load([extra_dir, '/', in_dirs(i).name, '.mat']); + gt_landmarks(:,:,int32(error)) = []; + preds(:,:,int32(error))=[]; + end + + vid_name = in_dirs(i).name; + save([output_dir, '/', vid_name], 'preds', 'gt_landmarks'); +end +% [pocr_error, err_pp_pocr] = compute_error( gts_all, preds_all); + + + diff --git a/matlab_version/experiments_300VW/DCLM_300VW.m b/matlab_version/experiments_300VW/DCLM_300VW.m new file mode 100644 index 00000000..d2feea45 --- /dev/null +++ b/matlab_version/experiments_300VW/DCLM_300VW.m @@ -0,0 +1,156 @@ +clear; +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +output_dir = './DCLM_res/'; + +%% select database and load bb initializations +db_root = 'D:\Datasets\300VW_Dataset_2015_12_14\300VW_Dataset_2015_12_14/'; +bb_root = '../..//matlab_runners/Feature Point Experiments/300VW_face_dets/'; +extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; + +in_dirs = dir(db_root); +in_dirs = in_dirs(3:end); +in_dirs = in_dirs(1:end-2); +preds_all = []; +gts_all = []; + +%% loading the patch experts and the PDM +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams); + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; pdm.M = double(M); pdm.E = double(E); pdm.V = double(V); +num_points = numel(pdm.M) / 3; + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [0,0,0,0]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +multi_view = true; +verbose = true; + +%% Select video +for i=1:numel(in_dirs) + + in_file_name = [db_root '/', in_dirs(i).name, '/vid.avi']; + + vid = VideoReader(in_file_name); + + bounding_boxes = dlmread([bb_root, in_dirs(i).name, '_dets.txt'], ','); + preds = []; + n_frames = size(bounding_boxes,1); + for f=1:n_frames + input_image = readFrame(vid); + + reset = true; + + %% Initialize from detected bounding boxes every 30 frames + if (mod(f-1, 30) == 0) + ind = min(f, size(bounding_boxes,1)); + bb = bounding_boxes(ind, :); + % If no face detected use the closest detected BB + if(bb(3) == 0) + ind_next = ind + find(bounding_boxes(ind+1:end,3), 1); + if(isempty(bounding_boxes(ind_next)) || bounding_boxes(ind_next,3)==0) + ind_next = find(bounding_boxes(1:ind,3), 1, 'last'); + end + bb = bounding_boxes(ind_next, :); + end + bb(3) = bb(1) + bb(3); + bb(4) = bb(2) + bb(4); + reset = true; + else + reset = false; + end + + % have a multi-view version for initialization, otherwise use + % previous shape + if(reset && multi_view) + clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; + clmParams.numPatchIters = 4; + clmParams.startScale = 1; + views = [0,0,0; 0,-45,0; -30,0,0; 0,45,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),g_param,l_param,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + clmParams.window_size = [23,23; 21,21; 19,19; 17,17]; + clmParams.numPatchIters = 3; + clmParams.startScale = 2; + [shape,g_param,l_param,lhood,lmark_lhood,view_used] = Fitting_from_bb(input_image, [], bb, pdm, patches, clmParams, 'gparam', g_param, 'lparam', l_param); + end + + preds = cat(3, preds, shape); + + %% plot the result + imshow(input_image); + hold on; + plot(shape(:,1), shape(:,2), '.r'); +% rectangle('Position', [bb(2), bb(1), bb(4), bb(3)]); + hold off; + drawnow expose + + end + + %% Grab the ground truth + fps_all = dir([db_root, '/', in_dirs(i).name, '/annot/*.pts']); + gt_landmarks = zeros([68, 2, size(fps_all)]); + for k = 1:size(fps_all) + + gt_landmarks_frame = dlmread([db_root, '/', in_dirs(i).name, '/annot/', fps_all(k).name], ' ', 'A4..B71'); + gt_landmarks(:,:,k) = gt_landmarks_frame; + end + + if(size(gt_landmarks,3) ~= size(preds,3)) + fprintf('something went wrong with vid %d\n', i); + end + + % Remove unreliable frames + if(exist([extra_dir, '/', in_dirs(i).name, '.mat'], 'file')) + load([extra_dir, '/', in_dirs(i).name, '.mat']); + gt_landmarks(:,:,int32(error)) = []; + preds(:,:,int32(error))=[]; + end + + vid_name = in_dirs(i).name; + save([output_dir, '/', vid_name], 'preds', 'gt_landmarks'); +end +% [pocr_error, err_pp_pocr] = compute_error( gts_all, preds_all); + + + diff --git a/matlab_version/experiments_300VW/Display_300VW_results.m b/matlab_version/experiments_300VW/Display_300VW_results.m new file mode 100644 index 00000000..1abf1fca --- /dev/null +++ b/matlab_version/experiments_300VW/Display_300VW_results.m @@ -0,0 +1,109 @@ +clear + +%% Gather predictions and ground truth +% d_loc = '300VW_features/'; +% extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; +% +% +% files_yt = dir([d_loc, '/*.txt']); +% preds_all = []; +% gts_all = []; +% confs = []; +% for i = 1:numel(files_yt) +% [~, name, ~] = fileparts(files_yt(i).name); +% pred_landmarks = dlmread([d_loc, files_yt(i).name], ',', 1, 0); +% conf = pred_landmarks(:,3); +% pred_landmarks = pred_landmarks(:,5:end); +% +% xs = pred_landmarks(:, 1:end/2); +% ys = pred_landmarks(:, end/2+1:end); +% pred_landmarks = zeros([size(xs,2), 2, size(xs,1)]); +% pred_landmarks(:,1,:) = xs'; +% pred_landmarks(:,2,:) = ys'; +% +% name = name(1:end-3); +% fps_all = dir([database_root, '/', name, '/annot/*.pts']); +% gt_landmarks = zeros(size(pred_landmarks)); +% for k = 1:size(fps_all) +% +% gt_landmarks_frame = dlmread([database_root, '/', name, '/annot/', fps_all(k).name], ' ', 'A4..B71'); +% gt_landmarks(:,:,k) = gt_landmarks_frame; +% end +% +% if(size(pred_landmarks,3) ~= size(fps_all)) +% fprintf('something wrong at vid %s, fps - %d, preds - %d\n', name, gt_landmarks); +% end +% +% % Remove unreliable frames +% if(exist([extra_dir, '/', name, '.mat'], 'file')) +% load([extra_dir, '/', name, '.mat']); +% gt_landmarks(:,:,int32(error)) = []; +% pred_landmarks(:,:,int32(error))=[]; +% conf(int32(error)) = []; +% end +% +% preds_all = cat(3, preds_all, pred_landmarks); +% gts_all = cat(3, gts_all, gt_landmarks); +% confs = cat(1, confs, conf); +% end +%% +[clnf_error, err_pp_clnf] = compute_error( gts_all - 0.5, preds_all); +preds_all_49 = preds_all; +preds_all_49(65,:,:) = []; +preds_all_49(61,:,:) = []; +preds_all_49 = preds_all_49(18:end,:,:); + +gts_all_49 = gts_all; +gts_all_49(65,:,:) = []; +gts_all_49(61,:,:) = []; +gts_all_49 = gts_all_49(18:end,:,:); + +[clnf_error49, err_pp_clnf49] = compute_error( gts_all_49 - 0.5, preds_all_49); + +filename = sprintf('results/300VW_CLNF'); +save(filename, 'clnf_error', 'clnf_error49'); + +%% +line_width = 2; + +% [error_x, error_y] = cummErrorCurve(clnf_error49); +% plot(error_x, error_y, 'DisplayName', 'OpenFace-49', 'LineWidth',line_width); +% hold on; + +load('results/300VW_SDM.mat'); +[error_x, error_y] = cummErrorCurve(sdm_error); +plot(error_x, error_y, 'DisplayName', 'SDM-49', 'LineWidth',line_width); +hold on; + +load('results/300VW_chehra.mat'); +[error_x, error_y] = cummErrorCurve(chehra_error); +plot(error_x, error_y, 'DisplayName', 'Chehra-49', 'LineWidth',line_width); + +load('results/300VW_pocr.mat'); +[error_x, error_y] = cummErrorCurve(pocr_error); +plot(error_x, error_y, 'DisplayName', 'PO-CR-49', 'LineWidth',line_width); + +% Make it look nice and print to a pdf +set(gca,'xtick',[0:0.01:0.08]) +xlim([0,0.08]); +xlabel('Size normalised shape RMS error of 49 landmarks','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +legend('show', 'Location', 'SouthEast'); +print -dpdf results/300VWres_49_114vids.pdf + +%% +line_width = 2; +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'OpenFace-68', 'LineWidth',line_width); +hold on; + +% Make it look nice and print to a pdf +set(gca,'xtick',[0:0.01:0.08]) +xlim([0,0.08]); +xlabel('Size normalised shape RMS error of 68 landmarks','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +legend('show', 'Location', 'SouthEast'); +print -dpdf results/300VWres_68_114vids.pdf + diff --git a/matlab_version/experiments_300VW/Display_300VW_results_cats.m b/matlab_version/experiments_300VW/Display_300VW_results_cats.m new file mode 100644 index 00000000..b7400fb9 --- /dev/null +++ b/matlab_version/experiments_300VW/Display_300VW_results_cats.m @@ -0,0 +1,320 @@ +clear + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +database_root = 'D:\Datasets\300VW_Dataset_2015_12_14\300VW_Dataset_2015_12_14/'; +pocr_dir = 'D:\Dropbox\Dropbox\AAM\3rd party models\PO-CR\300VW/'; +cfss_dir = 'C:\Users\tbaltrus\Documents\300W-CFSS/'; +%% Gather predictions and ground truth +d_loc = '300VW_features/'; +extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; + +cat_1 = [ 114, 124, 125, 126, 150, 158, 401, 402, 505, 506, 507, 508, 509, 510, 511, 514, 515, 518, 519, 520, 521, 522, 524, 525, 537, 538, 540, 541, 546, 547, 548]; +% cat_1 = [ 114, 124, 125, 126, 150, 158, 401, 402, 505, 506, 507, 508, 509, 510, 511, 514, 515, 518, 519, 520, 521, 522, 524, 525, 537]; +cat_2 = [203, 208, 211, 212, 213, 214, 218, 224, 403, 404, 405, 406, 407, 408, 409, 412, 550, 551, 553]; +cat_3 = [410, 411, 516, 517, 526, 528, 529, 530, 531, 533, 557, 558, 559, 562]; + +%% +load('results/cat_ids.mat'); + +%% + +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +pocr_preds = zeros(49,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_1 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([pocr_dir, num2str(i)]); + pocr_preds = cat(3, pocr_preds, preds); + + load([cfss_dir, num2str(i)]); + cfss_preds = cat(3, cfss_preds, preds); +end + +dclm_preds = dclm_preds([1:60,62:64,66:end],:,:); +labels = labels([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +dclm_preds = dclm_preds(18:end,:,:); + +dclm_error = compute_error(labels, dclm_preds); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_preds = clnf_preds([1:60,62:64,66:end],:,:); +clnf_preds = clnf_preds(18:end,:,:); + +clnf_error = compute_error(labels, clnf_preds); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +load('results/300VW_SDM.mat'); +[error_x, error_y] = cummErrorCurve(sdm_error(inds_cat1)); +plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width); + +cfss_preds = cfss_preds([1:60,62:64,66:end],:,:); +cfss_preds = cfss_preds(18:end,:,:); + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +load('results/300VW_chehra.mat'); +[error_x, error_y] = cummErrorCurve(chehra_error(inds_cat1)); +plot(error_x, error_y, 'DisplayName', 'Chehra', 'LineWidth',line_width); + +pocr_error = compute_error(labels, pocr_preds); +[error_x, error_y] = cummErrorCurve(pocr_error); +plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width); + +clm_preds = clm_preds([1:60,62:64,66:end],:,:); +clm_preds = clm_preds(18:end,:,:); + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.01:0.07]) +xlim([0.01,0.07]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_49_cat1.pdf + +%% +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +pocr_preds = zeros(49,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_2 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([pocr_dir, num2str(i)]); + pocr_preds = cat(3, pocr_preds, preds); + + load([cfss_dir, num2str(i)]); + cfss_preds = cat(3, cfss_preds, preds); +end + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +dclm_preds = dclm_preds([1:60,62:64,66:end],:,:); +labels = labels([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +dclm_preds = dclm_preds(18:end,:,:); + +dclm_error = compute_error(labels, dclm_preds); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_preds = clnf_preds([1:60,62:64,66:end],:,:); +clnf_preds = clnf_preds(18:end,:,:); + +clnf_error = compute_error(labels, clnf_preds); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +load('results/300VW_SDM.mat'); +[error_x, error_y] = cummErrorCurve(sdm_error(inds_cat2)); +plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width); + +cfss_preds = cfss_preds([1:60,62:64,66:end],:,:); +cfss_preds = cfss_preds(18:end,:,:); + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + +load('results/300VW_chehra.mat'); +[error_x, error_y] = cummErrorCurve(chehra_error(inds_cat2)); +plot(error_x, error_y, 'DisplayName', 'Chehra', 'LineWidth',line_width); + +pocr_error = compute_error(labels, pocr_preds); +[error_x, error_y] = cummErrorCurve(pocr_error); +plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width); + +clm_preds = clm_preds([1:60,62:64,66:end],:,:); +clm_preds = clm_preds(18:end,:,:); + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.01:0.07]) +xlim([0.01,0.07]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_49_cat2.pdf + +%% +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +pocr_preds = zeros(49,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_3 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([pocr_dir, num2str(i)]); + pocr_preds = cat(3, pocr_preds, preds); + + load([cfss_dir, num2str(i)]); + cfss_preds = cat(3, cfss_preds, preds); +end + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +dclm_preds = dclm_preds([1:60,62:64,66:end],:,:); +labels = labels([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +dclm_preds = dclm_preds(18:end,:,:); + +dclm_error = compute_error(labels, dclm_preds ); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_preds = clnf_preds([1:60,62:64,66:end],:,:); +clnf_preds = clnf_preds(18:end,:,:); + +clnf_error = compute_error(labels, clnf_preds ); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +load('results/300VW_SDM.mat'); +[error_x, error_y] = cummErrorCurve(sdm_error(inds_cat3)); +plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width); + +cfss_preds = cfss_preds([1:60,62:64,66:end],:,:); +cfss_preds = cfss_preds(18:end,:,:); + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + +load('results/300VW_chehra.mat'); +[error_x, error_y] = cummErrorCurve(chehra_error(inds_cat3)); +plot(error_x, error_y, 'DisplayName', 'Chehra', 'LineWidth',line_width); + +pocr_error = compute_error(labels, pocr_preds); +[error_x, error_y] = cummErrorCurve(pocr_error); +plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width); + +clm_preds = clm_preds([1:60,62:64,66:end],:,:); +clm_preds = clm_preds(18:end,:,:); + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.01:0.07]) +xlim([0.01,0.07]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_49_cat3.pdf + +%% \ No newline at end of file diff --git a/matlab_version/experiments_300VW/Display_300VW_results_cats_68.m b/matlab_version/experiments_300VW/Display_300VW_results_cats_68.m new file mode 100644 index 00000000..a7dd5ed6 --- /dev/null +++ b/matlab_version/experiments_300VW/Display_300VW_results_cats_68.m @@ -0,0 +1,235 @@ +clear + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +database_root = 'D:\Datasets\300VW_Dataset_2015_12_14\300VW_Dataset_2015_12_14/'; +pocr_dir = 'D:\Dropbox\Dropbox\AAM\3rd party models\PO-CR\300VW/'; +cfss_dir = 'C:\Users\tbaltrus\Documents\300W-CFSS/'; +%% Gather predictions and ground truth +d_loc = '300VW_features/'; +extra_dir = 'D:\Datasets\300VW_Dataset_2015_12_14\extra'; + +cat_1 = [ 114, 124, 125, 126, 150, 158, 401, 402, 505, 506, 507, 508, 509, 510, 511, 514, 515, 518, 519, 520, 521, 522, 524, 525, 537, 538, 540, 541, 546, 547, 548]; +% cat_1 = [ 114, 124, 125, 126, 150, 158, 401, 402, 505, 506, 507, 508, 509, 510, 511, 514, 515, 518, 519, 520, 521, 522, 524, 525, 537]; +cat_2 = [203, 208, 211, 212, 213, 214, 218, 224, 403, 404, 405, 406, 407, 408, 409, 412, 550, 551, 553]; +%cat_3 = [410, 411, 516, 517, 526, 528, 529, 530, 531, 533, 557, 558, 559, 562]; +cat_3 = [410, 411, 516, 517, 526, 528]; + +%% +load('results/cat_ids.mat'); + +cfss_dir = 'C:\Users\tbaltrus\Documents\300W-CFSS/'; + +%% + +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_1 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([cfss_dir, num2str(i)]); + + cfss_preds = cat(3, cfss_preds, preds); +end + +dclm_error = compute_error(labels, dclm_preds); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_error = compute_error(labels, clnf_preds); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.02:0.09]) +xlim([0.01,0.09]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_68_cat1.pdf + +%% +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_2 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([cfss_dir, num2str(i)]); + + cfss_preds = cat(3, cfss_preds, preds); +end + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +dclm_error = compute_error(labels, dclm_preds); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_error = compute_error(labels, clnf_preds); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.02:0.09]) +xlim([0.01,0.09]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_68_cat2.pdf + +%% +dclm_preds = zeros(68,2,0); +clnf_preds = zeros(68,2,0); +cfss_preds = zeros(68,2,0); +clm_preds = zeros(68,2,0); + +labels = zeros(68,2,0); +% Load DCLM and CLNF results +for i=cat_3 + load(['DCLM_res/', num2str(i)]); + dclm_preds = cat(3, dclm_preds, preds); + + load(['CLNF_res/', num2str(i)]); + clnf_preds = cat(3, clnf_preds, preds); + + load(['CLM_res/', num2str(i)]); + clm_preds = cat(3, clm_preds, preds); + + labels = cat(3, labels, gt_landmarks); + + load([cfss_dir, num2str(i)]); + + cfss_preds = cat(3, cfss_preds, preds); +end + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +dclm_error = compute_error(labels, dclm_preds ); + +[error_x, error_y] = cummErrorCurve(dclm_error); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +clnf_error = compute_error(labels, clnf_preds ); + +[error_x, error_y] = cummErrorCurve(clnf_error); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +clm_error = compute_error(labels, clm_preds); + +[error_x, error_y] = cummErrorCurve(clm_error); +plot(error_x, error_y, 'DisplayName', 'CLM+', 'LineWidth',line_width); +hold on; + +cfss_error = compute_error(labels, cfss_preds - 1); + +[error_x, error_y] = cummErrorCurve(cfss_error); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + +% Make it look nice and print to a pdf +set(gca,'xtick',[0.01:0.02:0.09]) +xlim([0.01,0.09]); +xlabel('IOD normalized MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +ax=legend('show', 'Location', 'SouthEast'); +ax.FontSize = 50; +print -dpdf results/300VWres_68_cat3.pdf + +%% \ No newline at end of file diff --git a/matlab_version/experiments_300VW/compute_error.m b/matlab_version/experiments_300VW/compute_error.m new file mode 100644 index 00000000..8debb0f8 --- /dev/null +++ b/matlab_version/experiments_300VW/compute_error.m @@ -0,0 +1,41 @@ +function [ error_per_image, err_pp, err_pp_dim ] = compute_error( ground_truth_all, detected_points_all ) +%compute_error +% compute the average point-to-point Euclidean error normalized by the +% inter-ocular distance (measured as the Euclidean distance between the +% outer corners of the eyes) +% +% Inputs: +% grounth_truth_all, size: num_of_points x 2 x num_of_images +% detected_points_all, size: num_of_points x 2 x num_of_images +% Output: +% error_per_image, size: num_of_images x 1 + + +num_of_images = size(ground_truth_all,3); +num_of_points = size(ground_truth_all,1); + +error_per_image = zeros(num_of_images,1); +err_pp = zeros(num_of_images, num_of_points); +err_pp_dim = zeros(num_of_images, num_of_points, 2); + +for i =1:num_of_images + detected_points = detected_points_all(:,:,i); + ground_truth_points = ground_truth_all(:,:,i); + if(num_of_points == 66 || num_of_points == 68) + interocular_distance = norm(ground_truth_points(37,:)-ground_truth_points(46,:)); + else + interocular_distance = norm(ground_truth_points(37-17,:)-ground_truth_points(46-17,:)); + end + sum=0; + for j=1:num_of_points + sum = sum+norm(detected_points(j,:)-ground_truth_points(j,:)); + err_pp(i,j) = norm(detected_points(j,:)-ground_truth_points(j,:)); + err_pp_dim(i,j,1) = detected_points(j,1)-ground_truth_points(j,1); + err_pp_dim(i,j,2) = detected_points(j,2)-ground_truth_points(j,2); + end + error_per_image(i) = sum/(num_of_points*interocular_distance); + err_pp(i,:) = err_pp(i,:) ./ interocular_distance; + err_pp_dim(i,:) = err_pp_dim(i,:) ./ interocular_distance; +end + +end diff --git a/matlab_version/experiments_300VW/cummErrorCurve.m b/matlab_version/experiments_300VW/cummErrorCurve.m new file mode 100644 index 00000000..d0805ab2 --- /dev/null +++ b/matlab_version/experiments_300VW/cummErrorCurve.m @@ -0,0 +1,24 @@ +function [x, y] = cummErrorCurve( errorVec ) +%CUMMERRORCURVE Summary of this function goes here +% Detailed explanation goes here + + + spacing = 0.0005; + + sampling = [0:spacing:max(errorVec)]; + + x = sampling; + + y = zeros(numel(sampling,1)); + + for i=1:numel(sampling) + + y(i) = sum(errorVec < sampling(i)) / numel(errorVec); + end + + for i=1:200 + x = cat(2, x, x(end)+spacing); + y = cat(2, y, 1); + end +end + diff --git a/matlab_version/experiments_300VW/results/300VW_CLNF.mat b/matlab_version/experiments_300VW/results/300VW_CLNF.mat new file mode 100644 index 00000000..bf6063d0 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VW_CLNF.mat differ diff --git a/matlab_version/experiments_300VW/results/300VW_SDM.mat b/matlab_version/experiments_300VW/results/300VW_SDM.mat new file mode 100644 index 00000000..35c0e34f Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VW_SDM.mat differ diff --git a/matlab_version/experiments_300VW/results/300VW_chehra.mat b/matlab_version/experiments_300VW/results/300VW_chehra.mat new file mode 100644 index 00000000..5ad82d25 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VW_chehra.mat differ diff --git a/matlab_version/experiments_300VW/results/300VW_pocr.mat b/matlab_version/experiments_300VW/results/300VW_pocr.mat new file mode 100644 index 00000000..8a15fea5 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VW_pocr.mat differ diff --git a/matlab_version/experiments_300VW/results/300VWres_49_114vids.pdf b/matlab_version/experiments_300VW/results/300VWres_49_114vids.pdf new file mode 100644 index 00000000..fbfc8306 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_49_114vids.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_49_cat1.pdf b/matlab_version/experiments_300VW/results/300VWres_49_cat1.pdf new file mode 100644 index 00000000..bbb5eaba Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_49_cat1.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_49_cat2.pdf b/matlab_version/experiments_300VW/results/300VWres_49_cat2.pdf new file mode 100644 index 00000000..2136513b Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_49_cat2.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_49_cat3.pdf b/matlab_version/experiments_300VW/results/300VWres_49_cat3.pdf new file mode 100644 index 00000000..72e3f4e6 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_49_cat3.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_68_cat1.pdf b/matlab_version/experiments_300VW/results/300VWres_68_cat1.pdf new file mode 100644 index 00000000..ac0c57a4 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_68_cat1.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_68_cat2.pdf b/matlab_version/experiments_300VW/results/300VWres_68_cat2.pdf new file mode 100644 index 00000000..d600975b Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_68_cat2.pdf differ diff --git a/matlab_version/experiments_300VW/results/300VWres_68_cat3.pdf b/matlab_version/experiments_300VW/results/300VWres_68_cat3.pdf new file mode 100644 index 00000000..ffcbe6a8 Binary files /dev/null and b/matlab_version/experiments_300VW/results/300VWres_68_cat3.pdf differ diff --git a/matlab_version/experiments_300VW/results/cat_ids.mat b/matlab_version/experiments_300VW/results/cat_ids.mat new file mode 100644 index 00000000..31f638ce Binary files /dev/null and b/matlab_version/experiments_300VW/results/cat_ids.mat differ diff --git a/matlab_version/experiments_300VW/results/category_errors.mat b/matlab_version/experiments_300VW/results/category_errors.mat new file mode 100644 index 00000000..f6929b35 Binary files /dev/null and b/matlab_version/experiments_300VW/results/category_errors.mat differ diff --git a/matlab_version/experiments_JANUS/Display_dclm_results.m b/matlab_version/experiments_JANUS/Display_dclm_results.m index 9564f707..3bc654e9 100644 --- a/matlab_version/experiments_JANUS/Display_dclm_results.m +++ b/matlab_version/experiments_JANUS/Display_dclm_results.m @@ -14,20 +14,6 @@ axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); line_width = 6; hold on; -load('results/results_wild_clnf_general_final_inner.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,:); -shapes = experiments.shapes([1:60,62:64,66:end],:,:); -labels = labels(18:end,:,:); -% center the pixel -shapes = shapes(18:end,:,:); - -clnf_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clnf_error); - -plot(error_x, error_y, 'b','DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); -hold on; - load('results/results_wild_dclm_general.mat'); labels = experiments.labels([1:60,62:64,66:end],:,:); shapes = experiments.shapes([1:60,62:64,66:end],:,:); @@ -42,16 +28,59 @@ clnf_error = compute_error( labels, shapes); plot(error_x, error_y, 'r','DisplayName', 'DCLM', 'LineWidth',line_width); hold on; -load('results/JANUS_pocr.mat'); +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +load('results/JANUS-CFSS.mat'); % center the pixel +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end +shapes_all = shapes_all([1:60,62:64,66:end],:,:); +shapes_all = shapes_all(18:end,:,:); + +cfss_error = compute_error(labels, shapes_all(:,:,:) - 0.5); + +[error_x, error_y] = cummErrorCurve(cfss_error); + +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +load('results/JANUS_sdm.mat'); + +% center the pixel +shapes_all = experiments.shapes; + +sdm_error = compute_error(labels, shapes_all + 0.5); + +[error_x, error_y] = cummErrorCurve(sdm_error); + +plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width); +hold on; + +load('results/JANUS_pocr.mat'); shapes_all = experiments.shapes; pocr_error = compute_error(labels, shapes_all); [error_x, error_y] = cummErrorCurve(pocr_error); -plot(error_x, error_y, '.-g','DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width); +plot(error_x, error_y,'DisplayName', 'PO-CR', 'LineWidth',line_width); hold on; % load('results/intraface_wild_resize.mat'); @@ -112,9 +141,9 @@ hold on; % % plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); -set(gca,'xtick',[0:0.05:0.15]) -xlim([0,0.15]); -xlabel('Size normalised shape RMS error','FontName','Helvetica'); +set(gca,'xtick',[0:0.02:0.10]) +xlim([0.02,0.10]); +xlabel('Size normalised MSE','FontName','Helvetica'); ylabel('Proportion of images','FontName','Helvetica'); grid on % title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); @@ -139,18 +168,6 @@ axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); line_width = 6; hold on; -load('results/results_wild_clnf_general_final_inner.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,:); -% center the pixel -shapes = experiments.shapes([1:60,62:64,66:end],:,:); - -clnf_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clnf_error); -hold on; - -plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); - load('results/results_wild_dclm_general.mat'); labels = experiments.labels([1:60,62:64,66:end],:,:); % center the pixel @@ -161,7 +178,21 @@ clnf_error = compute_error( labels, shapes); [error_x, error_y] = cummErrorCurve(clnf_error); hold on; -plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width); +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); + +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); + + % load('results/zhu_wild.mat'); % @@ -200,9 +231,25 @@ plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width); % % plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); -set(gca,'xtick',[0:0.05:0.15]) -xlim([0,0.15]); -xlabel('Size normalised shape RMS error','FontName','Helvetica'); +load('results/JANUS-CFSS.mat'); + +% center the pixel +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end +cfss_error = compute_error(labels, shapes_all(:,:,:) - 0.5); + +[error_x, error_y] = cummErrorCurve(cfss_error); + +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +set(gca,'xtick',[0:0.02:0.10]) +xlim([0.02,0.10]); +xlabel('Size normalised MSE','FontName','Helvetica'); ylabel('Proportion of images','FontName','Helvetica'); grid on %title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); diff --git a/matlab_version/experiments_JANUS/Display_janus_results_cvpr2017.m b/matlab_version/experiments_JANUS/Display_janus_results_cvpr2017.m new file mode 100644 index 00000000..3bc654e9 --- /dev/null +++ b/matlab_version/experiments_JANUS/Display_janus_results_cvpr2017.m @@ -0,0 +1,260 @@ +clear + +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'r','DisplayName', 'DCLM', 'LineWidth',line_width); +hold on; + +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +% center the pixel +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); +hold on; + +load('results/JANUS-CFSS.mat'); + +% center the pixel +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end +shapes_all = shapes_all([1:60,62:64,66:end],:,:); +shapes_all = shapes_all(18:end,:,:); + +cfss_error = compute_error(labels, shapes_all(:,:,:) - 0.5); + +[error_x, error_y] = cummErrorCurve(cfss_error); + +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +load('results/JANUS_sdm.mat'); + +% center the pixel +shapes_all = experiments.shapes; + +sdm_error = compute_error(labels, shapes_all + 0.5); + +[error_x, error_y] = cummErrorCurve(sdm_error); + +plot(error_x, error_y, 'DisplayName', 'SDM', 'LineWidth',line_width); +hold on; + +load('results/JANUS_pocr.mat'); +shapes_all = experiments.shapes; + +pocr_error = compute_error(labels, shapes_all); + +[error_x, error_y] = cummErrorCurve(pocr_error); + +plot(error_x, error_y,'DisplayName', 'PO-CR', 'LineWidth',line_width); +hold on; + +% load('results/intraface_wild_resize.mat'); +% labels_all = labels_all(18:end,:,detected); +% % center the pixel +% shapes_all = shapes_all(18:end,:,detected) + 0.5; +% +% intraface_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(intraface_wild_error); +% +% plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width); +% hold on; + +% load('results/GNDPM_300W.mat'); +% % center the pixel +% shapes_all = shapes_all(:,:,detected) + 1; +% labels_all = labels_all(:,:,detected); +% +% gndpm_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(gndpm_wild_error); +% +% plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width); +% hold on; + + +% load('results/zhu_wild.mat'); +% labels_all = labels_all(18:end,:,detected); +% shapes_all = shapes_all(18:end,:,detected); +% +% zhu_wild_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(zhu_wild_error); +% +% plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width); + +% load('results/results_wild_clm.mat'); +% labels = experiments.labels([1:60,62:64,66:end],:,detected); +% shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +% labels = labels(18:end,:,:); +% % center the pixel +% shapes = shapes(18:end,:,:) + 0.5; +% +% clm_error = compute_error( labels, shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); + +% load('results/drmf_wild.mat'); +% labels_all = labels_all(18:end,:,detected); +% shapes_all = shapes_all(18:end,:,detected); +% +% drmf_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(drmf_error); +% +% plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.02:0.10]) +xlim([0.02,0.10]); +xlabel('Size normalised MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +leg = legend('show', 'Location', 'SouthEast'); +set(leg,'FontSize',30) + +print -dpdf results/Janus-no-outline.pdf +print -dpng results/Janus-no-outline.png + +%% +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM', 'LineWidth',line_width); + +load('results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:); + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); + + + +% load('results/zhu_wild.mat'); +% +% zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:)); +% +% [error_x, error_y] = cummErrorCurve(zhu_wild_error); +% +% plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width); +% +% load('results/yu_wild.mat'); +% +% yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); +% yu_wild_error(isnan(yu_wild_error)) = 1; +% yu_wild_error(isinf(yu_wild_error)) = 1; +% +% [error_x, error_y] = cummErrorCurve(yu_wild_error); +% +% plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); +% +% load('results/results_wild_clm.mat'); +% experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); +% % center the pixel +% experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; +% +% clm_error = compute_error( experiments(1).labels, experiments(1).shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); +% +% load('results/drmf_wild.mat'); +% +% drmf_error = compute_error(labels_all, shapes_all); +% +% [error_x, error_y] = cummErrorCurve(drmf_error); +% +% plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); + +load('results/JANUS-CFSS.mat'); + +% center the pixel +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end +cfss_error = compute_error(labels, shapes_all(:,:,:) - 0.5); + +[error_x, error_y] = cummErrorCurve(cfss_error); + +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +set(gca,'xtick',[0:0.02:0.10]) +xlim([0.02,0.10]); +xlabel('Size normalised MSE','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +%title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/Janus-full.pdf +print -dpng results/Janus-full.png diff --git a/matlab_version/experiments_JANUS/Script_CFSS_vis.m b/matlab_version/experiments_JANUS/Script_CFSS_vis.m new file mode 100644 index 00000000..f42f698b --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_CFSS_vis.m @@ -0,0 +1,79 @@ +function Script_CFSS_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + +inds = [50,78,151,195,220,619,883,237,258,299,384,442,526]; +images = images(inds); +labels = labels(inds,:,:); +detections = detections(inds,:); + +load('results/JANUS-CFSS.mat'); +output_root = './all_fit_cvpr/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + pred = estimatedPose(inds(i),:); + preds = cat(2, pred(1:68)', pred(69:end)'); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_cfss.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_JANUS/Script_CLNF_vis.m b/matlab_version/experiments_JANUS/Script_CLNF_vis.m new file mode 100644 index 00000000..60daf5eb --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_CLNF_vis.m @@ -0,0 +1,78 @@ +function Script_CLNF_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + +inds = [48,78,154,194,220,618,883,237,258,299,385,444,529]; +images = images(inds); +labels = labels(inds,:,:); +detections = detections(inds,:); + +load('results/results_wild_clnf_general_final_inner.mat'); +output_root = './all_fit_cvpr/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + preds = experiments.shapes(:,:,inds(i)); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_clnf.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_JANUS/Script_DCLM_vis.m b/matlab_version/experiments_JANUS/Script_DCLM_vis.m new file mode 100644 index 00000000..69b5b000 --- /dev/null +++ b/matlab_version/experiments_JANUS/Script_DCLM_vis.m @@ -0,0 +1,78 @@ +function Script_DCLM_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +root_test_data = 'D:/Datasets/janus_labeled'; + +[images, detections, labels] = Collect_JANUS_imgs(root_test_data); + +inds = [48,78,154,194,220,618,883,237,258,299,385,444,529]; +images = images(inds); +labels = labels(inds,:,:); +detections = detections(inds,:); + +load('results/results_wild_dclm_general.mat'); +output_root = './all_fit_cvpr/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + preds = experiments.shapes(:,:,inds(i)); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_dclm.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose + close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_JANUS/results/JANUS-CFSS.mat b/matlab_version/experiments_JANUS/results/JANUS-CFSS.mat new file mode 100644 index 00000000..363972f7 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/JANUS-CFSS.mat differ diff --git a/matlab_version/experiments_JANUS/results/JANUS_sdm.mat b/matlab_version/experiments_JANUS/results/JANUS_sdm.mat new file mode 100644 index 00000000..436fa9a6 Binary files /dev/null and b/matlab_version/experiments_JANUS/results/JANUS_sdm.mat differ diff --git a/matlab_version/experiments_JANUS/results/Janus-full.pdf b/matlab_version/experiments_JANUS/results/Janus-full.pdf index 38962545..a7fe757d 100644 Binary files a/matlab_version/experiments_JANUS/results/Janus-full.pdf and b/matlab_version/experiments_JANUS/results/Janus-full.pdf differ diff --git a/matlab_version/experiments_JANUS/results/Janus-full.png b/matlab_version/experiments_JANUS/results/Janus-full.png index 23c0dc7f..e3e1306f 100644 Binary files a/matlab_version/experiments_JANUS/results/Janus-full.png and b/matlab_version/experiments_JANUS/results/Janus-full.png differ diff --git a/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf b/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf index 1f9575aa..f4b01f2d 100644 Binary files a/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf and b/matlab_version/experiments_JANUS/results/Janus-no-outline.pdf differ diff --git a/matlab_version/experiments_JANUS/results/Janus-no-outline.png b/matlab_version/experiments_JANUS/results/Janus-no-outline.png index 85969f3a..45a49962 100644 Binary files a/matlab_version/experiments_JANUS/results/Janus-no-outline.png and b/matlab_version/experiments_JANUS/results/Janus-no-outline.png differ diff --git a/matlab_version/experiments_iccv_300w/Display_clnf_results_small.m b/matlab_version/experiments_iccv_300w/Display_clnf_results_small.m deleted file mode 100644 index 8c7f2b46..00000000 --- a/matlab_version/experiments_iccv_300w/Display_clnf_results_small.m +++ /dev/null @@ -1,73 +0,0 @@ -clear -scrsz = get(0,'ScreenSize'); -figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); - -set(figure1,'Units','Inches'); -pos = get(figure1,'Position'); -set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) - -% Create axes -axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); - -line_width = 4; -hold on; - -load('results/results_wild_clnf.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,:); -% center the pixel -shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 0.5; - -clnf_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clnf_error); -hold on; - -plot(error_x, error_y, 'r','DisplayName', 'OpenFace', 'LineWidth',line_width); - -load('results/zhu_wild.mat'); - -zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:)); - -[error_x, error_y] = cummErrorCurve(zhu_wild_error); - -plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width); - -load('results/yu_wild.mat'); - -yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); -yu_wild_error(isnan(yu_wild_error)) = 1; -yu_wild_error(isinf(yu_wild_error)) = 1; - -[error_x, error_y] = cummErrorCurve(yu_wild_error); - -plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); - -load('results/results_wild_clm.mat'); -experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); -% center the pixel -experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; - -clm_error = compute_error( experiments(1).labels, experiments(1).shapes); - -[error_x, error_y] = cummErrorCurve(clm_error); - -plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); - -load('results/drmf_wild.mat'); - -drmf_error = compute_error(labels_all, shapes_all); - -[error_x, error_y] = cummErrorCurve(drmf_error); - -plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); - -set(gca,'xtick',[0:0.05:0.15]) -xlim([0,0.15]); -xlabel('Size normalised shape RMS error','FontName','Helvetica'); -ylabel('Proportion of images','FontName','Helvetica'); -grid on -%title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); - -legend('show', 'Location', 'SouthEast'); - -print -dpdf results/in-the-wild-comparison.pdf diff --git a/matlab_version/experiments_iccv_300w/Display_dclm_results.m b/matlab_version/experiments_iccv_300w/Display_dclm_results.m index 4cdf6836..b8bd86ec 100644 --- a/matlab_version/experiments_iccv_300w/Display_dclm_results.m +++ b/matlab_version/experiments_iccv_300w/Display_dclm_results.m @@ -1,5 +1,6 @@ clear +inds = 338:1026; %% scrsz = get(0,'ScreenSize'); figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); @@ -14,136 +15,130 @@ axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); line_width = 6; hold on; -load('results/intraface_wild_resize.mat'); -labels_all = labels_all(18:end,:,:); -shapes_all = shapes_all(18:end,:,:); +load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:) + 1.0; -intraface_wild_error = compute_error(labels_all, shapes_all); +dclm_error = compute_error( labels, shapes); -% removing faces that were not detected by intraface for fairness -detected = intraface_wild_error < 1; +[error_x, error_y] = cummErrorCurve(dclm_error(inds)); -% load('results/results_wild_clnf.mat'); -% labels = experiments.labels([1:60,62:64,66:end],:,detected); -% shapes = experiments.shapes([1:60,62:64,66:end],:,detected); -% labels = labels(18:end,:,:); -% % center the pixel -% shapes = shapes(18:end,:,:) + 0.5; -% -% clnf_error = compute_error( labels, shapes); -% -% [error_x, error_y] = cummErrorCurve(clnf_error); -% -% plot(error_x, error_y, 'DisplayName', 'CLNF wild', 'LineWidth',line_width); -% hold on; +plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width); load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,detected); -shapes = experiments.shapes([1:60,62:64,66:end],:,detected); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); labels = labels(18:end,:,:); % center the pixel shapes = shapes(18:end,:,:) + 0.5; clnf_error = compute_error( labels, shapes); -[error_x, error_y] = cummErrorCurve(clnf_error); +[error_x, error_y] = cummErrorCurve(clnf_error(inds)); -plot(error_x, error_y, 'DisplayName', 'CLNF (ECCV 14)', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); hold on; -load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,detected); -shapes = experiments.shapes([1:60,62:64,66:end],:,detected); -labels = labels(18:end,:,:) - 1.0; -shapes = shapes(18:end,:,:); - -dclm_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(dclm_error); - -plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width); - - load('results/300W_pocr.mat'); % center the pixel shapes_all = experiments.shapes; -pocr_error = compute_error(labels, shapes_all(:,:,detected)); +pocr_error = compute_error(labels, shapes_all(:,:,:) + 1.0); -[error_x, error_y] = cummErrorCurve(pocr_error); +[error_x, error_y] = cummErrorCurve(pocr_error(inds)); -plot(error_x, error_y, 'DisplayName', 'PO-CR (CVPR 15)', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'PO-CR', 'LineWidth',line_width); hold on; -load('results/intraface_wild_resize.mat'); -labels_all = labels_all(18:end,:,detected); +load('results/300W-CFSS.mat'); + % center the pixel -shapes_all = shapes_all(18:end,:,detected) + 0.5; +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end +shapes_all = shapes_all([1:60,62:64,66:end],:,:); +shapes_all = shapes_all(18:end,:,:); -intraface_wild_error = compute_error(labels_all, shapes_all); +cfss_error = compute_error(labels, shapes_all(:,:,:) + 0.5); -[error_x, error_y] = cummErrorCurve(intraface_wild_error); +[error_x, error_y] = cummErrorCurve(cfss_error(inds)); -plot(error_x, error_y, '.-g','DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); +hold on; + +load('results/300W_sdm.mat'); + +% center the pixel +shapes_all = experiments.shapes; + +sdm_error = compute_error(labels, shapes_all(:,:,:)); +[error_x, error_y] = cummErrorCurve(sdm_error(inds)); + +plot(error_x, error_y, 'DisplayName', 'SDM (CVPR 13)', 'LineWidth',line_width); hold on; load('results/GNDPM_300W.mat'); % center the pixel -shapes_all = shapes_all(:,:,detected) + 1; -labels_all = labels_all(:,:,detected); +shapes_all = shapes_all(:,:,:) + 1; +labels_all = labels_all(:,:,:); gndpm_wild_error = compute_error(labels_all, shapes_all); -[error_x, error_y] = cummErrorCurve(gndpm_wild_error); +[error_x, error_y] = cummErrorCurve(gndpm_wild_error(inds)); -plot(error_x, error_y, '-.','DisplayName', 'GNDPM (CVPR 14)', 'LineWidth',line_width); +plot(error_x, error_y, 'DisplayName', 'GNDPM', 'LineWidth',line_width); hold on; load('results/zhu_wild.mat'); -labels_all = labels_all(18:end,:,detected); -shapes_all = shapes_all(18:end,:,detected); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); zhu_wild_error = compute_error(labels_all, shapes_all); -[error_x, error_y] = cummErrorCurve(zhu_wild_error); +[error_x, error_y] = cummErrorCurve(zhu_wild_error(inds)); -plot(error_x, error_y, '.-c','DisplayName', 'Tree based (CVPR 12)', 'LineWidth',line_width); +plot(error_x, error_y, 'c','DisplayName', 'Tree based', 'LineWidth',line_width); -load('results/results_wild_clm.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,detected); -shapes = experiments.shapes([1:60,62:64,66:end],:,detected); -labels = labels(18:end,:,:); -% center the pixel -shapes = shapes(18:end,:,:) + 0.5; - -clm_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clm_error); - -plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); +% load('results/results_wild_clm.mat'); +% labels = experiments.labels([1:60,62:64,66:end],:,:); +% shapes = experiments.shapes([1:60,62:64,66:end],:,:); +% labels = labels(18:end,:,:); +% % center the pixel +% shapes = shapes(18:end,:,:) + 0.5; +% +% clm_error = compute_error( labels, shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); load('results/drmf_wild.mat'); -labels_all = labels_all(18:end,:,detected); -shapes_all = shapes_all(18:end,:,detected); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); drmf_error = compute_error(labels_all, shapes_all); -[error_x, error_y] = cummErrorCurve(drmf_error); +[error_x, error_y] = cummErrorCurve(drmf_error(inds)); -plot(error_x, error_y, '-.k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); +plot(error_x, error_y, 'k','DisplayName', 'DRMF (CVPR 13)', 'LineWidth',line_width); -set(gca,'xtick',[0:0.05:0.15]) -xlim([0,0.08]); -xlabel('Size normalised shape RMS error','FontName','Helvetica'); +set(gca,'xtick',[0:0.01:0.1]) +xlim([0.01,0.07]); +xlabel('IOD normalised MSE','FontName','Helvetica'); ylabel('Proportion of images','FontName','Helvetica'); grid on % title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); leg = legend('show', 'Location', 'SouthEast'); set(leg,'FontSize',30) - +savefig('results/dclm-in-the-wild-no-outline.fig'); print -dpdf results/dclm-in-the-wild-clnf-no-outline.pdf print -dpng results/dclm-300W-no-outline.png %% @@ -160,18 +155,6 @@ axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); line_width = 6; hold on; -load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); -labels = experiments.labels([1:60,62:64,66:end],:,:); -% center the pixel -shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 1; - -clnf_error = compute_error( labels, shapes); - -[error_x, error_y] = cummErrorCurve(clnf_error); -hold on; - -plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); - load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); labels = experiments.labels([1:60,62:64,66:end],:,:); % center the pixel @@ -179,50 +162,79 @@ shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 0.5; dclm_error = compute_error( labels, shapes); -[error_x, error_y] = cummErrorCurve(dclm_error); +[error_x, error_y] = cummErrorCurve(dclm_error(inds)); plot(error_x, error_y, 'r', 'DisplayName', 'DCLM ', 'LineWidth',line_width); +load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +% center the pixel +shapes = experiments.shapes([1:60,62:64,66:end],:,:) + 1; + +clnf_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clnf_error(inds)); +hold on; + +plot(error_x, error_y, 'DisplayName', 'CLNF', 'LineWidth',line_width); + + +load('results/300W-CFSS.mat'); + +% center the pixel +shapes_all = zeros(68, 2, size(estimatedPose,1)); +for i = 1:size(estimatedPose,1) + + shapes_all(:,1,i) = estimatedPose(i,1:68)'; + shapes_all(:,2,i) = estimatedPose(i,69:end)'; +end + +cfss_error = compute_error(labels, shapes_all(:,:,:) + 0.5); + +[error_x, error_y] = cummErrorCurve(cfss_error(inds)); + +plot(error_x, error_y, 'DisplayName', 'CFSS', 'LineWidth',line_width); + load('results/zhu_wild.mat'); zhu_wild_error = compute_error(labels_all(:,:,:), shapes_all(:,:,:)); [error_x, error_y] = cummErrorCurve(zhu_wild_error); -plot(error_x, error_y, '.-c','DisplayName', 'Zhu et al.', 'LineWidth',line_width); +plot(error_x, error_y, '.-c','DisplayName', 'Tree based', 'LineWidth',line_width); -load('results/yu_wild.mat'); +% load('results/yu_wild.mat'); -yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); -yu_wild_error(isnan(yu_wild_error)) = 1; -yu_wild_error(isinf(yu_wild_error)) = 1; +% yu_wild_error = compute_error(lmark_dets_all(:,:,:)-1, shapes_all(:,:,:)); +% yu_wild_error(isnan(yu_wild_error)) = 1; +% yu_wild_error(isinf(yu_wild_error)) = 1; +% +% [error_x, error_y] = cummErrorCurve(yu_wild_error(inds)); +% +% plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); -[error_x, error_y] = cummErrorCurve(yu_wild_error); - -plot(error_x, error_y, 'xg','DisplayName', 'Yu et al.', 'LineWidth',line_width); - -load('results/results_wild_clm.mat'); -experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); -% center the pixel -experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; - -clm_error = compute_error( experiments(1).labels, experiments(1).shapes); - -[error_x, error_y] = cummErrorCurve(clm_error); - -plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); +% load('results/results_wild_clm.mat'); +% experiments(1).labels = experiments(1).labels([1:60,62:64,66:end],:,:); +% % center the pixel +% experiments(1).shapes = experiments(1).shapes([1:60,62:64,66:end],:,:) + 0.5; +% +% clm_error = compute_error( experiments(1).labels, experiments(1).shapes); +% +% [error_x, error_y] = cummErrorCurve(clm_error(inds)); +% +% plot(error_x, error_y, '--b','DisplayName', 'CLM+', 'LineWidth',line_width); load('results/drmf_wild.mat'); drmf_error = compute_error(labels_all, shapes_all); -[error_x, error_y] = cummErrorCurve(drmf_error); +[error_x, error_y] = cummErrorCurve(drmf_error(inds)); -plot(error_x, error_y, '-.k','DisplayName', 'DRMF', 'LineWidth',line_width); +plot(error_x, error_y, 'k','DisplayName', 'DRMF', 'LineWidth',line_width); -set(gca,'xtick',[0:0.05:0.15]) -xlim([0,0.15]); -xlabel('Size normalised shape RMS error','FontName','Helvetica'); +set(gca,'xtick',[0.01:0.02:0.15]) +xlim([0.01,0.15]); +xlabel('IOD normalised MSE','FontName','Helvetica'); ylabel('Proportion of images','FontName','Helvetica'); grid on %title('Fitting in the wild','FontSize',60,'FontName','Helvetica'); @@ -230,4 +242,5 @@ grid on legend('show', 'Location', 'SouthEast'); print -dpdf results/dclm-in-the-wild-comparison.pdf +savefig('results/dclm-in-the-wild-outline.fig'); print -dpng results/dclm-300W-outline.png \ No newline at end of file diff --git a/matlab_version/experiments_iccv_300w/Display_dlm_results_per_db.m b/matlab_version/experiments_iccv_300w/Display_dlm_results_per_db.m new file mode 100644 index 00000000..31cdd7c2 --- /dev/null +++ b/matlab_version/experiments_iccv_300w/Display_dlm_results_per_db.m @@ -0,0 +1,258 @@ +clear + +%% +inds = 1:337; +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_clnf.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'r','DisplayName', 'CLM + CCNF', 'LineWidth',line_width); + +load('results/intraface_wild_resize.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +intraface_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(intraface_wild_error); + +plot(error_x, error_y, '.-g','DisplayName', 'SDM [32]', 'LineWidth',line_width); + +load('results/zhu_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +zhu_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(zhu_wild_error); + +plot(error_x, error_y, '.-c','DisplayName', 'Tree based (p204) [28]', 'LineWidth',line_width); + +load('results/results_wild_clm.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clm_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, '--b','DisplayName', 'CLM + SVR', 'LineWidth',line_width); + +load('results/drmf_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +drmf_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(drmf_error); + +plot(error_x, error_y, '-.k','DisplayName', 'DRMF [27]', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/in-the-wild-clnf-no-outline_afw.pdf + +%% +inds = 562:696; + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_clnf.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'r','DisplayName', 'CLM + CCNF', 'LineWidth',line_width); + +load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,inds); +shapes = experiments.shapes([1:60,62:64,66:end],:,inds); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:) + 1.0; + +dclm_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(dclm_error); + +plot(error_x, error_y, 'DisplayName', 'DCLM ', 'LineWidth',line_width); + +load('results/intraface_wild_resize.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +intraface_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(intraface_wild_error); + +plot(error_x, error_y, '.-g','DisplayName', 'SDM [32]', 'LineWidth',line_width); + +load('results/zhu_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +zhu_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(zhu_wild_error); + +plot(error_x, error_y, '.-c','DisplayName', 'Tree based (p204) [28]', 'LineWidth',line_width); + +load('results/results_wild_clm.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clm_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, '--b','DisplayName', 'CLM + SVR', 'LineWidth',line_width); + +load('results/drmf_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +drmf_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(drmf_error); + +plot(error_x, error_y, '-.k','DisplayName', 'DRMF [27]', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/in-the-wild-clnf-no-outline_ibug.pdf + +%% +inds = [338:561,697:1026]; + +scrsz = get(0,'ScreenSize'); +figure1 = figure('Position',[20 50 3*scrsz(3)/4 0.9*scrsz(4)]); + +set(figure1,'Units','Inches'); +pos = get(figure1,'Position'); +set(figure1,'PaperPositionMode','Auto','PaperUnits','Inches','PaperSize',[pos(3), pos(4)]) + +% Create axes +axes1 = axes('Parent',figure1,'FontSize',40,'FontName','Helvetica'); + +line_width = 6; +hold on; + +load('results/results_wild_clnf.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clnf_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clnf_error); +hold on; + +plot(error_x, error_y, 'r','DisplayName', 'CLM + CCNF', 'LineWidth',line_width); + +load('results/intraface_wild_resize.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +intraface_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(intraface_wild_error); + +plot(error_x, error_y, '.-g','DisplayName', 'SDM [32]', 'LineWidth',line_width); + +load('results/zhu_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +zhu_wild_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(zhu_wild_error); + +plot(error_x, error_y, '.-c','DisplayName', 'Tree based (p204) [28]', 'LineWidth',line_width); + +load('results/results_wild_clm.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:); +shapes = shapes(18:end,:,:); + +clm_error = compute_error( labels(:,:,inds), shapes(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, '--b','DisplayName', 'CLM + SVR', 'LineWidth',line_width); + +load('results/drmf_wild.mat'); +labels_all = labels_all(18:end,:,:); +shapes_all = shapes_all(18:end,:,:); + +drmf_error = compute_error(labels_all(:,:,inds), shapes_all(:,:,inds)); + +[error_x, error_y] = cummErrorCurve(drmf_error); + +plot(error_x, error_y, '-.k','DisplayName', 'DRMF [27]', 'LineWidth',line_width); + +set(gca,'xtick',[0:0.05:0.15]) +xlim([0,0.15]); +xlabel('Size normalised shape RMS error','FontName','Helvetica'); +ylabel('Proportion of images','FontName','Helvetica'); +grid on +% title('Fitting in the wild without outline','FontSize',60,'FontName','Helvetica'); + +legend('show', 'Location', 'SouthEast'); + +print -dpdf results/in-the-wild-clnf-no-outline_lfpw_helen.pdf diff --git a/matlab_version/experiments_iccv_300w/Script_CFSS_vis.m b/matlab_version/experiments_iccv_300w/Script_CFSS_vis.m new file mode 100644 index 00000000..99ee8b5b --- /dev/null +++ b/matlab_version/experiments_iccv_300w/Script_CFSS_vis.m @@ -0,0 +1,81 @@ +function Script_CFSS_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +images = images(:); +labels = labels(:,:,:); + f = figure; +load('results/300W-CFSS.mat'); +output_root = './wild_fit_cfss/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + pred = estimatedPose(i,:); + preds = cat(2, pred(1:68)', pred(69:end)'); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_cfss.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose +% close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_iccv_300w/Script_CLNF_vis.m b/matlab_version/experiments_iccv_300w/Script_CLNF_vis.m new file mode 100644 index 00000000..f34a84ed --- /dev/null +++ b/matlab_version/experiments_iccv_300w/Script_CLNF_vis.m @@ -0,0 +1,80 @@ +function Script_CLNF_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +images = images(:); +labels = labels(:,:,:); + f = figure; +load('../experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat'); +output_root = './wild_fit_clnf/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + preds = experiments.shapes(:,:,i); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_clnf.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose +% close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_iccv_300w/Script_DCLM_vis.m b/matlab_version/experiments_iccv_300w/Script_DCLM_vis.m new file mode 100644 index 00000000..22ce33ad --- /dev/null +++ b/matlab_version/experiments_iccv_300w/Script_DCLM_vis.m @@ -0,0 +1,80 @@ +function Script_DCLM_vis() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +images = images(:); +labels = labels(:,:,:); + f = figure; +load('../experiments_in_the_wild/results/results_wild_dclm_general.mat'); +output_root = './wild_fit_dclm/'; +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + preds = experiments.shapes(:,:,i); + + actualShape = squeeze(labels(i,:,:)); + + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(v_points,1)) - min(actualShape(v_points,1)); + height = max(actualShape(v_points,2)) - min(actualShape(v_points,2)); + sz = (width+height)/2.0; + + img_min_x = max(int32(min(actualShape(v_points,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(v_points,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(v_points,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(v_points,2))) + height/3,height_img); + + preds(:,1) = preds(:,1) - double(img_min_x); + preds(:,2) = preds(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + scale = 600/sz; + image_orig = imresize(image_orig, scale); + + % valid points to draw (not to draw + % occluded ones) + +% f = figure('visible','off'); + + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + plot(preds(:,1) * scale, preds(:,2) * scale,'.r','MarkerSize', int32(40)); + plot(preds(:,1) * scale, preds(:,2) * scale,'.g','MarkerSize',int32(20)); +% plot(preds(:,1), preds(:,2),'.w','MarkerSize',40); + % plot(preds(:,1), preds(:,2),'.k','MarkerSize',30); + print(f, '-r80', '-dpng', sprintf('%s/%s%d_dclm.png', output_root, 'fit', i)); +% print(f, '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% close(f); + hold off; +% drawnow expose +% close(f); + catch warn + + end + +end \ No newline at end of file diff --git a/matlab_version/experiments_iccv_300w/errors.mat b/matlab_version/experiments_iccv_300w/errors.mat new file mode 100644 index 00000000..bf9d2ef8 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/errors.mat differ diff --git a/matlab_version/experiments_iccv_300w/results/300W-CFSS.mat b/matlab_version/experiments_iccv_300w/results/300W-CFSS.mat new file mode 100644 index 00000000..eeedc13b Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/300W-CFSS.mat differ diff --git a/matlab_version/experiments_iccv_300w/results/300W_sdm.mat b/matlab_version/experiments_iccv_300w/results/300W_sdm.mat new file mode 100644 index 00000000..d0679cb7 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/300W_sdm.mat differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png b/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png index 6abfef55..6701774f 100644 Binary files a/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png and b/matlab_version/experiments_iccv_300w/results/dclm-300W-no-outline.png differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png b/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png index 0200c003..2e6580c6 100644 Binary files a/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png and b/matlab_version/experiments_iccv_300w/results/dclm-300W-outline.png differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf index fbe7c1f2..1b0a42d2 100644 Binary files a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-clnf-no-outline.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf index 0d3c372c..a0ea8ca0 100644 Binary files a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-comparison.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-no-outline.fig b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-no-outline.fig new file mode 100644 index 00000000..aa9b08ac Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-no-outline.fig differ diff --git a/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-outline.fig b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-outline.fig new file mode 100644 index 00000000..b1dca450 Binary files /dev/null and b/matlab_version/experiments_iccv_300w/results/dclm-in-the-wild-outline.fig differ diff --git a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_afw.pdf b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_afw.pdf index 5a52f55d..2e4b59b1 100644 Binary files a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_afw.pdf and b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_afw.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_ibug.pdf b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_ibug.pdf index 85ab25bf..e80c8c0b 100644 Binary files a/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_ibug.pdf and b/matlab_version/experiments_iccv_300w/results/in-the-wild-clnf-no-outline_ibug.pdf differ diff --git a/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf b/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf index e60e5d2d..e4abb7f2 100644 Binary files a/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf and b/matlab_version/experiments_iccv_300w/results/in-the-wild-comparison.pdf differ diff --git a/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m b/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m index c0598216..8a5406fc 100644 --- a/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m +++ b/matlab_version/experiments_in_the_wild/Display_clnf_dclm_results.m @@ -39,6 +39,19 @@ clm_error = compute_error( labels, shapes); plot(error_x, error_y, 'DisplayName', 'DCLM', 'LineWidth',line_width); +load('results/results_wild_dclm_wild.mat'); +labels = experiments.labels([1:60,62:64,66:end],:,:); +shapes = experiments.shapes([1:60,62:64,66:end],:,:); +labels = labels(18:end,:,:) - 0.5; +shapes = shapes(18:end,:,:); + +clm_error = compute_error( labels, shapes); + +[error_x, error_y] = cummErrorCurve(clm_error); + +plot(error_x, error_y, 'DisplayName', 'DCLM2', 'LineWidth',line_width); + + set(gca,'xtick',[0:0.01:0.08]) xlim([0,0.08]); xlabel('Size normalised shape RMS error','FontName','Helvetica'); diff --git a/matlab_version/experiments_in_the_wild/Script_CLM_general_vis_resp.m b/matlab_version/experiments_in_the_wild/Script_CLM_general_vis_resp.m new file mode 100644 index 00000000..66d6b983 --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_CLM_general_vis_resp.m @@ -0,0 +1,82 @@ +function Script_CLM_general_vis_resp() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +to_vis = [29,51,75,79,81,9,112,146,152,172,199,204,230,234,235,241,251,256,263,272,279,340,342,348,358,362,394,478,484,486]; +images = images(to_vis); +detections = detections(to_vis,:); +labels = labels(to_vis,:); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [31,31;]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_Patch_Experts( '../models/general/', 'svr_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +out_dir_root = './patch_expert_responses/'; + +if(~exist(out_dir_root, 'dir')) + mkdir(out_dir_root); +end + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 1; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + out_dir = [out_dir_root, '/', num2str(to_vis(i))]; + if(~exist(out_dir, 'dir')) + mkdir(out_dir); + end + + Fitting_from_bb_vis(image, [], bbox, pdm, patches, clmParams, out_dir); + +end + +end diff --git a/matlab_version/experiments_in_the_wild/Script_CLNF_general_no_out.m b/matlab_version/experiments_in_the_wild/Script_CLNF_general_no_out.m index b595b0a8..d58e94a3 100644 --- a/matlab_version/experiments_in_the_wild/Script_CLNF_general_no_out.m +++ b/matlab_version/experiments_in_the_wild/Script_CLNF_general_no_out.m @@ -10,7 +10,7 @@ addpath('../models/'); if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; else - root_test_data = 'F:/Dropbox/Dropbox/AAM/test data/'; + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; end [images, detections, labels] = Collect_wild_imgs(root_test_data); @@ -26,7 +26,7 @@ clmParams.numPatchIters = size(clmParams.window_size,1); %% Fitting the model to the provided image -verbose = false; % set to true to visualise the fitting +verbose = true; % set to true to visualise the fitting output_root = './wild_fit_clnf/'; % the default PDM to use diff --git a/matlab_version/experiments_in_the_wild/Script_CLNF_general_vis_resp.m b/matlab_version/experiments_in_the_wild/Script_CLNF_general_vis_resp.m new file mode 100644 index 00000000..73843056 --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_CLNF_general_vis_resp.m @@ -0,0 +1,82 @@ +function Script_CLNF_general_vis_resp() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +to_vis = [29,51,75,79,81,9,112,146,152,172,199,204,230,234,235,241,251,256,263,272,279,340,342,348,358,362,394,478,484,486]; +images = images(to_vis); +detections = detections(to_vis,:); +labels = labels(to_vis,:); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [31,31;]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_Patch_Experts( '../models/general/', 'ccnf_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +out_dir_root = './patch_expert_responses/'; + +if(~exist(out_dir_root, 'dir')) + mkdir(out_dir_root); +end + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 1; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + out_dir = [out_dir_root, '/', num2str(to_vis(i))]; + if(~exist(out_dir, 'dir')) + mkdir(out_dir); + end + + Fitting_from_bb_vis(image, [], bbox, pdm, patches, clmParams, out_dir); + +end + +end diff --git a/matlab_version/experiments_in_the_wild/Script_DCLM_general_corrs.m b/matlab_version/experiments_in_the_wild/Script_DCLM_general_corrs.m new file mode 100644 index 00000000..5c535ba9 --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_DCLM_general_corrs.m @@ -0,0 +1,186 @@ +function Script_DCLM_general_corrs() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general_corrs.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +output_root = './wild_fit_dclm/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = false; +verbose = false; +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape; + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 50)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + img_min_x = max(int32(min(actualShape(:,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(:,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(:,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(:,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20); + plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_dclm_general_corr.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_in_the_wild/Script_DCLM_general_rmses.m b/matlab_version/experiments_in_the_wild/Script_DCLM_general_rmses.m new file mode 100644 index 00000000..7e54afd7 --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_DCLM_general_rmses.m @@ -0,0 +1,186 @@ +function Script_DCLM_general_rmses() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general_rmses.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +output_root = './wild_fit_dclm/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = false; +verbose = false; +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape; + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 50)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + img_min_x = max(int32(min(actualShape(:,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(:,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(:,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(:,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20); + plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); +% print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_dclm_general_rmse.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_in_the_wild/Script_DCLM_general_vis_resp.m b/matlab_version/experiments_in_the_wild/Script_DCLM_general_vis_resp.m new file mode 100644 index 00000000..3f072154 --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_DCLM_general_vis_resp.m @@ -0,0 +1,82 @@ +function Script_DCLM_general_vis_resp() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +to_vis = [29,51,75,79,81,9,112,146,152,172,199,204,230,234,235,241,251,256,263,272,279,340,342,348,358,362,394,478,484,486]; +images = images(to_vis); +detections = detections(to_vis,:); +labels = labels(to_vis,:); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [31,31;]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/general/', 'dccnf_patches_*_general.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +out_dir_root = './patch_expert_responses/'; + +if(~exist(out_dir_root, 'dir')) + mkdir(out_dir_root); +end + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 1; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + out_dir = [out_dir_root, '/', num2str(to_vis(i))]; + if(~exist(out_dir, 'dir')) + mkdir(out_dir); + end + + Fitting_from_bb_vis(image, [], bbox, pdm, patches, clmParams, out_dir); + +end + +end diff --git a/matlab_version/experiments_in_the_wild/Script_DCLM_general_wild.m b/matlab_version/experiments_in_the_wild/Script_DCLM_general_wild.m new file mode 100644 index 00000000..3071f31d --- /dev/null +++ b/matlab_version/experiments_in_the_wild/Script_DCLM_general_wild.m @@ -0,0 +1,186 @@ +function Script_DCLM_general_wild() + +addpath('../PDM_helpers/'); +addpath('../fitting/normxcorr2_mex_ALL'); +addpath('../fitting/'); +addpath('../CCNF/'); +addpath('../models/'); + +% Replace this with the location of in 300 faces in the wild data +if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) + root_test_data = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; +else + root_test_data = 'D:/Dropbox/Dropbox/AAM/test data/'; +end + +[images, detections, labels] = Collect_wild_imgs(root_test_data); + +%% loading the patch experts + +clmParams = struct; + +clmParams.window_size = [25,25; 23,23; 21,21; 21,21]; +clmParams.numPatchIters = size(clmParams.window_size,1); + +[patches] = Load_DCLM_Patch_Experts( '../models/wild/', 'dccnf_patches_*_wild.mat', [], [], clmParams); + +%% Fitting the model to the provided image + +output_root = './wild_fit_dclm/'; + +% the default PDM to use +pdmLoc = ['../models/pdm/pdm_68_aligned_wild.mat']; + +load(pdmLoc); + +pdm = struct; +pdm.M = double(M); +pdm.E = double(E); +pdm.V = double(V); + +clmParams.regFactor = [35, 27, 20, 20]; +clmParams.sigmaMeanShift = [1.25, 1.375, 1.5, 1.5]; +clmParams.tikhonov_factor = [2.5, 5, 7.5, 7.5]; + +clmParams.startScale = 1; +clmParams.num_RLMS_iter = 10; +clmParams.fTol = 0.01; +clmParams.useMultiScale = true; +clmParams.use_multi_modal = 1; +clmParams.multi_modal_types = patches(1).multi_modal_types; +clmParams.numPatchIters = 4; + +% for recording purposes +experiment.params = clmParams; + +num_points = numel(M)/3; + +shapes_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +labels_all = zeros(size(labels,2),size(labels,3), size(labels,1)); +lhoods = zeros(numel(images),1); +all_lmark_lhoods = zeros(num_points, numel(images)); +all_views_used = zeros(numel(images),1); + +% Use the multi-hypothesis model, as bounding box tells nothing about +% orientation +multi_view = true; +verbose = true; +tic +for i=1:numel(images) + + image = imread(images(i).img); + image_orig = image; + + if(size(image,3) == 3) + image = rgb2gray(image); + end + + bbox = detections(i,:); + + % have a multi-view version + if(multi_view) + + views = [0,0,0; 0,-30,0; -30,0,0; 0,30,0; 30,0,0]; + views = views * pi/180; + + shapes = zeros(num_points, 2, size(views,1)); + ls = zeros(size(views,1),1); + lmark_lhoods = zeros(num_points,size(views,1)); + views_used = zeros(num_points,size(views,1)); + + % Find the best orientation + for v = 1:size(views,1) + [shapes(:,:,v),~,~,ls(v),lmark_lhoods(:,v),views_used(v)] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'orientation', views(v,:)); + end + + [lhood, v_ind] = max(ls); + lmark_lhood = lmark_lhoods(:,v_ind); + + shape = shapes(:,:,v_ind); + view_used = views_used(v); + + else + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); + end + + all_lmark_lhoods(:,i) = lmark_lhood; + all_views_used(i) = view_used; + + shapes_all(:,:,i) = shape; + labels_all(:,:,i) = labels(i,:,:); + + if(mod(i, 200)==0) + fprintf('%d done\n', i ); + end + + lhoods(i) = lhood; + + if(verbose) + + actualShape = squeeze(labels(i,:,:)); + + [height_img, width_img,~] = size(image_orig); + width = max(actualShape(:,1)) - min(actualShape(:,1)); + height = max(actualShape(:,2)) - min(actualShape(:,2)); + + img_min_x = max(int32(min(actualShape(:,1))) - width/3,1); + img_max_x = min(int32(max(actualShape(:,1))) + width/3,width_img); + + img_min_y = max(int32(min(actualShape(:,2))) - height/3,1); + img_max_y = min(int32(max(actualShape(:,2))) + height/3,height_img); + + shape(:,1) = shape(:,1) - double(img_min_x); + shape(:,2) = shape(:,2) - double(img_min_y); + + image_orig = image_orig(img_min_y:img_max_y, img_min_x:img_max_x, :); + + % valid points to draw (not to draw + % occluded ones) + v_points = sum(squeeze(labels(i,:,:)),2) > 0; + +% f = figure('visible','off'); + f = figure; + try + if(max(image_orig(:)) > 1) + imshow(double(image_orig)/255, 'Border', 'tight'); + else + imshow(double(image_orig), 'Border', 'tight'); + end + axis equal; + hold on; + + plot(shape(v_points,1), shape(v_points,2),'.r','MarkerSize',20); + plot(shape(v_points,1), shape(v_points,2),'.b','MarkerSize',10); +% print(f, '-r80', '-dpng', sprintf('%s/%s%d.png', output_root, 'fit', i)); + print(f, '-djpeg', sprintf('%s/%s%d.jpg', output_root, 'fit', i)); +% close(f); + hold off; + close(f); + catch warn + + end + end + +end +toc + +experiment.errors_normed = compute_error(labels_all - 0.5, shapes_all); +experiment.lhoods = lhoods; +experiment.shapes = shapes_all; +experiment.labels = labels_all; +experiment.all_lmark_lhoods = all_lmark_lhoods; +experiment.all_views_used = all_views_used; +% save the experiment +if(~exist('experiments', 'var')) + experiments = experiment; +else + experiments = cat(1, experiments, experiment); +end +fprintf('experiment %d done: mean normed error %.3f median normed error %.4f\n', ... + numel(experiments), mean(experiment.errors_normed), median(experiment.errors_normed)); + +%% +output_results = 'results/results_wild_dclm_wild.mat'; +save(output_results, 'experiments'); + +end diff --git a/matlab_version/experiments_in_the_wild/results/dclm_results.png b/matlab_version/experiments_in_the_wild/results/dclm_results.png index c04fdfc1..f3b64fc4 100644 Binary files a/matlab_version/experiments_in_the_wild/results/dclm_results.png and b/matlab_version/experiments_in_the_wild/results/dclm_results.png differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat b/matlab_version/experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat index be592e64..f102d978 100644 Binary files a/matlab_version/experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat and b/matlab_version/experiments_in_the_wild/results/results_wild_clnf_general_final_inner.mat differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_corr.mat b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_corr.mat new file mode 100644 index 00000000..acd89bc8 Binary files /dev/null and b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_corr.mat differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_old.mat b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_old.mat new file mode 100644 index 00000000..4996d9f5 Binary files /dev/null and b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_old.mat differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_rmse.mat b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_rmse.mat new file mode 100644 index 00000000..627535fe Binary files /dev/null and b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_general_rmse.mat differ diff --git a/matlab_version/experiments_in_the_wild/results/results_wild_dclm_wild.mat b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_wild.mat new file mode 100644 index 00000000..48723360 Binary files /dev/null and b/matlab_version/experiments_in_the_wild/results/results_wild_dclm_wild.mat differ diff --git a/matlab_version/fitting/Fitting_from_bb.m b/matlab_version/fitting/Fitting_from_bb.m index a868f294..0aef371d 100644 --- a/matlab_version/fitting/Fitting_from_bb.m +++ b/matlab_version/fitting/Fitting_from_bb.m @@ -140,6 +140,7 @@ function [ shape2D, global_params, local_params, final_lhood, landmark_lhoods, v responses = PatchResponseDNN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:)); end + % If a depth image is provided compute patch experts around it as % well (unless it's the final iteration) if(~isempty(DepthImage) && (i ~= clmParams.numPatchIters)) diff --git a/matlab_version/fitting/Fitting_from_bb_vis.m b/matlab_version/fitting/Fitting_from_bb_vis.m new file mode 100644 index 00000000..86c67385 --- /dev/null +++ b/matlab_version/fitting/Fitting_from_bb_vis.m @@ -0,0 +1,163 @@ +function Fitting_from_bb_vis( Image, DepthImage, bounding_box, PDM, patchExperts, clmParams, out_dir, varargin) +%FITTING Summary of this function goes here +% Detailed explanation goes here + + % the bounding box format is [minX, minY, maxX, maxY]; + + % the mean model shape + M = PDM.M; + + num_points = numel(M) / 3; + + if(any(strcmp(varargin,'orientation'))) + orientation = varargin{find(strcmp(varargin, 'orientation'))+1}; + rot = Euler2Rot(orientation); + else + rot = eye(3); + orientation = [0;0;0]; + end + + rot_m = rot * reshape(M, num_points, 3)'; + width_model = max(rot_m(1,:)) - min(rot_m(1,:)); + height_model = max(rot_m(2,:)) - min(rot_m(2,:)); + + a = (((bounding_box(3) - bounding_box(1)) / width_model) + ((bounding_box(4) - bounding_box(2))/ height_model)) / 2; + + tx = (bounding_box(3) + bounding_box(1))/2; + ty = (bounding_box(4) + bounding_box(2))/2; + + % correct it so that the bounding box is just around the minimum + % and maximum point in the initialised face + tx = tx - a*(min(rot_m(1,:)) + max(rot_m(1,:)))/2; + ty = ty - a*(min(rot_m(2,:)) + max(rot_m(2,:)))/2; + + % visualisation of the initial state + %hold off;imshow(Image);hold on;plot(a*rot_m(1,:)+tx, a*rot_m(2,:)+ty,'.r');hold on;rectangle('Position', [bounding_box(1), bounding_box(2), bounding_box(3)-bounding_box(1), bounding_box(4)-bounding_box(2)]); + global_params = [a, 0, 0, 0, tx, ty]'; + global_params(2:4) = orientation; + + local_params = zeros(numel(PDM.E), 1); + + if(any(strcmp(varargin,'gparam'))) + global_params = varargin{find(strcmp(varargin, 'gparam'))+1}; + end + + if(any(strcmp(varargin,'lparam'))) + local_params = varargin{find(strcmp(varargin, 'lparam'))+1}; + end + + scale = clmParams.startScale; + + if(size(Image, 3) == 1) + GrayImage = Image; + else + GrayImage = rgb2gray(Image); + end + + [heightImg, widthImg] = size(GrayImage); + + % Some predefinitions for faster patch extraction + [xi, yi] = meshgrid(0:widthImg-1,0:heightImg-1); + xi = double(xi); + yi = double(yi); + + GrayImageDb = double(GrayImage); + + % multi iteration refinement using NU-RLMS in each one + i=1; + + current_patch_scaling = patchExperts(scale).trainingScale; + visibilities = patchExperts(scale).visibilities; + + view = GetView(patchExperts(scale).centers, global_params(2:4)); + + % The shape fitting is performed in the reference frame of the + % patch training scale + refGlobal = [current_patch_scaling, 0, 0, 0, 0, 0]'; + + % the reference shape + refShape = GetShapeOrtho(M, PDM.V, local_params, refGlobal); + + % shape around which the patch experts will be evaluated in the original image + [shape2D] = GetShapeOrtho(M, PDM.V, local_params, global_params); + shape2D_img = shape2D(:,1:2); + + % Create transform using a slightly modified version of Kabsch that + % takes scaling into account as well, in essence we get a + % similarity transform from current estimate to reference shape + [A_img2ref, T_img2ref, ~, ~] = AlignShapesWithScale(shape2D_img(:,1:2),refShape(:,1:2)); + + % Create a transform, from shape in image to reference shape + T = maketform('affine', [A_img2ref;T_img2ref]); + + shape_2D_ref = tformfwd(T, shape2D_img); + + % transform the current shape to the reference one, so we can + % interpolate + shape2D_in_ref = (A_img2ref * shape2D_img')'; + + sideSizeX = (clmParams.window_size(i,1) - 1)/2; + sideSizeY = (clmParams.window_size(i,2) - 1)/2; + + patches = zeros(size(shape2D_in_ref,1), clmParams.window_size(i,1) * clmParams.window_size(i,2)); + + Ainv = inv(A_img2ref); + + % extract patches on which patch experts will be evaluted + for l=1:size(shape2D_in_ref,1) + if(visibilities(view,l)) + + xs = (shape2D_in_ref(l,1)-sideSizeX):(shape2D_in_ref(l,1)+sideSizeX); + ys = (shape2D_in_ref(l,2)-sideSizeY):(shape2D_in_ref(l,2)+sideSizeY); + + [xs, ys] = meshgrid(xs, ys); + + pairs = [xs(:), ys(:)]; + + actualLocs = (Ainv * pairs')'; + + actualLocs(actualLocs(:,1) < 0,1) = 0; + actualLocs(actualLocs(:,2) < 0,2) = 0; + actualLocs(actualLocs(:,1) > widthImg - 1,1) = widthImg - 1; + actualLocs(actualLocs(:,2) > heightImg - 1,2) = heightImg - 1; + + [t_patch] = interp2_mine(xi, yi, GrayImageDb, actualLocs(:,1), actualLocs(:,2), 'bilinear'); + t_patch = reshape(t_patch, size(xs)); + + patches(l,:) = t_patch(:); + + end + end + + % Calculate patch responses, either SVR or CCNF + if(strcmp(patchExperts(scale).type, 'SVR')) + responses = PatchResponseSVM_multi_modal( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale).normalisationOptionsCol, clmParams, clmParams.window_size(i,:)); + + for r=1:numel(responses) + out_patch = reshape(patches(r,:)/255, size(xs)); + imwrite(out_patch, [out_dir, '/', num2str(r), '_a.png']); + imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_svr.png']); + end + + elseif(strcmp(patchExperts(scale).type, 'CCNF')) + responses = PatchResponseCCNF( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:)); + for r=1:numel(responses) + imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_lnf.png']); + end + elseif(strcmp(patchExperts(scale).type, 'DNN')) + responses = PatchResponseDNN( patches, patchExperts(scale).patch_experts(view,:), visibilities(view,:), patchExperts(scale), clmParams.window_size(i,:)); + for r=1:numel(responses) + imwrite(responses{r}/max(responses{r}(:)), [out_dir, '/', num2str(r), '_dnn.png']); + end + end + + + +end + + +function [id] = GetView(centers, rotation) + + [~,id] = min(sum((centers * pi/180 - repmat(rotation', size(centers,1), 1)).^2,2)); + +end \ No newline at end of file