diff --git a/matlab_version/demo/face_image_demo.m b/matlab_version/demo/face_image_demo.m index 654b0acb..80ad8973 100644 --- a/matlab_version/demo/face_image_demo.m +++ b/matlab_version/demo/face_image_demo.m @@ -16,6 +16,10 @@ addpath('../CCNF/'); clmParams.multi_modal_types = patches(1).multi_modal_types; +% Dependencies for face detection (MatConvNet), remove if not present +setup_mconvnet; +addpath('../face_detection/mtcnn/'); + %% root_dir = '../../samples/'; images = dir([root_dir, '*.jpg']); @@ -25,8 +29,11 @@ verbose = true; for img=1:numel(images) image_orig = imread([root_dir images(img).name]); + % MTCNN face detector + [bboxs, det_shapes, confidences] = detect_face_mtcnn(image_orig); + % First attempt to use the Matlab one (fastest but not as accurate, if not present use yu et al.) - [bboxs, det_shapes] = detect_faces(image_orig, {'cascade', 'yu'}); + % [bboxs, det_shapes] = detect_faces(image_orig, {'cascade', 'yu'}); % Zhu and Ramanan and Yu et al. are slower, but also more accurate % and can be used when vision toolbox is unavailable % [bboxs, det_shapes] = detect_faces(image_orig, {'yu', 'zhu'}); @@ -52,28 +59,14 @@ for img=1:numel(images) hold on; end - for i=1:size(bboxs,2) + for i=1:size(bboxs,1) % Convert from the initial detected shape to CLM model parameters, % if shape is available - bbox = bboxs(:,i); - - if(~isempty(det_shapes)) - shape = det_shapes(:,:,i); - inds = [1:60,62:64,66:68]; - M = pdm.M([inds, inds+68, inds+68*2]); - E = pdm.E; - V = pdm.V([inds, inds+68, inds+68*2],:); - [ a, R, T, ~, params, err, shapeOrtho] = fit_PDM_ortho_proj_to_2D(M, E, V, shape); - g_param = [a; Rot2Euler(R)'; T]; - l_param = params; + bbox = bboxs(i,:); - % Use the initial global and local params for clm fitting in the image - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams, 'gparam', g_param, 'lparam', l_param); - else - [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); - end + [shape,~,~,lhood,lmark_lhood,view_used] = Fitting_from_bb(image, [], bbox, pdm, patches, clmParams); % shape correction for matlab format shape = shape + 1; diff --git a/matlab_version/demo/face_video_demo.m b/matlab_version/demo/face_video_demo.m index 2241c686..ba84b2d2 100644 --- a/matlab_version/demo/face_video_demo.m +++ b/matlab_version/demo/face_video_demo.m @@ -33,6 +33,12 @@ od = cd('../face_validation/'); setup; cd(od); +% Setup the face detector (remove the setup mconvnet if not using +% MatConvNet) +setup_mconvnet; +addpath('../face_detection/mtcnn/'); + + %% for v=1:numel(vids) % load the video @@ -66,8 +72,9 @@ for v=1:numel(vids) image_orig = read(vr, i); if((~det && mod(i,4) == 0) || ~initialised) + [bboxs, det_shapes, confidences] = detect_face_mtcnn(image_orig); % First attempt to use the Matlab one (fastest but not as accurate, if not present use yu et al.) - [bboxs, det_shapes] = detect_faces(image_orig, {'cascade', 'yu'}); + % [bboxs, det_shapes] = detect_faces(image_orig, {'cascade', 'yu'}); % Zhu and Ramanan and Yu et al. are slower, but also more accurate % and can be used when vision toolbox is unavailable % [bboxs, det_shapes] = detect_faces(image_orig, {'yu', 'zhu'}); @@ -75,8 +82,8 @@ for v=1:numel(vids) if(~isempty(bboxs)) % Pick the biggest face for tracking - [~,ind] = max(bboxs(3,:) - bboxs(1,:)); - bbox = bboxs(:,ind); + [~,ind] = max(bboxs(:,3) - bboxs(:,1)); + bbox = bboxs(ind,:); % Discard overly small detections if(bbox(3) - bbox(1) > 40) @@ -84,39 +91,27 @@ for v=1:numel(vids) % Either infer the local and global shape parameters % from the detected landmarks or just using the % bounding box - if(~isempty(det_shapes)) - shape = det_shapes(:,:,ind); - - inds = [1:60,62:64,66:68]; - M = pdm.M([inds, inds+68, inds+68*2]); - E = pdm.E; - V = pdm.V([inds, inds+68, inds+68*2],:); - [ a, R, T, ~, params, err] = fit_PDM_ortho_proj_to_2D(M, E, V, shape); - g_param_n = [a; Rot2Euler(R)'; T]; - l_param_n = params; - else - num_points = numel(pdm.M) / 3; + num_points = numel(pdm.M) / 3; - M = reshape(pdm.M, num_points, 3); - width_model = max(M(:,1)) - min(M(:,1)); - height_model = max(M(:,2)) - min(M(:,2)); + M = reshape(pdm.M, num_points, 3); + width_model = max(M(:,1)) - min(M(:,1)); + height_model = max(M(:,2)) - min(M(:,2)); - a = (((bbox(3) - bbox(1)) / width_model) + ((bbox(4) - bbox(2))/ height_model)) / 2; + a = (((bbox(3) - bbox(1)) / width_model) + ((bbox(4) - bbox(2))/ height_model)) / 2; - tx = (bbox(3) + bbox(1))/2; - ty = (bbox(4) + bbox(2))/2; + tx = (bbox(3) + bbox(1))/2; + ty = (bbox(4) + bbox(2))/2; - % correct it so that the bounding box is just around the minimum - % and maximum point in the initialised face - tx = tx - a*(min(M(:,1)) + max(M(:,1)))/2; - ty = ty + a*(min(M(:,2)) + max(M(:,2)))/2; + % correct it so that the bounding box is just around the minimum + % and maximum point in the initialised face + tx = tx - a*(min(M(:,1)) + max(M(:,1)))/2; + ty = ty + a*(min(M(:,2)) + max(M(:,2)))/2; - % visualisation - g_param_n = [a, 0, 0, 0, tx, ty]'; + % visualisation + g_param_n = [a, 0, 0, 0, tx, ty]'; - l_param_n = zeros(size(pdm.E)); - end + l_param_n = zeros(size(pdm.E)); % If tracking has not started trust the detection if(~initialised) @@ -186,7 +181,7 @@ for v=1:numel(vids) end hold off; drawnow expose; - pause(0.05); + pause(0.01); if(record) frame = getframe; diff --git a/matlab_version/face_detection/detect_faces.m b/matlab_version/face_detection/detect_faces.m index 42cdc1d0..078ac069 100644 --- a/matlab_version/face_detection/detect_faces.m +++ b/matlab_version/face_detection/detect_faces.m @@ -4,8 +4,8 @@ function [ bboxes, shapes ] = detect_faces( image, types ) % image - the image to detect the faces on % type - cell array of the face detectors to use: 'zhu', 'yu', 'cascade' % OUTPUT: -% bboxes - a set of bounding boxes describing the detected faces 4 x -% num_faces, the format is [min_x; min_y; max_x; max_y]; +% bboxes - a set of bounding boxes describing the detected faces num_faces x +% 4, the format is [min_x; min_y; max_x; max_y]; % shapes - if the face detector detects landmarks as well, output them % n_points x 2 x num_faces @@ -57,6 +57,6 @@ function [ bboxes, shapes ] = detect_faces( image, types ) if(use_zhu && isempty(bboxes)) [bboxes, shapes] = Detect_tree_based_zhu(image); end - + bboxes = bboxes'' end diff --git a/matlab_version/face_detection/mtcnn/PReLU.m b/matlab_version/face_detection/mtcnn/PReLU.m index e573d901..4c345315 100644 --- a/matlab_version/face_detection/mtcnn/PReLU.m +++ b/matlab_version/face_detection/mtcnn/PReLU.m @@ -9,6 +9,10 @@ function [ out_map ] = PReLU( input_maps, PReLU_params ) % A more readable but slower version % in_map = input_maps(:,:,i,:); % in_map(in_map < 0) = in_map(in_map<0) * PReLU_params(i); + + % alternative +% out_map(:,:,i,:) = max(input_maps(:,:,i,:),0) + min(input_maps(:,:,i,:),0)*PReLU_params(i); + out_map(:,:,i,:) = input_maps(:,:,i,:) .* (PReLU_params(i) + (1 - PReLU_params(i)) * (input_maps(:,:,i,:) > 0)) ; end else diff --git a/matlab_version/face_detection/mtcnn/detect_face_mtcnn.m b/matlab_version/face_detection/mtcnn/detect_face_mtcnn.m index 21a6e36c..137f64b6 100644 --- a/matlab_version/face_detection/mtcnn/detect_face_mtcnn.m +++ b/matlab_version/face_detection/mtcnn/detect_face_mtcnn.m @@ -1,5 +1,10 @@ function [total_bboxes, lmarks, confidence] = detect_face_mtcnn(img, min_face_size) +% Check if MatConvNet is installed +if(~exist('vl_nnconv', 'file') == 3) + fprintf('Warning MatConvNet is not installed or not setup, face detection will be quite slow\n'); +end + height_orig = size(img,1); width_orig = size(img,2); @@ -211,5 +216,7 @@ new_txs = widths * -0.0075 + txs; new_tys = heights * 0.2459 + tys; total_bboxes = [new_txs, new_tys, new_txs + new_widths, new_tys + new_heights]; +total_bboxes = double(total_bboxes); +lmarks = double(lmarks); end \ No newline at end of file