From 5588742e322f7c55345fef3eb49b75c3b8b23bc5 Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Fri, 11 Aug 2017 13:49:55 -0400 Subject: [PATCH] 300W experiment with C++ data and sparse response maps. --- .gitignore | 1 + .../src/LandmarkDetectorUtils.cpp | 5 +- .../LandmarkDetector/src/Patch_experts.cpp | 9 +- .../run_OpenFace_feature_point_tests_300W.m | 2 + .../fitting/PatchResponseCEN_mirror.m | 84 +++++++++++++++++++ 5 files changed, 92 insertions(+), 9 deletions(-) create mode 100644 matlab_version/fitting/PatchResponseCEN_mirror.m diff --git a/.gitignore b/.gitignore index 3688a0d3..d8c8d9a5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ exe/FaceLandmarkVid/x64/ exe/FaceLandmarkImg/Debug/ OpenFace.sdf OpenFace.VC.opendb +matlab_runners/Feature Point Experiments/out_ceclm/ diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp index 7634f679..ffba8441 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorUtils.cpp @@ -432,7 +432,10 @@ namespace LandmarkDetector // Make sure the same number of images and bounding boxes is present, if any bounding boxes are defined if (input_bounding_boxes.size() > 0) { - assert(input_bounding_boxes.size() == input_image_files.size()); + if(input_bounding_boxes.size() != input_image_files.size()) + { + cout << "Warning, the input number of images does not match the input number of bounding boxes\n" << endl; + } } // Clear up the argument list diff --git a/lib/local/LandmarkDetector/src/Patch_experts.cpp b/lib/local/LandmarkDetector/src/Patch_experts.cpp index 7a73d614..9454b163 100644 --- a/lib/local/LandmarkDetector/src/Patch_experts.cpp +++ b/lib/local/LandmarkDetector/src/Patch_experts.cpp @@ -214,14 +214,7 @@ void Patch_experts::Response(vector >& patch_expert_responses, c // Get intensity response either from the SVR, CCNF, or CEN patch experts (prefer CEN as they are the most accurate so far) if (!cen_expert_intensity.empty()) { - if (scale <= 2) - { - cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i]); - } - else - { - cen_expert_intensity[scale][view_id][i].Response(area_of_interest, patch_expert_responses[i]); - } + cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i]); } else if (!ccnf_expert_intensity.empty()) { diff --git a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m index cd1036a2..f48c1b9c 100644 --- a/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m +++ b/matlab_runners/Feature Point Experiments/run_OpenFace_feature_point_tests_300W.m @@ -7,6 +7,8 @@ if(exist([getenv('USERPROFILE') '/Dropbox/AAM/test data/'], 'file')) database_root = [getenv('USERPROFILE') '/Dropbox/AAM/test data/']; elseif(exist('D:/Dropbox/Dropbox/AAM/test data/', 'file')) database_root = 'D:/Dropbox/Dropbox/AAM/test data/'; +elseif(exist('D:\Datasets\300W/', 'file')) + database_root = 'D:\Datasets\300W/'; else database_root = '/multicomp/datasets/300-W/'; end diff --git a/matlab_version/fitting/PatchResponseCEN_mirror.m b/matlab_version/fitting/PatchResponseCEN_mirror.m new file mode 100644 index 00000000..aec068c1 --- /dev/null +++ b/matlab_version/fitting/PatchResponseCEN_mirror.m @@ -0,0 +1,84 @@ +function [ responses ] = PatchResponseCEN_mirror(patches, patch_experts_class, visibilities, patchExperts, window_size) +% As frontal faces are roughly symmetrical can compute the responses for +% two patches at the same time using only one of the landmark patch experts + + normalisationOptions = patchExperts.normalisationOptionsCol; + patchSize = normalisationOptions.patchSize; + + responses = cell(size(patches, 1), 1); + empty = zeros(window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1); + + % These landmark responses can be computed together + mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;... + 32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;... + 61,65;62,64;68,66]; + + for i = 1:numel(patches(:,1)) + if visibilities(i) + % Do it only if not mirrored + if(isempty(find(mirror_inds(:,2)==i, 1))) + responses{i} = empty; + + col_norm = normalisationOptions.useNormalisedCrossCorr == 1; + + smallRegionVec = patches(i,:); + smallRegion = reshape(smallRegionVec, window_size(1), window_size(2)); + + patch = im2col_mine(smallRegion, patchSize)'; + + % Add the mirrored version as well (it will be applied the + % same way) + mirr_id = mirror_inds(find(mirror_inds(:,1)==i,1),2); + if(~isempty(mirr_id)) + responses{mirr_id} = empty; + smallRegionVec_mirr = patches(mirr_id,:); + smallRegion_mirr = reshape(smallRegionVec_mirr, window_size(1), window_size(2)); + patch_mirr = im2col_mine(fliplr(smallRegion_mirr), patchSize)'; + patch = cat(1, patch, patch_mirr); + end + + % Normalize + if(col_norm) + mean_curr = mean(patch, 2); + patch_normed = patch - repmat(mean_curr, 1, patchSize(1)* patchSize(2)); + + % Normalising the patches using the L2 norm + scaling = sqrt(sum(patch_normed.^2,2)); + scaling(scaling == 0) = 1; + + patch_normed = patch_normed ./ repmat(scaling, 1, 11 * 11); + + patch = patch_normed; + end + patch = patch'; + % Add bias + patch_normed = cat(1, ones(1, size(patch,2)), patch); + weights = patch_experts_class{i}; + % Where DNN will happen + for w =1:numel(weights)/2 + + % mult and bias + patch_normed = weights{(w-1)*2+1}' * patch_normed + repmat(weights{(w-1)*2+2}', 1, size(patch_normed,2)); + + if w < 3 + % patch_normed(patch_normed < 0) = 0; + patch_normed = max(0, patch_normed); + else + patch_normed = 1./(1+exp(-patch_normed)); + end + + end + % If no mirroring took place + if(isempty(mirr_id)) + responses{i}(:) = reshape(patch_normed', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1); + else + patch_normed_1 = patch_normed(1:end/2); + patch_normed_2 = patch_normed(end/2+1:end); + responses{i}(:) = reshape(patch_normed_1', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1); + responses{mirr_id}(:) = fliplr(reshape(patch_normed_2', window_size(1)-patchSize(1)+1, window_size(2)-patchSize(2)+1)); + end + end + end + end + +end