diff --git a/.gitignore b/.gitignore index 071561ae..564feb53 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ matlab_runners/Action Unit Experiments/out_DISFA/ matlab_runners/Action Unit Experiments/out_fera/ matlab_runners/Action Unit Experiments/out_SEMAINE/ matlab_runners/Gaze Experiments/mpii_out/ +*.vspx diff --git a/lib/local/LandmarkDetector/include/CEN_patch_expert.h b/lib/local/LandmarkDetector/include/CEN_patch_expert.h index bf4f86b8..145ec207 100644 --- a/lib/local/LandmarkDetector/include/CEN_patch_expert.h +++ b/lib/local/LandmarkDetector/include/CEN_patch_expert.h @@ -81,6 +81,9 @@ namespace LandmarkDetector // Faster version of the response that only considers a subset of the area_of_interest void ResponseSparse(const cv::Mat_ &area_of_interest, cv::Mat_ &response, cv::Mat_& mapMatrix); + void ResponseSparse_mirror(const cv::Mat_ &area_of_interest, cv::Mat_ &response, cv::Mat_& mapMatrix); + + void ResponseSparse_mirror_joint(const cv::Mat_ &area_of_interest_left, const cv::Mat_ &area_of_interest_right, cv::Mat_ &response_left, cv::Mat_ &response_right, cv::Mat_& mapMatrix); }; diff --git a/lib/local/LandmarkDetector/include/Patch_experts.h b/lib/local/LandmarkDetector/include/Patch_experts.h index 582aa984..80e0bde4 100644 --- a/lib/local/LandmarkDetector/include/Patch_experts.h +++ b/lib/local/LandmarkDetector/include/Patch_experts.h @@ -76,6 +76,9 @@ public: // Landmark visibilities for each scale and view vector > > visibilities; + cv::Mat_ mirror_inds; + cv::Mat_ mirror_views; + // Early termination calibration values, useful for CE-CLM model to speed up the multi-hypothesis setup vector early_term_weights; vector early_term_biases; diff --git a/lib/local/LandmarkDetector/model/cen_general.txt b/lib/local/LandmarkDetector/model/cen_general.txt index f76e85c9..1249c1d4 100644 --- a/lib/local/LandmarkDetector/model/cen_general.txt +++ b/lib/local/LandmarkDetector/model/cen_general.txt @@ -1,7 +1,7 @@ -PDM pdms/pdm_68_aligned_menpo.txt +PDM pdms/In-the-wild_aligned_PDM_68.txt Triangulations tris_68.txt -PatchesCEN patch_experts/cen_patches_0.25_menpo.dat -PatchesCEN patch_experts/cen_patches_0.35_menpo.dat -PatchesCEN patch_experts/cen_patches_0.50_menpo.dat -PatchesCEN patch_experts/cen_patches_1.00_menpo.dat -EarlyTermination early_term_cen.txt \ No newline at end of file +PatchesCEN patch_experts/cen_patches_0.25_of.dat +PatchesCEN patch_experts/cen_patches_0.35_of.dat +PatchesCEN patch_experts/cen_patches_0.50_of.dat +PatchesCEN patch_experts/cen_patches_1.00_of.dat +EarlyTermination early_term_cen_of.txt \ No newline at end of file diff --git a/lib/local/LandmarkDetector/model/early_term_cen.txt b/lib/local/LandmarkDetector/model/early_term_cen.txt deleted file mode 100644 index ea239e66..00000000 --- a/lib/local/LandmarkDetector/model/early_term_cen.txt +++ /dev/null @@ -1 +0,0 @@ -1.000 0.954 0.736 0.903 0.903 0.736 0.954 0.000 0.072 -0.279 -0.380 -0.380 -0.279 0.072 -0.270 -0.230 0.070 0.210 0.210 0.070 -0.230 \ No newline at end of file diff --git a/lib/local/LandmarkDetector/model/early_term_cen_of.txt b/lib/local/LandmarkDetector/model/early_term_cen_of.txt new file mode 100644 index 00000000..f7c39aca --- /dev/null +++ b/lib/local/LandmarkDetector/model/early_term_cen_of.txt @@ -0,0 +1 @@ +1.000 0.969 1.017 0.954 0.954 1.017 0.969 0.000 0.047 0.164 -0.375 -0.375 0.164 0.047 -0.270 -0.260 -0.170 0.250 0.250 -0.170 -0.260 \ No newline at end of file diff --git a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp index 69bba2aa..4d4c62d1 100644 --- a/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp +++ b/lib/local/LandmarkDetector/src/CEN_patch_expert.cpp @@ -108,10 +108,8 @@ void CEN_patch_expert::Read(ifstream &stream) if (num_layers == 0) { - // empty patch due to landmark being invisible at that orientation - - // read an empty int (due to the way things were written out) - stream.read((char*)&num_layers, 4); + // empty patch due to landmark being invisible at that orientation (or visible through mirroring) + stream.read((char*)&confidence, 8); return; } @@ -573,3 +571,83 @@ void CEN_patch_expert::ResponseSparse(const cv::Mat_ &area_of_interest, c response = response.reshape(1, response_height); response = response.t(); } + +//=========================================================================== +void CEN_patch_expert::ResponseSparse_mirror(const cv::Mat_ &area_of_interest, cv::Mat_ &response, cv::Mat_& mapMatrix) +{ + + int response_height = area_of_interest.rows - height + 1; + int response_width = area_of_interest.cols - width + 1; + + cv::flip(area_of_interest, area_of_interest, 1); + + // Extract im2col but in a sparse way and contrast normalize + im2colBiasSparseContrastNorm(area_of_interest, width, height, response); + response = response.t(); + + for (size_t layer = 0; layer < activation_function.size(); ++layer) + { + + // We are performing response = weights[layers] * response(t), but in OpenBLAS as that is significantly quicker than OpenCV + cv::Mat_ resp = response; + float* m1 = (float*)resp.data; + cv::Mat_ weight = weights[layer]; + float* m2 = (float*)weight.data; + + cv::Mat_ resp_blas(weight.rows, resp.cols); + float* m3 = (float*)resp_blas.data; + + // Perform matrix multiplication in OpenBLAS (fortran call) + float alpha1 = 1.0; + float beta1 = 0.0; + sgemm_("N", "N", &resp.cols, &weight.rows, &weight.cols, &alpha1, m1, &resp.cols, m2, &weight.cols, &beta1, m3, &resp.cols); + + // The above is a faster version of this, by calling the fortran version directly + //cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, resp.cols, weight.rows, weight.cols, 1, m1, resp.cols, m2, weight.cols, 0.0, m3, resp.cols); + + // Adding the bias (bit ugly, but the fastest way to do this) + response = resp_blas; + + float* data = (float*)response.data; + size_t height = response.rows; + size_t width = response.cols; + float* data_b = (float*)biases[layer].data; + for (size_t y = 0; y < height; ++y) + { + float bias = data_b[y]; + for (size_t x = 0; x < width; ++x) + { + float in = *data + bias; + *data++ = in; + } + } + + // Perform activation and add bias at the same time + if (activation_function[layer] == 0) // Sigmoid + { + + size_t resp_size = response.rows * response.cols; + + // Iterate over the data directly + float* data = (float*)response.data; + + for (size_t counter = 0; counter < resp_size; ++counter) + { + float in = *data; + *data++ = 1.0 / (1.0 + exp(-(in))); + } + + } + else if (activation_function[layer] == 2)// ReLU + { + cv::threshold(response, response, 0, 0, cv::THRESH_TOZERO); + } + + } + + response = response * mapMatrix; + response = response.t(); + response = response.reshape(1, response_height); + response = response.t(); + cv::flip(response, response, 1); +} \ No newline at end of file diff --git a/lib/local/LandmarkDetector/src/Patch_experts.cpp b/lib/local/LandmarkDetector/src/Patch_experts.cpp index 730c9f13..8d6a7cac 100644 --- a/lib/local/LandmarkDetector/src/Patch_experts.cpp +++ b/lib/local/LandmarkDetector/src/Patch_experts.cpp @@ -226,9 +226,15 @@ void Patch_experts::Response(vector >& patch_expert_responses, c // Get intensity response either from the SVR, CCNF, or CEN patch experts (prefer CEN as they are the most accurate so far) if (!cen_expert_intensity.empty()) { - - cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i], interp_mat); - + // For space and memory saving use a mirrored patch expert + if(!cen_expert_intensity[scale][view_id][i].biases.empty()) + { + cen_expert_intensity[scale][view_id][i].ResponseSparse(area_of_interest, patch_expert_responses[i], interp_mat); + } + else + { + cen_expert_intensity[scale][mirror_views.at(view_id)][mirror_inds.at(i)].ResponseSparse_mirror(area_of_interest, patch_expert_responses[i], interp_mat); + } // A slower, but slightly more accurate version //cen_expert_intensity[scale][view_id][i].Response(area_of_interest, patch_expert_responses[i]); @@ -546,6 +552,9 @@ void Patch_experts::Read_CEN_patch_experts(string expert_location, std::vector= 0.95) - count = i; - break; - end -end - -PC = PC(:,1:count); - -save('generic_face_rigid.mat', 'PC', 'means_norm', 'stds_norm'); diff --git a/matlab_version/AU_training/pca_generation/createSubFaceModels.m b/matlab_version/AU_training/pca_generation/create_face_pca_models.m similarity index 91% rename from matlab_version/AU_training/pca_generation/createSubFaceModels.m rename to matlab_version/AU_training/pca_generation/create_face_pca_models.m index 38f0f300..22b38bae 100644 --- a/matlab_version/AU_training/pca_generation/createSubFaceModels.m +++ b/matlab_version/AU_training/pca_generation/create_face_pca_models.m @@ -1,7 +1,7 @@ clear; %% CK+, FERA2011, and UNBC datasets -hog_dir = 'D:\Datasets/face_datasets/hog_aligned_rigid/'; +hog_dir = 'E:\Datasets/face_datasets/hog_aligned_rigid/'; hog_files = dir([hog_dir, '*.hog']); [appearance_data, valid_inds, vid_ids_train] = Read_HOG_files_small(hog_files, hog_dir); @@ -9,7 +9,7 @@ appearance_data = appearance_data(valid_inds,:); vid_ids_train = vid_ids_train(valid_inds,:); %% Bosphorus dataset -hog_dir = 'D:\Datasets/face_datasets/hog_aligned_rigid_b/'; +hog_dir = 'E:\Datasets/face_datasets/hog_aligned_rigid_b/'; hog_files = dir([hog_dir, '*.hog']); [appearance_data_bosph, valid_inds, vid_ids_train_bosph] = Read_HOG_files_small(hog_files, hog_dir); @@ -21,7 +21,7 @@ appearance_data = cat(1,appearance_data, appearance_data_bosph); vid_ids_train = cat(1,vid_ids_train, vid_ids_train_bosph); %% DISFA -hog_dir = 'D:\Datasets\DISFA\hog_aligned_rigid/'; +hog_dir = 'E:\Datasets\DISFA\hog_aligned_rigid/'; hog_files = dir([hog_dir, '*.hog']); [appearance_data_disfa, valid_inds, vid_ids_train_disfa] = Read_HOG_files_small(hog_files, hog_dir, 100); @@ -33,7 +33,7 @@ appearance_data = cat(1,appearance_data, appearance_data_disfa); vid_ids_train = cat(1,vid_ids_train, vid_ids_train_disfa); %% BP4D -hog_dir = 'D:\Datasets\FERA_2015\bp4d\processed_data/train/'; +hog_dir = 'E:\Datasets\FERA_2015\bp4d\processed_data/train/'; hog_files = dir([hog_dir, '*.hog']); [appearance_data_bp, valid_inds, vid_ids_train_bp] = Read_HOG_files_small(hog_files, hog_dir, 50); @@ -45,7 +45,7 @@ appearance_data = cat(1,appearance_data, appearance_data_bp); vid_ids_train = cat(1,vid_ids_train, vid_ids_train_bp); %% SEMAINE -hog_dir = 'D:\Datasets\FERA_2015\semaine\processed_data\train\'; +hog_dir = 'E:\Datasets\FERA_2015\semaine\processed_data\train\'; hog_files = dir([hog_dir, '*.hog']); [appearance_data_semaine, valid_inds, vid_ids_train_semaine] = Read_HOG_files_small(hog_files, hog_dir, 300); diff --git a/matlab_version/learn_error_mapping/early_term_cen_of.txt b/matlab_version/learn_error_mapping/early_term_cen_of.txt new file mode 100644 index 00000000..f7c39aca --- /dev/null +++ b/matlab_version/learn_error_mapping/early_term_cen_of.txt @@ -0,0 +1 @@ +1.000 0.969 1.017 0.954 0.954 1.017 0.969 0.000 0.047 0.164 -0.375 -0.375 0.164 0.047 -0.270 -0.260 -0.170 0.250 0.250 -0.170 -0.260 \ No newline at end of file diff --git a/matlab_version/learn_error_mapping/readme.txt b/matlab_version/learn_error_mapping/readme.txt index 4c90acb4..fef49698 100644 --- a/matlab_version/learn_error_mapping/readme.txt +++ b/matlab_version/learn_error_mapping/readme.txt @@ -9,4 +9,4 @@ Then: learn_error_pred_general.m learn_error_pred_menpo.m -To use in C++ also generate the early_term_cen.txt file using output_corrections.m \ No newline at end of file +To use in C++ also generate the early_term_cen_of.txt file using output_corrections.m, this is already generated so no need to re-run it. \ No newline at end of file diff --git a/matlab_version/models/cen/create_cen_experts_gen.m b/matlab_version/models/cen/create_cen_experts_gen.m index 4c6028f8..c5106ab7 100644 --- a/matlab_version/models/cen/create_cen_experts_gen.m +++ b/matlab_version/models/cen/create_cen_experts_gen.m @@ -109,5 +109,5 @@ for s=scales end trainingScale = str2num(s{1}); save(['cen_patches_', s{1} '_general.mat'], 'trainingScale', 'centers', 'visiIndex', 'patch_experts', 'normalisationOptions'); -% write_patch_expert_bin(['cen_patches_', s{1} '_general.dat'], trainingScale, centers, visiIndex, patch_experts); -end \ No newline at end of file + +end diff --git a/matlab_version/models/cen/crete_cen_experts_OF.m b/matlab_version/models/cen/crete_cen_experts_OF.m new file mode 100644 index 00000000..653e432c --- /dev/null +++ b/matlab_version/models/cen/crete_cen_experts_OF.m @@ -0,0 +1,58 @@ +clear; +%% +mirrorInds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;... +32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;... +61,65;62,64;68,66]; + +mirror_inds = [1:68]; +mirror_inds(mirrorInds(:,1)) = mirrorInds(:,2); +mirror_inds(mirrorInds(:,2)) = mirrorInds(:,1); + +scales = {'0.25', '0.35', '0.50', '1.00'}; + +for s=scales + + gen_experts = load(sprintf('cen_patches_%s_general.mat', s{1})); + load(sprintf('cen_patches_%s_menpo.mat', s{1})); + + for c=1:size(visiIndex,1) + for i=1:size(visiIndex,2) + % If present in general and not menpo replace + if(gen_experts.visiIndex(c,i) && ~visiIndex(c,i)) + visiIndex(c,i) = 1; + patch_experts.correlations(c,i) = gen_experts.patch_experts.correlations(c,i); + patch_experts.rms_errors(c,i) = gen_experts.patch_experts.rms_errors(c,i); + patch_experts.patch_experts(c,i) = gen_experts.patch_experts.patch_experts(c,i); + elseif(~visiIndex(c,i)) + patch_experts.correlations(c,i) = 0; + patch_experts.rms_errors(c,i) = 0; + patch_experts.patch_experts(c,i) = {[]}; + end + + end + end + % Work out the frontal view and remove mirror indices for it + [~, frontal] = min(mean(abs(bsxfun(@plus, centers, [0,0,0])'))); + + % First clean up the frontal view + patch_experts.patch_experts(frontal, mirrorInds(:,2)) = {[]}; + + % Work out which views have mirrors of each other, and keep only one set + % of them + n_views = size(visiIndex,1); + + mirror_view = 1:n_views; + + for i = 1:n_views + [~, mirror_view(i)] = min(mean(abs(bsxfun(@plus, centers, centers(i,:))'))); + end + + % Remove a set of mirror indices + to_rem = mirror_view < 1:n_views; + + patch_experts.patch_experts(to_rem, :) = {[]}; + + trainingScale = str2num(s{1}); + write_patch_expert_bin_simple(['cen_patches_', s{1} '_of.dat'], trainingScale, centers, visiIndex, patch_experts, mirror_inds - 1, mirror_view - 1); + +end \ No newline at end of file diff --git a/matlab_version/models/cen/readme.txt b/matlab_version/models/cen/readme.txt index bea82d24..527fa940 100644 --- a/matlab_version/models/cen/readme.txt +++ b/matlab_version/models/cen/readme.txt @@ -1,4 +1,7 @@ Scripts for creating CEN patch experts from already trained models. To create one from 300W + MultiPIE - create_cen_experts_gen.m -To create one from 300W + MultiPIE + Menpo - create_cen_experts_menpo.m \ No newline at end of file +To create one from 300W + MultiPIE + Menpo - create_cen_experts_menpo.m + +To create one used in OpenFace for C++ code: +create_cen_experts_OF.m (this uses both the general and menpo experts to create a joint one, with general ones used when menpo unavailable for that view) \ No newline at end of file diff --git a/matlab_version/models/cen/write_patch_expert_bin.m b/matlab_version/models/cen/write_patch_expert_bin.m index 86f05577..44f2aaf9 100644 --- a/matlab_version/models/cen/write_patch_expert_bin.m +++ b/matlab_version/models/cen/write_patch_expert_bin.m @@ -25,7 +25,7 @@ function write_patch_expert_bin(location, trainingScale, centers, visiIndex, pat for i=1:n_views for j=1:n_landmarks - % Write out that we're writing a dpn patch expert of 11x11 support region + % Write out that we're writing a CEN patch expert of 11x11 support region fwrite(patches_file, 6, 'int'); fwrite(patches_file, 11, 'int'); fwrite(patches_file, 11, 'int'); diff --git a/matlab_version/models/cen/write_patch_expert_bin_simple.m b/matlab_version/models/cen/write_patch_expert_bin_simple.m new file mode 100644 index 00000000..45c05766 --- /dev/null +++ b/matlab_version/models/cen/write_patch_expert_bin_simple.m @@ -0,0 +1,79 @@ +function write_patch_expert_bin_simple(location, trainingScale, centers, visiIndex, patch_experts, mirror_inds, mirror_view) + + patches_file = fopen(location, 'w'); + + [n_views, n_landmarks, ~] = size(patch_experts.correlations); + + % write out the scaling factor as this is what will be used when + % fitting on the window + fwrite(patches_file, trainingScale, 'float64'); + + fwrite(patches_file, n_views, 'int'); + + % Write out the information about the view's and centers here + for i=1:n_views + % this indicates that we're writing a 3x1 double matrix + writeMatrixBin(patches_file, centers(i,:)', 6); + end + + % Write out the visibilities + for i=1:n_views + % this indicates that we're writing a 3x1 double matrix + writeMatrixBin(patches_file, visiIndex(i,:)', 4); + end + + % Write out the mirror indices + writeMatrixBin(patches_file, mirror_inds, 4); + + % Write out the mirror views + writeMatrixBin(patches_file, mirror_view, 4); + + for i=1:n_views + for j=1:n_landmarks + + % Write out that we're writing a CEN patch expert of 11x11 support region + fwrite(patches_file, 6, 'int'); + fwrite(patches_file, 11, 'int'); + fwrite(patches_file, 11, 'int'); + + if(~visiIndex(i,j)) + % Write out that there won't be any neurons for this + % landmark + fwrite(patches_file, 0, 'int'); + fwrite(patches_file, patch_experts.correlations(i,j), 'float64'); % also writing out a fake correlation + else + num_layers = numel(patch_experts.patch_experts{i,j})/2; + + fwrite(patches_file, num_layers, 'int'); + + for n=1:num_layers + + % output the actual layer + + % Layer type, bias, weights + + % Type of layer, first two are relu, the final one is a + % sigmoid (0 - sigmoid, 1 - tanh_opt, 2 - ReLU) + if(n < 3) + fwrite(patches_file, 2, 'int'); + else + fwrite(patches_file, 0, 'int'); + end + + bias = patch_experts.patch_experts{i,j}{n*2}; + + weights = patch_experts.patch_experts{i,j}{n*2-1}; + + % the actual bias/weight matrix + writeMatrixBin(patches_file, bias, 5); + writeMatrixBin(patches_file, weights, 5); + end + + % finally write out the confidence + fwrite(patches_file, patch_experts.correlations(i,j), 'float64'); + + end + end + end + + fclose(patches_file);