diff --git a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp index 08e2776e..9d424fe0 100644 --- a/exe/FaceLandmarkImg/FaceLandmarkImg.cpp +++ b/exe/FaceLandmarkImg/FaceLandmarkImg.cpp @@ -334,7 +334,7 @@ int main (int argc, char **argv) LandmarkDetector::CLNF clnf_model(det_parameters.model_location); cout << "Model loaded" << endl; - cv::CascadeClassifier classifier(det_parameters.face_detector_location); + cv::CascadeClassifier classifier(det_parameters.haar_face_detector_location); dlib::frontal_face_detector face_detector_hog = dlib::get_frontal_face_detector(); // Loading the AU prediction models diff --git a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp index e3d2ab62..b21ec431 100644 --- a/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp +++ b/exe/FaceLandmarkVidMulti/FaceLandmarkVidMulti.cpp @@ -139,8 +139,8 @@ int main (int argc, char **argv) int num_faces_max = 4; LandmarkDetector::CLNF clnf_model(det_parameters[0].model_location); - clnf_model.face_detector_HAAR.load(det_parameters[0].face_detector_location); - clnf_model.face_detector_location = det_parameters[0].face_detector_location; + clnf_model.face_detector_HAAR.load(det_parameters[0].haar_face_detector_location); + clnf_model.haar_face_detector_location = det_parameters[0].haar_face_detector_location; clnf_models.reserve(num_faces_max); diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj index c06eb3c3..1bc2f3ed 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj @@ -190,6 +190,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c Use Use + Use Use @@ -248,6 +249,7 @@ xcopy /I /E /Y /D "$(SolutionDir)lib\3rdParty\OpenCV3.1\classifiers" "$(OutDir)c + diff --git a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj.filters b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj.filters index 34c4cdbb..a0e1f91b 100644 --- a/lib/local/LandmarkDetector/LandmarkDetector.vcxproj.filters +++ b/lib/local/LandmarkDetector/LandmarkDetector.vcxproj.filters @@ -34,6 +34,9 @@ source + + source + @@ -72,6 +75,9 @@ headers + + headers + diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h index 0f83b52c..99f8a0e3 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorModel.h @@ -89,7 +89,8 @@ public: // Haar cascade classifier for face detection cv::CascadeClassifier face_detector_HAAR; - string face_detector_location; + string haar_face_detector_location; + string mtcnn_face_detector_location; // A HOG SVM-struct based face detector dlib::frontal_face_detector face_detector_HOG; diff --git a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h index 77ed1683..eda3866a 100644 --- a/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h +++ b/lib/local/LandmarkDetector/include/LandmarkDetectorParameters.h @@ -90,7 +90,8 @@ struct FaceModelParameters // Also HAAR detector can detect smaller faces while HOG SVM is only capable of detecting faces at least 70px across enum FaceDetector{HAAR_DETECTOR, HOG_SVM_DETECTOR}; - string face_detector_location; + string haar_face_detector_location; + string mtcnn_face_detector_location; FaceDetector curr_face_detector; // Should the results be visualised and reported to console diff --git a/lib/local/LandmarkDetector/model/mtcnn_detector/MTCNN_detector.txt b/lib/local/LandmarkDetector/model/mtcnn_detector/MTCNN_detector.txt new file mode 100644 index 00000000..9a4f805b --- /dev/null +++ b/lib/local/LandmarkDetector/model/mtcnn_detector/MTCNN_detector.txt @@ -0,0 +1,3 @@ +PNet PNet.dat +RNet RNet.dat +ONet ONet.dat diff --git a/lib/local/LandmarkDetector/model/mtcnn_detector/ONet.dat b/lib/local/LandmarkDetector/model/mtcnn_detector/ONet.dat new file mode 100644 index 00000000..291c4462 Binary files /dev/null and b/lib/local/LandmarkDetector/model/mtcnn_detector/ONet.dat differ diff --git a/lib/local/LandmarkDetector/model/mtcnn_detector/PNet.dat b/lib/local/LandmarkDetector/model/mtcnn_detector/PNet.dat new file mode 100644 index 00000000..9550d39a Binary files /dev/null and b/lib/local/LandmarkDetector/model/mtcnn_detector/PNet.dat differ diff --git a/lib/local/LandmarkDetector/model/mtcnn_detector/RNet.dat b/lib/local/LandmarkDetector/model/mtcnn_detector/RNet.dat new file mode 100644 index 00000000..864e0dd9 Binary files /dev/null and b/lib/local/LandmarkDetector/model/mtcnn_detector/RNet.dat differ diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index bfe4fa3f..6f93387e 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -74,6 +74,11 @@ #define _USE_MATH_DEFINES #include +// Boost includes +#include +#include + + #ifndef M_PI #define M_PI 3.14159265358979323846 #endif @@ -85,6 +90,50 @@ FaceDetectorMTCNN::FaceDetectorMTCNN(const FaceDetectorMTCNN& other) : PNet(othe { } +CNN::CNN(const CNN& other) : cnn_layer_types(other.cnn_layer_types), cnn_max_pooling_layers(other.cnn_max_pooling_layers), cnn_convolutional_layers_bias(other.cnn_convolutional_layers_bias) +{ + this->cnn_convolutional_layers.resize(other.cnn_convolutional_layers.size()); + for (size_t l = 0; l < other.cnn_convolutional_layers.size(); ++l) + { + this->cnn_convolutional_layers[l].resize(other.cnn_convolutional_layers[l].size()); + + for (size_t i = 0; i < other.cnn_convolutional_layers[l].size(); ++i) + { + this->cnn_convolutional_layers[l][i].resize(other.cnn_convolutional_layers[l][i].size()); + + for (size_t k = 0; k < other.cnn_convolutional_layers[l][i].size(); ++k) + { + // Make sure the matrix is copied. + this->cnn_convolutional_layers[l][i][k] = other.cnn_convolutional_layers[l][i][k].clone(); + } + } + } + + this->cnn_fully_connected_layers_weights.resize(other.cnn_fully_connected_layers_weights.size()); + + for (size_t l = 0; l < other.cnn_fully_connected_layers_weights.size(); ++l) + { + // Make sure the matrix is copied. + this->cnn_fully_connected_layers_weights[l] = other.cnn_fully_connected_layers_weights[l].clone(); + } + + this->cnn_fully_connected_layers_biases.resize(other.cnn_fully_connected_layers_biases.size()); + + for (size_t l = 0; l < other.cnn_fully_connected_layers_biases.size(); ++l) + { + // Make sure the matrix is copied. + this->cnn_fully_connected_layers_biases[l] = other.cnn_fully_connected_layers_biases[l].clone(); + } + + this->cnn_prelu_layer_weights.resize(other.cnn_prelu_layer_weights.size()); + + for (size_t l = 0; l < other.cnn_prelu_layer_weights.size(); ++l) + { + // Make sure the matrix is copied. + this->cnn_prelu_layer_weights[l] = other.cnn_prelu_layer_weights[l].clone(); + } +} + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) { // Read in the number of rows, columns and the data type @@ -187,7 +236,7 @@ void CNN::Read(string location) cnn_fully_connected_layers_weights.push_back(weights); } - else if (layer_type == 4) + else if (layer_type == 3) { cv::Mat_ weights; ReadMatBin(cnn_stream, weights); diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp index bd0e4608..68238899 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorFunc.cpp @@ -314,8 +314,8 @@ bool LandmarkDetector::DetectLandmarksInVideo(const cv::Mat_ &grayscale_i // If the face detector has not been initialised read it in if(clnf_model.face_detector_HAAR.empty()) { - clnf_model.face_detector_HAAR.load(params.face_detector_location); - clnf_model.face_detector_location = params.face_detector_location; + clnf_model.face_detector_HAAR.load(params.haar_face_detector_location); + clnf_model.haar_face_detector_location = params.haar_face_detector_location; } cv::Point preference_det(-1, -1); @@ -529,8 +529,8 @@ bool LandmarkDetector::DetectLandmarksInImage(const cv::Mat_ &grayscale_i // If the face detector has not been initialised read it in if(clnf_model.face_detector_HAAR.empty()) { - clnf_model.face_detector_HAAR.load(params.face_detector_location); - clnf_model.face_detector_location = params.face_detector_location; + clnf_model.face_detector_HAAR.load(params.haar_face_detector_location); + clnf_model.haar_face_detector_location = params.haar_face_detector_location; } // Detect the face first diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp index c284868b..4900602c 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorModel.cpp @@ -67,8 +67,8 @@ CLNF::CLNF(string fname) // Copy constructor (makes a deep copy of CLNF) CLNF::CLNF(const CLNF& other): pdm(other.pdm), params_local(other.params_local.clone()), params_global(other.params_global), detected_landmarks(other.detected_landmarks.clone()), - landmark_likelihoods(other.landmark_likelihoods.clone()), patch_experts(other.patch_experts), landmark_validator(other.landmark_validator), face_detector_location(other.face_detector_location), - hierarchical_mapping(other.hierarchical_mapping), hierarchical_models(other.hierarchical_models), hierarchical_model_names(other.hierarchical_model_names), + landmark_likelihoods(other.landmark_likelihoods.clone()), patch_experts(other.patch_experts), landmark_validator(other.landmark_validator), haar_face_detector_location(other.haar_face_detector_location), + mtcnn_face_detector_location(other.mtcnn_face_detector_location), hierarchical_mapping(other.hierarchical_mapping), hierarchical_models(other.hierarchical_models), hierarchical_model_names(other.hierarchical_model_names), hierarchical_params(other.hierarchical_params), eye_model(other.eye_model) { this->detection_success = other.detection_success; @@ -78,9 +78,9 @@ CLNF::CLNF(const CLNF& other): pdm(other.pdm), params_local(other.params_local.c this->failures_in_a_row = other.failures_in_a_row; // Load the CascadeClassifier (as it does not have a proper copy constructor) - if(!face_detector_location.empty()) + if(!haar_face_detector_location.empty()) { - this->face_detector_HAAR.load(face_detector_location); + this->face_detector_HAAR.load(haar_face_detector_location); } // Make sure the matrices are allocated properly this->triangulations.resize(other.triangulations.size()); @@ -114,7 +114,8 @@ CLNF & CLNF::operator= (const CLNF& other) landmark_likelihoods =other.landmark_likelihoods.clone(); patch_experts = Patch_experts(other.patch_experts); landmark_validator = DetectionValidator(other.landmark_validator); - face_detector_location = other.face_detector_location; + haar_face_detector_location = other.haar_face_detector_location; + mtcnn_face_detector_location = other.mtcnn_face_detector_location; this->detection_success = other.detection_success; this->tracking_initialised = other.tracking_initialised; @@ -125,9 +126,9 @@ CLNF & CLNF::operator= (const CLNF& other) this->eye_model = other.eye_model; // Load the CascadeClassifier (as it does not have a proper copy constructor) - if(!face_detector_location.empty()) + if(!haar_face_detector_location.empty()) { - this->face_detector_HAAR.load(face_detector_location); + this->face_detector_HAAR.load(haar_face_detector_location); } // Make sure the matrices are allocated properly this->triangulations.resize(other.triangulations.size()); @@ -172,7 +173,8 @@ CLNF::CLNF(const CLNF&& other) landmark_likelihoods = other.landmark_likelihoods; patch_experts = other.patch_experts; landmark_validator = other.landmark_validator; - face_detector_location = other.face_detector_location; + haar_face_detector_location = other.haar_face_detector_location; + mtcnn_face_detector_location = other.mtcnn_face_detector_location; face_detector_HAAR = other.face_detector_HAAR; @@ -207,7 +209,8 @@ CLNF & CLNF::operator= (const CLNF&& other) landmark_likelihoods = other.landmark_likelihoods; patch_experts = other.patch_experts; landmark_validator = other.landmark_validator; - face_detector_location = other.face_detector_location; + haar_face_detector_location = other.haar_face_detector_location; + mtcnn_face_detector_location = other.mtcnn_face_detector_location; face_detector_HAAR = other.face_detector_HAAR; diff --git a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp index ae76f052..b6a6e3f0 100644 --- a/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp +++ b/lib/local/LandmarkDetector/src/LandmarkDetectorParameters.cpp @@ -86,7 +86,7 @@ FaceModelParameters::FaceModelParameters(vector &arguments) if (arguments[i].compare("-fdloc") ==0) { string face_detector_loc = arguments[i + 1]; - face_detector_location = face_detector_loc; + haar_face_detector_location = face_detector_loc; curr_face_detector = HAAR_DETECTOR; valid[i] = false; valid[i + 1] = false; @@ -209,6 +209,46 @@ FaceModelParameters::FaceModelParameters(vector &arguments) { std::cout << "Could not find the landmark detection model to load" << std::endl; } + + // Make sure face detector location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + model_path = boost::filesystem::path(haar_face_detector_location); + if (boost::filesystem::exists(model_path)) + { + haar_face_detector_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + haar_face_detector_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + haar_face_detector_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the HAAR face detector location" << std::endl; + } + + // Make sure face detector location is valid + // First check working directory, then the executable's directory, then the config path set by the build process. + model_path = boost::filesystem::path(mtcnn_face_detector_location); + if (boost::filesystem::exists(model_path)) + { + mtcnn_face_detector_location = model_path.string(); + } + else if (boost::filesystem::exists(root / model_path)) + { + mtcnn_face_detector_location = (root / model_path).string(); + } + else if (boost::filesystem::exists(config_path / model_path)) + { + mtcnn_face_detector_location = (config_path / model_path).string(); + } + else + { + std::cout << "Could not find the MTCNN face detector location" << std::endl; + } } void FaceModelParameters::init() @@ -262,7 +302,8 @@ void FaceModelParameters::init() reinit_video_every = 4; // Face detection - face_detector_location = "classifiers/haarcascade_frontalface_alt.xml"; + haar_face_detector_location = "classifiers/haarcascade_frontalface_alt.xml"; + mtcnn_face_detector_location = "model/mtcnn_detector/MTCNN_detector.txt"; quiet_mode = false; // By default use HOG SVM diff --git a/matlab_version/face_detection/mtcnn/convert_to_cpp/MTCNN_detector.txt b/matlab_version/face_detection/mtcnn/convert_to_cpp/MTCNN_detector.txt new file mode 100644 index 00000000..9a4f805b --- /dev/null +++ b/matlab_version/face_detection/mtcnn/convert_to_cpp/MTCNN_detector.txt @@ -0,0 +1,3 @@ +PNet PNet.dat +RNet RNet.dat +ONet ONet.dat diff --git a/matlab_version/face_detection/mtcnn/convert_to_cpp/ONet.dat b/matlab_version/face_detection/mtcnn/convert_to_cpp/ONet.dat new file mode 100644 index 00000000..291c4462 Binary files /dev/null and b/matlab_version/face_detection/mtcnn/convert_to_cpp/ONet.dat differ diff --git a/matlab_version/face_detection/mtcnn/convert_to_cpp/RNet.dat b/matlab_version/face_detection/mtcnn/convert_to_cpp/RNet.dat new file mode 100644 index 00000000..864e0dd9 Binary files /dev/null and b/matlab_version/face_detection/mtcnn/convert_to_cpp/RNet.dat differ diff --git a/matlab_version/face_detection/mtcnn/convert_to_cpp/Write_out_mtcnn.m b/matlab_version/face_detection/mtcnn/convert_to_cpp/Write_out_mtcnn.m index 20aaa714..7afbce88 100644 --- a/matlab_version/face_detection/mtcnn/convert_to_cpp/Write_out_mtcnn.m +++ b/matlab_version/face_detection/mtcnn/convert_to_cpp/Write_out_mtcnn.m @@ -39,4 +39,146 @@ cnn.layers{8} = struct; cnn.layers{8}.type = 'fc'; cnn.layers{8}.weights = {PNet_mlab.w, PNet_mlab.b}; -Write_CNN_to_binary('PNet.dat', cnn); \ No newline at end of file +Write_CNN_to_binary('PNet.dat', cnn); + +%% Next writing out the RNet +clear +load('../RNet_mlab.mat'); + +cnn = struct; +cnn.layers = cell(1,11); +cnn.layers{1} = struct; +cnn.layers{1}.type = 'conv'; +cnn.layers{1}.weights = {RNet_mlab.weights_conv1, RNet_mlab.biases_conv1}; + +cnn.layers{2} = struct; +cnn.layers{2}.type = 'prelu'; +cnn.layers{2}.weights = {RNet_mlab.prelu_weights_1}; + +cnn.layers{3} = struct; +cnn.layers{3}.type = 'max_pooling'; +cnn.layers{3}.weights = {}; +cnn.layers{3}.stride_x = 2; +cnn.layers{3}.stride_y = 2; +cnn.layers{3}.kernel_size_x = 3; +cnn.layers{3}.kernel_size_y = 3; + +cnn.layers{4} = struct; +cnn.layers{4}.type = 'conv'; +cnn.layers{4}.weights = {RNet_mlab.weights_conv2, RNet_mlab.biases_conv2}; + +cnn.layers{5} = struct; +cnn.layers{5}.type = 'prelu'; +cnn.layers{5}.weights = {RNet_mlab.prelu_weights_2}; + +cnn.layers{6} = struct; +cnn.layers{6}.type = 'max_pooling'; +cnn.layers{6}.weights = {}; +cnn.layers{6}.stride_x = 2; +cnn.layers{6}.stride_y = 2; +cnn.layers{6}.kernel_size_x = 3; +cnn.layers{6}.kernel_size_y = 3; + +cnn.layers{7} = struct; +cnn.layers{7}.type = 'conv'; +cnn.layers{7}.weights = {RNet_mlab.weights_conv3, RNet_mlab.biases_conv3}; + +cnn.layers{8} = struct; +cnn.layers{8}.type = 'prelu'; +cnn.layers{8}.weights = {RNet_mlab.prelu_weights_3}; + +cnn.layers{9} = struct; +cnn.layers{9}.type = 'fc'; +cnn.layers{9}.weights = {RNet_mlab.w_fc1, RNet_mlab.b_fc1}; + +cnn.layers{10} = struct; +cnn.layers{10}.type = 'prelu'; +cnn.layers{10}.weights = {RNet_mlab.prelu_fc1}; + +cnn.layers{11} = struct; +cnn.layers{11}.type = 'fc'; +cnn.layers{11}.weights = {RNet_mlab.w_fc2, RNet_mlab.b_fc2}; + +Write_CNN_to_binary('RNet.dat', cnn); + +%% Next writing out the ONet +clear +load('../ONet_mlab.mat'); + +cnn = struct; +cnn.layers = cell(1,14); +cnn.layers{1} = struct; +cnn.layers{1}.type = 'conv'; +cnn.layers{1}.weights = {ONet_mlab.weights_conv1, ONet_mlab.biases_conv1}; + +cnn.layers{2} = struct; +cnn.layers{2}.type = 'prelu'; +cnn.layers{2}.weights = {ONet_mlab.prelu_weights_1}; + +cnn.layers{3} = struct; +cnn.layers{3}.type = 'max_pooling'; +cnn.layers{3}.weights = {}; +cnn.layers{3}.stride_x = 2; +cnn.layers{3}.stride_y = 2; +cnn.layers{3}.kernel_size_x = 3; +cnn.layers{3}.kernel_size_y = 3; + +cnn.layers{4} = struct; +cnn.layers{4}.type = 'conv'; +cnn.layers{4}.weights = {ONet_mlab.weights_conv2, ONet_mlab.biases_conv2}; + +cnn.layers{5} = struct; +cnn.layers{5}.type = 'prelu'; +cnn.layers{5}.weights = {ONet_mlab.prelu_weights_2}; + +cnn.layers{6} = struct; +cnn.layers{6}.type = 'max_pooling'; +cnn.layers{6}.weights = {}; +cnn.layers{6}.stride_x = 2; +cnn.layers{6}.stride_y = 2; +cnn.layers{6}.kernel_size_x = 3; +cnn.layers{6}.kernel_size_y = 3; + +cnn.layers{7} = struct; +cnn.layers{7}.type = 'conv'; +cnn.layers{7}.weights = {ONet_mlab.weights_conv3, ONet_mlab.biases_conv3}; + +cnn.layers{8} = struct; +cnn.layers{8}.type = 'prelu'; +cnn.layers{8}.weights = {ONet_mlab.prelu_weights_3}; + +cnn.layers{9} = struct; +cnn.layers{9}.type = 'max_pooling'; +cnn.layers{9}.weights = {}; +cnn.layers{9}.stride_x = 2; +cnn.layers{9}.stride_y = 2; +cnn.layers{9}.kernel_size_x = 2; +cnn.layers{9}.kernel_size_y = 2; + +cnn.layers{10} = struct; +cnn.layers{10}.type = 'conv'; +cnn.layers{10}.weights = {ONet_mlab.weights_conv4, ONet_mlab.biases_conv4}; + +cnn.layers{11} = struct; +cnn.layers{11}.type = 'prelu'; +cnn.layers{11}.weights = {ONet_mlab.prelu_weights_4}; + +cnn.layers{12} = struct; +cnn.layers{12}.type = 'fc'; +cnn.layers{12}.weights = {ONet_mlab.w_fc1, ONet_mlab.b_fc1}; + +cnn.layers{13} = struct; +cnn.layers{13}.type = 'prelu'; +cnn.layers{13}.weights = {ONet_mlab.prelu_fc1}; + +cnn.layers{14} = struct; +cnn.layers{14}.type = 'fc'; +cnn.layers{14}.weights = {ONet_mlab.w_fc2, ONet_mlab.b_fc2}; + +Write_CNN_to_binary('ONet.dat', cnn); + +f = fopen('MTCNN_detector.txt', 'w'); +fprintf(f, 'PNet PNet.dat\r\n'); +fprintf(f, 'RNet RNet.dat\r\n'); +fprintf(f, 'ONet ONet.dat\r\n'); +fclose(f); \ No newline at end of file