From 7fe7bb4904fa3e9a0b5059725959dc9c08548e4c Mon Sep 17 00:00:00 2001 From: Tadas Baltrusaitis Date: Wed, 9 Aug 2017 15:04:14 -0400 Subject: [PATCH] Towards MTCNN c++ implementation. --- .../include/FaceDetectorMTCNN.h | 2 +- .../src/FaceDetectorMTCNN.cpp | 181 ++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/lib/local/LandmarkDetector/include/FaceDetectorMTCNN.h b/lib/local/LandmarkDetector/include/FaceDetectorMTCNN.h index 27e1ce40..6e07d516 100644 --- a/lib/local/LandmarkDetector/include/FaceDetectorMTCNN.h +++ b/lib/local/LandmarkDetector/include/FaceDetectorMTCNN.h @@ -82,7 +82,7 @@ namespace LandmarkDetector CNN(const CNN& other); // Given an image, orientation and detected landmarks output the result of the appropriate regressor - cv::Mat_ Inference(const cv::Mat_& intensity_img); + cv::Mat_ Inference(const cv::Mat_& input_img); // Reading in the model void Read(string location); diff --git a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp index 6f93387e..2cc3cb3b 100644 --- a/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp +++ b/lib/local/LandmarkDetector/src/FaceDetectorMTCNN.cpp @@ -83,6 +83,8 @@ #define M_PI 3.14159265358979323846 #endif +#include "LandmarkDetectorUtils.h" + using namespace LandmarkDetector; // Copy constructor @@ -134,6 +136,185 @@ CNN::CNN(const CNN& other) : cnn_layer_types(other.cnn_layer_types), cnn_max_poo } } +cv::Mat_ CNN::Inference(const cv::Mat_& input_img) +{ + if (input_img.channels() == 1) + { + cv::cvtColor(input_img, input_img, cv::COLOR_GRAY2BGR); + } + + int cnn_layer = 0; + int fully_connected_layer = 0; + int prelu_layer = 0; + int max_pool_layer = 0; + + vector > input_maps; + input_maps.push_back(input_img); + + vector > outputs; + + for (size_t layer = 0; layer < cnn_layer_types.size(); ++layer) + { + // Determine layer type + int layer_type = cnn_layer_types[layer]; + + // Convolutional layer + if (layer_type == 0) + { + outputs.clear(); + for (size_t in = 0; in < input_maps.size(); ++in) + { + cv::Mat_ input_image = input_maps[in]; + + // Useful precomputed data placeholders for quick correlation (convolution) + cv::Mat_ input_image_dft; + cv::Mat integral_image; + cv::Mat integral_image_sq; + + // TODO can TBB-ify this + for (size_t k = 0; k < cnn_convolutional_layers[cnn_layer][in].size(); ++k) + { + cv::Mat_ kernel = cnn_convolutional_layers[cnn_layer][in][k]; + + // The convolution (with precomputation) + cv::Mat_ output; + if (cnn_convolutional_layers_dft[cnn_layer][in][k].second.empty()) + { + std::map > precomputed_dft; + + LandmarkDetector::matchTemplate_m(input_image, input_image_dft, integral_image, integral_image_sq, kernel, precomputed_dft, output, CV_TM_CCORR); + + cnn_convolutional_layers_dft[cnn_layer][in][k].first = precomputed_dft.begin()->first; + cnn_convolutional_layers_dft[cnn_layer][in][k].second = precomputed_dft.begin()->second; + } + else + { + std::map > precomputed_dft; + precomputed_dft[cnn_convolutional_layers_dft[cnn_layer][in][k].first] = cnn_convolutional_layers_dft[cnn_layer][in][k].second; + LandmarkDetector::matchTemplate_m(input_image, input_image_dft, integral_image, integral_image_sq, kernel, precomputed_dft, output, CV_TM_CCORR); + } + + // Combining the maps + if (in == 0) + { + outputs.push_back(output); + } + else + { + outputs[k] = outputs[k] + output; + } + + } + + } + + for (size_t k = 0; k < cnn_convolutional_layers[cnn_layer][0].size(); ++k) + { + outputs[k] = outputs[k] + cnn_convolutional_layers_bias[cnn_layer][k]; + } + cnn_layer++; + } + if (layer_type == 1) + { + vector> outputs_sub; + + int stride_x = std::get<2>(cnn_max_pooling_layers[max_pool_layer]); + int stride_y = std::get<3>(cnn_max_pooling_layers[max_pool_layer]); + + int kernel_size_x = std::get<0>(cnn_max_pooling_layers[max_pool_layer]); + int kernel_size_y = std::get<1>(cnn_max_pooling_layers[max_pool_layer]); + + // Iterate over kernel height and width, based on stride + for (size_t in = 0; in < input_maps.size(); ++in) + { + int out_x = round((input_maps[in].cols - kernel_size_x) / stride_x) + 1; + int out_y = round((input_maps[in].rows - kernel_size_y) / stride_y) + 1; + + cv::Mat_ sub_out(out_y, out_x, 0.0); + cv::Mat_ in_map = input_maps[in]; + + for (int x = 0; x < input_maps[in].cols; x += stride_x) + { + for (int y = 0; y < input_maps[in].rows; y += stride_y) + { + float curr_max = -FLT_MAX; + for (int x_in = x; x_in < x + kernel_size_x; ++x_in) + { + for (int y_in = y; y_in < y + kernel_size_y; ++y_in) + { + float curr_val = in_map.at(y_in, x_in); + if (curr_val > curr_max) + { + curr_max = curr_val; + } + } + } + int x_in_out = floor(x / stride_x); + int y_in_out = floor(y / stride_y); + sub_out.at(y_in_out, x_in_out) = curr_max; + } + } + + outputs_sub.push_back(sub_out); + + } + outputs = outputs_sub; + } + if (layer_type == 2) + { + // Concatenate all the maps + cv::Mat_ input_concat = input_maps[0].t(); + input_concat = input_concat.reshape(0, 1); + + for (size_t in = 1; in < input_maps.size(); ++in) + { + cv::Mat_ add = input_maps[in].t(); + add = add.reshape(0, 1); + cv::hconcat(input_concat, add, input_concat); + } + + input_concat = input_concat * cnn_fully_connected_layers_weights[fully_connected_layer]; + input_concat = input_concat + cnn_fully_connected_layers_biases[fully_connected_layer].t(); + + outputs.clear(); + outputs.push_back(input_concat); + + fully_connected_layer++; + } + if (layer_type == 3) // PReLU, TODO + { + outputs.clear(); + for (size_t k = 0; k < input_maps.size(); ++k) + { + // Apply the ReLU + cv::threshold(input_maps[k], input_maps[k], 0, 0, cv::THRESH_TOZERO); + outputs.push_back(input_maps[k]); + + } + } + if (layer_type == 4) + { + outputs.clear(); + for (size_t k = 0; k < input_maps.size(); ++k) + { + // Apply the sigmoid + cv::exp(-input_maps[k], input_maps[k]); + input_maps[k] = 1.0 / (1.0 + input_maps[k]); + + outputs.push_back(input_maps[k]); + + } + } + // Set the outputs of this layer to inputs of the next + input_maps = outputs; + + } + + + return outputs[0]; + +} + void ReadMatBin(std::ifstream& stream, cv::Mat &output_mat) { // Read in the number of rows, columns and the data type